Initial commit

Data/missing.csv

@@ -0,0 +1,29 @@
+name,shares,price
+"AA",15,39.48
+"AXP",10,62.58
+"BA",5,98.31
+"C",,53.08
+"CAT",15,78.29
+"DD",10,50.75
+"DIS",50,N/A
+"GE",,37.23
+"GM",15,31.44
+"HD",20,37.67
+"HPQ",5,45.81
+"IBM",10,102.86
+"INTC",,21.84
+"JNJ",20,62.25
+"JPM",10,50.35
+"KO",5,51.65
+"MCD",,51.11
+"MMM",10,85.60
+"MO",,70.09
+"MRK",5,50.21
+"MSFT",20,30.08
+"PFE",,26.40
+"PG",5,62.79
+"T",10,40.03
+"UTX",8,69.81
+"VZ",,42.92
+"WMT",10,49.78
+"XOM",15,82.50

Data/portfolio.csv

@@ -0,0 +1,8 @@
+name,shares,price
+"AA",100,32.20
+"IBM",50,91.10
+"CAT",150,83.44
+"MSFT",200,51.23
+"GE",95,40.37
+"MSFT",50,65.10
+"IBM",100,70.44

Data/portfolio.dat

@@ -0,0 +1,7 @@
+AA 100 32.20
+IBM 50 91.10
+CAT 150 83.44
+MSFT 200 51.23
+GE 95 40.37
+MSFT 50 65.10
+IBM 100 70.44

Data/portfolio1.dat

@@ -0,0 +1,7 @@
+AA 100 32.20
+IBM 50 91.10
+CAT 150 83.44
+MSFT 200 51.23
+GE 95 40.37
+MSFT 50 65.10
+IBM 100 70.44

Data/portfolio2.csv

@@ -0,0 +1,5 @@
+name,shares,price
+"AA",50,27.10
+"HPQ",250,43.15
+"MSFT",25,50.15
+"GE",125,52.10

Data/portfolio2.dat

@@ -0,0 +1,4 @@
+AA  50 27.10
+HPQ  250 43.15
+MSFT 25 50.15
+GE 125 52.10

Data/portfolio3.csv

@@ -0,0 +1,28 @@
+"AA",15,39.48
+"AXP",10,62.58
+"BA",5,98.31
+"C",-,53.08
+"CAT",15,78.29
+"DD",10,50.75
+"DIS",-,N/A
+"GE",-,37.23
+"GM",15,31.44
+"HD",20,37.67
+"HPQ",5,45.81
+"IBM",10,102.86
+"INTC",-,21.84
+"JNJ",20,62.25
+"JPM",10,50.35
+"KO",5,51.65
+"MCD",-,51.11
+"MMM",10,85.60
+"MO",-,70.09
+"MRK",5,50.21
+"MSFT",20,30.08
+"PFE",-,26.40
+"PG",5,62.79
+"T",10,40.03
+"UTX",8,69.81
+"VZ",-,42.92
+"WMT",10,49.78
+"XOM",15,82.50

Data/portfolio3.dat

@@ -0,0 +1,28 @@
+AA 15 39.48
+AXP 10 62.58
+BA 5 98.31
+C - 53.08
+CAT 15 78.29
+DD 10 50.75
+DIS - N/A
+GE - 37.23
+GM 15 31.44
+HD 20 37.67
+HPQ 5 45.81
+IBM 10 102.86
+INTC - 21.84
+JNJ 20 62.25
+JPM 10 50.35
+KO 5 51.65
+MCD - 51.11
+MMM 10 85.60
+MO - 70.09
+MRK 5 50.21
+MSFT 20 30.08
+PFE - 26.40
+PG 5 62.79
+T 10 40.03
+UTX 8 69.81
+VZ - 42.92
+WMT 10 49.78
+XOM 15 82.50

Data/portfolio_noheader.csv

@@ -0,0 +1,7 @@
+"AA",100,32.20
+"IBM",50,91.10
+"CAT",150,83.44
+"MSFT",200,51.23
+"GE",95,40.37
+"MSFT",50,65.10
+"IBM",100,70.44

Data/prices.csv

@@ -0,0 +1,30 @@
+"AA",9.22
+"AXP",24.85
+"BA",44.85
+"BAC",11.27
+"C",3.72
+"CAT",35.46
+"CVX",66.67
+"DD",28.47
+"DIS",24.22
+"GE",13.48
+"GM",0.75
+"HD",23.16
+"HPQ",34.35
+"IBM",106.28
+"INTC",15.72
+"JNJ",55.16
+"JPM",36.90
+"KFT",26.11
+"KO",49.16
+"MCD",58.99
+"MMM",57.10
+"MRK",27.58
+"MSFT",20.89
+"PFE",15.19
+"PG",51.94
+"T",24.79
+"UTX",52.61
+"VZ",29.26
+"WMT",49.74
+"XOM",69.35

Data/stocksim.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python
+# stocksim.py
+#
+# Stock market simulator.  This simulator creates stock market
+# data and provides it in several different ways:
+#
+#    1. Makes periodic updates to a log file stocklog.dat
+#
+# The purpose of this module is to provide data to the user
+# in different ways in order to write interesting Python examples
+
+import math
+import time
+import threading
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+
+history_file = "dowstocks.csv"
+
+# Convert a time string such as "4:00pm" to minutes past midnight
+def minutes(tm):
+    am_pm = tm[-2:]
+    fields = tm[:-2].split(":")
+    hour = int(fields[0])
+    minute = int(fields[1])
+    if hour == 12:
+       hour = 0
+    if am_pm == 'pm':
+       hour += 12
+    return hour*60 + minute
+
+# Convert time in minutes to a format string
+def minutes_to_str(m):
+    frac,m = math.modf(m)
+    hours = m//60
+    minutes = m % 60
+    seconds = frac * 60
+    return "%02d:%02d.%02.f" % (hours,minutes,seconds)
+
+# Read the stock history file as a list of lists
+def read_history(filename):
+    result = []
+    for line in open(filename):
+        str_fields = line.strip().split(",")
+        fields = [eval(x) for x in str_fields]
+        fields[3] = minutes(fields[3])
+        result.append(fields)
+    return result
+
+# Format CSV record
+def csv_record(fields):
+    s = '"%s",%0.2f,"%s","%s",%0.2f,%0.2f,%0.2f,%0.2f,%d' % tuple(fields)
+    return s
+
+class StockTrack(object):
+    def __init__(self,name):
+        self.name    = name
+        self.history = []
+        self.price   = 0
+        self.time    = 0
+        self.index   = 0
+        self.open    = 0
+        self.low     = 0
+        self.high    = 0
+        self.volume  = 0
+        self.initial = 0
+        self.change  = 0
+        self.date    = ""
+    def add_data(self,record):
+        self.history.append(record)
+    def reset(self,time):
+        self.time = time
+        # Sort the history by time
+        self.history.sort(key=lambda t:t[3])
+        # Find the first entry who's time is behind the given time
+        self.index = 0
+        while self.index < len(self.history):
+            if self.history[self.index][3] > time:
+                break
+            self.index += 1
+        self.open = self.history[0][5]
+        self.initial = self.history[0][1] - self.history[0][4]
+        self.date = self.history[0][2]
+        self.update()
+        self.low = self.price
+        self.high = self.price
+
+    # Calculate interpolated value of a given field based on
+    # current time
+    def interpolate(self,field):
+        first = self.history[self.index][field]
+        next  = self.history[self.index+1][field]
+        first_t = self.history[self.index][3]
+        next_t = self.history[self.index+1][3]
+        try:
+            slope = (next - first)/(next_t-first_t)
+            return first + slope*(self.time - first_t)
+        except ZeroDivisionError:
+            return first
+
+    # Update all computed values
+    def update(self):
+        self.price = round(self.interpolate(1),2)
+        self.volume = int(self.interpolate(-1))
+        if self.price < self.low:
+            self.low = self.price
+        if self.price >= self.high:
+            self.high = self.price
+        self.change = self.price - self.initial
+        
+    # Increment the time by a delta
+    def incr(self,dt):
+        self.time += dt
+        if self.index < (len(self.history) - 2):
+            while self.index < (len(self.history) - 2) and self.time >= self.history[self.index+1][3]:
+                self.index += 1
+        self.update()
+
+    def make_record(self):
+        return [self.name,round(self.price,2),self.date,minutes_to_str(self.time),round(self.change,2),self.open,round(self.high,2),
+                round(self.low,2),self.volume]
+
+class MarketSimulator(object):
+    def __init__(self):
+        self.stocks = { }
+        self.prices = { }
+        self.time = 0
+        self.observers = []
+    def register(self,observer):
+        self.observers.append(observer)
+
+    def publish(self,record):
+        for obj in self.observers:
+            obj.update(record)
+    def add_history(self,filename):
+        hist = read_history(filename)
+        for record in hist:
+            if record[0] not in self.stocks:
+                self.stocks[record[0]] = StockTrack(record[0])
+            self.stocks[record[0]].add_data(record) 
+
+    def reset(self,time):
+        self.time = time
+        for s in list(self.stocks.values()):
+            s.reset(time)
+
+    # Run forever.  Dt is in seconds
+    def run(self,dt):
+        for s in self.stocks:
+            self.prices[s] = self.stocks[s].price
+            self.publish(self.stocks[s].make_record())
+        while self.time < 1000:
+            for s in self.stocks:
+                self.stocks[s].incr(dt/60.0)    # Increment is in minutes
+                if self.stocks[s].price != self.prices[s]:
+                    self.prices[s] = self.stocks[s].price
+                    self.publish(self.stocks[s].make_record())
+            time.sleep(dt)
+            self.time += (dt/60.0)
+
+
+class BasicPrinter(object):
+    def update(self,record):
+        print(csv_record(record))
+
+class LogPrinter(object):
+    def __init__(self,filename):
+        self.f = open(filename,"w")
+    def update(self,record):
+        self.f.write(csv_record(record)+"\n")
+        self.f.flush()
+
+m = MarketSimulator()
+m.add_history(history_file)
+m.reset(minutes("9:30am"))
+m.register(BasicPrinter())
+m.register(LogPrinter("stocklog.csv"))
+m.run(1)
+
+
+   

Data/words.txt

@@ -0,0 +1,6 @@
+look into my eyes
+look into my eyes
+the eyes the eyes the eyes
+not around the eyes 
+don't look around the eyes
+look into my eyes you're under

Exercises/README.md

@@ -0,0 +1,37 @@
+# Advanced Python Mastery
+
+Copyright (C) 2007-2023 
+David Beazley (dave@dabeaz.com) 
+http://www.dabeaz.com
+
+Welcome to the Python Mastery course. This
+directory, `pythonmaster` is where you find support files
+related to the class exercises.  It is also where you will be doing
+your work.
+
+This course requires the use of Python 3.6 or newer.  If you are
+using Python 2, most of the material still applies, but you will
+have to make minor code modifications here and there. 
+
+- link:PythonMastery.pdf[`PythonMastery.pdf`] is a PDF that contains
+all of the presentation slides.
+
+- The link:Exercises/index.html[`Exercises/`] folder is where you
+find all the class exercises.
+
+- The `Data/` folder is where you find data files, scripts, and
+other files used by the exercises.
+
+- The `Solutions/` folder contains complete solution code for
+various exercises.  Each problem has its own directory.  For example,
+the solution to exercise 3.2 can be found in the `Solution/3_2/` directory.
+
+Every attempt has been made to make sure exercises work.  However, it's 
+possible that you will find typos or minor mistakes. If you find any
+errors, please let me know so that I can fix them for future editions
+of the course.
+
+
+
+
+

Exercises/ex1_1.md

@@ -0,0 +1,93 @@
+\[ [Index](index.md) | []() | [Exercise 1.2](ex1_2.md) \]
+
+# Exercise 1.1
+
+*Objectives:*
+
+- Make sure Python is installed correctly on your machine
+- Start the interactive interpreter
+- Edit and run a small program
+
+*Files Created:* `art.py`
+
+## (a) Launch Python
+
+Start Python3 on your machine.  Make sure you can type simple
+statements such as the "hello world" program:
+
+```python
+>>> print('Hello World')
+Hello World
+>>>
+```
+
+In much of this course, you'll want to make sure you can work from
+the interactive REPL like this.   If you're working from a different
+environment such as IPython or Jupyter Notebooks, that's fine.
+
+## (b) Some Generative Art
+
+Create the following program and put it in a file called `art.py`:
+
+```python
+# art.py
+
+import sys
+import random
+
+chars = '\|/'
+
+def draw(rows, columns):
+    for r in rows:
+    	print(''.join(random.choice(chars) for _ in range(columns)))
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        raise SystemExit("Usage: art.py rows columns")
+    draw(int(sys.argv[1]), int(sys.argv[2]))
+```
+
+Make sure you can run this program from the command line or a terminal.
+
+```
+bash % python3 art.py 10 20
+```
+
+If you run the above command, you'll get a crash and traceback message.
+Go fix the problem and run the program again.  You should get output like
+this:
+
+```
+bash % python3 art.py 10 20
+||||/\||//\//\|||\|\
+///||\/||\//|\\|\\/\
+|\////|//|||\//|/\||
+|//\||\/|\///|\|\|/|
+|/|//|/|/|\\/\/\||//
+|\/\|\//\\//\|\||\\/
+|||\\\\/\\\|/||||\/|
+\\||\\\|\||||////\\|
+//\//|/|\\|\//\|||\/
+\\\|/\\|/|\\\|/|/\/|
+bash %
+```
+
+### Important Note
+
+It is absolutely essential that you are able to edit, run, and debug
+ordinary Python programs for the rest of this course.  The choice
+of editor, IDE, or operating system doesn't matter as long as you
+are able to experiment interactively and create normal Python source
+files that can execute from the command line.
+
+
+\[ [Solution](soln1_1.md) | [Index](index.md) | [Exercise 1.2](ex1_2.md) \]
+
+
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex1_2.md

@@ -0,0 +1,412 @@
+\[ [Index](index.md) | [Exercise 1.1](ex1_1.md) | [Exercise 1.3](ex1_3.md) \]
+
+
+# Exercise 1.2
+
+*Objectives:*
+
+- Manipulate various built-in Python objects
+
+*Files Created:* None
+
+## Part 1 : Numbers
+
+Numerical calculations work about like you would expect in Python.
+For example:
+
+```python
+>>> 3 + 4*5
+23
+>>> 23.45 / 1e-02
+2345.0
+>>>
+```
+
+Be aware that integer division is different in Python 2 and Python 3. 
+
+```python
+>>> 7 / 4      # In python 2, this truncates to 1
+1.75           
+>>> 7 // 4     # Truncating division
+1 
+>>> 
+```
+
+If you want Python 3 behavior in Python 2, do this:
+
+```python
+>>> from __future__ import division
+>>> 7 / 4
+1.75
+>>> 7 // 4      # Truncating division
+1
+>>>
+```
+
+Numbers have a small set of methods, many of which are actually quite
+recent and overlooked by even experienced Python programmers.  Try some of them.
+
+```python
+>>> x = 1172.5
+>>> x.as_integer_ratio()
+(2345, 2)
+>>> x.is_integer()
+False
+>>> y = 12345
+>>> y.numerator
+12345
+>>> y.denominator
+1
+>>> y.bit_length()
+14
+>>> 
+```
+
+## Part 2 : String Manipulation
+
+Define a string containing a series of stock ticker symbols like this:
+
+```python
+>>> symbols = 'AAPL IBM MSFT YHOO SCO'
+```
+
+Now, let's experiment with different string operations:
+
+### (a) Extracting individual characters and substrings
+
+Strings are arrays of characters.  Try extracting a few characters:
+
+```python
+>>> symbols[0]
+'A'
+>>> symbols[1]
+'A'
+>>> symbols[2]
+'P'
+>>> symbols[-1]        # Last character
+'O'
+>>> symbols[-2]        # 2nd from last character
+'C'
+>>>
+```
+
+Try taking a few slices:
+
+```python
+>>> symbols[:4]
+'AAPL'
+>>> symbols[-3:]
+'SCO'
+>>> symbols[5:8]
+'IBM'
+>>>
+```
+
+### (b) Strings as read-only objects
+
+Strings are read-only.   Verify this by trying to change the first character of `symbols` to a lower-case 'a'. 
+
+```python
+>>> symbols[0] = 'a'
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: 'str' object does not support item assignment
+>>> 
+```
+
+### (c) String concatenation
+
+Although string data is read-only, you can always reassign a variable to a newly created string.   
+Try the following statement which concatenates a new symbol "GOOG" to the end of `symbols`:
+
+```python
+>>> symbols += ' GOOG'             
+>>> symbols
+... look at the result ...
+```
+
+Now, try adding "HPQ" to the beginning of `symbols` like this:
+
+```python
+>>> symbols = 'HPQ ' + symbols     
+>>> symbols
+... look at the result ...
+```
+
+It should be noted in both of these examples, the original string `symbols` is _NOT_
+being modified "in place."  Instead, a completely new string is created.  The variable name `symbols` is
+just bound to the result.  Afterwards, the old string is destroyed since it's not being used anymore.
+
+
+### (d) Membership testing (substring testing)
+
+Experiment with the `in` operator to check for substrings.  At
+the interactive prompt, try these operations:
+
+```python
+>>> 'IBM' in symbols
+True
+>>> 'AA' in symbols
+True
+>>> 'CAT' in symbols
+False
+>>>
+```
+
+Make sure you understand why the check for "AA" returned `True`.
+
+### (e) String Methods
+
+At the Python interactive prompt, try experimenting with some of the
+string methods. 
+
+```python
+>>> symbols.lower()
+'hpq aapl ibm msft yhoo sco goog'
+>>> symbols       
+'HPQ AAPL IBM MSFT YHOO SCO GOOG'
+```
+
+Remember, strings are always read-only.  If you want to save the result of an operation, you
+need to place it in a variable:
+
+```python
+>>> lowersyms = symbols.lower()
+>>> lowersyms
+'hpq aapl ibm msft yhoo sco goog'
+>>>
+```
+
+Try some more operations:
+
+```python
+>>> symbols.find('MSFT')
+13
+>>> symbols[13:17]
+'MSFT'
+>>> symbols = symbols.replace('SCO','')
+>>> symbols
+'HPQ AAPL IBM MSFT YHOO  GOOG'
+>>>
+```
+
+## Part 3 : List Manipulation
+
+In the first part, you worked with strings containing stock symbols.  For example:
+
+```python
+>>> symbols = 'HPQ AAPL IBM MSFT YHOO  GOOG'
+>>>
+```
+
+Define the above variable and split it into a list of names using the `split()` operation of strings:
+
+```python
+>>> symlist = symbols.split()
+>>> symlist
+['HPQ', 'AAPL', 'IBM', 'MSFT', 'YHOO', 'GOOG' ]
+>>>
+```
+
+### (a) Extracting and reassigning list elements
+
+Lists work like arrays where you can look up and
+modify elements by numerical index.   Try a few lookups:
+
+```python
+>>> symlist[0]
+'HPQ'
+>>> symlist[1]
+'AAPL'
+>>> symlist[-1]
+'GOOG'
+>>> symlist[-2]
+'YHOO'
+>>>
+```
+
+Try reassigning one of the items:
+
+```python
+>>> symlist[2] = 'AIG'
+>>> symlist
+['HPQ', 'AAPL', 'AIG', 'MSFT', 'YHOO', 'GOOG' ]
+>>>
+```
+
+### (b) Looping over list items
+
+The `for` loop works by looping over data in a sequence such as a list.   Check this out
+by typing the following loop and watching what happens:
+
+```python
+>>> for s in symlist:
+        print('s =', s)
+
+... look at the output ...
+```
+
+### (c) Membership tests
+
+Use the `in` operator to check if `'AIG'`,`'AA'`, and `'CAT'` are in the list of symbols.
+
+```python
+>>> 'AIG' in symlist
+True
+>>> 'AA' in symlist
+False
+>>>
+```
+
+### (d) Appending, inserting, and deleting items
+
+Use the `append()` method to add the symbol `'RHT'` to end of `symlist`.  
+
+```python
+>>> symlist.append('RHT')
+>>> symlist
+['HPQ', 'AAPL', 'AIG', 'MSFT', 'YHOO', 'GOOG', 'RHT']
+>>> 
+```
+
+Use the `insert()` method to
+insert the symbol `'AA'` as the second item in the list.
+
+```python
+>>> symlist.insert(1,'AA')
+>>> symlist
+['HPQ', 'AA', 'AAPL', 'AIG', 'MSFT', 'YHOO', 'GOOG', 'RHT']
+>>>
+```
+
+Use the `remove()` method to remove `'MSFT'` from the list. 
+
+```python
+>>> symlist.remove('MSFT')
+>>> symlist
+['HPQ', 'AA', 'AAPL', 'AIG', 'YHOO', 'GOOG', 'RHT']
+```
+
+Try calling `remove()` again to see what happens if the item can't be found.
+
+```python
+>>> symlist.remove('MSFT')
+... watch what happens ...
+>>>
+```
+
+Use the `index()` method to find the position of `'YHOO'` in the list.
+
+```python
+>>> symlist.index('YHOO')
+4
+>>> symlist[4]
+'YHOO'
+>>>
+```
+
+### (e) List sorting
+
+Want to sort a list?  Use the `sort()` method.  Try it out:
+
+```python
+>>> symlist.sort()
+>>> symlist
+['AA', 'AAPL', 'AIG', 'GOOG', 'HPQ', 'RHT', 'YHOO']
+>>>
+```
+
+Want to sort in reverse?  Try this:
+
+```python
+>>> symlist.sort(reverse=True)
+>>> symlist
+['YHOO', 'RHT', 'HPQ', 'GOOG', 'AIG', 'AAPL', 'AA']
+>>>
+```
+
+Note: Sorting a list modifies its contents "in-place."  That is, the
+elements of the list are shuffled around, but no new list is created
+as a result.
+
+
+### (f) Lists of anything
+
+Lists can contain any kind of object, including other lists (e.g., nested
+lists).  Try this out:
+
+```python
+>>> nums = [101,102,103]
+>>> items = [symlist, nums]
+>>> items
+[['YHOO', 'RHT', 'HPQ', 'GOOG', 'AIG', 'AAPL', 'AA'], [101, 102, 103]]
+```
+
+Pay close attention to the above output.  `items` is a list
+with two elements. Each element is list.
+
+Try some nested list lookups:
+
+```python
+>>> items[0]
+['YHOO', 'RHT', 'HPQ', 'GOOG', 'AIG', 'AAPL', 'AA']
+>>> items[0][1]
+'RHT'
+>>> items[0][1][2]
+'T'
+>>> items[1]
+[101, 102, 103]
+>>> items[1][1]
+102
+>>>
+```
+
+## Part 4 : Dictionaries
+
+In last few parts, you've simply worked with stock symbols.   However,
+suppose you wanted to map stock symbols to other data such as the
+price?  Use a dictionary:
+
+```python
+>>> prices = { 'IBM': 91.1, 'GOOG': 490.1, 'AAPL':312.23 }
+>>>
+```
+
+A dictionary maps keys to values.  Here's how to access:
+
+```python
+>>> prices['IBM']
+91.1
+>>> prices['IBM'] = 123.45
+>>> prices['HPQ'] = 26.15
+>>> prices
+{'GOOG': 490.1, 'AAPL': 312.23, 'IBM': 123.45, 'HPQ': 26.15}
+>>>
+```
+
+To get a list of keys, use this:
+
+```python
+>>> list(prices)
+['GOOG', 'AAPL', 'IBM', 'HPQ']
+>>>
+```
+
+To delete a value, use `del`
+
+```python
+>>> del prices['AAPL']
+>>> prices
+{'GOOG': 490.1, 'IBM': 123.45, 'HPQ': 26.15}
+>>>
+```
+
+
+\[ [Solution](soln1_2.md) | [Index](index.md) | [Exercise 1.1](ex1_1.md) | [Exercise 1.3](ex1_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex1_3.md

@@ -0,0 +1,41 @@
+\[ [Index](index.md) | [Exercise 1.2](ex1_2.md) | [Exercise 1.4](ex1_4.md) \]
+
+# Exercise 1.3
+
+*Objectives:*
+
+- Review basic file I/O
+
+*Files Created:* `pcost.py`
+
+## (a) Working with files
+
+The file `Data/portfolio.dat` contains a list of lines with information
+on a portfolio of stocks.  The file looks like this:
+
+```
+AA 100 32.20
+IBM 50 91.10
+CAT 150 83.44
+MSFT 200 51.23
+GE 95 40.37
+MSFT 50 65.10
+IBM 100 70.44
+```
+
+The first column is the stock name, the second column is the number of
+shares, and the third column is the purchase price of a single share. 
+
+Write a program called `pcost.py` that opens this file, reads
+all lines, and calculates how much it cost to purchase all of the shares
+in the portfolio. To do this, compute the sum of the second column
+multiplied by the third column.
+
+\[ [Solution](soln1_3.md) | [Index](index.md) | [Exercise 1.2](ex1_2.md) | [Exercise 1.4](ex1_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex1_4.md

@@ -0,0 +1,87 @@
+\[ [Index](index.md) | [Exercise 1.3](ex1_3.md) | [Exercise 1.5](ex1_5.md) \]
+
+# Exercise 1.4
+
+*Objectives:*
+
+- Review of how to define simple functions
+- Exception handling
+
+*Files Created:* None
+
+*Files Modified:* `pcost.py`
+
+## (a) Defining a function
+
+Take the program `pcost.py` that you wrote in the last exercise and
+convert it into a function `portfolio_cost(filename)` that takes a
+filename as input, reads the portfolio data in that file, and returns
+the total cost of the portfolio as a floating point number. Once you
+written the function, have your program call the function by simply
+adding this statement at the end:
+
+```python
+print(portfolio_cost('Data/portfolio.dat'))
+```
+
+Run your program and make sure it produces the same output as
+before.
+
+
+## (b) Adding Error Handling
+
+When writing programs that process data, it is common to encounter
+errors related to bad data (malformed, missing fields, etc.).  Modify
+your `pcost.py` program to read the data file `Data/portfolio3.dat`
+and run it (hint: it should crash).
+
+Modify your function slightly so that it is able to recover from lines
+with bad data.  For example, the conversion functions `int()` and
+`float()` raise a `ValueError` exception if they can't convert the
+input.  Use `try` and `except` to catch and print a warning message
+about lines that can't be parsed.  For example:
+
+```
+Couldn't parse: 'C - 53.08\n'
+Reason: invalid literal for int() with base 10: '-'
+Couldn't parse: 'DIS - 34.20\n'
+Reason: invalid literal for int() with base 10: '-'
+...
+```
+
+Try running your program on the `Data/portfolio3.dat` file
+again.   It should run successfully despite printed warning messages.
+
+## (c) Interactive Experimentation
+
+Run your `pcost.py` program and call the
+`portfolio_cost()` function directly from the interactive
+interpreter.
+
+```python
+>>> portfolio_cost('Data/portfolio.dat')
+44671.15
+>>> portfolio_cost('Data/portfolio2.dat')
+19908.75
+>>>
+```
+
+Note: To do this, you might have to run python using the `-i`
+option.  For example:
+
+```
+bash % python3 -i pcost.py
+```
+
+We are going to be writing a lot of programs where you define
+functions and experiment interactively.  Make sure you know how to do
+this.
+
+\[ [Solution](soln1_4.md) | [Index](index.md) | [Exercise 1.3](ex1_3.md) | [Exercise 1.5](ex1_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex1_5.md

@@ -0,0 +1,53 @@
+\[ [Index](index.md) | [Exercise 1.4](ex1_4.md) | [Exercise 1.6](ex1_6.md) \]
+
+# Exercise 1.5
+
+*Objectives:*
+
+- Review of how to define a simple object
+
+*Files Created:* `stock.py`
+
+## (a) Defining a simple object
+
+Create a file `stock.py` and define the following class:
+
+```python
+class Stock:
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+    def cost(self):
+        return self.shares * self.price
+```
+
+Once you have done this, run your program and experiment with your new
+`Stock` object:
+
+```python
+>>> s = Stock('GOOG',100,490.10)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s.price
+490.1
+>>> s.cost()
+49010.0
+>>> print('%10s %10d %10.2f' % (s.name, s.shares, s.price))
+      GOOG        100     490.10
+>>> t = Stock('IBM', 50, 91.5)
+>>> t.cost()
+4575.0
+>>> 
+```
+
+\[ [Solution](soln1_5.md) | [Index](index.md) | [Exercise 1.4](ex1_4.md) | [Exercise 1.6](ex1_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex1_6.md

@@ -0,0 +1,85 @@
+\[ [Index](index.md) | [Exercise 1.5](ex1_5.md) | [Exercise 2.1](ex2_1.md) \]
+
+# Exercise 1.6
+
+*Objectives:*
+
+- Defining modules
+- Using the import statement
+
+*Files Created:* None
+
+**Note:**
+For this exercise involving modules, it is
+critically important to make sure you are running Python in a proper
+environment.  You may need to check the value of `sys.path` if you 
+can't get import statements to work.  Ask for assistance if everything
+seems broken.
+
+Before starting this exercise, first restart your Python interpreter session.  If using IDLE, click on
+the shell window and look for a menu option "Shell > Restart Shell".   You should get a message like this:
+
+```python
+>>> ##################== RESTART ##################==
+>>> 
+```
+
+If you are using Unix, simply exit Python and restart the interpreter.
+
+## (a) Using the import statement
+
+In previous exercises, you wrote two programs `pcost.py` and
+`stock.py`.   Use the `import` statement to load these
+programs and use their functionality:
+
+```python
+>>> import pcost
+44671.15
+>>> pcost.portfolio_cost('Data/portfolio2.dat')
+19908.75
+>>> from stock import Stock
+>>> s = Stock('GOOG', 100, 490.10)
+>>> s.name
+'GOOG'
+>>> s.cost()
+49010.0
+>>>
+```
+
+If you can't get the above statements to work, you might have placed
+your programs in a funny directory.  Make sure you are running Python
+in the same directory as your files or that the directory is included
+on `sys.path`.
+
+
+## (b) Main Module
+
+In your `pcost.py` program, the last statement called a
+function and printed out the result.  Modify the program so that this
+step only occurs if the program is run as the main program.  Now,
+try running the program two ways:
+
+First, run the program as main:
+
+```
+bash % python3 pcost.py
+44671.25
+bash %
+```
+
+Next, run the program as a library import.  You should not see any
+output.
+
+```python
+>>> import pcost
+>>>
+```
+
+\[ [Solution](soln1_6.md) | [Index](index.md) | [Exercise 1.5](ex1_5.md) | [Exercise 2.1](ex2_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_1.md

@@ -0,0 +1,195 @@
+\[ [Index](index.md) | [Exercise 1.6](ex1_6.md) | [Exercise 2.2](ex2_2.md) \]
+
+# Exercise 2.1
+
+*Objectives:*
+
+- Figure out the most memory-efficient way to store a lot of data.
+- Learn about different ways of representing records including tuples,
+dictionaries, classes, and named tuples.
+
+In this exercise, we look at different choices for representing data
+structures with an eye towards memory use and efficiency.  A lot of
+people use Python to perform various kinds of data analysis so knowing
+about different options and their tradeoffs is useful information.
+
+## (a) Stuck on the bus
+
+The file `Data/ctabus.csv` is a CSV file containing
+daily ridership data for the Chicago Transit Authority (CTA) bus
+system from January 1, 2001 to August 31, 2013.  It contains
+approximately 577000 rows of data.  Use Python to view a few lines
+of data to see what it looks like:
+
+```python
+>>> f = open('Data/ctabus.csv')
+>>> next(f)
+'route,date,daytype,rides\n'
+>>> next(f)
+'3,01/01/2001,U,7354\n'
+>>> next(f)
+'4,01/01/2001,U,9288\n'
+>>>
+```
+
+There are 4 columns of data.
+
+- route: Column 0.  The bus route name.
+- date: Column 1.  A date string of the form MM/DD/YYYY.
+- daytype: Column 2. A day type code (U=Sunday/Holiday, A=Saturday, W=Weekday)
+- rides: Column 3. Total number of riders (integer)
+
+The `rides` column records the total number of people who boarded a
+bus on that route on a given day. Thus, from the example, 7354 people
+rode the number 3 bus on January 1, 2001.
+
+## (b) Basic memory use of text
+
+Let's get a baseline of the memory required to work with this
+datafile.  First, restart Python and try a very simple experiment of
+simply grabbing the file and storing its data in a single string:
+
+```python
+>>> # --- RESTART 
+>>> import tracemalloc
+>>> f = open('Data/ctabus.csv')
+>>> tracemalloc.start()
+>>> data = f.read()
+>>> len(data)
+12361039
+>>> current, peak = tracemalloc.get_traced_memory()
+>>> current
+12369664
+>>> peak
+24730766
+>>> 
+```
+
+Your results might vary somewhat, but you should see current
+memory use in the range of 12MB with a peak of 24MB.
+
+What happens if you read the entire file into a list of strings
+instead?  Restart Python and try this:
+
+```python
+>>> # --- RESTART
+>>> import tracemalloc
+>>> f = open('Data/ctabus.csv')
+>>> tracemalloc.start()
+>>> lines = f.readlines()
+>>> len(lines)
+577564
+>>> current, peak = tracemalloc.get_traced_memory()
+>>> current
+45828030
+>>> peak
+45867371
+>>> 
+```
+
+You should see the memory use go up significantly into the range of 40-50MB.
+Point to ponder: what might be the source of that extra overhead?
+
+## (c) A List of Tuples
+
+In practice, you might read the data into a list and convert each line
+into some other data structure.  Here is a program `readrides.py` that
+reads the entire file into a list of tuples using the `csv` module:
+
+```python
+# readrides.py
+
+import csv
+
+def read_rides_as_tuples(filename):
+    '''
+    Read the bus ride data as a list of tuples
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = (route, date, daytype, rides)
+            records.append(record)
+    return records
+
+if __name__ == '__main__':
+    import tracemalloc
+    tracemalloc.start()
+    rows = read_rides_as_tuples('Data/ctabus.csv')
+    print('Memory Use: Current %d, Peak %d' % tracemalloc.get_traced_memory())
+```
+
+Run this program using `python3 -i readrides.py` and look at the
+resulting contents of `rows`. You should get a list of tuples like
+this:
+
+```python
+>>> len(rows)
+577563
+>>> rows[0]
+('3', '01/01/2001', 'U', 7354)
+>>> rows[1]
+('4', '01/01/2001', 'U', 9288)
+```
+
+Look at the resulting memory use. It should be substantially higher
+than in part (b).
+
+## (d) Memory Use of Other Data Structures
+
+Python has many different choices for representing data structures.
+For example:
+
+```python
+# A tuple
+row = (route, date, daytype, rides)
+
+# A dictionary
+row = {
+    'route': route,
+    'date': date,
+    'daytype': daytype,
+    'rides': rides,
+}
+
+# A class
+class Row:
+    def __init__(self, route, date, daytype, rides):
+        self.route = route
+        self.date = date
+        self.daytype = daytype
+        self.rides = rides
+
+# A named tuple
+from collections import namedtuple
+Row = namedtuple('Row', ['route', 'date', 'daytype', 'rides'])
+
+# A class with __slots__
+class Row:
+    __slots__ = ['route', 'date', 'daytype', 'rides']
+    def __init__(self, route, date, daytype, rides):
+        self.route = route
+        self.date = date
+        self.daytype = daytype
+        self.rides = rides
+```
+Your task is as follows:  Create different versions of the `read_rides()` function
+that use each of these data structures to represent a single row of data.
+Then, find out the resulting memory use of each option.   Find out which
+approach offers the most efficient storage if you were working with a lot 
+of data all at once.
+
+\[ [Solution](soln2_1.md) | [Index](index.md) | [Exercise 1.6](ex1_6.md) | [Exercise 2.2](ex2_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_2.md

@@ -0,0 +1,192 @@
+\[ [Index](index.md) | [Exercise 2.1](ex2_1.md) | [Exercise 2.3](ex2_3.md) \]
+
+# Exercise 2.2
+
+*Objectives:*
+
+- Work with various containers
+- List/Set/Dict Comprehensions
+- Collections module
+- Data analysis challenge
+
+Most Python programmers are generally familiar with lists, dictionaries,
+tuples, and other basic datatypes. In this exercise, we'll put that
+knowledge to work to solve various data analysis problems.
+
+## (a) Preliminaries
+
+To get started, let's review some basics with a slightly simpler dataset--
+a portfolio of stock holdings. Create a file `readport.py` and put this
+code in it:
+
+```python
+# readport.py
+
+import csv
+
+# A function that reads a file into a list of dicts
+def read_portfolio(filename):
+    portfolio = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            record = {
+                'name' : row[0],
+                'shares' : int(row[1]),
+                'price' : float(row[2])
+                }
+            portfolio.append(record)
+    return portfolio
+```
+
+This file reads some simple stock market data in the file `Data/portfolio.csv`.  Use
+the function to read the file and look at the results:
+
+```python
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> from pprint import pprint
+>>> pprint(portfolio)
+[{'name': 'AA', 'price': 32.2, 'shares': 100},
+ {'name': 'IBM', 'price': 91.1, 'shares': 50},
+ {'name': 'CAT', 'price': 83.44, 'shares': 150},
+ {'name': 'MSFT', 'price': 51.23, 'shares': 200},
+ {'name': 'GE', 'price': 40.37, 'shares': 95},
+ {'name': 'MSFT', 'price': 65.1, 'shares': 50},
+ {'name': 'IBM', 'price': 70.44, 'shares': 100}]
+>>>
+```
+
+In this data, each row consists of a stock name, a number of held
+shares, and a purchase price.   There are multiple entries for
+certain stock names such as MSFT and IBM.
+
+## (b) Comprehensions
+
+List, set, and dictionary comprehensions can be a useful tool for manipulating
+data.  For example, try these operations:
+
+```python
+>>> # Find all holdings more than 100 shares
+>>> [s for s in portfolio if s['shares'] > 100]
+[{'name': 'CAT', 'shares': 150, 'price': 83.44}, 
+ {'name': 'MSFT', 'shares': 200, 'price': 51.23}]
+
+>>> # Compute total cost (shares * price)
+>>> sum([s['shares']*s['price'] for s in portfolio])
+44671.15
+>>>
+
+>>> # Find all unique stock names (set)
+>>> { s['name'] for s in portfolio }
+{'MSFT', 'IBM', 'AA', 'GE', 'CAT'}
+>>>
+
+>>> # Count the total shares of each of stock
+>>> totals = { s['name']: 0 for s in portfolio }
+>>> for s in portfolio:
+        totals[s['name']] += s['shares']
+
+>>> totals
+{'AA': 100, 'IBM': 150, 'CAT': 150, 'MSFT': 250, 'GE': 95}
+>>> 
+```
+
+## (c) Collections
+
+The `collections` module has a variety of classes for more specialized data
+manipulation.  For example, the last example could be solved with a `Counter` like this:
+
+```python
+>>> from collections import Counter
+>>> totals = Counter()
+>>> for s in portfolio:
+        totals[s['name']] += s['shares']
+
+>>> totals
+Counter({'MSFT': 250, 'IBM': 150, 'CAT': 150, 'AA': 100, 'GE': 95})
+>>>
+```
+
+Counters are interesting in that they support other kinds of operations such as ranking
+and mathematics.  For example:
+
+```python
+>>> # Get the two most common holdings
+>>> totals.most_common(2)
+[('MSFT', 250), ('IBM', 150)]
+>>>
+
+>>> # Adding counters together
+>>> more = Counter()
+>>> more['IBM'] = 75
+>>> more['AA'] = 200
+>>> more['ACME'] = 30
+>>> more
+Counter({'AA': 200, 'IBM': 75, 'ACME': 30})
+>>> totals
+Counter({'MSFT': 250, 'IBM': 150, 'CAT': 150, 'AA': 100, 'GE': 95})
+>>> totals + more
+Counter({'AA': 300, 'MSFT': 250, 'IBM': 225, 'CAT': 150, 'GE': 95, 'ACME': 30})
+>>> 
+```
+
+The `defaultdict` object can be used to group data.  For example, suppose
+you want to make it easy to find all matching entries for a given name such as
+IBM.  Try this:
+
+```python
+>>> from collections import defaultdict
+>>> byname = defaultdict(list)
+>>> for s in portfolio:
+        byname[s['name']].append(s)
+
+>>> byname['IBM']
+[{'name': 'IBM', 'shares': 50, 'price': 91.1}, {'name': 'IBM', 'shares': 100, 'price': 70.44}]
+>>> byname['AA']
+[{'name': 'AA', 'shares': 100, 'price': 32.2}]
+>>>
+```
+
+The key feature that makes this work is that a defaultdict
+automatically initializes elements for you--allowing an insertion of a
+new element and an `append()` operation to be combined together.
+
+## (c) Data Analysis Challenge
+
+In the last exercise you just wrote some code to read CSV-data related
+to the Chicago Transit Authority.  For example, you can grab the data
+as dictionaries like this:
+
+```python
+>>> import readrides
+>>> rows = readrides.read_rides_as_dicts('Data/ctabus.csv')
+>>>
+```
+
+It would be a shame to do all of that work and then do nothing with
+the data.
+
+In this exercise, you task is this: write a program to answer the
+following three questions:
+
+1. How many bus routes exist in Chicago?
+
+2. How many people rode the number 22 bus on February 2, 2011?  What about any route on any date of your choosing?
+
+3. What is the total number of rides taken on each bus route?
+
+4. What five bus routes had the greatest ten-year increase in ridership from 2001 to 2011?
+
+You are free to use any technique whatsoever to answer the above
+questions as long as it's part of the Python standard library (i.e.,
+built-in datatypes, standard library modules, etc.). 
+
+\[ [Solution](soln2_2.md) | [Index](index.md) | [Exercise 2.1](ex2_1.md) | [Exercise 2.3](ex2_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_3.md

@@ -0,0 +1,365 @@
+\[ [Index](index.md) | [Exercise 2.2](ex2_2.md) | [Exercise 2.4](ex2_4.md) \]
+
+# Exercise 2.3
+
+*Objectives:*
+
+- Iterate like a pro
+
+*Files Modified:* None.
+
+Iteration is an essential Python skill.  In this exercise, we look at
+a number of common iteration idioms.
+
+Start the exercise by grabbing some rows of data from a CSV file.
+
+```python
+>>> import csv
+>>> f = open('Data/portfolio.csv')
+>>> f_csv = csv.reader(f)
+>>> headers = next(f_csv)
+>>> headers
+['name', 'shares', 'price']
+>>> rows = list(f_csv)
+>>> from pprint import pprint
+>>> pprint(rows)
+[['AA', '100', '32.20'],
+ ['IBM', '50', '91.10'],
+ ['CAT', '150', '83.44'],
+ ['MSFT', '200', '51.23'],
+ ['GE', '95', '40.37'],
+ ['MSFT', '50', '65.10'],
+ ['IBM', '100', '70.44']]
+>>>
+```
+
+## (a) Basic Iteration and Unpacking
+
+The `for` statement iterates over any sequence of data. For example:
+
+```python
+>>> for row in rows:
+        print(row)
+
+['AA', '100', '32.20']
+['IBM', '50', '91.10']
+['CAT', '150', '83.44']
+['MSFT', '200', '51.23']
+['GE', '95', '40.37']
+['MSFT', '50', '65.10']
+['IBM', '100', '70.44']
+>>>
+```
+
+Unpack the values into separate variables if you need to:
+
+```python
+>>> for name, shares, price in rows:
+        print(name, shares, price)
+
+AA 100 32.20
+IBM 50 91.10
+CAT 150 83.44
+MSFT 200 51.23
+GE 95 40.37
+MSFT 50 65.10
+IBM 100 70.44
+>>>
+```
+
+It's somewhat common to use `_` or `__` as a throw-away variable if you don't care
+about one or more of the values.  For example:
+
+```python
+>>> for name, _, price in rows:
+        print(name, price)
+
+AA 32.20
+IBM 91.10
+CAT 83.44
+MSFT 51.23
+GE 40.37
+MSFT 65.10
+IBM 70.44
+>>>
+```
+
+If you don't know how many values are being unpacked, you can use `*` as a wildcard.
+Try this experiment in grouping the data by name:
+
+```python
+>>> from collections import defaultdict
+>>> byname = defaultdict(list)
+>>> for name, *data in rows:
+        byname[name].append(data)
+
+>>> byname['IBM']
+[['50', '91.10'], ['100', '70.44']]
+>>> byname['CAT']
+[['150', '83.44']]
+>>> for shares, price in byname['IBM']:
+        print(shares, price)
+
+50 91.10
+100 70.44
+>>>
+```
+
+## (b) Counting with enumerate()
+
+`enumerate()` is a useful function if you ever need to keep a counter
+or index while iterating. For example, suppose you wanted an extra row
+number:
+
+```python
+>>> for rowno, row in enumerate(rows):
+        print(rowno, row)
+
+0 ['AA', '100', '32.20']
+1 ['IBM', '50', '91.10']
+2 ['CAT', '150', '83.44']
+3 ['MSFT', '200', '51.23']
+4 ['GE', '95', '40.37']
+5 ['MSFT', '50', '65.10']
+6 ['IBM', '100', '70.44']
+>>>
+```
+
+You can combine this with unpacking if you're careful about how you structure it:
+
+```python
+>>> for rowno, (name, shares, price) in enumerate(rows):
+        print(rowno, name, shares, price)
+
+0 AA 100 32.20
+1 IBM 50 91.10
+2 CAT 150 83.44
+3 MSFT 200 51.23
+4 GE 95 40.37
+5 MSFT 50 65.10
+6 IBM 100 70.44
+>>> 
+```
+
+## (c) Using the zip() function
+
+The `zip()` function is most commonly used to pair data.  For example,
+recall that you created a `headers` variable:
+
+```python
+>>> headers
+['name', 'shares', 'price']
+>>>
+```
+
+This might be useful to combine with the other row data:
+
+```python
+>>> row = rows[0]
+>>> row
+['AA', '100', '32.20']
+>>> for col, val in zip(headers, row):
+        print(col, val)
+
+name AA
+shares 100
+price 32.20
+>>>
+```
+
+Or maybe you can use it to make a dictionary:
+
+```python
+>>> dict(zip(headers, row))
+{'name': 'AA', 'shares': '100', 'price': '32.20'}
+>>>
+```
+
+Or maybe a sequence of dictionaries:
+
+```python
+>>> for row in rows:
+        record = dict(zip(headers, row))
+        print(record)
+
+{'name': 'AA', 'shares': '100', 'price': '32.20'}
+{'name': 'IBM', 'shares': '50', 'price': '91.10'}
+{'name': 'CAT', 'shares': '150', 'price': '83.44'}
+{'name': 'MSFT', 'shares': '200', 'price': '51.23'}
+{'name': 'GE', 'shares': '95', 'price': '40.37'}
+{'name': 'MSFT', 'shares': '50', 'price': '65.10'}
+{'name': 'IBM', 'shares': '100', 'price': '70.44'}
+>>>
+```
+
+## (d) Generator Expressions
+
+A generator expression is almost exactly the same as a list
+comprehension except that it does not create a list.  Instead, it
+creates an object that produces the results incrementally--typically
+for consumption by iteration. Try a simple example:
+
+```python
+>>> nums = [1,2,3,4,5]
+>>> squares = (x*x for x in nums)
+>>> squares
+<generator object <genexpr> at 0x37caa8>
+>>> for n in squares:
+        print(n)
+
+1
+4
+9
+16
+25
+>>>
+```
+
+You will notice that a generator expression can only be used once.
+Watch what happens if you do the for-loop again:
+
+```python
+>>> for n in squares:
+         print(n)
+
+>>>
+```
+
+You can manually get the results one-at-a-time if you use the
+`next()` function. Try this:
+
+```python
+>>> squares = (x*x for x in nums)
+>>> next(squares)
+1
+>>> next(squares)
+4
+>>> next(squares)
+9
+>>>
+```
+
+Keeping typing `next()` to see what happens when there is no
+more data.
+
+If the task you are performing is more complicated, you can
+still take advantage of generators by writing a generator function 
+and using the `yield` statement instead.
+For example:
+
+```python
+>>> def squares(nums):
+        for x in nums:
+            yield x*x
+
+>>> for n in squares(nums):
+        print(n)
+
+1
+4
+9
+16
+25
+>>>
+```
+
+We'll return to generator functions a little later in the course--for now,
+just view such functions as having the interesting property of feeding
+values to the `for`-statement.
+
+## (e) Generator Expressions and Reduction Functions
+
+Generator expressions are especially useful for feeding data into
+functions such as `sum()`, `min()`, `max()`,
+`any()`, etc.   Try some examples using the portfolio data from
+earlier.  Carefully observe that these examples are missing some
+extra square brackets ([]) that appeared when using list comprehensions.
+
+```python
+>>> from readport import read_portfolio
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> sum(s['shares']*s['price'] for s in portfolio)
+44671.15
+>>> min(s['shares'] for s in portfolio)
+50
+>>> any(s['name'] == 'IBM' for s in portfolio)
+True
+>>> all(s['name'] == 'IBM' for s in portfolio)
+False
+>>> sum(s['shares'] for s in portfolio if s['name'] == 'IBM')
+150
+>>>
+```
+
+Here is an subtle use of a generator expression in making comma
+separated values:
+
+```python
+>>> s = ('GOOG',100,490.10)
+>>> ','.join(s)
+... observe that it fails ...
+>>> ','.join(str(x) for x in s)    # This works
+'GOOG,100,490.1'
+>>>
+```
+
+The syntax in the above examples takes some getting used to, but the
+critical point is that none of the operations ever create a fully
+populated list of results.  This gives you a big memory savings.  However,
+you do need to make sure you don't go overboard with the syntax.
+
+## (f) Saving a lot of memory
+
+In link:ex2_1.html[Exercise 2.1] you wrote a function
+`read_rides_as_dicts()` that read the CTA bus data into a list of
+dictionaries.  Using it requires a lot of memory. For example,
+let's find the day on which the route 22 bus had the greatest
+ridership:
+
+```python
+>>> import tracemalloc
+>>> tracemalloc.start()
+>>> import readrides
+>>> rows = readrides.read_rides_as_dicts('Data/ctabus.csv')
+>>> rt22 = [row for row in rows if row['route'] == '22']
+>>> max(rt22, key=lambda row: row['rides'])
+{'date': '06/11/2008', 'route': '22', 'daytype': 'W', 'rides': 26896}
+>>> tracemalloc.get_traced_memory()
+... look at result. Should be around 220MB
+>>>
+```
+
+Now, let's try an example involving generators. Restart Python
+and try this:
+
+```python
+>>> # RESTART
+>>> import tracemalloc
+>>> tracemalloc.start()
+>>> import csv
+>>> f = open('Data/ctabus.csv')
+>>> f_csv = csv.reader(f)
+>>> headers = next(f_csv)
+>>> rows = (dict(zip(headers,row)) for row in f_csv)
+>>> rt22 = (row for row in rows if row['route'] == '22')
+>>> max(rt22, key=lambda row: int(row['rides']))
+{'date': '06/11/2008', 'route': '22', 'daytype': 'W', 'rides': 26896}
+>>> tracemalloc.get_traced_memory()
+... look at result. Should be a LOT smaller than before
+>>>
+```
+
+Keep in mind that you just processed the entire dataset as if it was
+stored as a sequence of dictionaries.  Yet, nowhere did you actually
+create and store a list of dictionaries.   Not all problems can be
+structured in this way, but if you can work with data in an
+iterative manner, generator expressions can save a huge amount of memory.
+
+\[ [Solution](soln2_3.md) | [Index](index.md) | [Exercise 2.2](ex2_2.md) | [Exercise 2.4](ex2_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_4.md

@@ -0,0 +1,449 @@
+\[ [Index](index.md) | [Exercise 2.3](ex2_3.md) | [Exercise 2.5](ex2_5.md) \]
+
+# Exercise 2.4
+
+*Objectives:*
+
+- Make a new primitive type
+
+In most programs, you use the primitive types such as `int`, `float`,
+and `str` to represent data.  However, you're not limited to just those
+types.  The standard library has modules such as the `decimal` and
+`fractions` module that implement new primitive types.  You can also
+make your own types as long as you understand the underlying protocols
+which make Python objects work.  In this exercise, we'll make a new
+primitive type.  There are a lot of little details to worry about, but
+this will give you a general sense for what's required.
+
+## (a) Mutable Integers
+
+Python integers are normally immutable.   However, suppose you wanted to
+make a mutable integer object.  Start off by making a class like this:
+
+```python
+# mutint.py
+
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+```
+
+Try it out:
+
+```python
+>>> a = MutInt(3)
+>>> a
+<__main__.MutInt object at 0x10e79d408>
+>>> a.value
+3
+>>> a.value = 42
+>>> a.value
+42
+>>> a + 10
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: unsupported operand type(s) for +: 'MutInt' and 'int'
+>>>
+```
+
+That's all very exciting except that nothing really works with this
+new `MutInt` object. Printing is horrible, none of the math
+operators work, and it's basically rather useless.  Well, except for
+the fact that its value is mutable--it does have that.
+
+## (b) Fixing output
+
+You can fix output by giving the object methods such as `__str__()`,
+`__repr__()`, and `__format__()`.  For example:
+
+```python
+# mint.py
+
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return str(self.value)
+
+    def __repr__(self):
+        return f'MutInt({self.value!r})'
+
+    def __format__(self, fmt):
+        return format(self.value, fmt)
+```
+
+Try it out:
+
+```python
+>>> a = MutInt(3)
+>>> print(a)
+3
+>>> a
+MutInt(3)
+>>> f'The value is {a:*^10d}'
+The value is ****3*****
+>>> a.value = 42
+>>> a
+MutInt(42)
+>>>
+```
+
+## (c) Math Operators
+
+You can make an object work with various math operators if you implement the
+appropriate methods for it.  However, it's your responsibility to
+recognize other types of data and implement the appropriate conversion
+code.  Modify the `MutInt` class by giving it an `__add__()` method
+as follows:
+
+```python
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __add__(self, other):
+        if isinstance(other, MutInt):
+            return MutInt(self.value + other.value)
+        elif isinstance(other, int):
+            return MutInt(self.value + other)
+        else:
+            return NotImplemented
+```
+
+With this change, you should find that you can add both integers and
+mutable integers. The result is a `MutInt` instance. Adding
+other kinds of numbers results in an error:
+
+```python
+>>> a = MutInt(3)
+>>> b = a + 10
+>>> b
+MutInt(13)
+>>> b.value = 23
+>>> c = a + b
+>>> c
+MutInt(26)
+>>> a + 3.5
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: unsupported operand type(s) for +: 'MutInt' and 'float'
+>>> 
+```
+
+One problem with the code is that it doesn't work when the order of operands
+is reversed.  Consider:
+
+```python
+>>> a + 10
+MutInt(13)
+>>> 10 + a
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: unsupported operand type(s) for +: 'int' and 'MutInt'
+>>> 
+```
+
+This is occurring because the `int` type has no knowledge of `MutInt`
+and it's confused. This can be fixed by adding an `__radd__()` method.  This
+method is called if the first attempt to call `__add__()` didn't work with the
+provided object.
+
+```python
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __add__(self, other):
+        if isinstance(other, MutInt):
+            return MutInt(self.value + other.value)
+        elif isinstance(other, int):
+            return MutInt(self.value + other)
+        else:
+            return NotImplemented
+
+    __radd__ = __add__    # Reversed operands
+```
+
+With this change, you'll find that addition works:
+
+```python
+>>> a = MutInt(3)
+>>> a + 10
+MutInt(13)
+>>> 10 + a
+MutInt(13)
+>>>
+```
+
+Since our integer is mutable, you can also make it recognize the in-place
+add-update operator `+=` by implementing the `__iadd__()` method:
+
+```python
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __iadd__(self, other):
+        if isinstance(other, MutInt):
+            self.value += other.value
+            return self
+        elif isinstance(other, int):
+            self.value += other
+            return self
+        else:
+            return NotImplemented
+```
+
+This allows for interesting uses like this:
+
+```python
+>>> a = MutInt(3)
+>>> b = a
+>>> a += 10
+>>> a
+MutInt(13)
+>>> b                 # Notice that b also changes
+MutInt(13)
+>>>
+```
+
+That might seem kind of strange that `b` also changes, but there are subtle features like
+this with built-in Python objects. For example:
+
+```python
+>>> a = [1,2,3]
+>>> b = a
+>>> a += [4,5]
+>>> a
+[1, 2, 3, 4, 5]
+>>> b
+[1, 2, 3, 4, 5]
+
+>>> c = (1,2,3)
+>>> d = c
+>>> c += (4,5)
+>>> c
+(1, 2, 3, 4, 5)
+>>> d                  # Explain difference from lists
+(1, 2, 3)
+>>> 
+```
+
+## (d) Comparisons
+
+One problem is that comparisons still don't work.  For example:
+
+```python
+>>> a = MutInt(3)
+>>> b = MutInt(3)
+>>> a == b
+False
+>>> a == 3
+False
+>>>
+```
+
+You can fix this by adding an `__eq__()` method.  Further methods such
+as `__lt__()`, `__le__()`, `__gt__()`, `__ge__()` can be used to
+implement other comparisons.  For example:
+
+```python
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+    def __eq__(self, other):
+        if isinstance(other, MutInt):
+            return self.value == other.value
+        elif isinstance(other, int):
+            return self.value == other
+        else:
+            return NotImplemented
+        
+    def __lt__(self, other):
+        if isinstance(other, MutInt):
+            return self.value < other.value
+        elif isinstance(other, int):
+            return self.value < other
+        else:
+            return NotImplemented
+```
+
+Try it:
+
+```python
+>>> a = MutInt(3)
+>>> b = MutInt(3)
+>>> a == b
+True
+>>> c = MutInt(4)
+>>> a < c
+True
+>>> a <= c
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: '<=' not supported between instances of 'MutInt' and 'MutInt'
+>>> 
+```
+
+The reason the `<=` operator is failing is that no `__le__()` method was provided.
+You could code it separately, but an easier way to get it is to use the `@total_ordering`
+decorator:
+
+```python
+from functools import total_ordering
+
+@total_ordering
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __eq__(self, other):
+        if isinstance(other, MutInt):
+            return self.value == other.value
+        elif isinstance(other, int):
+            return self.value == other
+        else:
+            return NotImplemented
+        
+    def __lt__(self, other):
+        if isinstance(other, MutInt):
+            return self.value < other.value
+        elif isinstance(other, int):
+            return self.value < other
+        else:
+            return NotImplemented
+```
+
+`@total_ordering` fills in the missing comparison methods for you as long as
+you minimally provide an equality operator and one of the other relations.
+
+## (e) Conversions
+
+Your new primitive type is almost complete.  You might want to give it 
+the ability to work with some common conversions. For example:
+
+```python
+>>> a = MutInt(3)
+>>> int(a)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: int() argument must be a string, a bytes-like object or a number, not 'MutInt'
+>>> float(a)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: float() argument must be a string, a bytes-like object or a number, not 'MutInt'
+>>>
+```
+
+You can give your class an `__int__()` and `__float__()` method to fix this:
+
+```python
+from functools import total_ordering
+
+@total_ordering
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __int__(self):
+        return self.value
+
+    def __float__(self):
+        return float(self.value)
+```
+
+Now, you can properly convert:
+
+```python
+>>> a = MutInt(3)
+>>> int(a)
+3
+>>> float(a)
+3.0
+>>>
+```
+
+As a general rule, Python never automatically converts data though. Thus, even though you
+gave the class an `__int__()` method, `MutInt` is still not going to work in all
+situations when an integer might be expected.  For example, indexing:
+
+```python
+>>> names = ['Dave', 'Guido', 'Paula', 'Thomas', 'Lewis']
+>>> a = MutInt(1)
+>>> names[a]
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: list indices must be integers or slices, not MutInt
+>>> 
+```
+
+This can be fixed by giving `MutInt` an `__index__()` method that produces an integer.
+Modify the class like this:
+
+```python
+from functools import total_ordering
+
+@total_ordering
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    ...
+
+    def __int__(self):
+        return self.value
+
+    __index__ = __int__     # Make indexing work
+```
+
+
+**Discussion**
+
+Making a new primitive datatype is actually one of the most complicated
+programming tasks in Python.   There are a lot of edge cases and low-level
+issues to worry about--especially with regard to how your type interacts
+with other Python types.  Probably the key thing to keep in mind is that
+you can customize almost every aspect of how an object interacts with the
+rest of Python if you know the underlying protocols.   If you're going to
+do this, it's advisable to look at the existing code for something similar
+to what you're trying to make.
+
+\[ [Solution](soln2_4.md) | [Index](index.md) | [Exercise 2.3](ex2_3.md) | [Exercise 2.5](ex2_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_5.md

@@ -0,0 +1,324 @@
+\[ [Index](index.md) | [Exercise 2.4](ex2_4.md) | [Exercise 2.6](ex2_6.md) \]
+
+# Exercise 2.5
+
+*Objectives:*
+
+- Look at memory allocation behavior of lists and dicts
+- Make a custom container
+
+*Files Created:* None
+
+## (a) List growth
+
+Python lists are highly optimized for performing `append()`
+operations. Each time a list grows, it grabs a larger chunk of memory
+than it actually needs with the expectation that more data will be
+added to the list later.  If new items are added and space is
+available, the `append()` operation stores the item without
+allocating more memory.
+
+Experiment with this feature of lists by using
+the `sys.getsizeof()` function on a list and appending a few
+more items.
+
+```python
+>>> import sys
+>>> items = []
+>>> sys.getsizeof(items)
+64
+>>> items.append(1)
+>>> sys.getsizeof(items)
+96
+>>> items.append(2)
+>>> sys.getsizeof(items)    # Notice how the size does not increase
+96
+>>> items.append(3)
+>>> sys.getsizeof(items)    # It still doesn't increase here
+96
+>>> items.append(4)
+>>> sys.getsizeof(items)    # Not yet.
+96
+>>> items.append(5)
+>>> sys.getsizeof(items)    # Notice the size has jumped
+128
+>>>
+```
+
+A list stores its items by reference.  So, the memory required for
+each item is a single memory address.  On a 64-bit machine, an address
+is typically 8 bytes.  However, if Python has been compiled for
+32-bits, it might be 4 bytes and the numbers for the above example
+will be half of what's shown.
+
+## (b) Dictionary/Class Growth
+
+Python dictionaries (and classes) allow up to 5 values to be stored
+before their reserved memory doubles.  Investigate by making a dictionary
+and adding a few more values to it:
+
+```python
+>>> row = { 'route': '22', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354 }
+>>> sys.getsizeof(row)
+>>> sys.getsizeof(row)
+240
+>>> row['a'] = 1
+>>> sys.getsizeof(row)
+240
+>>> row['b'] = 2
+>>> sys.getsizeof(row)
+368
+>>> 
+```
+
+Does the memory go down if you delete the item you just added?
+
+Food for thought:  If you are creating large numbers of records,
+representing each record as a dictionary might not be the most
+efficient approach--you could be paying a heavy price for the convenience
+of having a dictionary.   It might be better to consider the use of tuples,
+named tuples, or classes that define `__slots__`.
+
+## (c) Changing Your Orientation (to Columns)
+
+You can often save a lot of memory if you change your view of data.
+For example, what happens if you read all of the bus data into a
+columns using this function?
+
+```python
+# readrides.py
+
+...
+
+def read_rides_as_columns(filename):
+    '''
+    Read the bus ride data into 4 lists, representing columns
+    '''
+    routes = []
+    dates = []
+    daytypes = []
+    numrides = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            routes.append(row[0])
+            dates.append(row[1])
+            daytypes.append(row[2])
+            numrides.append(int(row[3]))
+    return dict(routes=routes, dates=dates, daytypes=daytypes, numrides=numrides)
+```
+
+In theory, this function should save a lot of memory.  Let's analyze it before trying it.
+
+First, the datafile contained 577563 rows of data where each row contained
+four values. If each row is stored as a dictionary, then those dictionaries
+are minimally 240 bytes in size.
+
+```python
+>>> nrows = 577563     # Number of rows in original file
+>>> nrows * 240
+138615120
+>>>
+```
+
+So, that's 138MB just for the dictionaries themselves.  This does not
+include any of the values actually stored in the dictionaries.
+
+By switching to columns, the data is stored in 4 separate lists.  
+Each list requires 8 bytes per item to store a pointer.  So, here's
+a rough estimate of the list requirements:
+
+```python
+>>> nrows * 4 * 8
+18482016
+>>>
+```
+
+That's about 18MB in list overhead.  So, switching to a column orientation
+should save about 120MB of memory solely from eliminating all of the extra information that
+needs to be stored in dictionaries. 
+
+Try using this function to read the bus data and look at the memory use.
+
+```python
+>>> import tracemalloc
+>>> tracemalloc.start()
+>>> columns = read_rides_as_columns('Data/ctabus.csv')
+>>> tracemalloc.get_traced_memory()
+... look at the result ...
+>>>
+```
+
+Does the result reflect the expected savings in memory from our rough calculations above?
+
+## (d) Making a Custom Container - The Great Fake Out
+
+Storing the data in columns offers a much better memory savings, but
+the data is now rather weird to work with.  In fact, none of our
+earlier analysis code from [Exercise 2.2](ex2_2.md) can work
+with columns. The reason everything is broken is that you've broken
+the data abstraction that was used in earlier exercises--namely the
+assumption that data is stored as a list of dictionaries.
+
+This can be fixed if you're willing to make a custom container object
+that "fakes" it. Let's do that.
+
+The earlier analysis code assumes the data is stored in a sequence of
+records.  Each record is represented as a dictionary.  Let's start
+by making a new "Sequence" class.  In this class, we store the
+four columns of data that were being using in the `read_rides_as_columns()`
+function.
+
+```python
+# readrides.py
+
+import collections
+...
+class RideData(collections.Sequence):
+    def __init__(self):
+        self.routes = []      # Columns
+        self.dates = []
+        self.daytypes = []
+        self.numrides = []
+```
+
+Try creating a `RideData` instance.  You'll find that it fails with an 
+error message like this:
+
+```python
+>>> records = RideData()
+Traceback (most recent call last):
+...
+TypeError: Can't instantiate abstract class RideData with abstract methods __getitem__, __len__
+>>>
+```
+
+Carefully read the error message. It tells us what we need to
+implement. Let's add a `__len__()` and `__getitem__()` method.  In the
+`__getitem__()` method, we'll make a dictionary.  In addition, we'll
+create an `append()` method that takes a dictionary and unpacks it
+into 4 separate `append()` operations.
+
+```python
+# readrides.py
+...
+
+class RideData(collections.Sequence):
+    def __init__(self):
+        # Each value is a list with all of the values (a column)
+        self.routes = []
+        self.dates = []
+        self.daytypes = []
+        self.numrides = []
+        
+    def __len__(self):
+        # All lists assumed to have the same length
+        return len(self.routes)
+
+    def __getitem__(self, index):
+        return { 'route': self.routes[index],
+                 'date': self.dates[index],
+                 'daytype': self.daytypes[index],
+                 'rides': self.numrides[index] }
+
+    def append(self, d):
+        self.routes.append(d['route'])
+        self.dates.append(d['date'])
+        self.daytypes.append(d['daytype'])
+        self.numrides.append(d['rides'])
+```
+
+If you've done this correctly, you should be able to drop this object into
+the previously written `read_rides_as_dicts()` function.  It involves
+changing only one line of code:
+
+```python
+# readrides.py
+...
+
+def read_rides_as_dicts(filename):
+    '''
+    Read the bus ride data as a list of dicts
+    '''
+    records = RideData()      # <--- CHANGE THIS
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = {
+                'route': route, 
+                'date': date, 
+                'daytype': daytype, 
+                'rides' : rides
+                }
+            records.append(record)
+    return records
+```
+
+If you've done this right, old code should work exactly as it did before.
+For example:
+
+```python
+>>> rows = readrides.read_rides_as_dicts('Data/ctabus.csv')
+>>> rows
+<readrides.RideData object at 0x10f5054a8>
+>>> len(rows)
+577563
+>>> rows[0]
+{'route': '3', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354}
+>>> rows[1]
+{'route': '4', 'date': '01/01/2001', 'daytype': 'U', 'rides': 9288}
+>>> rows[2]
+{'route': '6', 'date': '01/01/2001', 'daytype': 'U', 'rides': 6048}
+>>> 
+```
+
+Run your earlier CTA code from [Exercise 2.2](ex2_2.md).  It
+should work without modification, but use substantially less memory.
+
+## (e) Challenge
+
+What happens when you take a slice of ride data?
+
+```python
+>>> r = rows[0:10]
+>>> r
+... look at result ...
+>>>
+```
+
+It's probably going to look a little crazy.  Can you modify 
+the `RideData` class so that it produces a proper slice that
+looks like a list of dictionaries? For example, like this:
+
+```python
+>>> rows = readrides.read_rides_as_columns('Data/ctabus.csv')
+>>> rows
+<readrides.RideData object at 0x10f5054a8>
+>>> len(rows)
+577563
+>>> r = rows[0:10]
+>>> r
+<readrides.RideData object at 0x10f5068c8>
+>>> len(r)
+10
+>>> r[0]
+{'route': '3', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354}
+>>> r[1]
+{'route': '4', 'date': '01/01/2001', 'daytype': 'U', 'rides': 9288}
+>>>
+```
+
+\[ [Solution](soln2_5.md) | [Index](index.md) | [Exercise 2.4](ex2_4.md) | [Exercise 2.6](ex2_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex2_6.md

@@ -0,0 +1,303 @@
+\[ [Index](index.md) | [Exercise 2.5](ex2_5.md) | [Exercise 3.1](ex3_1.md) \]
+
+# Exercise 2.6
+
+*Objectives:*
+
+- Explore the power of having first-class objects.
+- Better understand Python's memory model
+
+*Files Created:* `reader.py`
+
+In previous exercises, you wrote various functions for reading CSV data in different
+files.  Surely this is a problem that could be generalized in some way.  In this
+exercise, we explore that idea.
+
+## (a) First-class Data
+
+In the file `Data/portfolio.csv`, you read data organized as columns
+that look like this:
+
+```python
+"AA",100,32.20
+"IBM",50,91.10
+...
+```
+
+In previous code, this data was processed by hard-coding all of the
+type conversions.  For example:
+
+```python
+rows = csv.reader(f)
+for row in rows:
+    name   = row[0]
+    shares = int(row[1])
+    price  = float(row[2])
+```
+
+This kind of conversion can also be performed in a more clever manner
+using some list operations. Make a Python list that contains the
+conversions you want to carry out on each column:
+
+```python
+>>> coltypes = [str, int, float]
+>>>
+```
+
+The reason you can even create this list is that everything in Python
+is "first-class."  So, if you want to have a list of functions, that's
+fine. 
+
+Now, read a row of data from the above file:
+
+```python
+>>> import csv
+>>> f = open('Data/portfolio.csv')
+>>> rows = csv.reader(f)
+>>> headers = next(rows)
+>>> row = next(rows)
+>>> row
+['AA', '100', '32.20']
+>>>
+```
+
+Zip the column types with the row and look at the result:
+
+```python
+>>> r = list(zip(coltypes, row))
+>>> r
+[(<class 'str'>, 'AA'), (<class 'int'>, '100'), (<class 'float'>,'32.20')]
+>>>
+```
+
+You will notice that this has paired a type conversion with a value.  For example, `int` is
+paired with the value `'100'`.   Now, try this:
+
+```python
+>>> record = [func(val) for func, val in zip(coltypes, row)]
+>>> record
+['AA', 100, 32.2]
+>>>
+```
+
+Make sure you understand what's happening in the above code.  In the
+loop, the `func` variable is one of the type conversion functions
+(e.g., `str`, `int`, etc.) and the `val` variable is one of the values
+like `'AA'`, `'100'`.  The expression `func(val)` is converting
+a value (kind of like a type cast).
+
+You can take it a step further and make dictionaries by using the column
+headers.   For example:
+
+```python
+>>> dict(zip(headers, record))
+{'name': 'AA', 'shares': 100, 'price': 32.2}
+>>>
+```
+
+If you prefer, you can perform all of these steps at once using a
+dictionary comprehension:
+
+```python
+>>> { name:func(val) for name, func, val in zip(headers, coltypes, row) }
+{'name': 'AA', 'shares': 100, 'price': 32.2}
+>>>
+```
+
+## (b) Making a Parsing Utility Function
+
+Create a new file `reader.py`.  In that file, define a
+utility function `read_csv_as_dicts()` that reads a file of CSV
+data into a list of dictionaries where the user specifies
+the type conversions for each column.
+
+Here is how it should work:
+
+```python
+>>> import reader
+>>> portfolio = reader.read_csv_as_dicts('Data/portfolio.csv', [str,int,float])
+>>> for s in portfolio:
+         print(s)
+
+{'name': 'AA', 'shares': 100, 'price': 32.2}
+{'name': 'IBM', 'shares': 50, 'price': 91.1}
+{'name': 'CAT', 'shares': 150, 'price': 83.44}
+{'name': 'MSFT', 'shares': 200, 'price': 51.23}
+{'name': 'GE', 'shares': 95, 'price': 40.37}
+{'name': 'MSFT', 'shares': 50, 'price': 65.1}
+{'name': 'IBM', 'shares': 100, 'price': 70.44}
+>>>
+```
+
+Or, if you wanted to read the CTA data:
+
+```python
+>>> rows = reader.read_csv_as_dicts('Data/ctabus.csv', [str,str,str,int])
+>>> len(rows)
+577563
+>>> rows[0]
+{'daytype': 'U', 'route': '3', 'rides': 7354, 'date': '01/01/2001'}
+>>> 
+```
+
+## (c) Memory Revisited
+
+In the CTA bus data, we determined that there were 181 unique bus routes.
+
+```python
+>>> routes = { row['route'] for row in rows }
+>>> len(routes)
+181
+>>>
+```
+
+Question:  How many unique route string objects are contained in the ride data?
+Instead of building a set of routes, build a set of object ids instead:
+
+```python
+>>> routeids = { id(row['route']) for row in rows }
+>>> len(routeids)
+542305
+>>>
+```
+
+Think about this for a moment--there are only 181 distinct route
+names, but the resulting list of dictionaries contains 542305
+different route strings.  Maybe this is something that could be fixed
+with a bit of caching or object reuse.  As it turns out, Python has
+a function that can be used to cache strings, `sys.intern()`.  For example:
+
+```python
+>>> a = 'hello world'
+>>> b = 'hello world'
+>>> a is b
+False
+>>> import sys
+>>> a = sys.intern(a)
+>>> b = sys.intern(b)
+>>> a is b
+True
+>>>
+```
+
+Restart Python and try this:
+
+```python
+>>> # ------------------ RESTART ---------```
+>>> import tracemalloc
+>>> tracemalloc.start()
+>>> from sys import intern
+>>> import reader
+>>> rows = reader.read_csv_as_dicts('Data/ctabus.csv', [intern, str, str, int])
+>>> routeids = { id(row['route']) for row in rows }
+>>> len(routeids)
+181
+>>>
+```
+
+Take a look at the memory use.  
+
+```python
+>>> tracemalloc.get_traced_memory()
+... look at result ...
+>>> 
+```
+
+The memory should drop quite a bit.  Observation:  There is a lot of repetition 
+involving dates as well.   What happens if you also cache the date strings?
+
+```python
+>>> # ------------------ RESTART ---------```
+>>> import tracemalloc
+>>> tracemalloc.start()
+>>> from sys import intern
+>>> import reader
+>>> rows = reader.read_csv_as_dicts('Data/ctabus.csv', [intern, intern, str, int])
+>>> tracemalloc.get_traced_memory()
+... look at result ...
+>>> 
+```
+
+## (d) Special Challenge Project
+
+In [Exercise 2.5](ex2_5.md), we created a class `RideData` that
+stored all of the bus data in columns, but actually presented the data
+to a user as a sequence of dictionaries.  It saved a lot of memory
+through various forms of magic.
+
+Can you generalize that idea?   Specifically, can you make a general
+purpose function `read_csv_as_columns()` that works like this:
+
+```python
+>>> data = read_csv_as_columns('Data/ctabus.csv', types=[str, str, str, int])
+>>> data
+<__main__.DataCollection object at 0x102b45048>
+>>> len(data)
+577563
+>>> data[0]
+{'route': '3', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354}
+>>> data[1]
+{'route': '4', 'date': '01/01/2001', 'daytype': 'U', 'rides': 9288}
+>>> data[2]
+{'route': '6', 'date': '01/01/2001', 'daytype': 'U', 'rides': 6048}
+>>> 
+```
+
+This function is supposed to be general purpose--you can give it any file and
+a list of the column types and it will read the data.  The data is read into
+a class `DataCollection` that stores the data as columns internally.  The data
+presents itself as a sequence of dictionaries when accessed however.
+
+Try using this function with the string interning trick in the last part. How
+much memory does it take to store all of the ride data now?   Can you still use
+this function with your earlier CTA analysis code?
+
+## (e) Deep Thought
+
+In this exercise, you have written two functions, `read_csv_as_dicts()` and
+`read_csv_as_columns()`.  These functions present data to the user in the same way.
+For example:
+
+```python
+>>> data1 = read_csv_as_dicts('Data/ctabus.csv', [str, str, str, int])
+>>> len(data1)
+577563
+>>> data1[0]
+{'route': '3', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354}
+>>> data1[1]
+{'route': '4', 'date': '01/01/2001', 'daytype': 'U', 'rides': 9288}
+>>>
+
+>>> data2 = read_csv_as_columns('Data/ctabus.csv', [str, str, str, int])
+>>> len(data2)
+577563
+>>> data2[0]
+{'route': '3', 'date': '01/01/2001', 'daytype': 'U', 'rides': 7354}
+>>> data2[1]
+{'route': '4', 'date': '01/01/2001', 'daytype': 'U', 'rides': 9288}
+>>>
+```
+
+In fact, you can use either function in the CTA data analysis code
+that you wrote.  Yet, under the covers completely different things are
+happening.  The `read_csv_as_columns()` function is storing the data
+in a different representation.  It's relying on Python sequence
+protocols (magic methods) to present information to you in a more useful
+way.
+
+This is really part of a much bigger programming concept of "Data
+Abstraction".  When writing programs, the way in which data is
+presented is often more important than how the data is actually put
+together under the hood.  Although we're presenting the data as a 
+sequence of dictionaries, there's a great deal of flexibility in 
+how that actually happens behind the scenes.  That's a powerful
+idea and something to think about when writing your own programs.
+
+\[ [Solution](soln2_6.md) | [Index](index.md) | [Exercise 2.5](ex2_5.md) | [Exercise 3.1](ex3_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_1.md

@@ -0,0 +1,102 @@
+\[ [Index](index.md) | [Exercise 2.6](ex2_6.md) | [Exercise 3.2](ex3_2.md) \]
+
+# Exercise 3.1
+
+*Objectives:*
+
+- Define a simple class
+
+*Files Modified:* `stock.py`
+
+In link:ex1_5.html[Exercise 1.5], you defined a simple class
+`Stock` for representing a holding of stock.  In this exercise,
+we're simply going to add a few features to that class as well as
+write some utility functions.
+
+## (a) Adding a new method
+
+Add a new method `sell(nshares)` to Stock that sells a certain number
+of shares by decrementing the share count.  Have it work like this:
+
+```python
+>>> s = Stock('GOOG',100,490.10)
+>>> s.shares
+100
+>>> s.sell(25)
+>>> s.shares
+75
+>>>
+```
+
+## (b) Reading a portfolio
+
+Add a function `read_portfolio()` to your `stock.py` program that
+reads a file of portfolio data into a list of `Stock` objects. Here's how it should work:
+
+```python
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> for s in portfolio:
+        print(s)
+
+<__main__.Stock object at 0x3902f0>
+<__main__.Stock object at 0x390270>
+<__main__.Stock object at 0x390330>
+<__main__.Stock object at 0x390370>
+<__main__.Stock object at 0x3903b0>
+<__main__.Stock object at 0x3903f0>
+<__main__.Stock object at 0x390430>
+>>>
+```
+
+You already wrote a similar function as part of
+link:ex2_3.html[Exercise 2.3].  Design discussion: Should
+`read_portfolio()` be a separate function or part of the class
+definition?
+
+## (c) Printing a Table
+
+Table the data read in part (b) and use it to make a nicely formatted
+table. For example:
+
+```python
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> for s in portfolio:
+           print('%10s %10d %10.2f' % (s.name, s.shares, s.price))
+
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+>>>
+```
+
+Take this code and put it in a function `print_portfolio()` that 
+produces the same output, but additionally adds some table headers.
+For example:
+
+```python
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> print_portfolio(portfolio)
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+>>>
+```
+
+\[ [Solution](soln3_1.md) | [Index](index.md) | [Exercise 2.6](ex2_6.md) | [Exercise 3.2](ex3_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_2.md

@@ -0,0 +1,170 @@
+\[ [Index](index.md) | [Exercise 3.1](ex3_1.md) | [Exercise 3.3](ex3_3.md) \]
+
+# Exercise 3.2
+
+*Objectives:*
+
+- Learn about attribute access
+- Learn how use getattr(), setattr(), and related functions.
+- Experiment with bound methods.
+
+
+*Files Created:* `tableformat.py`
+
+## (a) The Three Operations
+
+The entire Python object system consists of just three core operations: getting, setting, and deleting
+of attributes.  Normally, these are accessed via the dot (.) like this:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name    #  get
+'GOOG'
+>>> s.shares = 50    # set
+>>> del s.shares     # delete
+>>>
+```
+
+The three operations are also available as functions.  For example:
+
+```python
+>>> getattr(s, 'name')            # Same as s.name
+'GOOG'
+>>> setattr(s, 'shares', 50)      # Same as s.shares = 50
+>>> delattr(s, 'shares')          # Same as del s.shares
+>>>
+```
+
+An additional function `hasattr()` can be used to probe an object for the existence of an attribute:
+
+```python
+>>> hasattr(s, 'name')
+True
+>>> hasattr(s, 'blah')
+False
+>>>
+```
+
+## (b) Using getattr()
+
+The `getattr()` function is extremely useful for
+writing code that processes objects in an extremely generic way.  To
+illustrate, consider this example which prints out a set of
+user-defined attributes:
+
+```python
+>>> s= Stock('GOOG', 100, 490.1)
+>>> fields = ['name','shares','price']
+>>> for name in fields:
+           print(name, getattr(s, name))
+
+name GOOG
+shares 100
+price 490.1
+>>>
+```
+
+## (c) Table Output
+
+In link:ex3_1.html[Exercise 3.1], you wrote a function `print_portfolio()`
+that made a nicely formatted table.   That function was custom tailored
+to a list of `Stock` objects.  However, it can be completely generalized
+to work with any list of objects using the technique in part (b).
+
+Create a new module called `tableformat.py`.  In that program,
+write a function `print_table()` that takes a sequence (list) of objects,
+a list of attribute names, and prints a nicely formatted table. For example:
+
+```python
+>>> import stock
+>>> import tableformat
+>>> portfolio = stock.read_portfolio('Data/portfolio.csv')
+>>> tableformat.print_table(portfolio, ['name','shares','price'])
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100       32.2
+       IBM         50       91.1
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50       65.1
+       IBM        100      70.44
+
+>>> tableformat.print_table(portfolio,['shares','name'])
+    shares       name
+---------- ---------- 
+       100         AA
+        50        IBM
+       150        CAT
+       200       MSFT
+        95         GE
+        50       MSFT
+       100        IBM
+>>> 
+```
+
+For simplicity, just have the `print_table()` function print each field in
+a 10-character wide column.
+
+## (d) Bound Methods
+
+It may be surprising, but method calls are layered onto the machinery used
+for simple attributes.  Essentially, a method is an attribute that 
+executes when you add the required parentheses () to call it like a function.  For
+example:
+
+```python
+>>> s = Stock('GOOG',100,490.10)
+>>> s.cost           # Looks up the method
+<bound method Stock.cost of <__main__.Stock object at 0x409530>>
+>>> s.cost()         # Looks up and calls the method
+49010.0
+
+>>> # Same operations using getattr()
+>>> getattr(s, 'cost')
+<bound method Stock.cost of <__main__.Stock object at 0x409530>>
+>>> getattr(s, 'cost')()
+49010.0
+>>> 
+```
+
+A bound method is attached to the object where it came from.   If that
+object is modified, the method will see the modifications.  You can
+view the original object by inspecting the `__self__` attribute
+of the method.
+
+```python
+>>> c = s.cost
+>>> c()
+49010.0
+>>> s.shares = 75
+>>> c()
+36757.5
+>>> c.__self__
+<__main__.Stock object at 0x409530>
+>>> c.__func__
+<function cost at 0x37cc30>
+>>> c.__func__(c.__self__)      # This is what happens behind the scenes of calling c()
+36757.5
+>>>
+```
+
+Try it with the `sell()` method just to make sure you
+understand the mechanics:
+
+```python
+>>> f = s.sell
+>>> f.__func__(f.__self__, 25)     # Same as s.sell(25)
+>>> s.shares
+50
+>>>
+```
+
+\[ [Solution](soln3_2.md) | [Index](index.md) | [Exercise 3.1](ex3_1.md) | [Exercise 3.3](ex3_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_3.md

@@ -0,0 +1,234 @@
+\[ [Index](index.md) | [Exercise 3.2](ex3_2.md) | [Exercise 3.4](ex3_4.md) \]
+
+# Exercise 3.3
+
+*Objectives:*
+
+- Learn about class variables and class methods
+
+*Files Modified:* `stock.py`, `reader.py`
+
+Instances of the `Stock` class defined in the previous exercise are
+normally created as follows:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>>
+```
+
+However, the `read_portfolio()` function also creates instances from rows
+of data read from files.  For example, code such as the following is used:
+
+```python
+>>> import csv
+>>> f = open('Data/portfolio.csv')
+>>> rows = csv.reader(f)
+>>> headers = next(rows)
+>>> row = next(rows)
+>>> row
+['AA', '100', '32.20']
+>>> s = Stock(row[0], int(row[1]), float(row[2]))
+>>>
+```
+
+## (a) Alternate constructors
+
+Perhaps the creation of a `Stock` from a row of raw data is better handled
+by an alternate constructor.   Modify the `Stock` class so that it has
+a `types` class variable and `from_row()` class method like this:
+
+```python
+class Stock:
+    types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls.types, row)]
+        return cls(*values)
+    ...
+```
+
+Here's how to test it:
+
+```python
+>>> row = ['AA', '100', '32.20']
+>>> s = Stock.from_row(row)
+>>> s.name
+'AA'
+>>> s.shares
+100
+>>> s.price
+32.2
+>>> s.cost()
+3220.0000000000005
+>>>
+```
+
+Carefully observe how the string values in the row have been converted to a proper type.
+
+## (b) Class variables and inheritance
+
+Class variables such as `types` are sometimes used as a customization mechanism
+when inheritance is used.  For example, in the `Stock` class, the types can be
+easily changed in a subclass.  Try this example which changes the `price` attribute
+to a `Decimal` instance (which is often better suited to financial calculations):
+
+```python
+>>> from decimal import Decimal
+>>> class DStock(Stock):
+        types = (str, int, Decimal)
+
+>>> row = ['AA', '100', '32.20']
+>>> s = DStock.from_row(row)
+>>> s.price
+Decimal('32.20')
+>>> s.cost()
+Decimal('3220.0')
+>>>
+```
+
+**Design Discussion**
+
+The problem being addressed in this exercise concerns the conversion of data read
+from a file.  Would it make sense to perform these conversions in the `__init__()` 
+method of the `Stock` class instead?  For example:
+
+```python
+class Stock:
+    def __init__(self, name, shares, price):
+        self.name = str(name)
+        self.shares = int(shares)
+        self.price = float(price)
+```
+
+By doing this, you would convert a row of data as follows:
+
+```python
+>>> row = ['AA', '100', '32.20']
+>>> s = Stock(*row)
+>>> s.name
+'AA'
+>>> s.shares
+100
+>>> s.price
+32.2
+>>>
+```
+
+Is this good or bad?  What are your thoughts?  On the whole, I think
+it's a questionable design since it mixes two different things
+together--the creation of an instance and the conversion of data.
+Plus, the implicit conversions in `__init__()` limit flexibility and
+might introduce weird bugs if a user isn't paying careful attention.
+
+## (c) Reading Objects
+
+Change the `read_portfolio()` function to use the new `Stock.from_row()`
+method that you wrote. 
+
+Point of thought:  Which version of code do you like better?  The
+version of `read_portfolio()` that performed the type conversions or
+the one that performs conversions in the `Stock.from_row()` method?
+Think about data abstraction.
+
+Can you modify `read_portfolio()` to create objects using a class other
+than `Stock`?  For example, can the user select the class that they want?  
+
+## (d) Generalizing
+
+A useful feature of class-methods is that you can use them to put a
+highly uniform instance creation interface on a wide variety of
+classes and write general purpose utility functions that use them.
+
+Earlier, you created a file `reader.py` that had some functions for
+reading CSV data.  Add the following `read_csv_as_instances()`
+function to the file which accepts a class as input and uses the class
+`from_row()` method to create a list of instances:
+
+```python
+# reader.py
+...
+
+def read_csv_as_instances(filename, cls):
+    '''
+    Read a CSV file into a list of instances
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            records.append(cls.from_row(row))
+    return records
+```
+
+Get rid of the `read_portfolio()` function--you don't need that anymore.  If you want to
+read a list of `Stock` objects, do this:
+
+```python
+>>> # Read a portfolio of Stock instances
+>>> from reader import read_csv_as_instances
+>>> from stock import Stock
+>>> portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+>>> portfolio
+[<__main__.Stock object at 0x100674748>, 
+<__main__.Stock object at 0x1006746d8>, 
+<__main__.Stock object at 0x1006747b8>, 
+<__main__.Stock object at 0x100674828>, 
+<__main__.Stock object at 0x100674898>, 
+<__main__.Stock object at 0x100674908>, 
+<__main__.Stock object at 0x100674978>]
+>>>
+```
+
+Here is another example of how you might use `read_csv_as_instances()` with a completely different class:
+
+```python
+>>> class Row:
+         def __init__(self, route, date, daytype, numrides):
+             self.route = route
+             self.date = date
+             self.daytype = daytype
+             self.numrides = numrides
+         @classmethod
+         def from_row(cls, row):
+             return cls(row[0], row[1], row[2], int(row[3]))
+
+>>> rides = read_csv_as_instances('Data/ctabus.csv', Row)
+>>> len(rides)
+577563
+>>>
+```
+
+**Discussion**
+
+This exercise illustrates the two most common uses of class variables
+and class methods.  Class variables are often used to hold a global
+parameter (e.g., a configuration setting) that is shared across all
+instances.  Sometimes subclasses will inherit from the base class and
+override the setting to change behavior.
+
+Class methods are most commonly used to implement alternate
+constructors as shown.  A common way to spot such class methods is to
+look for the word "from" in the name.  For example, here is an example
+on built-in dictionaries:
+
+```python
+>>> d = dict.fromkeys(['a','b','c'], 0)     # class method
+>>> d
+{'a': 0, 'c': 0, 'b': 0}
+>>>
+```
+
+\[ [Solution](soln3_3.md) | [Index](index.md) | [Exercise 3.2](ex3_2.md) | [Exercise 3.4](ex3_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_4.md

@@ -0,0 +1,136 @@
+\[ [Index](index.md) | [Exercise 3.3](ex3_3.md) | [Exercise 3.5](ex3_5.md) \]
+
+# Exercise 3.4
+
+*Objectives:*
+
+- Learn how to encapsulate object internals using private
+  attributes, properties, and slots
+
+*Files Modified:* `stock.py`
+
+## (a) Private attributes
+
+As a general rule, attributes that are internal to a class should have a leading underscore.
+In the previous exercise, the `Stock` class had a `types` class variable that was
+used for converting rows of data.  Change the code so that this variable has a leading
+underscore on it.
+
+## (b) Properties for computed attributes
+
+Earlier, you defined a class `Stock`.  For example:
+
+```python
+>>> s = Stock('GOOG',100,490.10)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s.price
+490.1
+>>> s.cost()
+49010.0
+>>>
+```
+
+Using a property, turn `cost()` into an attribute that no longer requires the parentheses.   For example:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.cost               # Property. Computes the cost
+49010.0
+>>> 
+```
+
+## (c) Enforcing Validation Rules
+
+Using properties and private attributes, modify the `shares` attribute
+of the `Stock` class so that it can only be assigned a non-negative
+integer value.    In addition, modify the `price` attribute so that it
+can only be assigned a non-negative floating point value.
+
+The new object should work almost exactly the same as
+the old one except for extra type and value checking.
+
+```python
+>>> s = Stock('GOOG', 100, 490.10)
+>>> s.shares = 50          # OK
+>>> s.shares = '50'
+Traceback (most recent call last):
+...
+TypeError: Expected integer
+>>> s.shares = -10
+Traceback (most recent call last):
+...
+ValueError: shares must be >= 0
+
+>>> s.price = 123.45       # OK
+>>> s.price = '123.45'
+Traceback (most recent call last):
+...
+TypeError: Expected float
+>>> s.price = -10.0
+Traceback (most recent call last):
+...
+ValueError: price must be >= 0
+>>>
+```
+
+## (d) Adding `__slots__`
+
+Modify your new `Stock` class to use `__slots__`.  You will find that
+you have to use a different set of attribute names than
+before--specifically, you will have to list the private attribute
+names (e.g., if a property is storing a value in an attribute
+`_shares`, that is the name you list in `__slots__`).  Verify that the
+class still works and that you can no longer add new attributes.
+
+```python
+>>> s = Stock('GOOG', 100, 490.10)
+>>> s.spam = 42
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+AttributeError: 'Stock' object has no attribute 'spam'
+>>>
+```
+
+## (e) Reconciling Types
+
+In the current `Stock` class, there is a `_types` class variable
+that gives conversions when reading from a file, but there are also
+properties that are enforcing types.  Who is in charge of this show?
+Fix the property definitions so that they use the types specified
+in the `_types` class variable.  Make sure the properties work
+when types are changed via subclassing.   For example:
+
+```python
+>>> from decimal import Decimal
+>>> class DStock(Stock):
+        _types = (str, int, Decimal)
+
+>>> s = DStock('AA', 50, Decimal('91.1'))
+>>> s.price = 92.3
+Traceback (most recent call last):
+...
+TypeError: Expected a Decimal
+>>> 
+```
+
+**Discussion**
+
+The resulting `Stock` class at the end of this exercise is a muddled
+mess of properties, type checking, constructors, and other details.
+Imagine how unpleasant it would be to maintain code that featured
+dozens or hundreds of such class definitions.
+
+We're going to figure out how to simplify things considerably, but it's
+going to take some time and some more advanced techniques. Stay tuned.
+
+\[ [Solution](soln3_4.md) | [Index](index.md) | [Exercise 3.3](ex3_3.md) | [Exercise 3.5](ex3_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_5.md

@@ -0,0 +1,197 @@
+\[ [Index](index.md) | [Exercise 3.4](ex3_4.md) | [Exercise 3.6](ex3_6.md) \]
+
+# Exercise 3.5
+
+*Objectives:*
+
+- Learn how to use inheritance to write extensible code.
+- See a practical use of inheritance by writing a program that must
+output data in a variety of user-selectable formats such as plain-text,
+CSV, and HTML.
+
+*Files Modified:* `tableformat.py`
+
+One major use of classes in Python is in writing code that be
+extended/adapted in various ways.  To illustrate, in
+link:ex3_2.html[Exercise 3.2] you created a function `print_table()`
+that made tables.  You used this to make output from the `portfolio`
+list. For example:
+
+```python
+>>> import stock
+>>> import reader
+>>> import tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> tableformat.print_table(portfolio, ['name','shares','price'])
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100       32.2
+       IBM         50       91.1
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50       65.1
+       IBM        100      70.44
+>>> 
+```
+
+Suppose you wanted the `print_table()` function to be able to
+make tables in any number of output formats such as CSV, XML, HTML,
+Excel, etc.  Trying to modify the function to support all of those
+output formats at once would be painful.  A better way to do this
+involves moving the output-related formatting code to a class and using
+inheritance to implement different output formats.
+
+## (a) Defining a generic formatter class
+
+Add the following class definition to the `tableformat.py` file:
+
+```python
+class TableFormatter:
+    def headings(self, headers):
+        raise NotImplementedError()
+
+    def row(self, rowdata):
+        raise NotImplementedError()
+```
+
+Now, modify the `print_table()` function so that it accepts a `TableFormatter` instance
+and invokes methods on it to produce output:
+
+```python
+def print_table(records, fields, formatter):
+    formatter.headings(fields)
+    for r in records:
+        rowdata = [getattr(r, fieldname) for fieldname in fields]
+        formatter.row(rowdata)
+```
+
+These two classes are meant to be used together.  For example:
+
+```
+>>> import stock, reader, tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> formatter = tableformat.TableFormatter()
+>>> tableformat.print_table(portfolio, ['name', 'shares', 'price'], formatter)
+Traceback (most recent call last):
+...
+NotImplementedError
+>>>
+```
+
+For now, it doesn't do much of anything interesting.  You'll fix this in the next section.
+
+## (b) Implementing a concrete formatter
+
+The `TableFormatter` isn't meant to be used by itself. Instead, it is merely a base 
+for other classes that will implement the formatting.  Add the following class to
+`tableformat.py`:
+
+```python
+class TextTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print(' '.join('%10s' % h for h in headers))
+        print(('-'*10 + ' ')*len(headers))
+    
+    def row(self, rowdata):
+        print(' '.join('%10s' % d for d in rowdata))
+```
+
+Now, use your new class as follows:
+
+```python
+>>> import stock, reader, tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> formatter = tableformat.TextTableFormatter()
+>>> tableformat.print_table(portfolio, ['name','shares','price'], formatter)
+     name     shares      price
+---------- ---------- ---------- 
+        AA        100       32.2
+       IBM         50       91.1
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50       65.1
+       IBM        100      70.44
+>>> 
+```
+
+## (c) Adding More Implementations
+
+Create a class `CSVTableFormatter` that allows output to be generated in CSV format:
+
+```python
+>>> import stock, reader, tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> formatter = tableformat.CSVTableFormatter()
+>>> tableformat.print_table(portfolio, ['name','shares','price'], formatter)
+name,shares,price
+AA,100,32.2
+IBM,50,91.1
+CAT,150,83.44
+MSFT,200,51.23
+GE,95,40.37
+MSFT,50,65.1
+IBM,100,70.44
+>>>
+```
+
+Create a class `HTMLTableFormatter` that generates output in HTML format:
+
+```python
+>>> import stock, reader, tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> formatter = tableformat.HTMLTableFormatter()
+>>> tableformat.print_table(portfolio, ['name','shares','price'], formatter)
+<tr> <th>name</th> <th>shares</th> <th>price</th> </tr>
+<tr> <td>AA</td> <td>100</td> <td>32.2</td> </tr>
+<tr> <td>IBM</td> <td>50</td> <td>91.1</td> </tr>
+<tr> <td>CAT</td> <td>150</td> <td>83.44</td> </tr>
+<tr> <td>MSFT</td> <td>200</td> <td>51.23</td> </tr>
+<tr> <td>GE</td> <td>95</td> <td>40.37</td> </tr>
+<tr> <td>MSFT</td> <td>50</td> <td>65.1</td> </tr>
+<tr> <td>IBM</td> <td>100</td> <td>70.44</td> </tr>
+>>>
+```
+
+## (d) Making it Easier To Choose
+
+One problem with using inheritance is the added complexity of picking
+different classes to use (e.g., remembering the names, using the right
+`import` statements, etc.).   A factory function can simplify this.  Add
+a function `create_formatter()` to your `tableformat.py` file that 
+allows a user to more easily make a formatter by specifying a format such as `'text'`, `'csv'`, or `'html'`.   For example:
+
+```python
+>>> from tableformat import create_formatter, print_table
+>>> formatter = create_formatter('html')
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+<tr> <th>name</th> <th>shares</th> <th>price</th> </tr>
+<tr> <td>AA</td> <td>100</td> <td>32.2</td> </tr>
+<tr> <td>IBM</td> <td>50</td> <td>91.1</td> </tr>
+<tr> <td>CAT</td> <td>150</td> <td>83.44</td> </tr>
+<tr> <td>MSFT</td> <td>200</td> <td>51.23</td> </tr>
+<tr> <td>GE</td> <td>95</td> <td>40.37</td> </tr>
+<tr> <td>MSFT</td> <td>50</td> <td>65.1</td> </tr>
+<tr> <td>IBM</td> <td>100</td> <td>70.44</td> </tr>
+>>>
+```
+
+**Discussion**
+
+The `TableFormatter` class in this exercise is an example of something known
+as an "Abstract Base Class."   It's not something that's meant to be used directly.
+Instead, it's serving as a kind of interface specification for a program component--in
+this case the various output formats.   Essentially, the code that produces the table
+will be programmed against the abstract base class with the expectation that a user
+will provide a suitable implementation.  As long as all of the required methods
+have been implemented, it should all just "work" (fingers crossed).
+
+\[ [Solution](soln3_5.md) | [Index](index.md) | [Exercise 3.4](ex3_4.md) | [Exercise 3.6](ex3_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_6.md

@@ -0,0 +1,173 @@
+\[ [Index](index.md) | [Exercise 3.5](ex3_5.md) | [Exercise 3.7](ex3_7.md) \]
+
+# Exercise 3.6
+
+*Objectives:*
+
+- Learn how to customize the behavior of objects by redefining special methods.
+- Change the way that user-defined objects get printed
+- Make objects comparable
+- Create a context manager
+
+*Files Created:* None
+
+*Files Modified:* `stock.py`
+
+## (a) Better output for representing objects
+
+All Python objects have two string representations.  The first
+representation is created by string conversion via `str()`
+(which is called by `print`).   The string representation is
+usually a nicely formatted version of the object meant for humans.
+The second representation is a code representation of the object
+created by `repr()` (or simply by viewing a value in the
+interactive shell).   The code representation typically shows you the
+code that you have to type to get the object.  Here is an example
+that illustrates using dates:
+
+```python
+>>> from datetime import date
+>>> d = date(2008, 7, 5)
+>>> print(d)              # uses str()
+2008-07-05
+>>> d    # uses repr()
+datetime.date(2008, 7, 5)
+>>>
+```
+
+There are several techniques for obtaining the `repr()` string
+in output:
+
+```python
+>>> print('The date is', repr(d))
+The date is datetime.date(2008, 7, 5)
+>>> print(f'The date is {d!r}')
+The date is datetime.date(2008, 7, 5)
+>>> print('The date is %r' % d)
+The date is datetime.date(2008, 7, 5)
+>>> 
+```
+
+Modify the `Stock` object that you've created so that
+the `__repr__()` method
+produces more useful output.  For example:
+
+```python
+>>> goog = Stock('GOOG', 100, 490.10)
+>>> goog
+Stock('GOOG', 100, 490.1)
+>>>
+```
+
+See what happens when you read a portfolio of stocks and view the
+resulting list after you have made these changes.  For example:
+
+```python
+>>> import stock, reader
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> portfolio
+[Stock('AA', 100, 32.2), Stock('IBM', 50, 91.1), Stock('CAT', 150, 83.44), Stock('MSFT', 200, 51.23), 
+ Stock('GE', 95, 40.37), Stock('MSFT', 50, 65.1), Stock('IBM', 100, 70.44)]
+>>>
+```
+
+## (b) Making objects comparable
+
+What happens if you create two identical `Stock` objects and try to compare them? Find out:
+
+```python
+>>> a = Stock('GOOG', 100, 490.1)
+>>> b = Stock('GOOG', 100, 490.1)
+>>> a == b
+False
+>>>
+```
+
+You can fix this by giving the `Stock` class an `__eq__()` method.  For example:
+
+```python
+class Stock:
+    ...
+    def __eq__(self, other):
+        return isinstance(other, Stock) and ((self.name, self.shares, self.price) == 
+                                             (other.name, other.shares, other.price))
+    ...
+```
+
+Make this change and try comparing two objects again. 
+
+## (c) A Context Manager
+
+In link:ex3_5.html[Exercise 3.5], you made it possible for users to make
+nicely formatted tables.  For example:
+
+```python
+>>> from tableformat import create_formatter
+>>> formatter = create_formatter('text')
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100       32.2
+       IBM         50       91.1
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50       65.1
+       IBM        100      70.44
+>>>
+```
+
+One issue with the code is that all tables are printed to standard out
+(`sys.stdout`).  Suppose you wanted to redirect the output to a file
+or some other location.  In the big picture, you might modify all of
+the table formatting code to allow a different output file.  However,
+in a pinch, you could also solve this with a context manager.
+
+Define the following context manager:
+
+```python
+>>> import sys
+>>> class redirect_stdout:
+        def __init__(self, out_file):
+            self.out_file = out_file
+        def __enter__(self):
+            self.stdout = sys.stdout
+            sys.stdout = self.out_file
+            return self.out_file
+        def __exit__(self, ty, val, tb):
+            sys.stdout = self.stdout
+```
+
+This context manager works by making a temporary patch to `sys.stdout` to cause
+all output to redirect to a different file.  On exit, the patch is reverted. 
+Try it out:
+
+```python
+>>> from tableformat import create_formatter
+>>> formatter = create_formatter('text')
+>>> with redirect_stdout(open('out.txt', 'w')) as file:
+        tableformat.print_table(portfolio, ['name','shares','price', formatter)
+        file.close()
+
+>>> # Inspect the file
+>>> print(open('out.txt').read())
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100       32.2
+       IBM         50       91.1
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50       65.1
+       IBM        100      70.44
+>>>
+```
+
+\[ [Solution](soln3_6.md) | [Index](index.md) | [Exercise 3.5](ex3_5.md) | [Exercise 3.7](ex3_7.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_7.md

@@ -0,0 +1,143 @@
+\[ [Index](index.md) | [Exercise 3.6](ex3_6.md) | [Exercise 3.8](ex3_8.md) \]
+
+# Exercise 3.7
+
+*Objectives:*
+
+- Type checking and interfaces
+- Abstract base classes
+
+*Files Modified:* `tableformat.py`
+
+In link:ex3_5.html[Exercise 3.5], we modified the `tableformat.py` file to have a `TableFormatter`
+class and to use various subclasses for different output formats.  In this exercise, we extend that
+code a bit more.
+
+## (a) Interfaces and type checking
+
+Modify the `print_table()` function  so that it checks if the
+supplied formatter instance inherits from `TableFormatter`.  If
+not, raise a `TypeError`.
+
+Your new code should catch situations like this:
+
+```python
+>>> import stock, reader, tableformat
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> class MyFormatter:
+        def headings(self,headers): pass
+        def row(self,rowdata): pass
+
+>>> tableformat.print_table(portfolio, ['name','shares','price'], MyFormatter())
+Traceback (most recent call last):
+...
+TypeError: Expected a TableFormatter
+>>> 
+```
+
+Adding a check like this might add some degree of safety to the program.   However you should
+still be aware that type-checking is rather weak in Python.  There is no guarantee that the
+object passed as a formatter will work correctly even if it happens to inherit from the
+proper base class.  This next part addresses that issue.
+
+## (b) Abstract Base Classes
+
+Modify the `TableFormatter` base class so that it is defined as a proper
+abstract base class using the `abc` module.   Once you have done that, try
+this experiment:
+
+```python
+>>> class NewFormatter(TableFormatter):
+        def headers(self, headings):
+            pass
+        def row(self, rowdata):
+            pass
+
+>>> f = NewFormatter()
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: Can't instantiate abstract class NewFormatter with abstract methods headings
+>>> 
+```
+
+Here, the abstract base class caught a spelling error in the class--the fact that
+the `headings()` method was incorrectly given as `headers()`.
+
+## (c) Algorithm Template Classes
+
+The file `reader.py` contains two functions, `read_csv_as_dicts()` and `read_csv_as_instances()`.
+Both of those functions are almost identical--there is just one tiny bit of code that's
+different.  Maybe that code could be consolidated into a class definition of some sort.
+Add the following class to the `reader.py` file:
+
+```python
+# reader.py
+
+
+import csv
+from abc import ABC, abstractmethod
+
+class CSVParser(ABC):
+
+    def parse(self, filename):
+        records = []
+        with open(filename) as f:
+            rows = csv.reader(f)
+            headers = next(rows)
+            for row in rows:
+                record = self.make_record(headers, row)
+                records.append(record)
+        return records
+
+    @abstractmethod
+    def make_record(self, headers, row):
+        pass
+```
+
+This code provides a shell (or template) of the CSV parsing functionality.   To use it, you subclass it, add
+any additional attributes you might need, and redefine the `make_record()` method. For example:
+
+```python
+class DictCSVParser(CSVParser):
+    def __init__(self, types):
+        self.types = types
+
+    def make_record(self, headers, row):
+        return { name: func(val) for name, func, val in zip(headers, self.types, row) }
+
+class InstanceCSVParser(CSVParser):
+    def __init__(self, cls):
+        self.cls = cls
+
+    def make_record(self, headers, row):
+        return self.cls.from_row(row)
+```
+
+Add the above classes to the `reader.py` file.  Here's how you would use one of them:
+
+```python
+>>> from reader import DictCSVParser
+>>> parser = DictCSVParser([str, int, float])
+>>> port = parser.parse('Data/portfolio.csv')
+>>>
+```
+
+It works, but it's kind of annoying.  To fix this, reimplement the `read_csv_as_dicts()` and
+`read_csv_as_instances()` functions to use these classes.   Your refactored code should work
+exactly the same way that it did before.  For example:
+
+```python
+>>> import reader
+>>> import stock
+>>> port = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>>
+```
+
+\[ [Solution](soln3_7.md) | [Index](index.md) | [Exercise 3.6](ex3_6.md) | [Exercise 3.8](ex3_8.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex3_8.md

@@ -0,0 +1,249 @@
+\[ [Index](index.md) | [Exercise 3.7](ex3_7.md) | [Exercise 4.1](ex4_1.md) \]
+
+# Exercise 3.8
+
+*Objectives:*
+
+- Learn about mixin classes and cooperative inheritance
+
+*Files Modified:* `tableformat.py`
+
+## (a) The Trouble with Column Formatting
+
+If you go all the way back to link:ex3_1.txt[Exercise 3.1], you 
+wrote a function `print_portfolio()` that produced a table like this:
+
+```python
+>>> portfolio = read_portfolio('Data/portfolio.csv')
+>>> print_portfolio(portfolio)
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+>>>
+```
+
+The `print_table()` function developed in the last several exercises
+almost replaces this functionality--almost.  The one problem that it
+has is that it can't precisely format the content of each column.  For
+example, notice how the values in the `price` column are precisely
+formatted with 2 decimal points.  The `TableFormatter` class and
+related subclasses can't do that.
+
+One way to fix it would be to modify the `print_table()` function to
+accept an additional formats argument.  For example, maybe something
+like this:
+
+```python
+>>> def print_table(records, fields, formats, formatter):
+        formatter.headings(fields)
+        for r in records:
+            rowdata = [(fmt % getattr(r, fieldname))
+	               for fieldname,fmt in zip(fields,formats)]
+            formatter.row(rowdata)
+
+>>> import stock, reader
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> from tableformat import TextTableFormatter
+>>> formatter = TextTableFormatter()
+>>> print_table(portfolio, 
+                ['name','shares','price'],
+                ['%s','%d','%0.2f'],
+                formatter)
+
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+>>>
+```
+
+Yes, you could modify `print_table()` like this, but is that the right
+place to do it?  The whole idea of all of the `TableFormatter` classes
+is that they could be used in different kinds of applications.  Column
+formatting is something that could be useful elsewhere, not just
+in the `print_table()` function.
+
+Another possible approach might be to change the interface to the
+`TableFormatter` class in some way.   For example, maybe adding a third
+method to apply formatting.
+
+```python
+class TableFormatter:
+    def headings(self, headers):
+        ...
+    def format(self, rowdata):
+        ...
+    def row(self, rowdata):
+        ...
+```
+
+The problem here is that any time you change the interface on a class,
+you're going to have to refactor all of the existing code to work with
+it.  Specifically, you'd have to modify all of the already written
+`TableFormatter` subclasses and all of the code written to use them.
+Let's not do that.
+
+As an alternative, a user could use inheritance to customize a
+specific formatter in order to inject some formatting into it. For
+example, try this experiment:
+
+```python
+>>> from tableformat import TextTableFormatter, print_table
+>>> class PortfolioFormatter(TextTableFormatter):
+        def row(self, rowdata):
+            formats = ['%s','%d','%0.2f']
+            rowdata = [(fmt % d) for fmt, d in zip(formats, rowdata)]
+            super().row(rowdata)
+
+>>> formatter = PortfolioFormatter()
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+>>>
+```
+Yes, that works, but it's also a bit clumsy and weird.  The user has
+to pick a specific formatter to customize.  On top of that, they have
+to implement the actual column formatting code themselves.  Surely
+there is a different way to do this.
+
+## (b) Going Sideways
+
+In the `tableformat.py` file, add the following class definition:
+
+```python
+class ColumnFormatMixin:
+    formats = []
+    def row(self, rowdata):
+        rowdata = [(fmt % d) for fmt, d in zip(self.formats, rowdata)]
+        super().row(rowdata)
+```
+
+This class contains a single method `row()` that applies formatting to
+the row contents.  A class variable `formats` is used to hold the format
+codes.  This class is used via multiple inheritance.  For example:
+
+```python
+>>> import stock, reader
+>>> portfolio = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> from tableformat import TextTableFormatter, ColumnFormatMixin, print_table
+>>> class PortfolioFormatter(ColumnFormatMixin, TextTableFormatter):
+        formats = ['%s', '%d', '%0.2f']
+
+>>> formatter = PortfolioFormatter()
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      name     shares      price
+---------- ---------- ---------- 
+        AA        100      32.20
+       IBM         50      91.10
+       CAT        150      83.44
+      MSFT        200      51.23
+        GE         95      40.37
+      MSFT         50      65.10
+       IBM        100      70.44
+```
+
+This whole approach works because the `ColumnFormatMixin` class is
+meant to be mixed together with another class that provides the
+required `row()` method.   
+
+Make another class that makes a formatter print the table headers in all-caps:
+
+```python
+class UpperHeadersMixin:
+    def headings(self, headers):
+        super().headings([h.upper() for h in headers])
+```
+
+Try it out and notice that the headers are now uppercase:
+
+```python
+>>> from tableformat import TextTableFormatter, UpperHeadersMixin
+>>> class PortfolioFormatter(UpperHeadersMixin, TextTableFormatter):
+        pass
+
+>>> formatter = PortfolioFormatter()
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      NAME     SHARES      PRICE 
+---------- ---------- ---------- 
+        AA        100       32.2 
+       IBM         50       91.1 
+       CAT        150      83.44 
+      MSFT        200      51.23 
+        GE         95      40.37 
+      MSFT         50       65.1 
+       IBM        100      70.44 
+>>>
+```
+
+This is really the whole idea on "mixins."  The creator of a library
+can provide a basic set of classes such as `TextTableFormatter`,
+`CSVTableFormatter`, and so forth to start.  Then, a collection of
+add-on classes can be provided to make those classes behave in
+different ways.
+
+## (c) Making it Sane
+
+Using mixins can be a useful tool for framework builders for reducing 
+the amount of code that needs to be written.  However, forcing users
+to remember how to properly compose classes and use multiple inheritance can
+fry their brains.  In link:ex3_5.html[Exercise 3.5], you wrote a 
+function `create_formatter()` that made it easier to create a custom
+formatter.  Take that function and extend it to understand a few optional
+arguments related to the mixin classes. For example:
+
+```python
+>>> from tableformat import create_formatter
+>>> formatter = create_formatter('csv', column_formats=['"%s"','%d','%0.2f'])
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+name,shares,price
+"AA",100,32.20
+"IBM",50,91.10
+"CAT",150,83.44
+"MSFT",200,51.23
+"GE",95,40.37
+"MSFT",50,65.10
+"IBM",100,70.44
+
+>>> formatter = create_formatter('text', upper_headers=True)
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      NAME     SHARES      PRICE 
+---------- ---------- ---------- 
+        AA        100       32.2 
+       IBM         50       91.1 
+       CAT        150      83.44 
+      MSFT        200      51.23 
+        GE         95      40.37 
+      MSFT         50       65.1 
+       IBM        100      70.44 
+>>>
+```
+
+Under the covers the `create_formatter()` function will properly compose
+the classes and return a proper `TableFormatter` instance.
+
+\[ [Solution](soln3_8.md) | [Index](index.md) | [Exercise 3.7](ex3_7.md) | [Exercise 4.1](ex4_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex4_1.md

@@ -0,0 +1,167 @@
+\[ [Index](index.md) | [Exercise 3.8](ex3_8.md) | [Exercise 4.2](ex4_2.md) \]
+
+# Exercise 4.1
+
+*Objectives:*
+
+- Learn more about how objects are represented.
+- Learn how attribute assignment and lookup works.
+- Better understand the role of a class definition
+
+*Files Created:* None
+
+*Files Modified:* None
+
+Start this exercise, by going back to a simple version of the `Stock` class you created. 
+At the interactive prompt, define a 
+new class called `SimpleStock` that looks like this:
+
+```python
+>>> class SimpleStock:
+        def __init__(self, name, shares, price):
+            self.name = name
+            self.shares = shares
+            self.price = price
+        def cost(self):
+            return self.shares * self.price
+
+>>>
+```
+
+Once you have defined this class, create a few instances.
+
+```python
+>>> goog = SimpleStock('GOOG',100,490.10)
+>>> ibm  = SimpleStock('IBM',50, 91.23)
+>>>
+```
+
+## (a) Representation of Instances
+
+At the interactive shell, inspect the underlying dictionaries of the two instances you created:
+
+```python
+>>> goog.__dict__
+... look at the output ...
+>>> ibm.__dict__
+... look at the output ...
+>>>
+```
+
+## (b) Modification of Instance Data
+
+Try setting a new attribute on one of the above instances:
+
+```python
+>>> goog.date = "6/11/2007"
+>>> goog.__dict__
+... look at output ...
+>>> ibm.__dict__
+... look at output ...
+>>>
+```
+
+In the above output, you'll notice that the `goog` instance has
+a attribute `date` whereas the `ibm` instance does not.
+It is important to note that Python really doesn't place any
+restrictions on attributes.  For example, the attributes of an
+instance are not limited to those set up in the `__init__()`
+method.
+
+Instead of setting an attribute, try placing a new value directly into
+the `__dict__` object:
+
+```python
+>>> goog.__dict__['time'] = '9:45am'
+>>> goog.time
+'9:45am'
+>>>
+```
+
+Here, you really notice the fact that an instance is a layer on top of a dictionary.
+
+
+## (c) The role of classes
+
+The definitions that make up a class definition are shared by all
+instances of that class.  Notice, that all instances have a link back
+to their associated class:
+
+```python
+>>> goog.__class__
+... look at output ...
+>>> ibm.__class__
+... look at output ...
+>>> 
+```
+
+Try calling a method on the instances:
+
+```python
+>>> goog.cost()
+49010.0
+>>> ibm.cost()
+4561.5
+>>>
+```
+
+Notice that the name 'cost' is not defined in either `goog.__dict__` or `ibm.__dict__`.   Instead, it is being supplied by the
+class dictionary.  Try this:
+
+```python
+>>> SimpleStock.__dict__['cost']
+... look at output ...
+>>>
+```
+
+Try calling the `cost()` method directly through the dictionary:
+
+```python
+>>> SimpleStock.__dict__['cost'](goog)
+49010.00
+>>> SimpleStock.__dict__['cost'](ibm)
+4561.5
+>>>
+```
+
+Notice how you are calling the function defined in the class definition and how the `self` argument
+gets the instance.
+
+If you add a new value to the class, it becomes a class variable that's visible to all instances.  Try it:
+
+```python
+>>> SimpleStock.spam = 42
+>>> ibm.spam
+42
+>>> goog.spam
+42
+>>>
+```
+
+Observe that `spam` is not part of the instance dictionary.
+
+```python
+>>> ibm.__dict__
+... look at the output ...
+>>>
+```
+
+Instead, it's part of the class dictionary:
+
+```python
+>>> SimpleStock.__dict__['spam']
+42
+>>>
+```
+
+Essentially this is all a class really is--it's a collection of values shared by instances.
+
+
+\[ [Solution](soln4_1.md) | [Index](index.md) | [Exercise 3.8](ex3_8.md) | [Exercise 4.2](ex4_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex4_2.md

@@ -0,0 +1,288 @@
+\[ [Index](index.md) | [Exercise 4.1](ex4_1.md) | [Exercise 4.3](ex4_3.md) \]
+
+# Exercise 4.2
+
+*Objectives:*
+
+- Learn more about the behavior of inheritance
+- Understand the behavior of super().
+- More cooperative inheritance.
+
+*Files Created:* `validate.py`
+
+
+## (a) The directions of inheritance
+
+Python has two different "directions" of inheritance.  The first is
+found in the concept of "single inheritance" where a series
+of classes inherit from a single parent. For example, try this example:
+
+```python
+>>> class A:
+        def spam(self):
+            print('A.spam')
+
+>>> class B(A):
+        def spam(self):
+            print('B.spam')
+            super().spam()
+
+>>> class C(B):
+        def spam(self):
+            print('C.spam')
+            super().spam()
+
+
+>>> C.__mro__
+(<class '__main__.C'>, <class '__main__.B'>, <class '__main__.A'>, <class 'object'>)
+>>> c = C()
+>>> c.spam()
+C.spam
+B.spam
+A.spam
+>>> 
+```
+
+Observe that the `__mro__` attribute of class `C` encodes all of its ancestors in
+order.  When you invoke the `spam()` method, it walks the MRO class-by-class up
+the hierarchy.
+
+With multiple inheritance, you get a different kind of inheritance that
+allows different classes to be composed together.   Try this example:
+
+```
+>>> class Base:
+        def spam(self):
+            print('Base.spam')
+
+>>> class X(Base):
+        def spam(self):
+            print('X.spam')
+            super().spam()
+
+>>> class Y(Base):
+        def spam(self):
+            print('Y.spam')
+            super().spam()
+
+>>> class Z(Base):
+        def spam(self):
+            print('Z.spam')
+            super().spam()
+
+>>>
+```
+
+Notice that all of the classes above inherit from a common parent `Base`.
+However, the classes `X`, `Y`, and `Z` are not directly related 
+to each other (there is no inheritance chain linking those classes together).
+
+However, watch what happens in multiple inheritance:
+
+```python
+>>> class M(X,Y,Z):
+        pass
+
+>>> M.__mro__
+(<class '__main__.M'>, <class '__main__.X'>, <class '__main__.Y'>, <class '__main__.Z'>, <class '__main__.Base'>, <class 'object'>)
+>>> m = M()
+>>> m.spam()
+X.spam
+Y.spam
+Z.spam
+Base.spam
+>>>
+```
+
+Here, you see all of the classes stack together in the order supplied by the subclass.
+Suppose the subclass rearranges the class order:
+
+```python
+>>> class N(Z,Y,X):
+        pass
+
+>>> N.__mro__
+(<class '__main__.N'>, <class '__main__.Z'>, <class '__main__.Y'>, <class '__main__.X'>, <class '__main__.Base'>, <class 'object'>)
+>>> n = N()
+>>> n.spam()
+Z.spam
+Y.spam
+X.spam
+Base.spam
+>>>
+```
+
+Here, you see the order of the parents flip around.  Carefully pay attention to what `super()`
+is doing in both cases.  It doesn't delegate to the immediate parent of each class--instead,
+it moves to the next class on the MRO.   Not only that, the exact order is controlled
+by the child. This is pretty weird.
+
+Also notice that the common parent `Base` serves to terminate the chain of `super()` operations.
+Specifically, the `Base.spam()` method does not call any further methods. It also appears at
+the end of the MRO since it is the parent to all of the classes being composed together.
+
+## (b) Build a Value Checker
+
+In link:ex3_4.html[Exercise 3.4], you added some properties to the `Stock` class that
+checked attributes for different types and values (e.g., shares had to be a positive
+integer).  Let's play with that idea a bit.  Start by creating a file `validate.py` and
+defining the following base class:
+
+```python
+# validate.py
+class Validator:
+    @classmethod
+    def check(cls, value):
+        return value
+```
+
+Now, let's make some classes for type checking:
+
+```python
+class Typed(Validator):
+    expected_type = object
+    @classmethod
+    def check(cls, value):
+        if not isinstance(value, cls.expected_type):
+            raise TypeError(f'Expected {cls.expected_type}')
+        return super().check(value)
+
+class Integer(Typed):
+    expected_type = int
+
+class Float(Typed):
+    expected_type = float
+
+class String(Typed):
+    expected_type = str
+```
+
+Here's how you use these classes (Note: the use of `@classmethod` allows us to 
+avoid the extra step of creating instances which we don't really need):
+
+```python
+>>> Integer.check(10)
+10
+>>> Integer.check('10')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in check
+    raise TypeError(f'Expected {cls.expected_type}')
+TypeError: Expected <class 'int'>
+>>> String.check('10')
+'10'
+>>>
+```
+
+You could use the validators in a function. For example:
+
+```python
+>>> def add(x, y):
+        Integer.check(x)
+        Integer.check(y)
+        return x + y
+
+>>> add(2, 2)
+4
+>>> add('2', '3')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "<stdin>", line 2, in add
+  File "validate.py", line 11, in check
+    raise TypeError(f'Expected {cls.expected_type}')
+TypeError: Expected <class 'int'>
+>>>
+```
+
+Now, make some more classes for different kinds of domain checking:
+
+```python
+class Positive(Validator):
+    @classmethod
+    def check(cls, value):
+        if value < 0:
+            raise ValueError('Expected >= 0')
+        return super().check(value)
+
+class NonEmpty(Validator):
+    @classmethod
+    def check(cls, value):
+        if len(value) == 0:
+            raise ValueError('Must be non-empty')
+        return super().check(value)
+```
+
+Where is all of this going?   Let's start composing classes together with multiple inheritance like toy blocks:
+
+```python
+class PositiveInteger(Integer, Positive):
+    pass
+
+class PositiveFloat(Float, Positive):
+    pass
+
+class NonEmptyString(String, NonEmpty):
+    pass
+```
+
+Essentially, you're taking existing validators and composing them
+together into new ones. Madness!  However, let's use them to validate
+some things now:
+
+```python
+>>> PositiveInteger.check(10)
+10
+>>> PositiveInteger.check('10')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+    raise TypeError(f'Expected {cls.expected_type}')
+TypeError: Expected <class 'int'>
+>>> PositiveInteger.check(-10)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+    raise ValueError('Expected >= 0')
+ValueError: Must be >= 0
+
+
+>>> NonEmptyString.check('hello')
+'hello'
+>>> NonEmptyString.check('')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+    raise ValueError('Must be non-empty')
+ValueError: Must be non-empty
+>>>
+```
+
+At this point, your head is probably fully exploded.  However, the problem of composing
+different bits of code together is one that arises in real-world programs. Cooperative
+multiple inheritance is one of the tools that can be used to organize it. 
+
+## (c) Using your validators
+
+Your validators can be used to add value checking to functions and classes.  For
+example, perhaps the validators could be used in the properties of `Stock`:
+
+```python
+class Stock:
+    ...
+    @property
+    def shares(self):
+        return self._shares
+
+    @shares.setter
+    def shares(self, value):
+        self._shares = PositiveInteger.check(value)
+    ...
+```
+
+Copy the `Stock` class from `stock.py` change it to use the validators in the property
+code for `shares` and `price`.
+
+\[ [Solution](soln4_2.md) | [Index](index.md) | [Exercise 4.1](ex4_1.md) | [Exercise 4.3](ex4_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex4_3.md

@@ -0,0 +1,235 @@
+\[ [Index](index.md) | [Exercise 4.2](ex4_2.md) | [Exercise 4.4](ex4_4.md) \]
+
+# Exercise 4.3
+
+*Objectives:*
+
+- Learn about descriptors
+
+*Files Created:* `descrip.py`
+
+*Files Modified:* `validate.py`
+
+## (a) Descriptors in action
+
+Earlier, you created a class `Stock` that made use of
+slots, properties, and other features.   All of these features
+are implemented using the descriptor protocol.  See it in
+action by trying this simple experiment.
+
+First, create a stock object, and try looking up a few attributes:
+
+```python
+>>> s = Stock('GOOG', 100, 490.10)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>>
+```
+
+Now, notice that these attributes are in the class dictionary.
+
+```python
+>>> Stock.__dict__.keys()
+['sell', '__module__', '__weakref__', 'price', '_price', 'shares', '_shares',
+'__slots__', 'cost', '__repr__', '__doc__', '__init__']
+>>> 
+```
+
+Try these steps which illustrate how descriptors get and set values on an instance:
+
+```python
+>>> q = Stock.__dict__['shares']
+>>> q.__get__(s)
+100
+>>> q.__set__(s,75)
+>>> s.shares
+75
+>>> q.__set__(s, '75')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "stock.py", line 23, in shares
+    raise TypeError('Expected an integer')
+TypeError: Expected an integer
+>>>
+```
+
+The execution of `__get__()` and `__set__()` occurs automatically whenever you access instances.
+
+
+## (b) Make your own descriptor
+
+Define the descriptor class from the notes:
+
+```python
+# descrip.py
+
+class Descriptor:
+    def __init__(self, name):
+        self.name = name
+    def __get__(self, instance, cls):
+        print('%s:__get__' % self.name)
+    def __set__(self, instance, value):
+        print('%s:__set__ %s' % (self.name, value))
+    def __delete__(self, instance):
+        print('%s:__delete__' % self.name)
+```
+
+Now, try defining a simple class that uses this descriptor:
+
+```python
+>>> class Foo:
+        a = Descriptor('a')
+        b = Descriptor('b')
+        c = Descriptor('c')
+
+>>> f = Foo()
+>>> f
+<__main__.Foo object at 0x38e130> <class __main__.Foo>
+>>> f.a
+a:__get__
+>>> f.b
+b:__get__
+>>> f.a = 23
+a:__set__ 23
+>>> del f.a
+a:__delete__
+>>>
+```
+
+Ponder the fact that you have captured the dot-operator for a
+specific attribute.
+
+## (c) From Validators to Descriptors
+
+In the previous exercise, you wrote a series of classes that could perform checking.
+For example:
+
+```python
+>>> PositiveInteger.check(10)
+10
+>>> PositiveInteger.check('10')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+    raise TypeError('Expected %s' % cls.expected_type)
+TypeError: expected <class 'int'>
+>>> PositiveInteger.check(-10)
+```
+
+You can extend this to descriptors by making a simple change to the `Validator` base
+class.  Change it to the following:
+
+```python
+# validate.py
+
+class Validator:
+    def __init__(self, name):
+        self.name = name
+
+    @classmethod
+    def check(cls, value):
+        return value
+
+    def __set__(self, instance,	value):
+        instance.__dict__[self.name] = self.check(value)
+```
+
+Note: The lack of the `__get__()` method in the descriptor means that
+Python will use its default implementation of attribute lookup.  This
+requires that the supplied name matches the name used in the instance
+dictionary.
+
+No other changes should be necessary.   Now, try modifying the `Stock` class to
+use the validators as descriptors like this:
+
+```python
+class Stock:
+    name   = String('name')
+    shares = PositiveInteger('shares')
+    price  = PositiveFloat('price')
+
+    def __init__(self,name,shares,price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+You'll find that your class works the same way as before, involves much
+less code, and gives you all of the desired checking:
+
+```python
+>>> s = Stock('GOOG', 100, 490.10)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s.shares = 75
+>>> s.shares = '75'
+... TypeError ...
+>>> s.shares = -50
+... ValueError ...
+>>>
+```
+
+This is pretty cool.  Descriptors have allowed you to greatly simplify the implementation
+of the `Stock` class.  This is the real power of descriptors--you get low level control
+over the dot and can use it to do amazing things.
+
+## (d) Fixing the Names
+
+One annoying thing about descriptors is the redundant name specification. For example:
+
+```python
+class Stock:
+    ...
+    shares = PositiveInteger('shares')
+    ...
+```
+
+We can fix that.  Change the top-level `Validator` class to include a `__set_name__()` method
+like this:
+
+```python
+# validate.py
+
+class Validator:
+    def __init__(self, name=None):
+        self.name = name
+
+    def __set_name__(self, cls, name):
+        self.name = name
+
+    @classmethod
+    def check(cls, value):
+        return value
+
+    def __set__(self, instance,	value):
+        instance.__dict__[self.name] = self.check(value)
+```
+
+Now, try rewriting your `Stock` class so that it looks like this:
+
+```python
+class Stock:
+    name   = String()
+    shares = PositiveInteger()
+    price  = PositiveFloat()
+
+    def __init__(self,name,shares,price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+Ah, much nicer.   Be aware that this ability to set the name is a Python 3.6
+feature however.  It won't work on older versions.
+
+\[ [Solution](soln4_3.md) | [Index](index.md) | [Exercise 4.2](ex4_2.md) | [Exercise 4.4](ex4_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex4_4.md

@@ -0,0 +1,157 @@
+\[ [Index](index.md) | [Exercise 4.3](ex4_3.md) | [Exercise 5.1](ex5_1.md) \]
+
+# Exercise 4.4
+
+*Objectives:*
+
+- Learn about customizing attribute access
+- Delegation vs. inheritance
+
+## (a) Slots vs. setattr
+
+In previous exercises, `__slots__` was used to list the instance
+attributes on a class.  The primary purpose of slots is to optimize
+the use of memory.  A secondary effect is that it strictly limits the
+allowed attributes to those listed.  A downside of slots is that it
+often interacts strangely with other parts of Python (for example,
+classes using slots can't be used with multiple inheritance).  For
+that reason, you really shouldn't use slots except in special cases.
+
+If you really wanted to limit the set of allowed attributes, an
+alternate way to do it would be to define a `__setattr__()` method.
+Try this experiment:
+
+```python
+>>> class Stock:
+        def __init__(self, name, shares, price):
+            self.name = name
+            self.shares = shares
+            self.price = price
+        def __setattr__(self, name, value):
+            if name not in { 'name', 'shares', 'price' }:
+                raise AttributeError('No attribute %s' % name)
+            super().__setattr__(name, value)
+
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name
+'GOOG'
+>>> s.shares = 75
+>>> s.share = 50
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "<stdin>", line 8, in __setattr__
+AttributeError: No attribute share
+>>> 
+```
+
+In this example, there are no slots, but the `__setattr__()` method still restricts
+attributes to those in a predefined set.   You'd probably need to
+think about how this approach might interact with inheritance (e.g., if subclasses wanted
+to add new attributes, they'd probably need to redefine `__setattr__()` to make it work).
+
+## (b) Proxies
+
+A proxy class is a class that wraps around an existing class and provides a similar interface.
+Define the following class which makes a read-only layer around an existing object:
+
+```python
+>>> class Readonly:
+        def __init__(self, obj):
+            self.__dict__['_obj'] = obj
+        def __setattr__(self, name, value):
+            raise AttributeError("Can't set attribute")
+        def __getattr__(self, name):
+            return getattr(self._obj, name)
+
+>>>
+```
+
+To use the class, you simply wrap it around an existing instance:
+
+```python
+>>> from stock import Stock
+>>> s = Stock('GOOG', 100, 490.1)
+>>> p = Readonly(s)
+>>> p.name
+'GOOG'
+>>> p.shares
+100
+>>> p.cost
+49010.0
+>>> p.shares = 50
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "<stdin>", line 8, in __setattr__
+AttributeError: Can't set attribute
+>>> 
+```
+
+## (c) Delegation as an alternative to inheritance
+
+Delegation is sometimes used as an alternative to inheritance. The idea is almost the
+same as the proxy class you defined in part (b).  Try defining the following class:
+
+```python
+>>> class Spam:
+        def a(self):
+            print('Spam.a')
+        def b(self):
+            print('Spam.b')
+
+>>>
+```
+
+Now, make a class that wraps around it and redefines some of the methods:
+
+```python
+>>> class MySpam:
+        def __init__(self):
+            self._spam = Spam()
+        def a(self):
+            print('MySpam.a')
+            self._spam.a()
+        def c(self):
+            print('MySpam.c')
+        def __getattr__(self, name):
+            return getattr(self._spam, name)
+
+>>> s = MySpam()
+>>> s.a()
+MySpam.a
+Spam.a
+>>> s.b()
+Spam.b
+>>> s.c()
+MySpam.c
+>>>
+```
+
+Carefully notice that the resulting class looks very similar to
+inheritance.  For example the `a()` method is doing something similar
+to the `super()` call.  The method `b()` is picked up via the
+`__getattr__()` method which delegates to the internally held `Spam`
+instance.
+
+**Discussion**
+
+The `__getattr__()` method is commonly defined on classes that act as
+wrappers around other objects.  However, you have to be aware that the
+process of wrapping another object in this manner often introduces
+other complexities.  For example, the wrapper object might break
+type-checking if any other part of the application is using the
+`isinstance()` function. 
+
+Delegating methods through `__getattr__()` also doesn't work with special
+methods such as `__getitem__()`, `__enter__()`, and so forth.   If a class
+makes extensive use of such methods, you'll have to provide similar functions
+in your wrapper class. 
+
+
+\[ [Solution](soln4_4.md) | [Index](index.md) | [Exercise 4.3](ex4_3.md) | [Exercise 5.1](ex5_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_1.md

@@ -0,0 +1,178 @@
+\[ [Index](index.md) | [Exercise 4.4](ex4_4.md) | [Exercise 5.2](ex5_2.md) \]
+
+# Exercise 5.1
+
+*Objectives:*
+
+- Explore a few definitional aspects of functions/methods
+- Making functions more flexible
+- Type hints
+
+In [Exercise 2.6](ex2_6.md) you wrote a `reader.py` module that
+had a function for reading a CSV into a list of dictionaries.  For example:
+
+```python
+>>> import reader
+>>> port = reader.read_csv_as_dicts('Data/portfolio.csv', [str,int,float])
+>>>
+```
+
+We later expanded to that code to work with instances in
+[Exercise 3.3](ex3_3.md): 
+
+```python
+>>> import reader
+>>> from stock import Stock
+>>> port = reader.read_csv_as_instances('Data/portfolio.csv', Stock)
+>>>
+```
+
+Eventually the code was refactored into a collection of classes
+involving inheritance in [Exercise 3.7](ex3_7.md).  However,
+the code has become rather complex and convoluted.
+
+## (a) Back to Basics
+
+Start by reverting the changes related to class definitions.  Rewrite
+the `reader.py` file so that it contains the two basic functions that
+you had before you messed it up with classes:
+
+```python
+# reader.py
+
+import csv
+
+def read_csv_as_dicts(filename, types):
+    '''
+    Read CSV data into a list of dictionaries with optional type conversion
+    '''
+    records = []
+    with open(filename) as file:
+        rows = csv.reader(file)
+        headers = next(rows)
+        for row in rows:
+            record = { name: func(val) 
+                       for name, func, val in zip(headers, types, row) }
+            records.append(record)
+    return records
+
+def read_csv_as_instances(filename, cls):
+    '''
+    Read CSV data into a list of instances
+    '''
+    records = []
+    with open(filename) as file:
+        rows = csv.reader(file)
+        headers = next(rows)
+        for row in rows:
+            record = cls.from_row(row)
+            records.append(record)
+    return records
+```
+
+Make sure the code still works as it did before:
+
+```python
+>>> import reader
+>>> port = reader.read_csv_as_dicts('Data/portfolio.csv', [str, int, float])
+>>> port
+[{'name': 'AA', 'shares': 100, 'price': 32.2}, {'name': 'IBM', 'shares': 50, 'price': 91.1}, 
+ {'name': 'CAT', 'shares': 150, 'price': 83.44}, {'name': 'MSFT', 'shares': 200, 'price': 51.23}, 
+ {'name': 'GE', 'shares': 95, 'price': 40.37}, {'name': 'MSFT', 'shares': 50, 'price': 65.1}, 
+ {'name': 'IBM', 'shares': 100, 'price': 70.44}]
+>>> import stock
+>>> port = reader.read_csv_as_instances('Data/portfolio.csv', stock.Stock)
+>>> port
+[Stock('AA', 100, 32.2), Stock('IBM', 50, 91.1), Stock('CAT', 150, 83.44), 
+ Stock('MSFT', 200, 51.23), Stock('GE', 95, 40.37), Stock('MSFT', 50, 65.1), 
+ Stock('IBM', 100, 70.44)]
+>>>
+```
+
+## (b) Thinking about Flexibility
+
+Right now, the two functions in `reader.py` are hard-wired to work
+with filenames that are passed directly to `open()`.  Refactor the
+code so that it works with any iterable object that produces lines.
+To do this, create two new functions `csv_as_dicts(lines, types)` and
+`csv_as_instances(lines, cls)` that convert any iterable sequence of
+lines.  For example:
+
+```python
+>>> file = open('Data/portfolio.csv')
+>>> port = reader.csv_as_dicts(file, [str, int, float])
+>>> port
+[{'name': 'AA', 'shares': 100, 'price': 32.2}, {'name': 'IBM', 'shares': 50, 'price': 91.1}, 
+ {'name': 'CAT', 'shares': 150, 'price': 83.44}, {'name': 'MSFT', 'shares': 200, 'price': 51.23}, 
+ {'name': 'GE', 'shares': 95, 'price': 40.37}, {'name': 'MSFT', 'shares': 50, 'price': 65.1}, 
+ {'name': 'IBM', 'shares': 100, 'price': 70.44}]
+>>>
+```
+
+The whole point of doing this is to make it possible to work with different
+kinds of input sources.  For example:
+
+```python
+>>> import gzip
+>>> import stock
+>>> file = gzip.open('Data/portfolio.csv.gz')
+>>> port = reader.csv_as_instances(file, stock.Stock)
+>>> port
+[Stock('AA', 100, 32.2), Stock('IBM', 50, 91.1), Stock('CAT', 150, 83.44), 
+ Stock('MSFT', 200, 51.23), Stock('GE', 95, 40.37), Stock('MSFT', 50, 65.1), 
+ Stock('IBM', 100, 70.44)]
+>>>
+```
+
+To maintain backwards compatibility with older code, write functions
+`read_csv_as_dicts()` and `read_csv_as_instances()` that take a
+filename as before.  These functions should call `open()` on the
+supplied filename and use the new `csv_as_dicts()` or
+`csv_as_instances()` functions on the resulting file.
+
+## (c) Design Challenge: CSV Headers
+
+The code assumes that the first line of CSV data always contains
+column headers.  However, this isn't always the case. For example, the
+file `Data/portfolio_noheader.csv` contains data, but no column
+headers.
+
+How would you refactor the code to accommodate missing column headers, having
+them supplied manually by the caller instead?
+
+## (d) API Challenge: Type hints
+
+Functions can have optional type-hints attached to arguments and return values.
+For example:
+
+```python
+def add(x:int, y:int) -> int:
+    return x + y
+```
+
+The `typing` module has additional classes for expressing more complex kinds of
+types including containers.  For example:
+
+```python
+from typing import List
+
+def sum_squares(nums: List[int]) -> int:
+    total = 0
+    for n in nums:
+        total += n*n
+    return total
+```
+
+Your challenge: Modify the code in `reader.py` so that all functions
+have type hints.  Try to make the type-hints as accurate as possible.
+To do this, you may need to consult the documentation for the
+[typing module](https://docs.python.org/3/library/typing.html).
+
+\[ [Solution](soln5_1.md) | [Index](index.md) | [Exercise 4.4](ex4_4.md) | [Exercise 5.2](ex5_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_2.md

@@ -0,0 +1,126 @@
+\[ [Index](index.md) | [Exercise 5.1](ex5_1.md) | [Exercise 5.3](ex5_3.md) \]
+
+# Exercise 5.2
+
+*Objectives:*
+
+- Returning values from functions
+
+In this exercise, we briefly look at problems related to returning values from functions.
+At first glance, it seems like this should be straightforward, but there are some
+subtle problems that arise.
+
+## (a) Returning Multiple Values
+
+Suppose you were writing code to parse configuration files consisting of lines like this:
+
+    name=value
+
+Write a function `parse_line(line)` that takes such a line and returns both the associated name and 
+value.  The common convention for returning multiple values is to return them in a tuple.  For example:
+
+```python
+>>> parse_line('email=guido@python.org')
+('email', 'guido@python.org')
+>>> name, val = parse_line('email=guido@python.org')
+>>> name
+'email'
+>>> val
+'guido@python.org'
+>>>
+```
+
+## (b) Returning Optional Values
+
+Sometimes a function might return an optional value--possibly as a mechanism for indicating 
+success or failure.  The most common convention is to use `None` as a representation for 
+a missing value.  Modify the `parse_line()` function above so that it either returns a tuple 
+on success or `None` on bad data.  For example:
+
+```python
+>>> parse_line('email=guido@python.org')
+('email', 'guido@python.org')
+>>> parse_line('spam')       # Returns None
+>>>
+```
+
+Design discussion:  Would it be better for the `parse_line()` function to raise an exception
+on malformed data?
+
+## (c) Futures
+
+Sometimes Python code executes concurrently via threads or processes.  To illustrate, try
+this example:
+
+```python
+>>> import time
+>>> def worker(x, y):
+        print('About to work')
+        time.sleep(20)
+        print('Done')
+        return x + y
+
+>>> worker(2, 3)     # Normal function call
+About to work  
+Done
+5
+>>>
+```
+
+Now, launch `worker()` into a separate thread:
+
+```python
+>>> import threading
+>>> t = threading.Thread(target=worker, args=(2, 3))
+>>> t.start()
+About to work
+>>>
+Done
+```
+
+Carefully notice that the result of the calculation appears nowhere. Not only that, you don't even
+know when it's going to be completed.   There is a certain coordination problem here. The
+convention for handling this case is to wrap the result of a function in a `Future`.  A
+`Future` represents a future result. Here's how it works:
+
+```python
+>>> from concurrent.futures import Future
+>>> # Wrapper around the function to use a future
+>>> def do_work(x, y, fut):
+        fut.set_result(worker(x,y))
+
+>>> fut = Future()
+>>> t = threading.Thread(target=do_work, args=(2, 3, fut))
+>>> t.start()
+About to work  
+>>> result = fut.result()
+Done
+>>> result
+5
+>>>
+```
+
+You'll see this kind of pattern a lot of if working with thread pools, processes, and other
+constructs.  For example:
+
+```python
+>>> from concurrent.futures import ThreadPoolExecutor
+>>> pool = ThreadPoolExecutor()
+>>> fut = pool.submit(worker, 2, 3)
+About to work
+>>> fut
+<Future at 0x102157080 state=running>
+>>> fut.result()
+Done
+5
+>>>
+```
+
+\[ [Solution](soln5_2.md) | [Index](index.md) | [Exercise 5.1](ex5_1.md) | [Exercise 5.3](ex5_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_3.md

@@ -0,0 +1,101 @@
+\[ [Index](index.md) | [Exercise 5.2](ex5_2.md) | [Exercise 5.4](ex5_4.md) \]
+
+# Exercise 5.3
+
+*Objectives:*
+
+- Higher order functions
+
+*Files Modified:* `reader.py`
+
+## (a) Using higher-order functions
+
+At the moment, the `reader.py` program consists of two core functions, `csv_as_dicts()` and
+`csv_as_instances()`.   The code in these two functions is almost identical.  For example:
+
+```python
+def csv_as_dicts(lines, types, *, headers=None):
+    '''
+    Convert lines of CSV data into a list of dictionaries
+    '''
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    for row in rows:
+        record = { name: func(val)
+                   for name, func, val in zip(headers, types, row) }
+        records.append(record)
+    return records
+
+def csv_as_instances(lines, cls, *, headers=None):
+    '''
+    Convert lines of CSV data into a list of instances
+    '''
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    for row in rows:
+        record = cls.from_row(row)
+        records.append(record)
+    return records
+```
+
+Unify the core of these functions into a single function `convert_csv()` that accepts a user-defined
+conversion function as an argument.  For example:
+
+```python
+>>> def make_dict(headers, row):
+        return dict(zip(headers, row))
+
+>>> lines = open('Data/portfolio.csv')
+>>> convert_csv(lines, make_dict)
+[{'name': 'AA', 'shares': '100', 'price': '32.20'}, {'name': 'IBM', 'shares': '50', 'price': '91.10'}, 
+ {'name': 'CAT', 'shares': '150', 'price': '83.44'}, {'name': 'MSFT', 'shares': '200', 'price': '51.23'}, 
+ {'name': 'GE', 'shares': '95', 'price': '40.37'}, {'name': 'MSFT', 'shares': '50', 'price': '65.10'}, 
+ {'name': 'IBM', 'shares': '100', 'price': '70.44'}]
+>>>
+```
+
+Rewrite the `csv_as_dicts()` and `csv_as_instances()` functions in terms of the new `convert_csv()`
+function.
+
+## (b) Mapping
+
+One of the most common operations in functional programming is the `map()` operation that maps a function
+to the values in a sequence.  Python has a built-in `map()` function that does this.  For
+example:
+
+```python
+>>> nums = [1,2,3,4]
+>>> squares = map(lambda x: x*x, nums)
+>>> for n in squares:
+        print(n)
+
+1
+4
+9
+16
+>>>
+```
+
+`map()` produces an iterator so if you want a list, you'll need to create it explicitly:
+
+```python
+>>> squares = list(map(lambda x: x*x, nums))
+>>> squares
+[1, 4, 9, 16]
+>>>
+```
+
+Try to use `map()` in your `convert_csv()` function.
+
+\[ [Solution](soln5_3.md) | [Index](index.md) | [Exercise 5.2](ex5_2.md) | [Exercise 5.4](ex5_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_4.md

@@ -0,0 +1,153 @@
+\[ [Index](index.md) | [Exercise 5.3](ex5_3.md) | [Exercise 5.5](ex5_5.md) \]
+
+# Exercise 5.4
+
+*Objectives:*
+
+- Learn more about closures
+
+In this section, we look briefly at a few of the more unusual aspects of
+closures.
+
+## (a) Closures as a data structure
+
+One potential use of closures is as a tool for data encapsulation.  Try this
+example:
+
+```python
+def counter(value):
+    def incr():
+        nonlocal value
+        value += 1
+        return value
+
+    def decr():
+        nonlocal value
+        value -= 1
+        return value
+
+    return incr, decr
+```
+
+This code defines two inner functions that manipulate a value.  Try it out:
+
+```python
+>>> up, down = counter(0)
+>>> up()
+1
+>>> up()
+2
+>>> up()
+3
+>>> down()
+2
+>>> down()
+1
+>>>
+```
+
+Notice how there is no class definition involved here.  Moreover,
+there is no global variable either.  Yet, the `up()` and `down()`
+functions are manipulating some "behind the scenes" value.  It's
+fairly magical.
+
+## (b) Closures as a code generator
+
+In [Exercise 4.3](ex4_3.md), you developed a collection of
+descriptor classes that allowed type-checking of object attributes.
+For example:
+
+```python
+
+class Stock:
+    name = String()
+    shares = Integer()
+    price = Float()
+```
+
+This kind of thing can also be implemented using closures.  Define a file
+``typedproperty.py`` and put the following code in it:
+
+```python
+# typedproperty.py
+
+def typedproperty(name, expected_type):
+    private_name = '_' + name
+
+    @property
+    def value(self):
+        return getattr(self, private_name)
+
+    @value.setter
+    def value(self, val):
+        if not isinstance(val, expected_type):
+            raise TypeError(f'Expected {expected_type}')
+        setattr(self, private_name, val)
+   
+    return value
+```
+
+This look pretty wild, but the function is effectively making code.  You'd use it in
+a class definition like this:
+
+```python
+from typedproperty import typedproperty
+
+class Stock:
+    name = typedproperty('name', str)
+    shares = typedproperty('shares', int)
+    price = typedproperty('price', float)
+
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+Verify that this class performs type-checking in the same way as the
+descriptor code.
+
+Add function `String()`, `Integer()`, and `Float()` to the `typedproperty.py` file
+so that you can write the following code:
+
+```python
+from typedproperty import String, Integer, Float
+
+class Stock:
+    name = String('name')
+    shares = Integer('shares')
+    price = Float('price')
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+## (c) Challenge: Eliminating names
+
+Modify the `typedproperty.py` code so that attribute names are no-longer required:
+
+```python
+from typedproperty import String, Integer, Float
+
+class Stock:
+    name = String()
+    shares = Integer()
+    price = Float()
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+Hint: To do this, recall the `__set_name__()` method of descriptor objects that 
+gets called when descriptors are placed in a class definition.
+
+\[ [Solution](soln5_4.md) | [Index](index.md) | [Exercise 5.3](ex5_3.md) | [Exercise 5.5](ex5_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_5.md

@@ -0,0 +1,97 @@
+\[ [Index](index.md) | [Exercise 5.4](ex5_4.md) | [Exercise 5.6](ex5_6.md) \]
+
+# Exercise 5.5
+
+*Objectives:*
+
+- Learn more about exception handling and logging
+
+*Files Modified:* `reader.py`
+
+In the `reader.py` file, there is a central function `convert_csv()` that does
+most of the work.  This function crashes if you run it on data with missing or
+bad data.  For example:
+
+```python
+>>> port = read_csv_as_dicts('Data/missing.csv', types=[str, int, float])
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "reader.py", line 24, in read_csv_as_dicts
+    return csv_as_dicts(file, types, headers=headers)
+  File "reader.py", line 13, in csv_as_dicts
+    lambda headers, row: { name: func(val) for name, func, val in zip(headers, types, row) })
+  File "reader.py", line 9, in convert_csv
+    return list(map(lambda row: converter(headers, row), rows))
+  File "reader.py", line 9, in <lambda>
+    return list(map(lambda row: converter(headers, row), rows))
+  File "reader.py", line 13, in <lambda>
+    lambda headers, row: { name: func(val) for name, func, val in zip(headers, types, row) })
+  File "reader.py", line 13, in <dictcomp>
+    lambda headers, row: { name: func(val) for name, func, val in zip(headers, types, row) })
+ValueError: invalid literal for int() with base 10: ''
+>>> 
+```
+
+## (a) Catching Exceptions
+
+Instead of crashing on bad data, modify the code to issue a warning message
+instead. The final result should be a list of the rows that were successfully
+converted. For example:
+
+```python
+>>> port = read_csv_as_dicts('Data/missing.csv', types=[str, int, float])
+Row 4: Bad row: ['C', '', '53.08']
+Row 7: Bad row: ['DIS', '50', 'N/A']
+Row 8: Bad row: ['GE', '', '37.23']
+Row 13: Bad row: ['INTC', '', '21.84']
+Row 17: Bad row: ['MCD', '', '51.11']
+Row 19: Bad row: ['MO', '', '70.09']
+Row 22: Bad row: ['PFE', '', '26.40']
+Row 26: Bad row: ['VZ', '', '42.92']
+>>> len(port)
+20
+>>>
+```
+
+Note:  Making this change may be a bit tricky because of your previous use of the `map()`
+built-in function. You may have to abandon that approach since there's no easy way to catch
+and handle exceptions in `map()`.
+
+## (b) Logging
+
+Modify the code so that warning messages are issued using the `logging` module.  In 
+addition, give optional debugging information indicating the reason for failure.
+For example:
+
+```python
+>>> import reader
+>>> import logging
+>>> logging.basicConfig(level=logging.DEBUG)
+>>> port = reader.read_csv_as_dicts('Data/missing.csv', types=[str, int, float])
+WARNING:reader:Row 4: Bad row: ['C', '', '53.08']
+DEBUG:reader:Row 4: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 7: Bad row: ['DIS', '50', 'N/A']
+DEBUG:reader:Row 7: Reason: could not convert string to float: 'N/A'
+WARNING:reader:Row 8: Bad row: ['GE', '', '37.23']
+DEBUG:reader:Row 8: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 13: Bad row: ['INTC', '', '21.84']
+DEBUG:reader:Row 13: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 17: Bad row: ['MCD', '', '51.11']
+DEBUG:reader:Row 17: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 19: Bad row: ['MO', '', '70.09']
+DEBUG:reader:Row 19: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 22: Bad row: ['PFE', '', '26.40']
+DEBUG:reader:Row 22: Reason: invalid literal for int() with base 10: ''
+WARNING:reader:Row 26: Bad row: ['VZ', '', '42.92']
+DEBUG:reader:Row 26: Reason: invalid literal for int() with base 10: ''
+>>>
+```
+
+\[ [Solution](soln5_5.md) | [Index](index.md) | [Exercise 5.4](ex5_4.md) | [Exercise 5.6](ex5_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex5_6.md

@@ -0,0 +1,104 @@
+\[ [Index](index.md) | [Exercise 5.5](ex5_5.md) | [Exercise 6.1](ex6_1.md) \]
+
+# Exercise 5.6
+
+*Objectives:*
+
+- Learn how to use Python's unittest module
+
+*Files Created:* `teststock.py`
+
+In this exercise, you will explore the basic mechanics of using
+Python's `unittest` modules.
+
+## (a) Preliminaries
+
+In previous exercises, you created a file `stock.py` that contained
+a `Stock` class.   In a separate file, `teststock.py`, define the following
+testing code:
+
+```python
+# teststock.py
+
+import unittest
+import stock
+
+class TestStock(unittest.TestCase):
+    def test_create(self):
+        s = Stock('GOOG', 100, 490.1)
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+
+if __name__ == '__main__':
+    unittest.main()
+```
+
+Make sure you can run the file:
+
+```
+ bash % python3 teststock.py
+ .
+ ------------------------------------------------------------------```
+ Ran 1 tests in 0.001s
+
+ OK
+ bash %
+```
+
+## (b) Unit testing
+
+Using the code in `teststock.py` as a guide, extend the `TestStock` class
+with tests for the following:
+
+- Test that you can create a `Stock` using keyword arguments such as `Stock(name='GOOG',shares=100,price=490.1)`.
+- Test that the `cost` property returns a correct value
+- Test that the `sell()` method correctly updates the shares.
+- Test that the `from_row()` class method creates a new instance from good data.
+- Test that the `__repr__()` method creates a proper representation string.
+- Test the comparison operator method `__eq__()`
+
+## (c) Unit tests with expected errors
+
+Suppose you wanted to write a unit test that checks for an exception.
+Here is how you can do it:
+
+```python
+class TestStock(unittest.TestCase):
+    ...
+    def test_bad_shares(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(TypeError):
+             s.shares = '50'
+    ...
+```
+
+Using this test as a guide, write unit tests for the following failure modes:
+
+- Test that setting `shares` to a string raises a `TypeError`
+- Test that setting `shares` to a negative number raises a `ValueError`
+- Test that setting `price` to a string raises a `TypeError`
+- Test that setting `price` to a negative number raises a `ValueError`
+- Test that setting a non-existent attribute `share` raises an `AttributeError`
+
+In total, you should have around a dozen unit tests when you're done.
+
+**Important Note**
+
+For later use in the course, you will want to have a fully working
+`stock.py` and `teststock.py` file.  Save your work in progress if you
+have to, but you are strongly encouraged to copy the code from
+`Solutions/5_6` if things are still broken at this point.
+
+We're going to use the `teststock.py` file as a tool for improving the `Stock` code
+later.  You'll want it on hand to make sure that the new code behaves the same way
+as the old code.
+
+\[ [Solution](soln5_6.md) | [Index](index.md) | [Exercise 5.5](ex5_5.md) | [Exercise 6.1](ex6_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex6_1.md

@@ -0,0 +1,195 @@
+\[ [Index](index.md) | [Exercise 5.6](ex5_6.md) | [Exercise 6.2](ex6_2.md) \]
+
+# Exercise 6.1
+
+*Objectives:*
+
+- Learn more about function argument passing conventions
+
+*Files Created:* `structure.py`, `stock.py`
+
+
+**IMPORTANT NOTE**
+
+This exercise is going to start a long road of rewriting the `stock.py` file in a more
+sane way.   Before doing anything, copy your work in `stock.py` to a new file
+`orig_stock.py`.  
+
+We're going to recreate the `Stock` class from scratch using some new techniques.
+Make sure you have your unit tests from link:ex5_4.html[Exercise 5.4] handy. You'll want those.
+
+If you define a function, you probably already know that it can be
+called using a mix of positional or keyword arguments.  For example:
+
+```python
+>>> def foo(x, y, z):
+        return x + y + z
+
+>>> foo(1, 2, 3)
+6
+>>> foo(1, z=3, y=2)
+6
+>>>
+```
+
+You may also know that you can pass sequences and dictionaries as
+function arguments using the * and ** syntax.  For example:
+
+```python
+>>> args = (1, 2, 3)
+>>> foo(*args)
+6
+>>> kwargs = {'y':2, 'z':3 }
+>>> foo(1,**kwargs)
+6
+>>>
+```
+
+In addition to that, you can write functions that accept any number of
+positional or keyword arguments using the * and ** syntax.  For
+example:
+
+```python
+>>> def foo(*args):
+        print(args)
+
+>>> foo(1,2)
+(1, 2)
+>>> foo(1,2,3,4,5)
+(1, 2, 3, 4, 5)
+>>> foo()
+()
+>>>
+>>> def bar(**kwargs):
+        print(kwargs)
+
+>>> bar(x=1,y=2)
+{'y': 2, 'x': 1}
+>>> bar(x=1,y=2,z=3)
+{'y': 2, 'x': 1, 'z': 3}
+>>> bar()
+{}
+>>> 
+```
+
+Variable argument functions are sometimes useful as a technique for
+reducing or simplifying the amount of code you need to type.  In this
+exercise, we'll explore that idea for simple data structures.
+
+## (a) Simplified Data Structures
+
+In earlier exercises, you defined a class representing a stock like
+this:
+
+```python
+class Stock:
+    def __init__(self,name,shares,price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+```
+
+Focus on the `__init__()` method---isn't that a lot of
+code to type each time you want to populate a structure?   What if you
+had to define dozens of such structures in your program?
+
+In a file `structure.py`, define a base class
+`Structure` that allows the user to define simple
+data structures as follows:
+
+```python
+class Stock(Structure):
+    _fields = ('name','shares','price')
+
+class Date(Structure):
+    _fields = ('year', 'month', 'day')
+```
+  
+The `Structure` class should define an `__init__()`
+method that takes any number of arguments and which looks for the
+presence of a `_fields` class variable.  Have the method
+populate the instance from the attribute names in `_fields`
+and values passed to `__init__()`.
+
+Here is some sample code to test your implementation:
+
+```python
+>>> s = Stock('GOOG',100,490.1)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s.price
+490.1
+>>> s = Stock('AA',50)
+Traceback (most recent call last):
+...
+TypeError: Expected 3 arguments
+>>>
+```
+
+## (b) Making a Useful Representation
+
+Modify the `Structure` class so that it produces a nice 
+representation when `repr()` is used.  For example:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s
+Stock('GOOG',100,490.1)
+>>>
+```
+
+## (c) Restricting Attribute Names
+
+Give the `Structure` class a `__setattr__()` method that restricts
+the allowed set of attributes to those listed in the `_fields` variable.
+However, it should still allow any "private" attribute (e.g., name starting
+with `_` to be set). 
+
+For example:
+
+```python
+>>> s = Stock('GOOG',100,490.1)
+>>> s.shares = 50
+>>> s.share = 50
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "structure.py", line 13, in __setattr__
+    raise AttributeError('No attribute %s' % name)
+AttributeError: No attribute share
+>>> s._shares = 100     # Private attribute. OK
+>>>
+```
+
+## (d) Starting Over
+
+Create a new file `stock.py` (or delete all of your previous code). Start over by defining `Stock` as follows:
+
+```python
+# stock.py
+
+from structure import Structure
+
+class Stock(Structure):
+    _fields = ('name', 'shares', 'price')
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+Once you've done this, run your `teststock.py` unit tests.   You should get a lot of failures, but at least a
+handful of the tests should pass.
+
+\[ [Solution](soln6_1.md) | [Index](index.md) | [Exercise 5.6](ex5_6.md) | [Exercise 6.2](ex6_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex6_2.md

@@ -0,0 +1,197 @@
+\[ [Index](index.md) | [Exercise 6.1](ex6_1.md) | [Exercise 6.3](ex6_3.md) \]
+
+# Exercise 6.2
+
+*Objectives:*
+
+- Learn more about scoping rules
+- Learn some scoping tricks
+
+*Files modified:* `structure.py`, `stock.py`
+
+In the last exercise, you create a class `Structure` that made it easy to define
+data structures.  For example:
+
+```python
+class Stock(Structure):
+    _fields = ('name','shares','price')
+```
+
+This works fine except that a lot of things are pretty weird about the `__init__()`
+function.   For example, if you ask for help using `help(Stock)`, you don't get 
+any kind of useful signature.  Also, keyword argument passing doesn't work. For
+example:
+
+```python
+>>> help(Stock)
+... look at output ...
+
+>>> s = Stock(name='GOOG', shares=100, price=490.1)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: __init__() got an unexpected keyword argument 'price'
+>>> 
+```
+
+In this exercise, we're going to look at a different approach to the problem.
+
+## (a) Show me your locals
+
+First, try an experiment by defining the following class:
+
+```python
+>>> class Stock:
+        def __init__(self, name, shares, price):
+            print(locals())
+
+>>>
+```
+
+Now, try running this:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+{'self': <__main__.Stock object at 0x100699b00>, 'price': 490.1, 'name': 'GOOG', 'shares': 100}
+>>>
+```
+
+Notice how the locals dictionary contains all of the arguments passed
+to `__init__()`. That's interesting.  Now, define the following function
+and class definitions:
+
+```python
+>>> def _init(locs):
+        self = locs.pop('self')
+        for name, val in locs.items():
+            setattr(self, name, val)
+
+>>> class Stock:
+        def __init__(self, name, shares, price):
+            _init(locals())
+```
+
+In this code, the `_init()` function is used to automatically
+initialize an object from a dictionary of passed local variables.
+You'll find that `help(Stock)` and keyword arguments work perfectly.
+
+```python
+>>> s = Stock(name='GOOG', price=490.1, shares=50)
+>>> s.name
+'GOOG'
+>>> s.shares
+50
+>>> s.price
+490.1
+>>>
+```
+
+## (b) Frame Hacking
+
+One complaint about the last part is that the `__init__()` function
+now looks pretty weird with that call to `locals()` inserted into it.
+You can get around that though if you're willing to do a bit of stack
+frame hacking.  Try this variant of the `_init()` function:
+
+```python
+>>> import sys
+>>> def _init():
+        locs = sys._getframe(1).f_locals   # Get callers local variables
+        self = locs.pop('self')
+        for name, val in locs.items():
+            setattr(self, name, val)
+>>>
+```
+
+In this code, the local variables are extracted from the stack frame of the caller.
+Here is a modified class definition:
+
+```python
+>>> class Stock:
+        def __init__(self, name, shares, price):
+            _init()
+
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>>
+```
+
+At this point, you're probably feeling rather disturbed.  Yes, you just wrote a function that reached
+into the stack frame of another function and examined its local variables.    
+
+## (c) Putting it Together
+
+Taking the ideas in the first two parts, delete the `__init__()` method that was originally part of the
+`Structure` class.  Next, add an `_init()` method like this:
+
+```python
+# structure.py
+import sys
+
+class Structure:
+    ...
+    @staticmethod
+    def _init():
+        locs = sys._getframe(1).f_locals
+        self = locs.pop('self')
+        for name, val in locs.items():
+            setattr(self, name, val)
+    ...
+```
+
+Note: The reason this is defined as a `@staticmethod` is that the `self` argument
+is obtained from the locals--there's no need to additionally have it passed as
+an argument to the method itself (admittedly this is a bit subtle).
+
+Now, modify your `Stock` class so that it looks like the following:
+
+```python
+# stock.py
+from structure import Structure
+
+class Stock(Structure):
+    _fields = ('name','shares','price')
+    def __init__(self, name, shares, price):
+        self._init()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= shares
+```
+
+Verify that the class works properly, supports keyword arguments, and has a 
+proper help signature.
+
+```python
+>>> s = Stock(name='GOOG', price=490.1, shares=50)
+>>> s.name
+'GOOG'
+>>> s.shares
+50
+>>> s.price
+490.1
+>>> help(Stock)
+... look at the output ...
+>>>
+```
+
+Run your unit tests in `teststock.py` again.  You should see at least one more test pass. Yay!
+
+At this point, it's going to look like we just took a giant step backwards.  Not
+only do the classes need the `__init__()` method, they also need the `_fields`
+variable for some of the other methods to work (`__repr__()` and `__setattr__()`).  Plus,
+the use of `self._init()` looks pretty hacky.    We'll work on this, but be patient.
+
+\[ [Solution](soln6_2.md) | [Index](index.md) | [Exercise 6.1](ex6_1.md) | [Exercise 6.3](ex6_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex6_3.md

@@ -0,0 +1,152 @@
+\[ [Index](index.md) | [Exercise 6.2](ex6_2.md) | [Exercise 6.4](ex6_4.md) \]
+
+# Exercise 6.3
+
+*Objectives:*
+
+- Learn how to inspect the internals of functions
+
+*Files Modified:* `structure.py`
+
+## (a) Inspecting functions
+
+Define a simple function:
+
+```python
+>>> def add(x,y):
+       'Adds two things'
+       return x+y
+
+>>>
+```
+
+Do a `dir()` on the function to look at its attributes.
+
+```python
+>>> dir(add)
+... look at the result ...
+>>>
+```
+
+Get some basic information such as the function name, defining module name, and documentation string.
+
+```python
+>>> add.__name__
+'add'
+>>> add.__module__
+'__main__'
+>>> add.__doc__
+'Adds two things'
+>>>
+```
+
+The `__code__` attribute of a function has low-level information about
+the function implementation.  See if you can look at this and
+determine the number of required arguments and names of local
+variables.
+
+## (b) Using the inspect module
+
+Use the inspect module to get calling information about the function:
+
+```python
+>>> import inspect
+>>> sig = inspect.signature(add)
+>>> sig
+<Signature (x, y)>
+>>> sig.parameters
+mappingproxy(OrderedDict([('x', <Parameter "x">), ('y', <Parameter "y">)]))
+>>> tuple(sig.parameters)
+('x', 'y')
+>>>
+```
+
+## (c) Putting it Together
+
+In link:ex6_1.html[Exercise 6.1], you created a class `Structure`
+that defined a generalized `__init__()`, `__setattr__()`, and `__repr__()`
+method.   That class required a user to define a `_fields` class
+variable like this:
+
+```python
+class Stock(Structure):
+    _fields = ('name','shares','price')
+```
+
+The problem with this class is that the `__init__()` function didn't
+have a useful argument signature for the purposes of help and
+keyword argument passing.  In link:ex6_2.html[Exercise 6.2], you 
+did a sneaky trick involving a special `self._init()` function. For example:
+
+```python
+class Stock(Structure):
+    _fields = ('name', 'shares', 'price')
+    def __init__(self, name, shares, price):
+        self._init()
+    ...
+```
+
+This gave a useful signature, but now the class is just weird because
+the user has to provide both the `_fields` variable and the `__init__()` method.
+
+Your task is to eliminate the `_fields` variable using some function
+inspection techniques.  First, notice that you can get the argument
+signature from `Stock` as follows:
+
+```python
+>>> import inspect
+>>> sig = inspect.signature(Stock)
+>>> tuple(sig.parameters)
+('name', 'shares', 'price')
+>>>
+```
+
+Perhaps you could set the `_fields` variable from the argument signature
+of `__init__()`.   Add a class method `set_fields(cls)` to `Structure` that
+inspects the `__init__()` function, and sets the `_fields`
+variable appropriately.   You should use your new function like this:
+
+```python
+class Stock(Structure):
+    def __init__(self, name, shares, price):
+        self._init()
+
+    ...
+
+Stock.set_fields()
+```
+
+The resulting class should work the same way as before:
+
+```python
+>>> s = Stock(name='GOOG', shares=100, price=490.1)
+>>> s
+Stock('GOOG',100,490.1)
+>>> s.shares = 50
+>>> s.share = 50
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "structure.py", line 12, in __setattr__
+    raise AttributeError('No attribute %s' % name)
+AttributeError: No attribute share
+>>> 
+```
+
+Verify the slightly modified `Stock` class with your unit tests again.  There will still
+be failures, but nothing should change from the previous exercise.
+
+At this point, it's all still a bit "hacky", but you're making 
+progress.  You have a Stock structure class with a useful `__init__()`
+function, there is a useful representation string, and the
+`__setattr__()` method restricts the set of attribute names.   The
+extra step of having to invoke `set_fields()` is a bit odd, but we'll
+get back to that.
+
+\[ [Solution](soln6_3.md) | [Index](index.md) | [Exercise 6.2](ex6_2.md) | [Exercise 6.4](ex6_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex6_4.md

@@ -0,0 +1,145 @@
+\[ [Index](index.md) | [Exercise 6.3](ex6_3.md) | [Exercise 6.5](ex6_5.md) \]
+
+# Exercise 6.4
+
+*Objectives:*
+
+- Learn to create code with `exec()`
+
+## (a) Experiment with exec()
+
+Define a fragment of Python code in a string and try running it:
+
+```python
+>>> code = '''
+for i in range(n):
+    print(i, end=' ')
+'''
+>>> n = 10
+>>> exec(code)
+0 1 2 3 4 5 6 7 8 9
+>>>
+```
+
+That's interesting, but executing random code fragments is not
+especially useful. A more interesting use of `exec()` is in making
+code such as functions, methods, or classes.  Try this example in
+which we make an `__init__()` function for a class.
+
+```python
+>>> class Stock:
+        _fields = ('name', 'shares', 'price')
+
+>>> argstr = ','.join(Stock._fields)
+>>> code = f'def __init__(self, {argstr}):\n'
+>>> for name in Stock._fields:
+        code += f'    self.{name} = {name}\n'
+>>> print(code)
+def __init__(self, name,shares,price):
+    self.name = name
+    self.shares = shares
+    self.price = price
+
+>>> locs = { }
+>>> exec(code, locs)
+>>> Stock.__init__ = locs['__init__']
+
+>>> # Now try the class
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s.price
+490.1
+>>> 
+```
+
+In this example, an `__init__()` function is made directly from the `_fields` variable.  
+There are no weird hacks involving a special `_init()` method or stack frames.
+
+## (b) Creating an `__init__()` function
+
+In link:ex6_3.txt[Exercise 6.3], you wrote code that inspected the
+signature of the `__init__()` method to set the attribute names
+in a `_fields` class variable.   For example:
+
+```python
+class Stock(Structure):
+    def __init__(self, name, shares, price):
+        self._init()
+
+Stock.set_fields()
+```
+
+Instead of inspecting the `__init__()` method, write a class method
+`create_init(cls)` that creates an `__init__()` method from the value of
+`_fields`.  Use the `exec()` function to do this as shown above. 
+Here's how a user will use it:
+
+```python
+class Stock(Structure):
+    _fields = ('name', 'shares', 'price')
+
+Stock.create_init()
+```
+
+The resulting class should work exactly the name way as before:
+
+```python
+>>> s = Stock(name='GOOG', shares=100, price=490.1)
+>>> s
+Stock('GOOG',100,490.1)
+>>> s.shares = 50
+>>> s.share = 50
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "structure.py", line 12, in __setattr__
+    raise AttributeError('No attribute %s' % name)
+AttributeError: No attribute share
+>>> 
+```
+
+Modify the `Stock` class in progress to use the `create_init()` function as shown.  
+Verify with your unit tests as before.
+
+While you're at it, get rid of the `_init()` and `set_fields()`
+methods on the `Structure` class--that approach was kind of weird.
+
+## (c) Named Tuples
+
+In link:ex2_1.html[Exercise 2.1], you experimented with `namedtuple` objects
+in the `collections` module.  Just to refresh your memory, here is how
+they worked:
+
+```python
+>>> from collections import namedtuple
+>>> Stock = namedtuple('Stock', ['name', 'shares', 'price'])
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name
+'GOOG'
+>>> s.shares
+100
+>>> s[1]
+100
+>>>
+```
+
+Under the covers, the `namedtuple()` function is creating code as a string
+and executing it using `exec()`.   Look at the code and marvel:
+
+```python
+>>> import inspect
+>>> print(inspect.getsource(namedtuple))
+... look at the output ...
+>>>
+```
+
+\[ [Solution](soln6_4.md) | [Index](index.md) | [Exercise 6.3](ex6_3.md) | [Exercise 6.5](ex6_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex6_5.md

@@ -0,0 +1,147 @@
+\[ [Index](index.md) | [Exercise 6.4](ex6_4.md) | [Exercise 7.1](ex7_1.md) \]
+
+# Exercise 6.5
+
+*Objectives:*
+
+- Learn how to define a proper callable object
+
+Files Modified :  `validate.py`
+
+Back in [Exercise 4.3](ex4_3.md), you created a series of `Validator` classes
+for performing different kinds of type and value checks.  For example:
+
+```python
+>>> from validate import Integer
+>>> Integer.check(1)
+>>> Integer.check('hello')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "validate.py", line 21, in check
+    raise TypeError(f'Expected {cls.expected_type}')
+TypeError: Expected <class 'int'>
+>>>
+```
+
+You could use the validators in functions like this:
+
+```python
+>>> def add(x, y):
+        Integer.check(x)
+        Integer.check(y)
+        return x + y
+
+>>>
+```
+
+In this exercise, we're going to take it just one step further. 
+
+## (a) Creating a Callable Object
+
+In the file `validate.py`, start by creating a class like this:
+
+```python
+# validate.py
+...
+
+class ValidatedFunction:
+    def __init__(self, func):
+        self.func = func
+
+    def __call__(self, *args, **kwargs):
+        print('Calling', self.func)
+        result = self.func(*args, **kwargs)
+        return result
+```
+
+Test the class by applying it to a function:
+
+```python
+>>> def add(x, y):
+        return x + y
+
+>>> add = ValidatedFunction(add)
+>>> add(2, 3)
+Calling <function add at 0x1014df598>
+5
+>>>
+```
+
+## (b) Enforcement
+
+Modify the `ValidatedFunction` class so that it enforces value checks
+attached via function annotations.  For example:
+
+```python
+>>> def add(x: Integer, y:Integer):
+        return x + y
+>>> add = ValidatedFunction(add)
+>>> add(2,3)
+5
+>>> add('two','three')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "validate.py", line 67, in __call__
+    self.func.__annotations__[name].check(val)
+  File "validate.py", line 21, in check
+    raise TypeError(f'Expected {cls.expected_type}')
+TypeError: expected <class 'int'>
+>>>>
+```
+
+Hint: To do this, play around with signature binding. Use the `bind()`
+method of `Signature` objects to bind function arguments to argument
+names.  Then cross reference this information with the
+`__annotations__` attribute to get the different validator classes.
+
+Keep in mind, you're making an object that looks like a function, but
+it's really not.  There is magic going on behind the scenes.
+
+## (c) Use as a Method (Challenge)
+
+A custom callable often presents problems if used as a custom method.
+For example, try this:
+
+```python
+class Stock:
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares:Integer):
+        self.shares -= nshares
+    sell = ValidatedFunction(sell)     # Fails
+```
+
+You'll find that the wrapped `sell()` fails miserably:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.sell(10)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "validate.py", line 64, in __call__
+    bound = self.signature.bind(*args, **kwargs)
+  File "/usr/local/lib/python3.6/inspect.py", line 2933, in bind
+    return args[0]._bind(args[1:], kwargs)
+  File "/usr/local/lib/python3.6/inspect.py", line 2848, in _bind
+    raise TypeError(msg) from None
+TypeError: missing a required argument: 'nshares'
+>>> 
+```
+
+Bonus: Figure out why it fails--but don't spend too much time fooling around with it.
+
+\[ [Solution](soln6_5.md) | [Index](index.md) | [Exercise 6.4](ex6_4.md) | [Exercise 7.1](ex7_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_1.md

@@ -0,0 +1,137 @@
+\[ [Index](index.md) | [Exercise 6.5](ex6_5.md) | [Exercise 7.2](ex7_2.md) \]
+
+# Exercise 7.1
+
+*Objectives:*
+
+- Learn how to define a simple decorator functions.
+
+*Files Created:* `logcall.py`
+
+*Files Modifie:* `validate.py`
+
+## (a) Your First Decorator
+
+To start with decorators, write a _very_ simple decorator
+function that simply prints out a message each time a function is
+called.  Create a file `logcall.py` and define the following
+function:
+
+```python
+# logcall.py
+
+def logged(func):
+    print('Adding logging to', func.__name__)
+    def wrapper(*args, **kwargs):
+        print('Calling', func.__name__)
+        return func(*args, **kwargs)
+    return wrapper
+```
+
+Now, make a separate file `sample.py` and apply it to a
+few function definitions:
+
+```python
+# sample.py
+
+from logcall import logged
+
+@logged
+def add(x,y):
+    return x+y
+
+@logged
+def sub(x,y):
+    return x-y
+```
+
+Test your code as follows:
+
+```python
+>>> import sample
+Adding logging to add
+Adding logging to sub
+>>> sample.add(3,4)
+Calling add
+7
+>>> sample.sub(2,3)
+Calling sub
+-1
+>>> 
+```
+
+## (b) A Real Decorator
+
+In [Exercise 6.6](ex6_6.md), you created a callable class `ValidatedFunction` that
+enforced type annotations.  Rewrite this class as a decorator function called `validated`.
+It should allow you to write code like this:
+
+```python
+from validate import Integer, validated
+
+@validated
+def add(x: Integer, y:Integer) -> Integer:
+    return x + y
+
+@validated
+def pow(x: Integer, y:Integer) -> Integer:
+    return x ** y
+```
+
+Here's how the decorated functions should work:
+
+```python
+>>> add(2, 3)
+5
+>>> add('2', '3')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "validate.py", line 75, in wrapper
+    raise TypeError('Bad Arguments\n' + '\n'.join(errors))
+TypeError: Bad Arguments
+    x: Expected <class 'int'>
+    y: Expected <class 'int'>
+
+>>> pow(2, 3)
+8
+>>> pow(2, -1)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "validate.py", line 83, in wrapper
+    raise TypeError(f'Bad return: {e}') from None
+TypeError: Bad return: Expected <class 'int'>
+>>>
+```
+
+Your decorator should try to patch up the exceptions so that they
+show more useful information as shown.  Also, the `@validated`
+decorator should work in classes (you don't need to do anything special).
+
+```python
+class Stock:
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    @validated
+    def sell(self, nshares:PositiveInteger):
+        self.shares -= nshares
+```
+
+Note:  This part doesn't involve a lot of code, but there are a lot of low-level
+fiddly bits.  The solution will look almost the same as for Exercise 6.6.  Don't
+be shy about looking at solution code though.
+
+\[ [Solution](soln7_1.md) | [Index](index.md) | [Exercise 6.5](ex6_5.md) | [Exercise 7.2](ex7_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_2.md

@@ -0,0 +1,178 @@
+\[ [Index](index.md) | [Exercise 7.1](ex7_1.md) | [Exercise 7.3](ex7_3.md) \]
+
+# Exercise 7.2
+
+*Objectives:*
+
+- Decorator chaining
+- Defining decorators that accept arguments.
+
+*Files Modified:* `logcall.py`, `validate.py`
+
+## (a) Copying Metadata
+
+When a function gets wrapped by a decorator, you often lose
+information about the name of the function, documentation strings, and
+other details.  Verify this:
+
+```python
+>>> @logged
+    def add(x,y):
+        'Adds two things'
+        return x+y
+
+>>> add
+<function wrapper at 0x4439b0>
+>>> help(add)
+... look at the output ...
+>>>
+```
+
+Fix the definition of the `logged` decorator so that it copies
+function metadata properly.  To do this, use the `@wraps(func)`
+decorator as shown in the notes.
+
+After you're done, make sure the decorator preserves the function name
+and doc string.
+
+```python
+>>> @logged
+    def add(x,y):
+        'Adds two things'
+        return x+y
+
+>>> add
+<function add at 0x4439b0>
+>>> add.__doc__
+'Adds two things'
+>>>
+```
+
+Fix the `@validated` decorator you wrote earlier so that it also preserves
+metadata using `@wraps(func)`.
+
+## (b) Your first decorator with arguments
+
+The `@logged` decorator you defined earlier always just
+prints a simple message with the function name.
+Suppose that you wanted the user to be able to specify a 
+custom message of some sort.
+
+Define a new decorator `@logformat(fmt)` that accepts
+a format string as an argument and uses `fmt.format(func=func)` to
+format a supplied function into a log message:
+
+```python
+# sample.py
+...
+from logcall import logformat
+
+@logformat('{func.__code__.co_filename}:{func.__name__}')
+def mul(x,y):
+    return x*y
+```
+
+To do this, you need to define a decorator that takes an argument.
+This is what it should look like when you test it:
+
+```python
+>>> import sample
+Adding logging to add
+Adding logging to sub
+Adding logging to mul
+>>> sample.add(2,3)
+Calling add
+5
+>>> sample.mul(2,3)
+sample.py:mul
+6
+>>>
+```
+
+To further simplify the code, show how you can define the original `@logged` decorator
+using the the `@logformat` decorator. 
+
+## (c) Multiple decorators and methods
+
+Things can get a bit dicey when decorators are applied to methods in a
+class.  Try applying your `@logged` decorator to the methods in the
+following class.
+
+```python
+class Spam:
+    @logged
+    def instance_method(self):
+        pass
+
+    @logged
+    @classmethod
+    def class_method(cls):
+        pass
+
+    @logged
+    @staticmethod
+    def static_method():
+        pass
+
+    @logged
+    @property
+    def property_method(self):
+        pass
+```
+
+Does it even work at all? (hint: no). Is there any way to fix the code so
+that it works?  For example, can you make it so the following example
+works?
+
+```python
+>>> s = Spam()
+>>> s.instance_method()
+instance_method
+>>> Spam.class_method()
+class_method
+>>> Spam.static_method()
+static_method
+>>> s.property_method
+property_method
+>>>
+```
+
+## (d) Validation (Redux)
+
+In the last exercise, you wrote a `@validated` decorator that enforced
+type annotations. For example:
+
+```python
+@validated
+def add(x: Integer, y:Integer) -> Integer:
+    return x + y
+```
+
+Make a new decorator `@enforce()` that enforces types specified
+via keyword arguments to the decorator instead.  For example:
+
+```python
+@enforce(x=Integer, y=Integer, return_=Integer)
+def add(x, y):
+    return x + y
+```
+
+The resulting behavior of the decorated function should be identical.
+Note: Make the `return_` keyword specify the return type.  `return` is
+a Python reserved word so you have to pick a slightly different name.
+
+
+**Discussion**
+
+Writing robust decorators is often a lot harder than it looks.
+Recommended reading:
+
+
+\[ [Solution](soln7_2.md) | [Index](index.md) | [Exercise 7.1](ex7_1.md) | [Exercise 7.3](ex7_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_3.md

@@ -0,0 +1,261 @@
+\[ [Index](index.md) | [Exercise 7.2](ex7_2.md) | [Exercise 7.4](ex7_4.md) \]
+
+# Exercise 7.3
+
+*Objectives:*
+
+- Learn about class decorators
+- Descriptors revisited
+
+*Files Modified:* `validate.py`, `structure.py`
+
+This exercise is going to pull together a bunch of topics we've
+developed over the last few days. Hang on to your hat.
+
+## (a) Descriptors Revisited
+
+In link:ex4_3.html[Exercise 4.3] you defined some descriptors that
+allowed a user to define classes with type-checked attributes like
+this:
+
+```python
+from validate import String, PositiveInteger, PositiveFloat
+
+class Stock:
+    name   = String()
+    shares = PositiveInteger()
+    price  = PositiveFloat()
+    ...
+```
+
+Modify your `Stock` class so that it includes the above descriptors
+and now looks like this (see link:ex6_4.html[Exercise 6.4]):
+
+```python
+# stock.py
+
+from structure import Structure
+from validate import String, PositiveInteger, PositiveFloat
+
+class Stock(Structure):
+    _fields = ('name', 'shares', 'price')
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+
+Stock.create_init()
+```
+
+Run the unit tests in `teststock.py`.  You should see a significant
+number of tests passing with the addition of type checking.
+Excellent.
+
+## (b) Using Class Decorators to Fill in Details
+
+An annoying aspect of the above code is there are extra details such as 
+`_fields` variable and the final step of `Stock.create_init()`.  A lot
+of this could be packaged into a class decorator instead.
+
+In the file `structure.py`, make a class decorator `@validate_attributes`
+that examines the class body for instances of Validators and fills in
+the `_fields` variable.  For example:
+
+```python
+# structure.py
+
+from validate import Validator
+
+def validate_attributes(cls):
+    validators = []
+    for name, val in vars(cls).items():
+        if isinstance(val, Validator):
+            validators.append(val)
+    cls._fields = [val.name for val in validators]
+    return cls
+```
+
+This code relies on the fact that class dictionaries are ordered
+starting in Python 3.6.  Thus, it will encounter the different
+`Validator` descriptors in the order that they're listed.  Using this
+order, you can then fill in the `_fields` variable.   This allows
+you to write code like this:
+
+```python
+# stock.py
+
+from structure import Structure, validate_attributes
+from validate import String, PositiveInteger, PositiveFloat
+
+@validate_attributes
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+
+Stock.create_init()
+```
+
+Once you've got this working, modify the `@validate_attributes`
+decorator to additionally perform the final step of calling
+`Stock.create_init()`.   This will reduce the class to the
+following:
+
+```python
+# stock.py
+
+from structure import Structure, validate_attributes
+from validate import String, PositiveInteger, PositiveFloat
+
+@validate_attributes
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+## (c) Applying Decorators via Inheritance
+
+Having to specify the class decorator itself is kind of annoying.  Modify the
+`Structure` class with the following `__init_subclass__()` method:
+
+```python
+# structure.py
+
+class Structure:
+    ...
+    @classmethod
+    def __init_subclass__(cls):
+        validate_attributes(cls)
+```
+
+Once you've made this change, you should be able to drop the decorator entirely and
+solely rely on inheritance. It's inheritance plus some hidden magic!
+
+```python
+# stock.py
+
+from structure import Structure
+from validate import String, PositiveInteger, PositiveFloat
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+Now, the code is really starting to go places. In fact, it almost
+looks normal.  Let's keep pushing it.
+
+## (d) Row Conversion
+
+One missing feature from the `Structure` class is a `from_row()` method that
+allows it to work with earlier CSV reading code.   Let's fix that.  Give the
+`Structure` class a `_types` class variable and the following class method:
+
+```python
+# structure.py
+
+class Structure:
+    _types = ()
+    ...
+    @classmethod
+    def from_row(cls, row):
+        rowdata = [ func(val) for func, val in zip(cls._types, row) ]
+        return cls(*rowdata)
+    ...
+```
+
+Modify the `@validate_attributes` decorator so that it examines the
+various validators for an `expected_type` attribute and uses it to 
+fill in the `_types` variable above.
+
+Once you've done this, you should be able to do things like this:
+
+```python
+>>> s = Stock.from_row(['GOOG', '100', '490.1'])
+>>> s
+Stock('GOOG', 100, 490.1)
+>>> import reader
+>>> port = reader.read_csv_as_instances('Data/portfolio.csv', Stock)
+>>>
+```
+
+## (e) Method Argument Checking
+
+Remember that `@validated` decorator you wrote in the last part?
+Let's modify the `@validate_attributes` decorator so that any method
+in the class with annotations gets wrapped by `@validated`
+automatically.  This allows you to put enforced annotations on methods
+such as the `sell()` method:
+
+```python
+# stock.py
+
+from structure import Structure
+from validate import String, PositiveInteger, PositiveFloat
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+```
+
+You'll find that `sell()` now enforces the argument.
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.sell(25)
+>>> s.sell(-25)
+Traceback (most recent call last):
+  ...
+TypeError: Bad Arguments
+  nshares: must be >= 0
+>>>
+```
+
+Yes, this starting to get very interesting now.  The combination of a class decorator and
+inheritance is a powerful force.
+
+\[ [Solution](soln7_3.md) | [Index](index.md) | [Exercise 7.2](ex7_2.md) | [Exercise 7.4](ex7_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_4.md

@@ -0,0 +1,177 @@
+\[ [Index](index.md) | [Exercise 7.3](ex7_3.md) | [Exercise 7.5](ex7_5.md) \]
+
+# Exercise 7.4
+
+*Objectives:*
+
+- Learn about the low-level steps involved in creating a class
+
+*Files Modified:* `validate.py`, `structure.py`
+
+In this exercise, we look at the mechanics of how classes are actually
+created. 
+
+## (a) Class creation
+
+Recall, from earlier exercises, we defined a simple class
+`Stock` that looked like this:
+
+```python
+class Stock:
+    def __init__(self,name,shares,price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+    def cost(self):
+        return self.shares*self.price
+    def sell(self,nshares):
+        self.shares -= nshares
+```
+
+What we're going to do here is create the class manually.  Start out
+by just defining the methods as normal Python functions.
+
+```python
+>>> def __init__(self,name,shares,price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+>>> def cost(self):
+        return self.shares*self.price
+
+>>> def sell(self,nshares):
+        self.shares -= nshares
+
+>>>
+```
+
+Next, make a methods dictionary:
+
+```python
+>>> methods = {
+         '__init__' : __init__,
+         'cost' : cost,
+         'sell' : sell } 
+
+>>>
+```
+
+Finally, create the `Stock` class object:
+
+```python
+>>> Stock = type('Stock',(object,),methods)
+>>> s = Stock('GOOG',100,490.10)
+>>> s.name
+'GOOG'
+>>> s.cost()
+49010.0
+>>> s.sell(25)
+>>> s.shares
+75
+>>>
+```
+
+Congratulations, you just created a class.   A class is really nothing
+more than a name, a tuple of base classes, and a dictionary holding
+all of the class contents. `type()` is a constructor that
+creates a class for you if you supply these three parts.
+
+## (b) Typed structures
+
+In the `structure.py` file, define the following function:
+
+```python
+# structure.py
+
+...
+def typed_structure(clsname, **validators):
+    cls = type(clsname, (Structure,), validators)
+    return cls
+```
+
+This function is somewhat similar to the `namedtuple()` function in that it creates a class. Try it out:
+
+```python
+>>> from validate import String, PositiveInteger, PositiveFloat
+>>> from structure import typed_structure
+>>> Stock = typed_structure('Stock', name=String(), shares=PositiveInteger(), price=PositiveFloat())
+>>> s = Stock('GOOG', 100, 490.1)
+>>> s.name
+'GOOG'
+>>> s
+Stock('GOOG', 100, 490.1)
+>>>
+```
+
+You might find the seams of your head starting to pull apart about now.
+
+## (c) Making a lot of classes
+
+
+There are other situations where direct usage of the `type()` constructor might be advantageous.
+Consider this bit of code:
+
+```python
+# validate.py
+...
+
+class Typed(Validator):
+    expected_type = object
+    @classmethod
+    def check(cls, value):
+        if not isinstance(value, cls.expected_type):
+            raise TypeError(f'expected {cls.expected_type}')
+        super().check(value)
+
+class Integer(Typed):
+    expected_type = int
+
+class Float(Typed):
+    expected_type = float
+
+class String(Typed):
+    expected_type = str
+...
+```
+
+Wow is the last part of that annoying and repetitive.  Change it
+to use a table of desired type classes like this:
+
+```python
+# validate.py
+...
+
+_typed_classes = [
+    ('Integer', int),
+    ('Float', float),
+    ('String', str) ]
+
+globals().update((name, type(name, (Typed,), {'expected_type':ty}))
+                 for name, ty in _typed_classes)
+```
+
+Now, if you want to have more type classes, you just add them to the
+table:
+
+```python
+_typed_classes = [
+    ('Integer', int),
+    ('Float', float),
+    ('Complex', complex),
+    ('Decimal', decimal.Decimal),
+    ('List', list),
+    ('Bool', bool),
+    ('String', str) ]
+```
+
+Admit it, that's kind of cool and saves a lot of typing (at the keyboard).
+
+\[ [Solution](soln7_4.md) | [Index](index.md) | [Exercise 7.3](ex7_3.md) | [Exercise 7.5](ex7_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_5.md

@@ -0,0 +1,81 @@
+\[ [Index](index.md) | [Exercise 7.4](ex7_4.md) | [Exercise 7.6](ex7_6.md) \]
+
+# Exercise 7.5
+
+*Objectives:*
+
+- Create your first metaclass
+
+*Files Created:* `mymeta.py`
+
+## (a) Create your first metaclass
+
+Create a file called `mymeta.py`
+and put the following code in it (from the slides):
+
+```python
+# mymeta.py
+
+class mytype(type):
+    @staticmethod
+    def __new__(meta, name, bases, __dict__):
+        print("Creating class :", name)
+        print("Base classes   :", bases)
+        print("Attributes     :", list(__dict__))
+        return super().__new__(meta, name, bases, __dict__)
+
+class myobject(metaclass=mytype):
+    pass
+```
+
+Once you've done this, define a class that inherits from
+`myobject` instead of object.  For example:
+
+```python
+class Stock(myobject):
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+    def cost(self):
+        return self.shares * self.price
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+Try running your code and creating instances of `Stock`.  See
+what happens. You should see the print statements from your
+`mytype` running once when the `Stock` class is
+defined.
+
+What happens if you inherit from `Stock`?
+
+```python
+class MyStock(Stock):
+    pass
+```
+
+You should still see your metaclass at work.  Metaclasses are "sticky" in that they
+get applied across an entire inheritance hierarchy.
+
+**Discussion**
+
+Why would you want to do something like this?
+The main power of a metaclass is that it gives a programmer the ability
+to capture details about classes just prior to their creation. For
+example, in the `__new__()` method, you are given all of the
+basic details including the name of the class, base classes, and
+methods data.   If you inspect this data, you can perform various
+types of diagnostic checks.  If you're more daring, you can modify the
+data and change what gets placed in the class definition when it is
+created.   Needless to say, there are many opportunities for horrible
+diabolical evil.
+
+\[ [Solution](soln7_5.md) | [Index](index.md) | [Exercise 7.4](ex7_4.md) | [Exercise 7.6](ex7_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex7_6.md

@@ -0,0 +1,165 @@
+\[ [Index](index.md) | [Exercise 7.5](ex7_5.md) | [Exercise 8.1](ex8_1.md) \]
+
+# Exercise 7.6
+
+*Objectives:*
+
+- Metaclasses in action
+- Explode your brain
+
+*Files Modified:* `structure.py`, `validate.py`
+
+## (a) The Final Frontier
+
+In link:ex7_3.html[Exercise 7.3], we made it possible to define type-checked structures as follows:
+
+```python
+from validate import String, PositiveInteger, PositiveFloat
+from structure import Structure
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+```
+
+There are a lot of things going on under the covers.  However, one annoyance
+concerns all of those type-name imports at the top (e.g., `String`, `PositiveInteger`, etc.).
+That's just the kind of thing that might lead to a `from validate import *` statement.
+One interesting thing about a metaclass is that it can be used to control
+the process by which a class gets defined.  This includes managing the
+environment of a class definition itself.  Let's tackle those imports.
+
+The first step in managing all of the validator names is to collect
+them.   Go to the file `validate.py` and modify the `Validator` base
+class with this extra bit of code involving `__init_subclass__()` again:
+
+```python
+# validate.py
+
+class Validator:
+    ...
+
+    # Collect all derived classes into a dict
+    validators = { }
+    @classmethod
+    def __init_subclass__(cls):
+        cls.validators[cls.__name__] = cls
+```
+
+That's not much, but it's creating a little namespace of all of the `Validator`
+subclasses.  Take a look at it:
+
+```python
+>>> from validate import Validator
+>>> Validator.validators
+{'Float': <class 'validate.Float'>,
+ 'Integer': <class 'validate.Integer'>,
+ 'NonEmpty': <class 'validate.NonEmpty'>,
+ 'NonEmptyString': <class 'validate.NonEmptyString'>,
+ 'Positive': <class 'validate.Positive'>,
+ 'PositiveFloat': <class 'validate.PositiveFloat'>,
+ 'PositiveInteger': <class 'validate.PositiveInteger'>,
+ 'String': <class 'validate.String'>,
+ 'Typed': <class 'validate.Typed'>}
+>>>
+```
+
+Now that you've done that, let's inject this namespace into namespace
+of classes defined from `Structure`. Define the following metaclass:
+
+```python
+# structure.py
+...
+
+from validate import Validator
+from collections import ChainMap
+
+class StructureMeta(type):
+    @classmethod
+    def __prepare__(meta, clsname, bases):
+        return ChainMap({}, Validator.validators)
+        
+    @staticmethod
+    def __new__(meta, name, bases, methods):
+        methods = methods.maps[0]
+        return super().__new__(meta, name, bases, methods)
+
+class Structure(metaclass=StructureMeta):
+    ...
+```
+
+In this code, the `__prepare__()` method is making a special `ChainMap` mapping that consists
+of an empty dictionary and a dictionary of all of the defined validators.  The empty dictionary
+that's listed first is going to collect all of the definitions made inside the class body.
+The `Validator.validators` dictionary is going to make all of the type definitions available 
+to for use as descriptors or argument type annotations.
+
+The `__new__()` method discards extra the validator dictionary and
+passes the remaining definitions onto the type constructor.  It's
+ingenious, but it lets you drop the annoying imports:
+
+```python
+# stock.py
+
+from structure import Structure
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+```
+
+## (b) Stare in Amazement
+
+Try running your `teststock.py` unit tests across this new file. Most of them should be
+passing now.   For kicks, try your `Stock` class with some of the earlier code
+for tableformatting and reading data.  It should all work.
+
+```python
+>>> from stock import Stock
+>>> from reader import read_csv_as_instances
+>>> portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+>>> portfolio
+[Stock('AA',100,32.2), Stock('IBM',50,91.1), Stock('CAT',150,83.44), Stock('MSFT',200,51.23), Stock('GE',95,40.37), Stock('MSFT',50,65.1), Stock('IBM',100,70.44)]
+>>> from tableformat import create_formatter, print_table
+>>> formatter = create_formatter('text')
+>>> print_table(portfolio, ['name','shares','price'], formatter)
+      name     shares      price 
+---------- ---------- ---------- 
+        AA        100       32.2 
+       IBM         50       91.1 
+       CAT        150      83.44 
+      MSFT        200      51.23 
+        GE         95      40.37 
+      MSFT         50       65.1 
+       IBM        100      70.44 
+>>> 
+```
+
+Again, marvel at the final `stock.py` file and observe how clean the
+code looks.  Just try not think about everything that is happening
+under the hood with the `Structure` base class.
+
+\[ [Solution](soln7_6.md) | [Index](index.md) | [Exercise 7.5](ex7_5.md) | [Exercise 8.1](ex8_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_1.md

@@ -0,0 +1,258 @@
+\[ [Index](index.md) | [Exercise 7.6](ex7_6.md) | [Exercise 8.2](ex8_2.md) \]
+
+# Exercise 8.1
+
+*Objectives:*
+
+- Learn how to customize iteration using generators
+
+*Files Modified:* `structure.py`
+
+*Files Created:* `follow.py`
+
+## (a) A Simple Generator
+
+If you ever find yourself wanting to customize iteration, you should
+always think generator functions.  They're easy to write---simply make
+a function that carries out the desired iteration logic and uses `yield`
+to emit values.
+
+For example, try this generator that allows you to iterate over a
+range of numbers with fractional steps (something not supported by
+the `range()` builtin):
+
+```python
+>>> def frange(start,stop,step):
+        while start < stop:
+            yield start
+            start += step
+
+>>> for x in frange(0, 2, 0.25):
+        print(x, end=' ')
+
+0 0.25 0.5 0.75 1.0 1.25 1.5 1.75
+>>>
+```
+
+Iterating on a generator is a one-time operation. For example, here's
+what happen if you try to iterate twice:
+
+```python
+>>> f = frange(0, 2, 0.25)
+>>> for x in f:
+        print(x, end=' ')
+
+0 0.25 0.5 0.75 1.0 1.25 1.5 1.75
+>>> for x in f:
+        print(x, end=' ')
+
+>>>
+```
+
+If you want to iterate over the same sequence, you need to recreate the generator
+by calling `frange()` again.    Alternative, you could package everything into a class:
+
+```python
+>>> class FRange:
+        def __init__(self, start, stop, step):
+            self.start = start
+            self.stop = stop
+            self.step = step
+        def __iter__(self):
+            n = self.start
+            while n < self.stop:
+                yield n
+                n += self.step
+
+>>> f = FRange(0, 2, 0.25)
+>>> for x in f:
+        print(x, end=' ')
+
+0 0.25 0.5 0.75 1.0 1.25 1.5 1.75
+>>> for x in f:
+        print(x, end=' ')
+
+0 0.25 0.5 0.75 1.0 1.25 1.5 1.75
+>>>
+```
+
+## (b) Adding Iteration to Objects
+
+If you've created a custom class, you can make it support iteration by
+defining an `__iter__()` special method.  `__iter__()` returns an
+iterator as a result.  As shown in the previous example, an easy way
+to do it is to define `__iter__()` as a generator.
+
+In earlier exercises, you defined a `Structure` base class.
+Add an `__iter__()` method to this class that produces the attribute values
+in order. For example:
+
+```python
+class Structure(metaclass=StructureMeta):
+    ...
+    def __iter__(self):
+        for name in self._fields:
+            yield getattr(self, name)
+    ...
+```
+
+Once you've done this, you should be able to iterate over the instance
+attributes like this:
+
+```python
+>>> from stock import Stock
+>>> s = Stock('GOOG', 100, 490.1)
+>>> for val in s:
+        print(val)
+GOOG
+100
+490.1
+>>>
+```
+
+## (c) The Surprising Power of Iteration
+
+Python uses iteration in ways you might not expect.   Once you've added `__iter__()`
+to the `Structure` class, you'll find that it is easy to do all sorts of new 
+operations. For example, conversions to sequences and unpacking:
+
+```python
+>>> s = Stock('GOOG', 100, 490.1)
+>>> list(s)
+['GOOG', 100, 490.1]
+>>> tuple(s)
+('GOOG', 100, 490.1)
+>>> name, shares, price = s
+>>> name
+'GOOG'
+>>> shares
+100
+>>> price
+490.1
+>>> 
+```
+
+While we're at it, we can now add a comparison operator to our `Structure`
+class:
+
+```python
+# structure.py
+class Structure(metaclass=StructureMeta):
+    ...
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and tuple(self) == tuple(other)
+    ...
+```
+
+You should now be able to compare objects:
+
+```python
+>>> a = Stock('GOOG', 100, 490.1)
+>>> b = Stock('GOOG', 100, 490.1)
+>>> a == b
+True
+>>>
+```
+
+Try running your `teststock.py` unit tests again.  Everything should be passing now.
+Excellent.
+
+## (d) Monitoring a streaming data source
+
+Generators can also be a useful way to simply produce a stream of
+data.  In this part, we'll explore this idea by writing a generator to
+watch a log file.  To start, follow the next instructions carefully.
+
+The program `Data/stocksim.py` is a program that
+simulates stock market data.  As output, the program constantly writes
+real-time data to a file `stocklog.csv`.  In a
+command window (not IDLE) go into the `Data/` directory and run this program:
+
+```
+% python3 stocksim.py
+```
+
+If you are on Windows, just locate the `stocksim.py` program and
+double-click on it to run it.  Now, forget about this program (just
+let it run).  Again, just let this program run in the background---it
+will run for several hours (you shouldn't need to worry about it).
+
+Once the above program is running, let's write a little program to
+open the file, seek to the end, and watch for new output.  Create a
+file `follow.py` and put this code in it:
+
+```python
+# follow.py
+import os
+import time
+f = open('Data/stocklog.csv')
+f.seek(0, os.SEEK_END)   # Move file pointer 0 bytes from end of file
+
+while True:
+    line = f.readline()
+    if line == '':
+        time.sleep(0.1)   # Sleep briefly and retry
+        continue
+    fields = line.split(',')
+    name = fields[0].strip('"')
+    price = float(fields[1])
+    change = float(fields[4])
+    if change < 0:
+        print('%10s %10.2f %10.2f' % (name, price, change))
+```
+
+If you run the program, you'll see a real-time stock ticker.  Under the covers,
+this code is kind of like the Unix `tail -f` command that's used to watch a log file.
+
+**Note:** The use of the `readline()` method in this example is
+somewhat unusual in that it is not the usual way of reading lines from
+a file (normally you would just use a `for`-loop).  However, in
+this case, we are using it to repeatedly probe the end of the file to
+see if more data has been added (`readline()` will either
+return new data or an empty string).
+
+If you look at the code carefully, the first part of the code is
+producing lines of data whereas the statements at the end of the
+`while` loop are consuming the data.  A major feature of generator
+functions is that you can move all of the data production code into a
+reusable function.
+
+Modify the code so that the file-reading is performed by 
+a generator function `follow(filename)`.   Make it so the following code
+works:
+
+```python
+>>> for line in follow('Data/stocklog.csv'):
+          print(line, end='')
+   
+... Should see lines of output produced here ...
+```
+
+Modify the stock ticker code so that it looks like this:
+
+```python
+for line in follow('Data/stocklog.csv'):
+    fields = line.split(',')
+    name = fields[0].strip('"')
+    price = float(fields[1])
+    change = float(fields[4])
+    if change < 0:
+        print('%10s %10.2f %10.2f' % (name, price, change))
+```
+
+**Discussion**
+
+Something very powerful just happened here.  You moved an interesting iteration pattern
+(reading lines at the end of a file) into its own little function.   The `follow()` function
+is now this completely general purpose utility that you can use in any program.  For
+example, you could use it to watch server logs, debugging logs, and other similar data sources.
+That's kind of cool.
+
+\[ [Solution](soln8_1.md) | [Index](index.md) | [Exercise 7.6](ex7_6.md) | [Exercise 8.2](ex8_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_2.md

@@ -0,0 +1,137 @@
+\[ [Index](index.md) | [Exercise 8.1](ex8_1.md) | [Exercise 8.3](ex8_3.md) \]
+
+# Exercise 8.2
+
+*Objectives:*
+
+- Using generators to set up processing pipelines
+
+*Files Created:* `ticker.py`
+
+**Note**
+
+For this exercise the `stocksim.py` program should still be
+running in the background.  You're going to use the `follow()`
+function you wrote in the previous exercise.
+
+## (a) Setting up a processing pipeline
+
+A major power of generators is that they allow you to create programs
+that set up processing pipelines--much like pipes on Unix systems.
+Experiment with this concept by performing these steps:
+
+```python
+>>> from follow import follow
+>>> import csv
+>>> lines = follow('Data/stocklog.csv')
+>>> rows = csv.reader(lines)
+>>> for row in rows:
+        print(row)
+
+['BA', '98.35', '6/11/2007', '09:41.07', '0.16', '98.25', '98.35', '98.31', '158148']
+['AA', '39.63', '6/11/2007', '09:41.07', '-0.03', '39.67', '39.63', '39.31', '270224']
+['XOM', '82.45', '6/11/2007', '09:41.07', '-0.23', '82.68', '82.64', '82.41', '748062']
+['PG', '62.95', '6/11/2007', '09:41.08', '-0.12', '62.80', '62.97', '62.61', '454327']
+...
+```
+
+Well, that's interesting.  What you're seeing here is that the output of the
+`follow()` function has been piped into the `csv.reader()` function and we're
+now getting a sequence of split rows.   
+
+## (b) Making more pipeline components
+
+In a file `ticker.py`, define the following class (using your structure code from before) and set up
+a pipeline:
+
+```python
+# ticker.py
+
+from structure import Structure
+
+class Ticker(Structure):
+    name = String()
+    price = Float()
+    date = String()
+    time = String()
+    change = Float()
+    open = Float()
+    high = Float()
+    low = Float()
+    volume = Integer()
+
+if __name__ == '__main__':
+    from follow import follow
+    import csv
+    lines = follow('Data/stocklog.csv')
+    rows = csv.reader(lines)
+    records = (Ticker.from_row(row) for row in rows)
+    for record in records:
+        print(record)
+```
+
+When you run this, you should see some output like this:
+
+    Ticker('IBM',103.53,'6/11/2007','09:53.59',0.46,102.87,103.53,102.77,541633)
+    Ticker('MSFT',30.21,'6/11/2007','09:54.01',0.16,30.05,30.21,29.95,7562516)
+    Ticker('AA',40.01,'6/11/2007','09:54.01',0.35,39.67,40.15,39.31,576619)
+    Ticker('T',40.1,'6/11/2007','09:54.08',-0.16,40.2,40.19,39.87,1312959)
+
+## (c) Keep going
+
+Oh, you can do better than that.  Let's plug this into your table generation code. Change
+the program to the following:
+
+```python
+# ticker.py
+...
+
+if __name__ == '__main__':
+    from follow import follow
+    import csv
+    from tableformat import create_formatter, print_table
+
+    formatter = create_formatter('text')
+
+    lines = follow('Data/stocklog.csv')
+    rows = csv.reader(lines)
+    records = (Ticker.from_row(row) for row in rows)
+    negative = (rec for rec in records if rec.change < 0)
+    print_table(negative, ['name','price','change'], formatter)
+```
+
+This should produce some output that looks like this:
+
+          name      price     change 
+    ---------- ---------- ---------- 
+             C      53.12      -0.21 
+           UTX      70.04      -0.19 
+           AXP      62.86      -0.18 
+           MMM      85.72      -0.22 
+           MCD      51.38      -0.03 
+           WMT      49.85      -0.23 
+            KO       51.6      -0.07 
+           AIG      71.39      -0.14 
+            PG      63.05      -0.02 
+            HD      37.76      -0.19 
+
+Now, THAT is crazy! And pretty awesome.
+
+**Discussion**
+
+Some lessons learned: You can create various generator functions and
+chain them together to perform processing involving data-flow
+pipelines.  
+
+A good mental model for generator functions might be Lego blocks.
+You can make a collection of small iterator patterns and start
+stacking them together in various ways.  It can be an extremely powerful way to program.
+
+\[ [Solution](soln8_2.md) | [Index](index.md) | [Exercise 8.1](ex8_1.md) | [Exercise 8.3](ex8_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_3.md

@@ -0,0 +1,141 @@
+\[ [Index](index.md) | [Exercise 8.2](ex8_2.md) | [Exercise 8.4](ex8_4.md) \]
+
+# Exercise 8.3
+
+*Objectives:*
+
+- Using coroutines to set up processing pipelines
+
+*Files Created:* `cofollow.py`, `coticker.py`
+
+**Note**
+
+For this exercise the `stocksim.py` program should still be
+running in the background.  
+
+In link:ex8_2.html[Exercise 8.2] you wrote some code that used
+generators to set up a processing pipeline.  A key aspect of that
+program was the idea of data flowing between generator functions.  A
+very similar kind of dataflow can be set up using coroutines.  The
+only difference is that with a coroutine, you send data into different
+processing elements as opposed to pulling data out with a for-loop.
+
+## (a) A coroutine example
+
+Getting started with coroutines can be a little tricky.  Here is an
+example program that performs the same task as
+link:ex8_2.html[Exercise 8.2], but with coroutines.  Take this program
+and copy it into a file called `cofollow.py`.
+
+```python
+# cofollow.py
+import os
+import time
+
+# Data source
+def follow(filename,target):
+    with open(filename,'r') as f:
+        f.seek(0,os.SEEK_END)
+        while True:
+            line = f.readline()
+            if line != '':
+                target.send(line)
+            else:
+                time.sleep(0.1)
+
+# Decorator for coroutine functions
+from functools import wraps
+
+def consumer(func):
+    @wraps(func)
+    def start(*args,**kwargs):
+        f = func(*args,**kwargs)
+        f.send(None)
+        return f
+    return start
+
+# Sample coroutine
+@consumer
+def printer():
+    while True:
+        item = yield     # Receive an item sent to me
+        print(item)
+
+# Example use
+if __name__ == '__main__':
+    follow('Data/stocklog.csv',printer())
+```
+
+Run this program and make sure produces output..   Make sure you understand how the different pieces are hooked together.
+
+## (b) Build some pipeline components
+
+In a file `coticker.py`, build a series of pipeline components that carry out the same tasks as
+the `ticker.py` program in link:ex8_2.html[Exercise 8.2].  Here is the implementation of the
+various pieces.
+
+```python
+# coticker.py
+from structure import Structure
+
+class Ticker(Structure):
+    name = String()
+    price =Float()
+    date = String()
+    time = String()
+    change = Float()
+    open = Float()
+    high = Float()
+    low = Float()
+    volume = Integer()
+
+from cofollow import consumer, follow
+from tableformat import create_formatter
+import csv
+
+# This one is tricky. See solution for notes about it
+@consumer
+def to_csv(target):
+    def producer():
+        while True:
+            yield line
+
+    reader = csv.reader(producer())
+    while True:
+        line = yield
+        target.send(next(reader))
+
+@consumer
+def create_ticker(target):
+    while True:
+        row = yield
+        target.send(Ticker.from_row(row))
+
+@consumer
+def negchange(target):
+    while True:
+        record = yield
+        if record.change < 0:
+            target.send(record)
+
+@consumer
+def ticker(fmt, fields):
+    formatter = create_formatter(fmt)
+    formatter.headings(fields)
+    while True:
+        rec = yield
+        row = [getattr(rec, name) for name in fields]
+        formatter.row(row)
+```
+
+Your challenge: Write the main program that hooks all of these components together to 
+generate the same stock ticker as in the previous exercise.
+
+\[ [Solution](soln8_3.md) | [Index](index.md) | [Exercise 8.2](ex8_2.md) | [Exercise 8.4](ex8_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_4.md

@@ -0,0 +1,153 @@
+\[ [Index](index.md) | [Exercise 8.3](ex8_3.md) | [Exercise 8.5](ex8_5.md) \]
+
+# Exercise 8.4
+
+*Objectives:*
+
+- Managing what happens at the `yield` statements
+
+*Files Modified:* `follow.py`, `cofollow.py`
+
+## (a) Closing a Generator
+
+A common question concerning generators is their lifetime and garbage
+collection. For example, the `follow()` generator runs forever in
+an infinite `while` loop. What happens if the iteration loop that's
+driving it stops?  Also, is there anyway to prematurely terminate the 
+generator?
+
+Modify the `follow()` function so that all of the code is enclosed in
+a `try-except` block like this:
+
+```python
+def follow(filename):
+    try:
+        with open(filename,'r') as f:
+            f.seek(0,os.SEEK_END)
+            while True:
+                 line = f.readline()
+                 if line == '':
+                     time.sleep(0.1)    # Sleep briefly to avoid busy wait
+                     continue
+                 yield line
+    except GeneratorExit:
+        print('Following Done')
+```
+
+Now, try a few experiments:
+
+```python
+>>> from follow import follow
+>>> # Experiment: Garbage collection of a running generator
+>>> f = follow('Data/stocklog.csv')
+>>> next(f)
+'"MO",70.29,"6/11/2007","09:30.09",-0.01,70.25,70.30,70.29,365314\n'
+>>> del f
+Following Done
+>>> # Experiment: Closing a generator
+>>> f = follow('Data/stocklog.csv')
+>>> for line in f:
+        print(line,end='')
+        if 'IBM' in line:
+            f.close()
+
+"VZ",42.91,"6/11/2007","09:34.28",-0.16,42.95,42.91,42.78,210151
+"HPQ",45.76,"6/11/2007","09:34.29",0.06,45.80,45.76,45.59,257169
+"GM",31.45,"6/11/2007","09:34.31",0.45,31.00,31.50,31.45,582429
+...
+"IBM",102.86,"6/11/2007","09:34.44",-0.21,102.87,102.86,102.77,147550
+Following Done
+>>> for line in f:
+        print(line, end='')    # No output: generator is done
+
+>>> 
+```
+
+In these experiments you can see that a `GeneratorExit` exception is
+raised when a generator is garbage-collected or explicitly closed via
+its `close()` method. 
+
+One additional area of exploration is whether or not you can resume
+iteration on a generator if you break out of a for-loop.  For example,
+try this:
+
+```python
+>>> f = follow('Data/stocklog.csv')
+>>> for line in f:
+        print(line,end='')
+        if 'IBM' in line:
+            break
+
+"CAT",78.36,"6/11/2007","09:37.19",-0.16,78.32,78.36,77.99,237714
+"VZ",42.99,"6/11/2007","09:37.20",-0.08,42.95,42.99,42.78,268459
+...
+"IBM",102.91,"6/11/2007","09:37.31",-0.16,102.87,102.91,102.77,190859
+>>> # Resume iteration
+>>> for line in f:
+        print(line,end='')
+        if 'IBM' in line:
+            break
+
+"AA",39.58,"6/11/2007","09:39.28",-0.08,39.67,39.58,39.31,243159
+"HPQ",45.94,"6/11/2007","09:39.29",0.24,45.80,45.94,45.59,408919
+...
+"IBM",102.95,"6/11/2007","09:39.44",-0.12,102.87,102.95,102.77,225350
+>>> del f
+Following Done
+>>>
+```
+
+In general, you can break out of running iteration and resume it later
+if you need to.  You just need to make sure the generator object isn't
+forcefully closed or garbage collected somehow.
+
+## (b) Raising Exceptions
+
+In the file `cofollow.py`, you created a coroutine `printer()`.  Modify the
+code to catch and report exceptions like this:
+
+```python
+# cofollow.py
+...
+@consumer
+def printer():
+    while True:
+        try:
+            item = yield
+            print(item)
+        except Exception as e:
+            print('ERROR: %r' % e)
+```
+
+Now, try an experiment:
+
+```python
+>>> from cofollow import printer
+>>> p = printer()
+>>> p.send('hello')
+hello
+>>> p.send(42)
+42
+>>> p.throw(ValueError('It failed'))
+ERROR: ValueError('It failed',)
+>>> try:
+        int('n/a')
+    except ValueError as e:
+        p.throw(e)
+
+ERROR: ValueError("invalid literal for int() with base 10: 'n/a'",)
+>>> 
+```
+
+Notice how the running generator is not terminated by the exception. This
+is merely allowing the `yield` statement to signal an error instead of
+receiving a value.
+
+\[ [Solution](soln8_4.md) | [Index](index.md) | [Exercise 8.3](ex8_3.md) | [Exercise 8.5](ex8_5.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_5.md

@@ -0,0 +1,225 @@
+\[ [Index](index.md) | [Exercise 8.4](ex8_4.md) | [Exercise 8.6](ex8_6.md) \]
+
+# Exercise 8.5
+
+*Objectives:*
+
+- Learn about managed generators
+
+*Files Created:* `multitask.py`, `server.py`
+
+A generator or coroutine function can never execute without being
+driven by some other code.  For example, a generator used for
+iteration doesn't do anything unless iteration is actually carried out
+using a for-loop.  Similarly, a collection of coroutines won't run
+unless their `send()` method is invoked somehow.
+
+In advanced applications of generators, it is possible to drive 
+generators in various unusual ways.  In this exercise, we look at a
+few examples.
+
+## (a) Generators as tasks
+
+If a file `multitask.py`, define the following code:
+
+```python
+# multitask.py
+
+from collections import deque
+
+tasks = deque()
+def run():
+    while tasks:
+        task = tasks.popleft()
+        try:
+            task.send(None)
+            tasks.append(task)
+        except StopIteration:
+            print('Task done')
+```
+
+This code implements a tiny task scheduler that runs generator functions.
+Try it by running it on the following functions.
+
+```python
+# multitask.py
+...
+
+def countdown(n):
+    while n > 0:
+        print('T-minus', n)
+        yield
+        n -= 1
+
+def countup(n):
+    x = 0
+    while x < n:
+        print('Up we go', x)
+        yield
+        x += 1
+
+if __name__ == '__main__':
+    tasks.append(countdown(10))
+    tasks.append(countdown(5))
+    tasks.append(countup(20))
+    run()
+```
+
+When you run this, you should see output from all of the generators
+interleaved together.  For example:
+
+```python
+T-minus 10
+T-minus 5
+Up we go 0
+T-minus 9
+T-minus 4
+Up we go 1
+T-minus 8
+T-minus 3
+Up we go 2
+T-minus 7
+T-minus 2
+Up we go 3
+T-minus 6
+T-minus 1
+Up we go 4
+T-minus 5
+Task done
+Up we go 5
+T-minus 4
+Up we go 6
+T-minus 3
+Up we go 7
+T-minus 2
+Up we go 8
+T-minus 1
+Up we go 9
+Task done
+Up we go 10
+Up we go 11
+Up we go 12
+Up we go 13
+Up we go 14
+Up we go 15
+Up we go 16
+Up we go 17
+Up we go 18
+Up we go 19
+Task done
+```
+
+That's interesting, but not especially compelling.  Move on to the next example.
+
+## (b) Generators as Tasks Serving Network Connections
+
+Create a file `server.py` and put the following code into it:
+
+```python
+# server.py
+
+from socket import *
+from select import select
+from collections import deque
+
+tasks = deque()
+recv_wait = {}   #  sock -> task
+send_wait = {}   #  sock -> task
+
+def run():
+    while any([tasks, recv_wait, send_wait]):
+        while not tasks:
+            can_recv, can_send, _ = select(recv_wait, send_wait, [])
+            for s in can_recv:
+                tasks.append(recv_wait.pop(s))
+            for s in can_send:
+                tasks.append(send_wait.pop(s))
+        task = tasks.popleft()
+        try:
+            reason, resource = task.send(None)
+            if reason == 'recv':
+                recv_wait[resource] = task
+            elif reason == 'send':
+                send_wait[resource] = task
+            else:
+                raise RuntimeError('Unknown reason %r' % reason)
+        except StopIteration:
+            print('Task done')
+```
+
+This code is a slightly more complicated version of the task scheduler in
+part (a).  It will require a bit of study, but the idea is that not only
+will each task yield, it will indicate a reason for doing so (receiving or
+sending).  Depending on the reason, the task will move over to a waiting
+area.  The scheduler then runs any available tasks or waits for I/O
+events to occur when nothing is left to do.
+
+It's all a bit tricky perhaps, but add the following code which implements
+a simple echo server:
+
+```python
+# server.py
+...
+
+def tcp_server(address, handler):
+    sock = socket(AF_INET, SOCK_STREAM)
+    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+    sock.bind(address)
+    sock.listen(5)
+    while True:
+        yield 'recv', sock
+        client, addr = sock.accept()
+        tasks.append(handler(client, addr))
+        
+def echo_handler(client, address):
+    print('Connection from', address)
+    while True:
+        yield 'recv', client
+        data = client.recv(1000)
+        if not data:
+            break
+        yield 'send', client
+        client.send(b'GOT:' + data)
+    print('Connection closed')
+
+if __name__ == '__main__':
+    tasks.append(tcp_server(('',25000), echo_handler))
+    run()
+```
+
+Run this server in its own terminal window.   In another terminal, connect to it using a command such as `telnet` or `nc`.   For example:
+
+```
+bash % nc localhost 25000
+Hello
+Got: Hello
+World
+Got: World
+```
+
+If you don't have access to `nc` or `telnet` you can also use Python itself:
+
+```
+bash % python3 -m telnetlib localhost 25000
+Hello
+Got: Hello
+World
+Got: World
+```
+
+If it's working, you should see output being echoed back to you. Not only that,
+if you connect multiple clients, they'll all operate concurrently.
+
+This tricky use of generators is not something that you would
+likely have to code directly.  However, they are used in certain advanced
+packages such as `asyncio` that was added to the standard
+library in Python 3.4.
+
+\[ [Solution](soln8_5.md) | [Index](index.md) | [Exercise 8.4](ex8_4.md) | [Exercise 8.6](ex8_6.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex8_6.md

@@ -0,0 +1,200 @@
+\[ [Index](index.md) | [Exercise 8.5](ex8_5.md) | [Exercise 9.1](ex9_1.md) \]
+
+# Exercise 8.6
+
+*Objectives:*
+
+- Learn about delegating generators
+
+*Files Modified:* `cofollow.py`, `server.py`
+
+One potential issue in code that relies on generators is the problem
+of hiding details from the user and writing libraries.  A lot of low-level
+mechanics are generally required to drive everything and it's often rather
+awkward to directly expose it to users.
+
+Starting in Python 3.3, a new `yield from` statement can be used to
+delegate generators to another function.  It is a useful way to
+clean-up code that relies on generators.
+
+## (a) Example: Receiving messages
+
+In link:ex8_3.html[Exercise 8.3], we looked at the definitions of coroutines.
+Coroutines were functions that you sent data to.  For example:
+
+```python
+>>> from cofollow import consumer
+>>> @consumer
+    def printer():
+        while True:
+            item = yield
+            print('Got:', item)
+
+>>> p = printer()
+>>> p.send('Hello')
+Got: Hello
+>>> p.send('World')
+Got: World
+>>>
+```
+
+At the time, it might have been interesting to use `yield` to receive a 
+value.  However, if you really look at the code, it looks pretty weird--a
+bare `yield` like that?  What's going on there?
+
+In the `cofollow.py` file, define the following function:
+
+```python
+def receive(expected_type):
+    msg = yield
+    assert isinstance(msg, expected_type), 'Expected type %s' % (expected_type)
+    return msg
+```
+
+This function receives a message, but then verifies that it is of an expected
+type.  Try it:
+
+```python
+>>> from cofollow import consumer, receive
+>>> @consumer
+    def print_ints():
+        while True:
+             val = yield from receive(int)
+             print('Got:', val)
+
+>>> p = print_ints()
+>>> p.send(42)
+Got: 42
+>>> p.send(13)
+Got: 13
+>>> p.send('13')
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  ...
+AssertionError: Expected type <class 'int'>
+>>> 
+```
+
+From a readability point of view, the `yield from receive(int)` statement
+is a bit more descriptive--it indicates that the function will yield until
+it receives a message of a given type. 
+
+Now, modify all of the coroutines in `coticker.py` to use the new `receive()`
+function and make sure the code from link:ex8_3.html[Exercise 8.3] still
+works.
+
+## (b) Wrapping a Socket
+
+In the previous exercise, you wrote a simple network echo server using
+generators.  The code for the server looked like this:
+
+```python
+def tcp_server(address, handler):
+    sock = socket(AF_INET, SOCK_STREAM)
+    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+    sock.bind(address)
+    sock.listen(5)
+    while True:
+        yield 'recv', sock
+        client, addr = sock.accept()
+        tasks.append(handler(client, addr))
+        
+def echo_handler(client, address):
+    print('Connection from', address)
+    while True:
+        yield 'recv', client
+        data = client.recv(1000)
+        if not data:
+            break
+        yield 'send', client
+        client.send(b'GOT:', data)
+    print('Connection closed')
+```
+
+Create a class `GenSocket` that cleans up the `yield` statements and
+allows the server to be rewritten more simply as follows:
+
+```python
+def tcp_server(address, handler):
+    sock = GenSocket(socket(AF_INET, SOCK_STREAM))
+    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+    sock.bind(address)
+    sock.listen(5)
+    while True:
+        client, addr = yield from sock.accept()
+        tasks.append(handler(client, addr))
+        
+def echo_handler(client, address):
+    print('Connection from', address)
+    while True:
+        data = yield from client.recv(1000)
+        if not data:
+            break
+        yield from client.send(b'GOT:', data)
+    print('Connection closed')
+```
+
+## (c) Async/Await
+
+Take the `GenSocket` class you just wrote and wrap all of the methods 
+that use `yield` with the `@coroutine` decorator from the `types` module.
+
+```python
+from types import coroutine
+...
+
+class GenSocket:
+    def __init__(self, sock):
+        self.sock = sock
+
+    @coroutine
+    def accept(self):
+        yield 'recv', self.sock
+        client, addr = self.sock.accept()
+        return GenSocket(client), addr
+
+    @coroutine
+    def recv(self, maxsize):
+        yield 'recv', self.sock
+        return self.sock.recv(maxsize)
+
+    @coroutine
+    def send(self, data):
+        yield 'send', self.sock
+        return self.sock.send(data)
+
+    def __getattr__(self, name):
+        return getattr(self.sock, name)
+```
+
+Now, rewrite your server code to use `async` functions and `await` statements like this:
+
+```python
+async def tcp_server(address, handler):
+    sock = GenSocket(socket(AF_INET, SOCK_STREAM))
+    sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+    sock.bind(address)
+    sock.listen(5)
+    while True:
+        client, addr = await sock.accept()
+        tasks.append(handler(client, addr))
+        
+async def echo_handler(client, address):
+    print('Connection from', address)
+    while True:
+        data = await client.recv(1000)
+        if not data:
+            break
+        await client.send(b'GOT:', data)
+    print('Connection closed')
+```
+
+
+\[ [Solution](soln8_6.md) | [Index](index.md) | [Exercise 8.5](ex8_5.md) | [Exercise 9.1](ex9_1.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex9_1.md

@@ -0,0 +1,211 @@
+\[ [Index](index.md) | [Exercise 8.6](ex8_6.md) | [Exercise 9.2](ex9_2.md) \]
+
+# Exercise 9.1
+
+*Objectives:*
+
+- A review of module basics
+
+This exercise is about some of the more tricky details of library modules. 
+Start this exercise by creating a very simple library module:
+
+```python
+# simplemod.py
+
+x = 42        # A global variable
+
+# A simple function
+def foo():   
+    print('x is', x)
+
+# A simple class
+class Spam:
+    def yow(self):
+        print('Yow!')
+
+# A scripting statement
+print('Loaded simplemod')
+```
+    
+##  (a) Module Loading and System Path
+
+Try importing the module you just created:
+
+```python
+>>> import simplemod
+Loaded simplemod
+>>> simplemod.foo()
+x is 42
+>>>
+```
+    
+If this failed with an `ImportError`, your path setting is
+flaky.  Look at the value of `sys.path` and fix it.
+
+```python
+>>> import sys
+>>> sys.path
+... look at the result ...
+>>>
+```
+    
+## (b) Repeated Module Loading
+    
+Make sure you understand that modules are only loaded
+once.  Try a repeated import and notice how you do not see
+the output from the `print` function:
+
+```python
+>>> import simplemod
+>>>
+```
+
+Try changing the value of `x` and see that a repeated import
+has no effect.
+
+```python
+>>> simplemod.x
+42
+>>> simplemod.x = 13
+>>> simplemod.x
+13
+>>> import simplemod
+>>> simplemod.x
+13
+>>>
+```
+
+Use `importlib.reload()` if you want to force a module to reload.
+
+```python
+>>> import importlib
+>>> importlib.reload(simplemod)
+Loaded simplemod
+<module 'simplemod' from 'simplemod.py'>
+>>> simplemod.x
+42
+>>>
+```
+    
+`sys.modules` is a dictionary of all loaded modules.  Take
+a look at it, delete your module, and try a repeated import.
+
+```python
+>>> sys.modules
+... look at output ...
+>>> sys.modules['simplemod']
+<module 'simplemod' from 'simplemod.py'>
+>>> del sys.modules['simplemod']
+>>> import simplemod
+Loaded simplemod
+>>>
+```
+
+## (c) from module import
+
+Restart Python and import a selected symbol from a module.
+
+```python
+>>> ############### [ RESTART ] ###############
+>>> from simplemod import foo
+Loaded simplemod
+>>> foo()
+x is 42
+>>>
+```
+
+Notice how this loaded the entire module (observe the output of
+the print function and how the `x` variable is used).
+
+When you use `from`, the module object itself is not
+visible.  For example:
+
+```python
+>>> simplemod.foo()
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+NameError: name 'simplemod' is not defined
+>>> 
+```
+
+Make sure you understand that when you export things from a module,
+they are simply name references.  For example, try this and explain:
+
+```python
+>>> from simplemod import x,foo
+>>> x
+42
+>>> foo()
+x is 42
+>>> x = 13
+>>> foo()
+x is 42                   # !! Please explain
+>>> x
+13
+>>>
+```
+
+## (d) Broken reload()
+
+Create an instance:
+
+```python
+>>> import simplemod
+>>> s = simplemod.Spam()
+>>> s.yow()
+Yow!
+>>>
+```
+
+Now, go to the `simplemod.py` file and change the implementation of `Spam.yow()` to the
+following:
+
+```python
+# simplemod.py
+...
+
+class Spam:
+    def yow(self):
+        print('More Yow!')
+```
+
+Now, watch what happens on a reload.  Do not restart Python for this part.
+
+```python
+>>> importlib.reload(simplemod)
+Loaded simplemod
+<module 'simplemod' from 'simplemod.py'>
+>>> s.yow()
+'Yow!'
+>>> t = simplemod.Spam()
+>>> t.yow()
+'More Yow!'
+>>>
+```
+
+Notice how you have two instances of `Spam`, but they're using different implementations
+of the `yow()` method.  Yes, actually both versions of code are loaded at the same time.
+You'll find other oddities as well.  For example:
+
+```python
+>>> s
+<simplemod.Spam object at 0x1006940b8>
+>>> isinstance(s, simplemod.Spam)
+False
+>>> isinstance(t, simplemod.Spam)
+True
+>>>
+```
+
+Bottom line:  It's probably best not to rely on reloading for anything important. 
+It might be fine if you're just trying to debug some things (as long as you're
+aware of its limitations and dangers).
+
+\[ [Solution](soln9_1.md) | [Index](index.md) | [Exercise 8.6](ex8_6.md) | [Exercise 9.2](ex9_2.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex9_2.md

@@ -0,0 +1,67 @@
+\[ [Index](index.md) | [Exercise 9.1](ex9_1.md) | [Exercise 9.3](ex9_3.md) \]
+
+# Exercise 9.2
+
+*Objectives:*
+
+- Learn how to create a Python package
+
+**Note**
+
+This exercise mostly just involves copying files on the file system.
+There shouldn't be a lot of coding.
+
+## (a) Making a Package
+
+In previous exercises, you created the following files that were related to
+type-checked structures, reading data, and making tables:
+
+- `structure.py`
+- `validate.py`
+- `reader.py`
+- `tableformat.py`
+
+Your task is to take all of these files and move them into a package called `structly`.
+To do that, follow these steps:
+
+- Make a directory called `structly`
+- Make an empty file `__init__.py` and put it in the `structly` directory
+- Move the files `structure.py`, `validate.py`, `reader.py`, and `tableformat.py` into the `structly` directory.
+- Fix any import statements between modules (specifically, the `structure` module depends on `validate`).
+
+Once you've done that, modify the `stock.py` program so that it looks exactly like this
+and that it works:
+
+```python
+# stock.py
+
+from structly.structure import Structure
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+
+if __name__ == '__main__':
+    from structly.reader import read_csv_as_instances
+    from structly.tableformat import create_formatter, print_table
+    portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+    formatter = create_formatter('text')
+    print_table(portfolio, ['name','shares','price'], formatter)
+```
+
+\[ [Solution](soln9_2.md) | [Index](index.md) | [Exercise 9.1](ex9_1.md) | [Exercise 9.3](ex9_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex9_3.md

@@ -0,0 +1,208 @@
+\[ [Index](index.md) | [Exercise 9.2](ex9_2.md) | [Exercise 9.4](ex9_4.md) \]
+
+# Exercise 9.3
+
+*Objectives:*
+
+- Learn about controlling symbols and combining submodules
+- Learn about module splitting
+
+One potentially annoying aspect of packages is that they complicate
+import statements.   For example, in the `stock.py` program, you now
+have import statements such as the following:
+
+```python
+from structly.structure import Structure
+from structly.reader import read_csv_as_instances
+from structly.tableformat import create_formatter, print_table
+```
+
+If the package is meant to be used as a unified whole, it might be
+more sane (and easier) to consolidate everything into a single top
+level package.   Let's do that:
+
+## (a) Controlling Exported Symbols
+
+Modify all of the submodules in the `structly` package so that they explicitly
+define an `__all__` variable which exports selected symbols.  Specifically:
+
+- `structure.py` should export `Structure`
+- `reader.py` should export all of the various `read_csv_as_*()` functions
+- `tableformat.py` exports `create_formatter()` and `print_table()`
+
+Now, in the `__init__.py` file, unify all of the submodules like this:
+
+```python
+# structly/__init__.py
+
+from .structure import *
+from .reader import *
+from .tableformat import *
+```
+
+Once you have done this, you should be able to import everything from 
+a single logical module:
+
+```python
+# stock.py
+
+from structly import Structure
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+
+if __name__ == '__main__':
+    from structly import read_csv_as_instances, create_formatter, print_table
+    portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+    formatter = create_formatter('text')
+    print_table(portfolio, ['name','shares','price'], formatter)
+```
+
+## (b) Exporting Everything
+
+In the `structly/__init__.py`, define an `__all__` variable that contains all
+exported symbols.   Once you've done this, you should be able to simplify the
+`stock.py` file further:
+
+```python
+# stock.py
+
+from structly import *
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares: PositiveInteger):
+        self.shares -= nshares
+
+if __name__ == '__main__':
+    portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+    formatter = create_formatter('text')
+    print_table(portfolio, ['name','shares','price'], formatter)
+```
+
+As an aside, use of the `from module import *` statement is generally frowned upon
+the Python community--especially if you're not sure what you're doing.  That said,
+there are situations where it often makes sense. For example, if a package defines
+a large number of commonly used symbols or constants it might be useful to use it.
+
+
+## (c) Module Splitting
+
+The file `structly/tableformat.py` contains code for creating tables in different
+formats. Specifically:
+
+- A `TableFormatter` base class.
+- A `TextTableFormatter` class.
+- A `CSVTableFormatter` class.
+- A `HTMLTableFormatter` class.
+
+Instead of having all of these classes in a single `.py`
+file, maybe it would make sense to move each concrete formatter to
+its own file.  To do this, we're going to split the `tableformat.py`
+file into parts.  Follow these instructions carefully:
+
+First, remove the `structly/__pycache__` directory. 
+
+```
+% cd structly
+% rm -rf __pycache__
+```
+
+Next, create the directory `structly/tableformat`.  This directory
+must have exactly the same name as the module it is replacing
+(`tableformat.py`).
+
+```
+bash % mkdir tableformat
+bash %
+```
+
+Move the original `tableformat.py` file into the new
+`tableformat` directory and rename it to `formatter.py`. 
+
+```
+bash % mv tableformat.py tableformat/formatter.py
+bash %
+```
+
+In the `tableformat` directory, split the
+`tableformat.py` code into the following files and directories:
+
+- `formatter.py` - Contains the `TableFormatter` base class, mixins, and various functions.
+- `formats/text.py` - Contains the `TextTableFormatter` class.
+- `formats/csv.py` - Contains the `CSVTableFormatter` class.
+- `formats/html.py` - Contains the `HTMLTableFormatter` class.
+
+Add an `__init__.py` file to the `tableformat/` and `tableformat/formats`
+directories.  Have the `tableformat/__init__.py` export the same
+symbols that the original `tableformat.py` file exported.
+
+After you have made all of these changes, you should have a package
+structure that looks like this:
+
+```
+structly/
+      __init__.py
+      validate.py
+      reader.py
+      structure.py
+      tableformat/
+           __init__.py
+           formatter.py
+           formats/
+               __init__.py
+               text.py
+               csv.py
+               html.py
+```
+
+To users, everything should work exactly as it did before.  For example, your
+prior `stock.py` file should work:
+
+```python
+# stock.py
+
+from structly import *
+
+class Stock(Structure):
+    name = String()
+    shares = PositiveInteger()
+    price = PositiveFloat()
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+
+if __name__ == '__main__':
+    portfolio = read_csv_as_instances('Data/portfolio.csv', Stock)
+    formatter = create_formatter('text')
+    print_table(portfolio, ['name','shares','price'], formatter)
+```
+
+\[ [Solution](soln9_3.md) | [Index](index.md) | [Exercise 9.2](ex9_2.md) | [Exercise 9.4](ex9_4.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/ex9_4.md

@@ -0,0 +1,193 @@
+\[ [Index](index.md) | [Exercise 9.3](ex9_3.md) | []() \]
+
+# Exercise 9.4
+
+*Objectives:*
+
+- Explore circular imports
+- Dynamic module imports
+
+In the last exercise, you split the `tableformat.py` file up into submodules.
+The last part of the resulting `tableformat/formatter.py` file has turned into a mess of imports.
+
+```python
+# tableformat.py
+...
+
+class TableFormatter(ABC):
+    @abstractmethod
+    def headings(self, headers):
+        pass
+
+    @abstractmethod
+    def row(self, rowdata):
+        pass
+
+from .formats.text import TextTableFormatter
+from .formats.csv import CSVTableFormatter
+from .formats.html import HTMLTableFormatter
+
+...
+
+def create_formatter(name, column_formats=None, upper_headers=False):
+    if name == 'text':
+        formatter_cls = TextTableFormatter
+    elif name == 'csv':
+        formatter_cls = CSVTableFormatter
+    elif name == 'html':
+        formatter_cls = HTMLTableFormatter
+    else:
+        raise RuntimeError('Unknown format %s' % name)
+
+    if column_formats:
+        class formatter_cls(ColumnFormatMixin, formatter_cls):
+              formats = column_formats
+
+    if upper_headers:
+        class formatter_cls(UpperHeadersMixin, formatter_cls):
+            pass
+
+    return formatter_cls()
+```
+
+The imports in the middle of the file are required because the `create_formatter()`
+function needs them to find the appropriate classes.   Really, the whole thing is a mess.
+
+## (a) Circular Imports
+
+Try moving the following import statements to the top of the `formatter.py` file:
+
+```python
+# formatter.py
+
+from .formats.text import TextTableFormatter
+from .formats.csv import CSVTableFormatter
+from .formats.html import HTMLTableFormatter
+
+class TableFormatter(ABC):
+    @abstractmethod
+    def headings(self, headers):
+        pass
+
+    @abstractmethod
+    def row(self, rowdata):
+        pass
+
+...
+```
+
+Observe that nothing works anymore.  Try running the `stock.py` program and
+notice the error about `TableFormatter` not being defined.   The order
+of import statements matters and you can't just move the imports anywhere
+you want.
+
+Move the import statements back where they were.  Sigh.
+
+## (b) Subclass Registration
+
+Try the following experiment and observe:
+
+```python
+>>> from structly.tableformat.formats.text import TextTableFormatter
+>>> TextTableFormatter.__module__
+'structly.tableformat.formats.text'
+>>> TextTableFormatter.__module__.split('.')[-1]
+'text'
+>>>
+```
+
+Modify the `TableFormatter` base class by adding a dictionary and an
+`__init_subclass__()` method:
+
+```python
+class TableFormatter(ABC):
+    _formats = { }
+
+    @classmethod
+    def __init_subclass__(cls):
+        name = cls.__module__.split('.')[-1]
+        TableFormatter._formats[name] = cls
+
+    @abstractmethod
+    def headings(self, headers):
+        pass
+
+    @abstractmethod
+    def row(self, rowdata):
+        pass
+```
+
+This makes the parent class track all of its subclasses.  Check it out:
+
+```python
+>>> from structly.tableformat.formatter import TableFormatter
+>>> TableFormatter._formats
+{'text': <class 'structly.tableformat.formats.text.TextTableFormatter'>, 
+ 'csv': <class 'structly.tableformat.formats.csv.CSVTableFormatter'>, 
+ 'html': <class 'structly.tableformat.formats.html.HTMLTableFormatter'>}
+>>>
+```
+
+Modify the `create_formatter()` function to look up the class in this dictionary
+instead:
+
+```python
+def create_formatter(name, column_formats=None, upper_headers=False):
+    formatter_cls = TableFormatter._formats.get(name)
+    if not formatter_cls:
+        raise RuntimeError('Unknown format %s' % name)
+
+    if column_formats:
+        class formatter_cls(ColumnFormatMixin, formatter_cls):
+              formats = column_formats
+
+    if upper_headers:
+        class formatter_cls(UpperHeadersMixin, formatter_cls):
+            pass
+
+    return formatter_cls()
+```
+
+Run the `stock.py` program.  Make sure it still works after you've made these changes.
+Just a note that all of the import statements are still there.  You've mainly
+just cleaned up the code a bit and eliminated the hard-wired class names.
+
+## (c) Dynamic Imports
+
+You're now ready for the final frontier.   Delete the following import statements
+altogether:
+
+```python
+# formatter.py
+...
+
+from .formats.text import TextTableFormatter     # DELETE
+from .formats.csv import CSVTableFormatter       # DELETE
+from .formats.html import HTMLTableFormatter     # DELETE
+...
+```
+
+Run your `stock.py` code again--it should fail with an error. It knows nothing about the
+text formatter.  Fix it by adding this tiny fragment of code to `create_formatter()`:
+
+```python
+def create_formatter(name, column_formats=None, upper_headers=False):
+    if name not in TableFormatter._formats:
+        __import__(f'{__package__}.formats.{name}')
+    ...
+```
+
+This code attempts a dynamic import of a formatter module if nothing is known about the
+name.   The import alone (if it works) will register the class with the `_formats` 
+dictionary and everything will just work.   Magic!
+
+Try running the `stock.py` code and make sure it works afterwards.
+
+\[ [Solution](soln9_4.md) | [Index](index.md) | [Exercise 9.3](ex9_3.md) \]
+
+----
+`>>>` Advanced Python Mastery  
+`...` A course by [dabeaz](https://www.dabeaz.com)  
+`...` Copyright 2007-2023  
+
+![](https://i.creativecommons.org/l/by-sa/4.0/88x31.png). This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-sa/4.0/)

Exercises/index.md

@@ -0,0 +1,95 @@
+# Advanced Python Mastery Exercises
+
+Copyright (C) 2007-2023  
+David M. Beazley  
+https://www.dabeaz.com
+
+This page contains links to all of the course exercises.  For the best experience,
+the exercises should be worked in order as later exercises often build upon earlier
+exercises.  The [`Solutions/`](../Solutions) directory contains fully worked
+out solutions to every exercise.  You can use this if you get stuck or you
+need to reset your code to a working state.
+
+Before beginning, you should fork/clone the GitHub repo so that you have a
+local copy on your machine.  Exercises assume that all of your work will
+take place in the top-level `python-mastery` directory.
+
+Associated course presentation slides can be found at [PythonMastery.pdf](../PythonMastery.pdf).
+
+## 1. Python Review
+
+- [Exercise 1.1](ex1_1.md)
+- [Exercise 1.2](ex1_2.md)
+- [Exercise 1.3](ex1_3.md)
+- [Exercise 1.4](ex1_4.md)
+- [Exercise 1.5](ex1_5.md)
+- [Exercise 1.6](ex1_6.md)
+
+## 2. Data Handling
+
+- [Exercise 2.1](ex2_1.md)
+- [Exercise 2.2](ex2_2.md)
+- [Exercise 2.3](ex2_3.md)
+- [Exercise 2.4](ex2_4.md)
+- [Exercise 2.5](ex2_5.md)
+- [Exercise 2.6](ex2_6.md)
+
+## 3. Classes and Objects
+
+- [Exercise 3.1](ex3_1.md)
+- [Exercise 3.2](ex3_2.md)
+- [Exercise 3.3](ex3_3.md)
+- [Exercise 3.4](ex3_4.md)
+- [Exercise 3.5](ex3_5.md)
+- [Exercise 3.6](ex3_6.md)
+- [Exercise 3.7](ex3_7.md)
+- [Exercise 3.8](ex3_8.md)
+
+## 4. Inside Python Objects
+
+- [Exercise 4.1](ex4_1.md)
+- [Exercise 4.2](ex4_2.md)
+- [Exercise 4.3](ex4_3.md)
+- [Exercise 4.4](ex4_4.md)
+
+## 5. Functions, Errors, and Testing
+
+- [Exercise 5.1](ex5_1.md)
+- [Exercise 5.2](ex5_2.md)
+- [Exercise 5.3](ex5_3.md)
+- [Exercise 5.4](ex5_4.md)
+- [Exercise 5.5](ex5_5.md)
+- [Exercise 5.6](ex5_6.md)
+
+## 6. Working with Code
+
+- [Exercise 6.1](ex6_1.md)
+- [Exercise 6.2](ex6_2.md)
+- [Exercise 6.3](ex6_3.md)
+- [Exercise 6.4](ex6_4.md)
+- [Exercise 6.5](ex6_5.md)
+
+## 7. Metaprogramming
+
+- [Exercise 7.1](ex7_1.md)
+- [Exercise 7.2](ex7_2.md)
+- [Exercise 7.3](ex7_3.md)
+- [Exercise 7.4](ex7_4.md)
+- [Exercise 7.5](ex7_5.md)
+- [Exercise 7.6](ex7_6.md)
+
+## 8. Iterators, Generators, and Coroutines
+
+- [Exercise 8.1](ex8_1.md)
+- [Exercise 8.2](ex8_2.md)
+- [Exercise 8.3](ex8_3.md)
+- [Exercise 8.4](ex8_4.md)
+- [Exercise 8.5](ex8_5.md)
+- [Exercise 8.6](ex8_6.md)
+
+## 9. Modules and Packages
+
+- [Exercise 9.1](ex9_1.md)
+- [Exercise 9.2](ex9_2.md)
+- [Exercise 9.3](ex9_3.md)
+- [Exercise 9.4](ex9_4.md)

Exercises/soln1_1.md

@@ -0,0 +1,6 @@
+# Exercise 1.1 - Solution
+
+Nothing here.  Just follow along with the exercise.
+
+
+[Back](ex1_1.md)

Exercises/soln1_2.md

@@ -0,0 +1,6 @@
+# Exercise 1.2 - Solution
+
+Solution code is shown in the exercise.
+
+
+[Back](ex1_2.md)

Exercises/soln1_3.md

@@ -0,0 +1,19 @@
+# Exercise 1.3 - Solution
+
+```python
+# pcost.py
+
+total_cost = 0.0
+
+with open('Data/portfolio.dat', 'r') as f:
+    for line in f:
+        fields = line.split()
+        nshares = int(fields[1])
+        price = float(fields[2])
+        total_cost = total_cost + nshares * price
+
+print(total_cost)
+```
+
+
+[Back](ex1_3.md)

Exercises/soln1_4.md

@@ -0,0 +1,46 @@
+# Exercise 1.4 - Solution
+
+## (a) Defining a function
+
+```python
+# pcost.py
+
+def portfolio_cost(filename):
+    total_cost = 0.0
+    with open(filename) as f:
+        for line in f:
+            fields = line.split()
+            nshares = int(fields[1])
+            price = float(fields[2])
+            total_cost = total_cost + nshares * price
+    return total_cost
+
+print(portfolio_cost('Data/portfolio.dat'))
+```
+
+## (b) Adding some error handling
+
+```python
+# pcost.py
+
+def portfolio_cost(filename):
+    total_cost = 0.0
+    with open(filename) as f:
+        for line in f:
+            fields = line.split()
+            try:
+                nshares = int(fields[1])
+                price = float(fields[2])
+                total_cost = total_cost + nshares*price
+
+            # This catches errors in int() and float() conversions above
+            except ValueError as e:
+                print("Couldn't parse:", repr(line))
+                print("Reason:", e)
+    return total_cost
+
+print(portfolio_cost('Data/portfolio3.dat'))
+```
+
+
+[Back](ex1_4.md)

Exercises/soln1_5.md

@@ -0,0 +1,6 @@
+# Exercise 1.5 - Solution
+
+The solution is shown in the exercise description.
+
+
+[Back](ex1_5.md)

Exercises/soln1_6.md

@@ -0,0 +1,30 @@
+# Exercise 1.6 - Solution
+
+## (b) Main Module
+
+```python
+# pcost.py
+
+def portfolio_cost(filename):
+    total_cost = 0.0
+    with open(filename) as f:
+        for line in f:
+            fields = line.split()
+            try:
+                nshares = int(fields[1])
+                price = float(fields[2])
+                total_cost = total_cost + nshares * price
+
+            # This catches errors in int() and float() conversions above
+            except ValueError as e:
+                print("Couldn't parse:", line)
+                print("Reason:", e)
+
+    return total_cost
+
+if __name__ == '__main__':
+    print(portfolio_cost('Data/portfolio.dat'))
+```
+
+
+[Back](ex1_6.md)

Exercises/soln2_1.md

@@ -0,0 +1,88 @@
+# Exercise 2.1 - Solution
+
+```python
+# readrides.py
+
+import csv
+
+def read_rides_as_tuples(filename):
+    '''
+    Read the bus ride data as a list of tuples
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = (route, date, daytype, rides)
+            records.append(record)
+    return records
+
+def read_rides_as_dicts(filename):
+    '''
+    Read the bus ride data as a list of dicts
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = {
+                'route': route, 
+                'date': date, 
+                'daytype': daytype, 
+                'rides' : rides
+                }
+            records.append(record)
+    return records
+
+class Row:
+    # Uncomment to see effect of slots
+    # __slots__ = ('route', 'date', 'daytype', 'rides')
+    def __init__(self, route, date, daytype, rides):
+        self.route = route
+        self.date = date
+        self.daytype = daytype
+        self.rides = rides
+
+# Uncomment to use a namedtuple instead
+#from collections import namedtuple
+#Row = namedtuple('Row',('route','date','daytype','rides'))
+
+def read_rides_as_instances(filename):
+    '''
+    Read the bus ride data as a list of instances
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = Row(route, date, daytype, rides)
+            records.append(record)
+    return records
+
+if __name__ == '__main__':
+    import tracemalloc
+    tracemalloc.start()
+    read_rides = read_rides_as_tuples # Change to as_dicts, as_instances, etc.
+    rides = read_rides("Data/ctabus.csv")
+
+    print('Memory Use: Current %d, Peak %d' % tracemalloc.get_traced_memory())
+```
+
+
+
+[Back](ex2_1.md)

Exercises/soln2_2.md

@@ -0,0 +1,23 @@
+# Exercise 2.2 - Solution
+
+There is no official solution for this exercise--you need rely on your
+current Python knowledge.  However, there are a few tips that can
+help.
+
+- For problems where you need to determine uniqueness, use a set. Sets aren't allowed to
+have duplicates.
+
+- If you need to tabulate data, consider using a dictionary that maps keys to a numeric
+count.  For example, to count rides on each bus route, you could make a dictionary that
+maps the route name to the total ride count for that route.  A `Counter` object from
+`collections` is perfect for this task.
+
+- If you need to keep data in order or sort data, store it in a list.
+
+- You can make Python data structures out of almost anything.  For
+example, dictionaries of sets, nested dictionaries, etc.  You might
+need to do this to answer questions 3 and 4.
+
+
+
+[Back](ex2_2.md)

Exercises/soln2_3.md

@@ -0,0 +1,7 @@
+# Exercise 2.3 - Solution
+
+Work through the exercise. Code is given in the description.
+
+
+
+[Back](ex2_3.md)

Exercises/soln2_4.md

@@ -0,0 +1,78 @@
+# Exercise 2.4 - Solution
+
+```python
+# mutint.py
+
+from functools import total_ordering
+
+@total_ordering
+class MutInt:
+    __slots__ = ['value']
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return str(self.value)
+    
+    def __repr__(self):
+        return f'MutInt({self.value!r})'
+
+    def __format__(self, fmt):
+        return format(self.value, fmt)
+
+    # Implement the "+" operator. Forward operands (MutInt + other)
+    def __add__(self, other):
+        if isinstance(other, MutInt):
+            return MutInt(self.value + other.value)
+        elif isinstance(other, int):
+            return MutInt(self.value + other)
+        else:
+            return NotImplemented
+
+    # Support for reversed operands (other + MutInt)
+    __radd__ = __add__
+
+    # Support for in-place update (MutInt += other)
+    def __iadd__(self, other):
+        if isinstance(other, MutInt):
+            self.value += other.value
+            return self
+        elif isinstance(other, int):
+            self.value += other
+            return self
+        else:
+            return NotImplemented
+
+    # Support for equality testing
+    def __eq__(self, other):
+        if isinstance(other, MutInt):
+            return self.value == other.value
+        elif isinstance(other, int):
+            return self.value == other
+        else:
+            return NotImplemented
+
+    # One relation is needed for @total_ordering decorator. It fills in others
+    def __lt__(self, other):
+        if isinstance(other, MutInt):
+            return self.value < other.value
+        elif isinstance(other, int):
+            return self.value < other
+        else:
+            return NotImplemented
+
+    # Conversions to int() and float()
+    def __int__(self):
+        return int(self.value)
+
+    def __float__(self):
+        return float(self.value)
+
+    # Support for indexing s[MutInt]
+    __index__ = __int__
+```
+
+
+
+[Back](ex2_4.md)

Exercises/soln2_5.md

@@ -0,0 +1,131 @@
+# Exercise 2.5 - Solution
+
+Here is a version of `readrides.py` with changes for parts (c) and (d).
+
+```python
+# readrides.py
+
+import csv
+
+def read_rides_as_tuples(filename):
+    '''
+    Read the bus ride data as a list of tuples
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = (route, date, daytype, rides)
+            records.append(record)
+    return records
+
+def read_rides_as_dicts(filename):
+    '''
+    Read the bus ride data as a list of dicts
+    '''
+    records = RideData()          # <---- CHANGED
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = {
+                'route': route, 
+                'date': date, 
+                'daytype': daytype, 
+                'rides' : rides
+                }
+            records.append(record)
+    return records
+
+class Row:
+    __slots__ = ('route', 'date', 'daytype', 'rides')
+    def __init__(self, route, date, daytype, rides):
+        self.route = route
+        self.date = date
+        self.daytype = daytype
+        self.rides = rides
+
+def read_rides_as_instances(filename):
+    '''
+    Read the bus ride data as a list of instances
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            route = row[0]
+            date = row[1]
+            daytype = row[2]
+            rides = int(row[3])
+            record = Row(route, date, daytype, rides)
+            records.append(record)
+    return records
+
+# Read as columns
+def read_rides_as_columns(filename):
+    '''
+    Read the bus ride data into 4 lists, representing columns
+    '''
+    routes = []
+    dates = []
+    daytypes = []
+    numrides = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headings = next(rows)     # Skip headers
+        for row in rows:
+            routes.append(row[0])
+            dates.append(row[1])
+            daytypes.append(row[2])
+            numrides.append(int(row[3]))
+    return dict(routes=routes, dates=dates, daytypes=daytypes, numrides=numrides)
+
+# The great "fake"
+
+import collections
+class RideData(collections.Sequence):
+    def __init__(self):
+        # Each value is a list with all of the values (a column)
+        self.routes = []
+        self.dates = []
+        self.daytypes = []
+        self.numrides = []
+        
+    def __len__(self):
+        # All lists assumed to have the same length
+        return len(self.routes)
+
+    def append(self, d):
+        self.routes.append(d['route'])
+        self.dates.append(d['date'])
+        self.daytypes.append(d['daytype'])
+        self.numrides.append(d['rides'])
+        
+    def __getitem__(self, index):
+        return { 'route': self.routes[index],
+                 'date': self.dates[index],
+                 'daytype': self.daytypes[index],
+                 'rides': self.numrides[index] }
+
+if __name__ == '__main__':
+    import tracemalloc
+    tracemalloc.start()
+    read_rides = read_rides_as_dicts # Change to as_dicts, as_instances, etc.
+    rides = read_rides("Data/ctabus.csv")
+
+    print('Memory Use: Current %d, Peak %d' % tracemalloc.get_traced_memory())
+```
+
+
+
+[Back](ex2_5.md)

Exercises/soln2_6.md

@@ -0,0 +1,25 @@
+# Exercise 2.6 - Solution
+
+```python
+# reader.py
+
+import csv
+from collections import defaultdict
+
+def read_csv_as_dicts(filename, types):
+    '''
+    Read a CSV file with column type conversion
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            record = { name: func(val) for name, func, val in zip(headers, types, row) }
+            records.append(record)
+    return records
+```
+
+
+
+[Back](ex2_6.md)

Exercises/soln3_1.md

@@ -0,0 +1,48 @@
+# Exercise 3.1 - Solution
+
+```python
+# stock.py
+
+class Stock:
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+
+def read_portfolio(filename):
+    '''
+    Read a CSV file of stock data into a list of Stocks
+    '''
+    import csv
+    portfolio = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            record = Stock(row[0], int(row[1]), float(row[2]))
+            portfolio.append(record)
+    return portfolio
+
+def print_portfolio(portfolio):
+    '''
+    Make a nicely formatted table showing stock data
+    '''
+    print('%10s %10s %10s' % ('name', 'shares', 'price'))
+    print(('-'*10 + ' ')*3)
+    for s in portfolio:
+        print('%10s %10d %10.2f' % (s.name, s.shares, s.price))
+
+if __name__ == '__main__':
+    portfolio = read_portfolio('Data/portfolio.csv')
+    print_portfolio(portfolio)
+```
+
+
+
+[Back](ex3_1.md)

Exercises/soln3_2.md

@@ -0,0 +1,18 @@
+# Exercise 3.2 - Solution
+
+(c) Table Formatter
+
+```python
+# tableformat.py
+
+# Print a table
+def print_table(records, fields):
+    print(' '.join('%10s' % fieldname for fieldname in fields))
+    print(('-'*10 + ' ')*len(fields))
+    for record in records:
+        print(' '.join('%10s' % getattr(record, fieldname) for fieldname in fields))
+```
+
+
+
+[Back](ex3_2.md)

Exercises/soln3_3.md

@@ -0,0 +1,84 @@
+# Exercise 3.3 - Solution
+
+```python
+# stock.py
+
+class Stock:
+    types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls.types, row)]
+        return cls(*values)
+
+def read_portfolio(filename):
+    '''
+    Read a CSV file of stock data into a list of Stocks
+    '''
+    import csv
+    portfolio = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            record = Stock.from_row(row)
+            portfolio.append(record)
+    return portfolio
+
+if __name__ == '__main__':
+    import tableformat
+    import reader
+    portfolio = read_portfolio('Data/portfolio.csv')
+    
+    # Generalized version
+    # portfolio = reader.read_csv_as_instances('Data/portfolio.csv', Stock)
+    tableformat.print_table(portfolio, ['name', 'shares', 'price'])
+```
+
+The `reader.py` file should now look like this:
+
+```python
+# reader.py
+
+import csv
+from collections import defaultdict
+
+def read_csv_as_dicts(filename, types):
+    '''
+    Read a CSV file into a list of dicts with column type conversion
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            record = { name: func(val) for name, func, val in zip(headers, types, row) }
+            records.append(record)
+    return records
+
+def read_csv_as_instances(filename, cls):
+    '''
+    Read a CSV file into a list of instances
+    '''
+    records = []
+    with open(filename) as f:
+        rows = csv.reader(f)
+        headers = next(rows)
+        for row in rows:
+            records.append(cls.from_row(row))
+    return records
+```
+
+
+
+[Back](ex3_3.md)

Exercises/soln3_4.md

@@ -0,0 +1,199 @@
+# Exercise 3.4 - Solution
+
+## (a) Private attributes
+
+```python
+# stock.py
+
+class Stock:
+    _types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls._types, row)]
+        return cls(*values)
+
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+
+## (b) Computed Attributes
+
+```python
+# stock.py
+
+class Stock:
+    _types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls._types, row)]
+        return cls(*values)
+    
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+## (c) Enforcing Type-Checking Rules
+
+```python
+# stock.py
+
+class Stock:
+    _types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls._types, row)]
+        return cls(*values)
+    
+    @property
+    def shares(self):
+        return self._shares
+    @shares.setter
+    def shares(self, value):
+        if not isinstance(value, int):
+            raise TypeError('Expected an integer')
+        if value < 0:
+            raise ValueError('shares must be >= 0')
+        self._shares = value
+
+    @property
+    def price(self):
+        return self._price
+    @price.setter
+    def price(self, value):
+        if not isinstance(value, float):
+            raise TypeError('Expected a float')
+        if value < 0:
+            raise ValueError('price must be >= 0')
+        self._price = value
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+## (d) Adding `__slots__`
+
+```python
+# stock.py
+
+class Stock:
+    __slots__ = ('name','_shares','_price')
+    _types = (str, int, float)
+
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls._types, row)]
+        return cls(*values)
+
+    @property
+    def shares(self):
+        return self._shares
+    @shares.setter
+    def shares(self, value):
+        if not isinstance(value, int):
+            raise TypeError('Expected integer')
+        if value < 0:
+            raise ValueError('shares must be >= 0')
+        self._shares = value
+
+    @property
+    def price(self):
+        return self._price
+    @price.setter
+    def price(self, value):
+        if not isinstance(value, float):
+            raise TypeError('Expected float')
+        if value < 0:
+            raise ValueError('price must be >= 0')
+        self._price = value
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+## (e) Reconciling types
+
+```python
+# stock.py
+
+class Stock:
+    __slots__ = ('name','_shares','_price')
+    _types = (str, int, float)
+    def __init__(self, name, shares, price):
+        self.name = name
+        self.shares = shares
+        self.price = price
+
+    @classmethod
+    def from_row(cls, row):
+        values = [func(val) for func, val in zip(cls._types, row)]
+        return cls(*values)
+
+    @property
+    def shares(self):
+        return self._shares
+    @shares.setter
+    def shares(self, value):
+        if not isinstance(value, self._types[1]):
+            raise TypeError(f'Expected {self._types[1].__name__}')
+        if value < 0:
+            raise ValueError('shares must be >= 0')
+        self._shares = value
+
+    @property
+    def price(self):
+        return self._price
+    @price.setter
+    def price(self, value):
+        if not isinstance(value, self._types[2]):
+            raise TypeError(f'Expected {self._types[2].__name__}')
+        if value < 0:
+            raise ValueError('price must be >= 0')
+        self._price = value
+
+    @property
+    def cost(self):
+        return self.shares * self.price
+
+    def sell(self, nshares):
+        self.shares -= nshares
+```
+
+
+
+[Back](ex3_4.md)

Exercises/soln3_5.md

@@ -0,0 +1,61 @@
+# Exercise 3.5 - Solution
+
+```python
+# tableformat.py
+
+def print_table(records, fields, formatter):
+    formatter.headings(fields)
+    for r in records:
+        rowdata = [getattr(r, fieldname) for fieldname in fields]
+        formatter.row(rowdata)
+
+class TableFormatter:
+    def headings(self, headers):
+        raise NotImplementedError()
+
+    def row(self, rowdata):
+        raise NotImplementedError()
+
+class TextTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print(' '.join('%10s' % h for h in headers))
+        print(('-'*10 + ' ')*len(headers))
+    
+    def row(self, rowdata):
+        print(' '.join('%10s' % d for d in rowdata))
+
+class CSVTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print(','.join(headers))
+
+    def row(self, rowdata):
+        print(','.join(str(d) for d in rowdata))
+
+class HTMLTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print('<tr>', end=' ')
+        for h in headers:
+            print('<th>%s</th>' % h, end=' ')
+        print('</tr>')
+
+    def row(self, rowdata):
+        print('<tr>', end=' ')
+        for d in rowdata:
+            print('<td>%s</td>' % d, end=' ')
+        print('</tr>')
+
+def create_formatter(name):
+    if name == 'text':
+        formatter_cls = TextTableFormatter
+    elif name == 'csv':
+        formatter_cls = CSVTableFormatter
+    elif name == 'html':
+        formatter_cls = HTMLTableFormatter
+    else:
+        raise RuntimeError('Unknown format %s' % name)
+    return formatter_cls()
+```
+
+
+
+[Back](ex3_5.md)

Exercises/soln3_6.md

@@ -0,0 +1,34 @@
+# Exercise 3.6 - Solution
+
+## (a) Better output for printing objects
+
+```python
+# stock.py
+
+class Stock:
+    ...
+
+    def __repr__(self):
+        # Note: The !r format code produces the repr() string
+        return f'{type(self).__name__}({self.name!r}, {self.shares!r}, {self.price!r})'
+    ...
+```
+
+## (b) Making Objects Comparable
+
+```python
+class Stock:
+    ...
+    def __eq__(self, other):
+        return isinstance(other, Stock) and ((self.name, self.shares, self.price) ==
+                                 (other.name, other.shares, other.price))
+    ...
+```
+
+## (c) Context Managers
+
+Code is given in the exercise.
+
+
+
+[Back](ex3_6.md)

Exercises/soln3_7.md

@@ -0,0 +1,85 @@
+
+# Exercise 3.7 - Solution
+
+## (a) Interfaces
+
+```python
+# tableformat.py
+
+def print_table(records, fields, formatter):
+    if not isinstance(formatter, TableFormatter):
+        raise TypeError('Expected a TableFormatter')
+
+    formatter.headings(fields)
+    for r in records:
+        rowdata = [getattr(r, fieldname) for fieldname in fields]
+        formatter.row(rowdata)
+...
+```
+
+## (b) Abstract Base Classes
+
+```python
+# tableformat.py
+from abc import ABC, abstractmethod
+
+class TableFormatter(ABC):
+    @abstractmethod
+    def headings(self, headers):
+        pass
+
+    @abstractmethod
+    def row(self, rowdata):
+        pass
+```
+
+## (c) Algorithm Template Classes
+
+```python
+# reader.py
+
+import csv
+from abc import ABC, abstractmethod
+
+class CSVParser(ABC):
+    def parse(self, filename):
+        records = []
+        with open(filename) as f:
+            rows = csv.reader(f)
+            headers = next(rows)
+            for row in rows:
+                record = self.make_record(headers, row)
+                records.append(record)
+        return records
+
+    @abstractmethod
+    def make_record(self, headers, row):
+        pass
+
+class DictCSVParser(CSVParser):
+    def __init__(self, types):
+        self.types = types
+
+    def make_record(self, headers, row):
+        return { name: func(val) for name, func, val in zip(headers, self.types, row) }
+
+class InstanceCSVParser(CSVParser):
+    def __init__(self, cls):
+        self.cls = cls
+
+    def make_record(self, headers, row):
+        return self.cls.from_row(row)
+
+def read_csv_as_dicts(filename, types):
+    parser = DictCSVParser(types)
+    return parser.parse(filename)
+
+def read_csv_as_instances(filename, cls):
+    parser = InstanceCSVParser(cls)
+    return parser.parse(filename)
+```
+
+
+
+
+[Back](ex3_7.md)

Exercises/soln3_8.md

@@ -0,0 +1,88 @@
+# Exercise 3.8 - Solution
+
+```python
+# tableformat.py
+from abc import ABC, abstractmethod
+
+def print_table(records, fields, formatter):
+    if not isinstance(formatter, TableFormatter):
+        raise RuntimeError('Expected a TableFormatter')
+
+    formatter.headings(fields)
+    for r in records:
+        rowdata = [getattr(r, fieldname) for fieldname in fields]
+        formatter.row(rowdata)
+
+class TableFormatter(ABC):
+    @abstractmethod
+    def headings(self, headers):
+        pass
+
+    @abstractmethod
+    def row(self, rowdata):
+        pass
+
+
+class TextTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print(' '.join('%10s' % h for h in headers))
+        print(('-'*10 + ' ')*len(headers))
+    
+    def row(self, rowdata):
+        print(' '.join('%10s' % d for d in rowdata))
+
+class CSVTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print(','.join(headers))
+
+    def row(self, rowdata):
+        print(','.join(str(d) for d in rowdata))
+
+class HTMLTableFormatter(TableFormatter):
+    def headings(self, headers):
+        print('<tr>', end=' ')
+        for h in headers:
+            print('<th>%s</th>' % h, end=' ')
+        print('</tr>')
+
+    def row(self, rowdata):
+        print('<tr>', end=' ')
+        for d in rowdata:
+            print('<td>%s</td>' % d, end=' ')
+        print('</tr>')
+
+
+class ColumnFormatMixin:
+    formats = []
+    def row(self, rowdata):
+        rowdata = [ (fmt % item) for fmt, item in zip(self.formats, rowdata)]
+        super().row(rowdata)
+
+class UpperHeadersMixin:
+    def headings(self, headers):
+        super().headings([h.upper() for h in headers])
+
+def create_formatter(name, column_formats=None, upper_headers=False):
+    if name == 'text':
+        formatter_cls = TextTableFormatter
+    elif name == 'csv':
+        formatter_cls = CSVTableFormatter
+    elif name == 'html':
+        formatter_cls = HTMLTableFormatter
+    else:
+        raise RuntimeError('Unknown format %s' % name)
+
+    if column_formats:
+        class formatter_cls(ColumnFormatMixin, formatter_cls):
+              formats = column_formats
+
+    if upper_headers:
+        class formatter_cls(UpperHeadersMixin, formatter_cls):
+            pass
+
+    return formatter_cls()
+```
+
+
+
+[Back](ex3_8.md)

Exercises/soln4_1.md

@@ -0,0 +1,7 @@
+# Exercise 4.1 - Solution
+
+No solution.  Just follow along.
+
+
+
+[Back](ex4_1.md)

Exercises/soln4_2.md

@@ -0,0 +1,93 @@
+# Exercise 4.2 - Solution
+
+Here is the validator code in its entirety:
+
+```python
+class Validator:
+    @classmethod
+    def check(cls, value):
+        return value
+
+class Typed(Validator):
+    expected_type = object
+    @classmethod
+    def check(cls, value):
+        if not isinstance(value, cls.expected_type):
+            raise TypeError(f'Expected {cls.expected_type}')
+        return super().check(value)
+
+class Integer(Typed):
+    expected_type = int
+
+class Float(Typed):
+    expected_type = float
+
+class String(Typed):
+    expected_type = str
+
+class Positive(Validator):
+    @classmethod
+    def check(cls, value):
+        if value < 0:
+            raise ValueError('Must be >= 0')
+        return super().check(value)
+
+class NonEmpty(Validator):
+    @classmethod
+    def check(cls, value):
+        if len(value) == 0:
+            raise ValueError('Must be non-empty')
+        return super().check(value)
+
+class PositiveInteger(Integer, Positive):
+    pass
+
+class PositiveFloat(Float, Positive):
+    pass
+
+class NonEmptyString(String, NonEmpty):
+    pass
+```
+
+## (c) Using the validators
+
+```python
+# validate.py
+...
+
+if __name__ == '__main__':
+    class Stock:
+        __slots__ = ('name', '_shares', '_price')
+        def __init__(self, name, shares, price):
+            self.name = name
+            self.shares = shares
+            self.price = price
+
+        def __repr__(self):
+            return f'Stock({self.name!r}, {self.shares!r}, {self.price!r})'
+
+        @property
+        def shares(self):
+            return self._shares
+        @shares.setter
+        def shares(self, value):
+            self._shares = PositiveInteger.check(value)
+
+        @property
+        def price(self):
+            return self._price
+        @price.setter
+        def price(self, value):
+            self._price = PositiveFloat.check(value)
+
+        @property
+        def cost(self):
+            return self.shares * self.price
+
+        def sell(self, nshares):
+            self.shares -= nshares
+```
+
+
+
+[Back](ex4_2.md)

Exercises/soln4_3.md

@@ -0,0 +1,87 @@
+# Exercise 4.3 - Solution
+
+```python
+class Validator:
+    def __init__(self, name=None):
+        self.name = name
+
+    def __set_name__(self, cls, name):
+        self.name = name
+
+    @classmethod
+    def check(cls, value):
+        return value
+
+    def __set__(self, instance, value):
+        instance.__dict__[self.name] = self.check(value)
+
+class Typed(Validator):
+    expected_type = object
+    @classmethod
+    def check(cls, value):
+        if not isinstance(value, cls.expected_type):
+            raise TypeError(f'expected {cls.expected_type}')
+        return super().check(value)
+
+class Integer(Typed):
+    expected_type = int
+
+class Float(Typed):
+    expected_type = float
+
+class String(Typed):
+    expected_type = str
+
+class Positive(Validator):
+    @classmethod
+    def check(cls, value):
+        if value < 0:
+            raise ValueError('must be >= 0')
+        return super().check(value)
+
+class NonEmpty(Validator):
+    @classmethod
+    def check(cls, value):
+        if len(value) == 0:
+            raise ValueError('must be non-empty')
+        return super().check(value)
+
+class PositiveInteger(Integer, Positive):
+    pass
+
+class PositiveFloat(Float, Positive):
+    pass
+
+class NonEmptyString(String, NonEmpty):
+    pass
+
+# Examples
+if __name__ == '__main__':
+    def add(x, y):
+        Integer.check(x)
+        Integer.check(y)
+        return x + y
+
+    class Stock:
+        name = NonEmptyString()
+        shares = PositiveInteger()
+        price = PositiveFloat()
+        def __init__(self,name,shares,price):
+            self.name = name
+            self.shares = shares
+            self.price = price
+
+        def __repr__(self):
+            return f'Stock({self.name!r}, {self.shares!r}, {self.price!r})'
+
+        @property
+        def cost(self):
+            return self.shares * self.price
+
+        def sell(self, nshares):
+            self.shares -= nshares
+```
+
+
+
+[Back](ex4_3.md)

Exercises/soln4_4.md

@@ -0,0 +1,7 @@
+# Exercise 4.4 - Solution
+
+Code is given in the exercise.
+
+
+
+[Back](ex4_4.md)

Exercises/soln5_1.md

@@ -0,0 +1,55 @@
+# Exercise 5.1 - Solution
+
+This is a partial solution that does not include type-hints. 
+
+```python
+# reader.py
+
+import csv
+
+def csv_as_dicts(lines, types, *, headers=None):
+    '''
+    Convert lines of CSV data into a list of dictionaries
+    '''
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    for row in rows:
+        record = { name: func(val)
+                   for name, func, val in zip(headers, types, row) }
+        records.append(record)
+    return records
+
+def csv_as_instances(lines, cls, *, headers=None):
+    '''
+    Convert lines of CSV data into a list of instances
+    '''
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    for row in rows:
+        record = cls.from_row(row)
+        records.append(record)
+    return records
+
+def read_csv_as_dicts(filename, types, *, headers=None):
+    '''
+    Read CSV data into a list of dictionaries with optional type conversion
+    '''
+    with open(filename) as file:
+        return csv_as_dicts(file, types, headers=headers)
+
+def read_csv_as_instances(filename, cls, *, headers=None):
+    '''
+    Read CSV data into a list of instances
+    '''
+    with open(filename) as file:
+        return csv_as_instances(file, cls, headers=headers)
+```
+
+
+
+
+[Back](ex5_1.md)

Exercises/soln5_2.md

@@ -0,0 +1,7 @@
+# Exercise 5.2 - Solution
+
+None. Just work through the exercise.
+
+
+
+[Back](ex5_2.md)

Exercises/soln5_3.md

@@ -0,0 +1,60 @@
+# Exercise 5.3 - Solution
+
+## (a) Higher order functions
+
+```python
+# reader.py
+
+import csv
+
+def convert_csv(lines, converter, *, headers=None):
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    for row in rows:
+        record = converter(headers, row)
+        records.append(record)
+    return records
+
+def csv_as_dicts(lines, types, *, headers=None):
+    return convert_csv(lines, 
+                       lambda headers, row: { name: func(val) for name, func, val in zip(headers, types, row) })
+
+def csv_as_instances(lines, cls, *, headers=None):
+    return convert_csv(lines,
+                       lambda headers, row: cls.from_row(row))
+
+def read_csv_as_dicts(filename, types, *, headers=None):
+    '''
+    Read CSV data into a list of dictionaries with optional type conversion
+    '''
+    with open(filename) as file:
+        return csv_as_dicts(file, types, headers=headers)
+
+def read_csv_as_instances(filename, cls, *, headers=None):
+    '''
+    Read CSV data into a list of instances
+    '''
+    with open(filename) as file:
+        return csv_as_instances(file, cls, headers=headers)
+```
+
+## (b) Using map()
+
+```python
+# reader.py
+
+import csv
+
+def convert_csv(lines, converter, *, headers=None):
+    records = []
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+    return map(lambda row: converter(headers, row), rows)
+```
+
+
+
+[Back](ex5_3.md)

Exercises/soln5_4.md

@@ -0,0 +1,39 @@
+# Exercise 5.4 - Solution
+
+```python
+# typedproperty.py
+
+def typedproperty(name, expected_type):
+    private_name = '_' + name
+
+    @property
+    def value(self):
+        return getattr(self, private_name)
+
+    @value.setter
+    def value(self, val):
+        if not isinstance(val, expected_type):
+            raise TypeError(f'Expected {expected_type}')
+        setattr(self, private_name, val)
+
+    return value
+
+
+String = lambda name: typedproperty(name, str)
+Integer = lambda name: typedproperty(name, int)
+Float = lambda name: typedproperty(name, float)
+
+# Example
+if __name__ == '__main__':
+    class Stock:
+        name = String('name')
+        shares = Integer('shares')
+        price = Float('price')
+        def __init__(self, name, shares, price):
+            self.name = name
+            self.shares = shares
+            self.price = price
+```
+
+
+[Back](ex5_4.md)

Exercises/soln5_5.md

@@ -0,0 +1,50 @@
+# Exercise 5.5 - Solution
+
+```python
+# reader.py
+
+import csv
+import logging
+
+log = logging.getLogger(__name__)
+
+def convert_csv(lines, converter, *, headers=None):
+    rows = csv.reader(lines)
+    if headers is None:
+        headers = next(rows)
+
+    records = []
+    for rowno, row in enumerate(rows, start=1):
+        try:
+            records.append(converter(headers, row))
+        except ValueError as e:
+            log.warning('Row %s: Bad row: %s', rowno, row)
+            log.debug('Row %s: Reason: %s', rowno, row)
+    return records
+
+def csv_as_dicts(lines, types, *, headers=None):
+    return convert_csv(lines, 
+                       lambda headers, row: { name: func(val) for name, func, val in zip(headers, types, row) })
+
+def csv_as_instances(lines, cls, *, headers=None):
+    return convert_csv(lines,
+                       lambda headers, row: cls.from_row(row))
+
+def read_csv_as_dicts(filename, types, *, headers=None):
+    '''
+    Read CSV data into a list of dictionaries with optional type conversion
+    '''
+    with open(filename) as file:
+        return csv_as_dicts(file, types, headers=headers)
+
+def read_csv_as_instances(filename, cls, *, headers=None):
+    '''
+    Read CSV data into a list of instances
+    '''
+    with open(filename) as file:
+        return csv_as_instances(file, cls, headers=headers)
+```
+
+
+
+[Back](ex5_5.md)

Exercises/soln5_6.md

@@ -0,0 +1,129 @@
+# Exercise 5.6 - Solution
+
+## (b) Unit testing
+
+```python
+# teststock.py
+
+import stock
+import unittest
+
+class TestStock(unittest.TestCase):
+    def test_create(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+
+    def test_create_keyword(self):
+        s = stock.Stock(name='GOOG', shares=100, price=490.1)
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+        
+    def test_cost(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(s.cost, 49010.0)
+
+    def test_sell(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        s.sell(25)
+        self.assertEqual(s.shares, 75)
+
+    def test_from_row(self):
+        s = stock.Stock.from_row(['GOOG','100','490.1'])
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+
+    def test_repr(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(repr(s), "Stock('GOOG', 100, 490.1)")
+
+    def test_eq(self):
+        a = stock.Stock('GOOG', 100, 490.1)
+        b = stock.Stock('GOOG', 100, 490.1)
+        self.assertTrue(a==b)
+
+if __name__ == '__main__':
+    unittest.main()
+```
+
+## (c) Testing for Errors
+
+```python
+# teststock.py
+
+import stock
+import unittest
+
+class TestStock(unittest.TestCase):
+    def test_create(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+
+    def test_create_keyword(self):
+        s = stock.Stock(name='GOOG', shares=100, price=490.1)
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+        
+    def test_cost(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(s.cost, 49010.0)
+
+    def test_sell(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        s.sell(25)
+        self.assertEqual(s.shares, 75)
+
+    def test_from_row(self):
+        s = stock.Stock.from_row(['GOOG','100','490.1'])
+        self.assertEqual(s.name, 'GOOG')
+        self.assertEqual(s.shares, 100)
+        self.assertEqual(s.price, 490.1)
+
+    def test_repr(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        self.assertEqual(repr(s), "Stock('GOOG', 100, 490.1)")
+
+    def test_eq(self):
+        a = stock.Stock('GOOG', 100, 490.1)
+        b = stock.Stock('GOOG', 100, 490.1)
+        self.assertTrue(a==b)
+
+    # Tests for failure conditions
+    def test_shares_badtype(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(TypeError):
+            s.shares = '50'
+
+    def test_shares_badvalue(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(ValueError):
+            s.shares = -50
+
+    def test_price_badtype(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(TypeError):
+            s.price = '45.23'
+
+    def test_price_badvalue(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(ValueError):
+            s.price = -45.23
+
+    def test_bad_attribute(self):
+        s = stock.Stock('GOOG', 100, 490.1)
+        with self.assertRaises(AttributeError):
+            s.share = 100
+
+if __name__ == '__main__':
+    unittest.main()
+```
+
+
+
+[Back](ex5_6.md)

Exercises/soln6_1.md

@@ -0,0 +1,43 @@
+# Exercise 6.1 - Solution
+
+## (a) Simplified Structures
+
+```python
+# structure.py
+
+class Structure:
+    _fields = ()
+    def __init__(self, *args):
+        if len(args) != len(self._fields):
+            raise TypeError('Expected %d arguments' % len(self._fields))
+        for name, arg in zip(self._fields,args):
+            setattr(self, name, arg)
+```
+
+## (b) Making a Useful Representation
+
+[source, python]
+```
+class Structure:
+    ...    
+    def __repr__(self):
+        return '%s(%s)' % (type(self).__name__,
+                           ', '.join(repr(getattr(self, name)) for name in self._fields))
+```
+
+## (c) Restricting Attribute Names
+
+[source, python]
+```
+class Structure:
+    ...    
+    def __setattr__(self, name, value):
+        if name.startswith('_') or name in self._fields:
+            super().__setattr__(name, value)
+        else:
+            raise AttributeError('No attribute %s' % name)
+```
+
+
+
+[Back](ex6_1.md)

Exercises/soln6_2.md

@@ -0,0 +1,32 @@
+# Exercise 6.2 - Solution
+
+```python
+# structure.py
+
+import sys
+
+class Structure:
+    _fields = ()
+
+    @staticmethod
+    def _init():
+        locs = sys._getframe(1).f_locals
+        self = locs.pop('self')
+        for name, val in locs.items():
+            setattr(self, name, val)
+
+    def __setattr__(self, name, value):
+        if name.startswith('_') or name in self._fields:
+            super().__setattr__(name, value)
+        else:
+            raise AttributeError('No attribute %s' % name)
+
+    def __repr__(self):
+        return '%s(%s)' % (type(self).__name__,
+                           ', '.join(repr(getattr(self, name)) for name in self._fields))
+```
+
+
+
+
+[Back](ex6_2.md)