renumbering chapters >= 19

20-futures/demo_executor_map.py

@@ -0,0 +1,31 @@
+"""
+Experiment with ``ThreadPoolExecutor.map``
+"""
+# tag::EXECUTOR_MAP[]
+from time import sleep, strftime
+from concurrent import futures
+
+def display(*args):  # <1>
+    print(strftime('[%H:%M:%S]'), end=' ')
+    print(*args)
+
+def loiter(n):  # <2>
+    msg = '{}loiter({}): doing nothing for {}s...'
+    display(msg.format('\t'*n, n, n))
+    sleep(n)
+    msg = '{}loiter({}): done.'
+    display(msg.format('\t'*n, n))
+    return n * 10  # <3>
+
+def main():
+    display('Script starting.')
+    executor = futures.ThreadPoolExecutor(max_workers=3)  # <4>
+    results = executor.map(loiter, range(5))  # <5>
+    display('results:', results)  # <6>
+    display('Waiting for individual results:')
+    for i, result in enumerate(results):  # <7>
+        display(f'result {i}: {result}')
+
+if __name__ == '__main__':
+    main()
+# end::EXECUTOR_MAP[]

20-futures/getflags/.gitignore

@@ -0,0 +1 @@
+flags/

20-futures/getflags/country_codes.txt

@@ -0,0 +1,8 @@
+AD AE AF AG AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BI BJ BN BO BR BS BT
+BW BY BZ CA CD CF CG CH CI CL CM CN CO CR CU CV CY CZ DE DJ DK DM DZ EC EE
+EG ER ES ET FI FJ FM FR GA GB GD GE GH GM GN GQ GR GT GW GY HN HR HT HU ID
+IE IL IN IQ IR IS IT JM JO JP KE KG KH KI KM KN KP KR KW KZ LA LB LC LI LK
+LR LS LT LU LV LY MA MC MD ME MG MH MK ML MM MN MR MT MU MV MW MX MY MZ NA
+NE NG NI NL NO NP NR NZ OM PA PE PG PH PK PL PT PW PY QA RO RS RU RW SA SB
+SC SD SE SG SI SK SL SM SN SO SR SS ST SV SY SZ TD TG TH TJ TL TM TN TO TR
+TT TV TW TZ UA UG US UY UZ VA VC VE VN VU WS YE ZA ZM ZW

20-futures/getflags/downloaded/.gitignore

@@ -0,0 +1 @@
+*.gif

20-futures/getflags/flags.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+"""Download flags of top 20 countries by population
+
+Sequential version
+
+Sample runs (first with new domain, so no caching ever)::
+
+    $ ./flags.py
+    BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN
+    20 downloads in 26.21s
+    $ ./flags.py
+    BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN
+    20 downloads in 14.57s
+
+
+"""
+
+# tag::FLAGS_PY[]
+import time
+from pathlib import Path
+from typing import Callable
+
+import requests  # <1>
+
+POP20_CC = ('CN IN US ID BR PK NG BD RU JP '
+            'MX PH VN ET EG DE IR TR CD FR').split()  # <2>
+
+BASE_URL = 'http://fluentpython.com/data/flags'       # <3>
+DEST_DIR = Path('downloaded')                         # <4>
+
+def save_flag(img: bytes, filename: str) -> None:     # <5>
+    (DEST_DIR / filename).write_bytes(img)
+
+def get_flag(cc: str) -> bytes:  # <6>
+    url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
+    resp = requests.get(url)
+    return resp.content
+
+def download_many(cc_list: list[str]) -> int:  # <7>
+    for cc in sorted(cc_list):                 # <8>
+        image = get_flag(cc)
+        save_flag(image, f'{cc}.gif')
+        print(cc, end=' ', flush=True)         # <9>
+    return len(cc_list)
+
+def main(downloader: Callable[[list[str]], int]) -> None:  # <10>
+    t0 = time.perf_counter()                               # <11>
+    count = downloader(POP20_CC)
+    elapsed = time.perf_counter() - t0
+    print(f'\n{count} downloads in {elapsed:.2f}s')
+
+if __name__ == '__main__':
+    main(download_many)     # <12>
+# end::FLAGS_PY[]

20-futures/getflags/flags2_asyncio.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+asyncio async/await version
+
+"""
+# tag::FLAGS2_ASYNCIO_TOP[]
+import asyncio
+from collections import Counter
+
+import aiohttp
+import tqdm  # type: ignore
+
+from flags2_common import main, HTTPStatus, Result, save_flag
+
+# default set low to avoid errors from remote site, such as
+# 503 - Service Temporarily Unavailable
+DEFAULT_CONCUR_REQ = 5
+MAX_CONCUR_REQ = 1000
+
+
+class FetchError(Exception):  # <1>
+    def __init__(self, country_code: str):
+        self.country_code = country_code
+
+
+async def get_flag(session: aiohttp.ClientSession,  # <2>
+                   base_url: str,
+                   cc: str) -> bytes:
+    url = f'{base_url}/{cc}/{cc}.gif'.lower()
+    async with session.get(url) as resp:
+        if resp.status == 200:
+            return await resp.read()
+        else:
+            resp.raise_for_status()  # <3>
+            return bytes()
+
+async def download_one(session: aiohttp.ClientSession,  # <4>
+                       cc: str,
+                       base_url: str,
+                       semaphore: asyncio.Semaphore,
+                       verbose: bool) -> Result:
+    try:
+        async with semaphore:  # <5>
+            image = await get_flag(session, base_url, cc)
+    except aiohttp.ClientResponseError as exc:
+        if exc.status == 404:               # <6>
+            status = HTTPStatus.not_found
+            msg = 'not found'
+        else:
+            raise FetchError(cc) from exc  # <7>
+    else:
+        save_flag(image, f'{cc}.gif')
+        status = HTTPStatus.ok
+        msg = 'OK'
+    if verbose and msg:
+        print(cc, msg)
+    return Result(status, cc)
+# end::FLAGS2_ASYNCIO_TOP[]
+
+# tag::FLAGS2_ASYNCIO_START[]
+async def supervisor(cc_list: list[str],
+                     base_url: str,
+                     verbose: bool,
+                     concur_req: int) -> Counter[HTTPStatus]:  # <1>
+    counter: Counter[HTTPStatus] = Counter()
+    semaphore = asyncio.Semaphore(concur_req)  # <2>
+    async with aiohttp.ClientSession() as session:
+        to_do = [download_one(session, cc, base_url, semaphore, verbose)
+                 for cc in sorted(cc_list)]  # <3>
+        to_do_iter = asyncio.as_completed(to_do)  # <4>
+        if not verbose:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <5>
+        for coro in to_do_iter:  # <6>
+            try:
+                res = await coro  # <7>
+            except FetchError as exc:  # <8>
+                country_code = exc.country_code  # <9>
+                try:
+                    error_msg = exc.__cause__.message  # type: ignore  # <10>
+                except AttributeError:
+                    error_msg = 'Unknown cause'  # <11>
+                if verbose and error_msg:
+                    print(f'*** Error for {country_code}: {error_msg}')
+                status = HTTPStatus.error
+            else:
+                status = res.status
+            counter[status] += 1  # <12>
+    return counter  # <13>
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[HTTPStatus]:
+    coro = supervisor(cc_list, base_url, verbose, concur_req)
+    counts = asyncio.run(coro)  # <14>
+
+    return counts
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# end::FLAGS2_ASYNCIO_START[]

20-futures/getflags/flags2_asyncio_executor.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+asyncio async/await version using run_in_executor for save_flag.
+
+"""
+
+import asyncio
+from collections import Counter
+
+import aiohttp
+import tqdm  # type: ignore
+
+from flags2_common import main, HTTPStatus, Result, save_flag
+
+# default set low to avoid errors from remote site, such as
+# 503 - Service Temporarily Unavailable
+DEFAULT_CONCUR_REQ = 5
+MAX_CONCUR_REQ = 1000
+
+
+class FetchError(Exception):
+    def __init__(self, country_code: str):
+        self.country_code = country_code
+
+
+async def get_flag(session: aiohttp.ClientSession,
+                   base_url: str,
+                   cc: str) -> bytes:
+    url = f'{base_url}/{cc}/{cc}.gif'.lower()
+    async with session.get(url) as resp:
+        if resp.status == 200:
+            return await resp.read()
+        else:
+            resp.raise_for_status()
+            return bytes()
+
+# tag::FLAGS2_ASYNCIO_EXECUTOR[]
+async def download_one(session: aiohttp.ClientSession,
+                       cc: str,
+                       base_url: str,
+                       semaphore: asyncio.Semaphore,
+                       verbose: bool) -> Result:
+    try:
+        async with semaphore:
+            image = await get_flag(session, base_url, cc)
+    except aiohttp.ClientResponseError as exc:
+        if exc.status == 404:
+            status = HTTPStatus.not_found
+            msg = 'not found'
+        else:
+            raise FetchError(cc) from exc
+    else:
+        loop = asyncio.get_running_loop()   # <1>
+        loop.run_in_executor(None,          # <2>
+            save_flag, image, f'{cc}.gif')  # <3>
+        status = HTTPStatus.ok
+        msg = 'OK'
+    if verbose and msg:
+        print(cc, msg)
+    return Result(status, cc)
+# end::FLAGS2_ASYNCIO_EXECUTOR[]
+
+async def supervisor(cc_list: list[str],
+                     base_url: str,
+                     verbose: bool,
+                     concur_req: int) -> Counter[HTTPStatus]:
+    counter: Counter[HTTPStatus] = Counter()
+    semaphore = asyncio.Semaphore(concur_req)
+    async with aiohttp.ClientSession() as session:
+        to_do = [download_one(session, cc, base_url, semaphore, verbose)
+                 for cc in sorted(cc_list)]
+
+        to_do_iter = asyncio.as_completed(to_do)
+        if not verbose:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
+        for coro in to_do_iter:
+            try:
+                res = await coro
+            except FetchError as exc:
+                country_code = exc.country_code
+                try:
+                    error_msg = exc.__cause__.message  # type: ignore
+                except AttributeError:
+                    error_msg = 'Unknown cause'
+                if verbose and error_msg:
+                    print(f'*** Error for {country_code}: {error_msg}')
+                status = HTTPStatus.error
+            else:
+                status = res.status
+
+            counter[status] += 1
+
+    return counter
+
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[HTTPStatus]:
+    coro = supervisor(cc_list, base_url, verbose, concur_req)
+    counts = asyncio.run(coro)  # <14>
+
+    return counts
+
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)

20-futures/getflags/flags2_common.py

@@ -0,0 +1,159 @@
+"""Utilities for second set of flag examples.
+"""
+
+import argparse
+import string
+import sys
+import time
+from collections import namedtuple, Counter
+from enum import Enum
+from pathlib import Path
+
+Result = namedtuple('Result', 'status data')
+
+HTTPStatus = Enum('HTTPStatus', 'ok not_found error')
+
+POP20_CC = ('CN IN US ID BR PK NG BD RU JP '
+            'MX PH VN ET EG DE IR TR CD FR').split()
+
+DEFAULT_CONCUR_REQ = 1
+MAX_CONCUR_REQ = 1
+
+SERVERS = {
+    'REMOTE': 'http://fluentpython.com/data/flags',
+    'LOCAL':  'http://localhost:8000/flags',
+    'DELAY':  'http://localhost:8001/flags',
+    'ERROR':  'http://localhost:8002/flags',
+}
+DEFAULT_SERVER = 'LOCAL'
+
+DEST_DIR = Path('downloaded')
+COUNTRY_CODES_FILE = Path('country_codes.txt')
+
+
+def save_flag(img: bytes, filename: str) -> None:
+    (DEST_DIR / filename).write_bytes(img)
+
+
+def initial_report(cc_list: list[str],
+                   actual_req: int,
+                   server_label: str) -> None:
+    if len(cc_list) <= 10:
+        cc_msg = ', '.join(cc_list)
+    else:
+        cc_msg = f'from {cc_list[0]} to {cc_list[-1]}'
+    print(f'{server_label} site: {SERVERS[server_label]}')
+    plural = 's' if len(cc_list) != 1 else ''
+    print(f'Searching for {len(cc_list)} flag{plural}: {cc_msg}')
+    if actual_req == 1:
+        print('1 connection will be used.')
+    else:
+        print(f'{actual_req} concurrent connections will be used.')
+
+
+def final_report(cc_list: list[str],
+                 counter: Counter[HTTPStatus],
+                 start_time: float) -> None:
+    elapsed = time.perf_counter() - start_time
+    print('-' * 20)
+    plural = 's' if counter[HTTPStatus.ok] != 1 else ''
+    print(f'{counter[HTTPStatus.ok]} flag{plural} downloaded.')
+    if counter[HTTPStatus.not_found]:
+        print(f'{counter[HTTPStatus.not_found]} not found.')
+    if counter[HTTPStatus.error]:
+        plural = 's' if counter[HTTPStatus.error] != 1 else ''
+        print(f'{counter[HTTPStatus.error]} error{plural}.')
+    print(f'Elapsed time: {elapsed:.2f}s')
+
+
+def expand_cc_args(every_cc: bool,
+                   all_cc: bool,
+                   cc_args: list[str],
+                   limit: int) -> list[str]:
+    codes: set[str] = set()
+    A_Z = string.ascii_uppercase
+    if every_cc:
+        codes.update(a+b for a in A_Z for b in A_Z)
+    elif all_cc:
+        text = COUNTRY_CODES_FILE.read_text()
+        codes.update(text.split())
+    else:
+        for cc in (c.upper() for c in cc_args):
+            if len(cc) == 1 and cc in A_Z:
+                codes.update(cc + c for c in A_Z)
+            elif len(cc) == 2 and all(c in A_Z for c in cc):
+                codes.add(cc)
+            else:
+                raise ValueError('*** Usage error: each CC argument '
+                                 'must be A to Z or AA to ZZ.')
+    return sorted(codes)[:limit]
+
+
+def process_args(default_concur_req):
+    server_options = ', '.join(sorted(SERVERS))
+    parser = argparse.ArgumentParser(
+        description='Download flags for country codes. '
+                    'Default: top 20 countries by population.')
+    parser.add_argument(
+        'cc', metavar='CC', nargs='*',
+        help='country code or 1st letter (eg. B for BA...BZ)')
+    parser.add_argument(
+        '-a', '--all', action='store_true',
+        help='get all available flags (AD to ZW)')
+    parser.add_argument(
+        '-e', '--every', action='store_true',
+        help='get flags for every possible code (AA...ZZ)')
+    parser.add_argument(
+        '-l', '--limit', metavar='N', type=int, help='limit to N first codes',
+        default=sys.maxsize)
+    parser.add_argument(
+        '-m', '--max_req', metavar='CONCURRENT', type=int,
+        default=default_concur_req,
+        help=f'maximum concurrent requests (default={default_concur_req})')
+    parser.add_argument(
+        '-s', '--server', metavar='LABEL', default=DEFAULT_SERVER,
+        help=f'Server to hit; one of {server_options} '
+             f'(default={DEFAULT_SERVER})')
+    parser.add_argument(
+        '-v', '--verbose', action='store_true',
+        help='output detailed progress info')
+    args = parser.parse_args()
+    if args.max_req < 1:
+        print('*** Usage error: --max_req CONCURRENT must be >= 1')
+        parser.print_usage()
+        # "standard" exit status codes:
+        # https://stackoverflow.com/questions/1101957/are-there-any-standard-exit-status-codes-in-linux/40484670#40484670
+        sys.exit(2)  # command line usage error
+    if args.limit < 1:
+        print('*** Usage error: --limit N must be >= 1')
+        parser.print_usage()
+        sys.exit(2)  # command line usage error
+    args.server = args.server.upper()
+    if args.server not in SERVERS:
+        print(f'*** Usage error: --server LABEL '
+              f'must be one of {server_options}')
+        parser.print_usage()
+        sys.exit(2)  # command line usage error
+    try:
+        cc_list = expand_cc_args(args.every, args.all, args.cc, args.limit)
+    except ValueError as exc:
+        print(exc.args[0])
+        parser.print_usage()
+        sys.exit(2)  # command line usage error
+
+    if not cc_list:
+        cc_list = sorted(POP20_CC)
+    return args, cc_list
+
+
+def main(download_many, default_concur_req, max_concur_req):
+    args, cc_list = process_args(default_concur_req)
+    actual_req = min(args.max_req, max_concur_req, len(cc_list))
+    initial_report(cc_list, actual_req, args.server)
+    base_url = SERVERS[args.server]
+    t0 = time.perf_counter()
+    counter = download_many(cc_list, base_url, args.verbose, actual_req)
+    assert sum(counter.values()) == len(cc_list), (
+        'some downloads are unaccounted for'
+    )
+    final_report(cc_list, counter, t0)

20-futures/getflags/flags2_sequential.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+Sequential version
+
+Sample run::
+
+    $ python3 flags2_sequential.py -s DELAY b
+    DELAY site: http://localhost:8002/flags
+    Searching for 26 flags: from BA to BZ
+    1 concurrent connection will be used.
+    --------------------
+    17 flags downloaded.
+    9 not found.
+    Elapsed time: 13.36s
+
+"""
+
+from collections import Counter
+
+import requests
+import tqdm  # type: ignore
+
+from flags2_common import main, save_flag, HTTPStatus, Result
+
+DEFAULT_CONCUR_REQ = 1
+MAX_CONCUR_REQ = 1
+
+# tag::FLAGS2_BASIC_HTTP_FUNCTIONS[]
+def get_flag(base_url: str, cc: str) -> bytes:
+    url = f'{base_url}/{cc}/{cc}.gif'.lower()
+    resp = requests.get(url)
+    if resp.status_code != 200:  # <1>
+        resp.raise_for_status()
+    return resp.content
+
+def download_one(cc: str, base_url: str, verbose: bool = False):
+    try:
+        image = get_flag(base_url, cc)
+    except requests.exceptions.HTTPError as exc:  # <2>
+        res = exc.response
+        if res.status_code == 404:
+            status = HTTPStatus.not_found  # <3>
+            msg = 'not found'
+        else:  # <4>
+            raise
+    else:
+        save_flag(image, f'{cc}.gif')
+        status = HTTPStatus.ok
+        msg = 'OK'
+
+    if verbose:  # <5>
+        print(cc, msg)
+
+    return Result(status, cc)  # <6>
+# end::FLAGS2_BASIC_HTTP_FUNCTIONS[]
+
+# tag::FLAGS2_DOWNLOAD_MANY_SEQUENTIAL[]
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  _unused_concur_req: int) -> Counter[int]:
+    counter: Counter[int] = Counter()  # <1>
+    cc_iter = sorted(cc_list)  # <2>
+    if not verbose:
+        cc_iter = tqdm.tqdm(cc_iter)  # <3>
+    for cc in cc_iter:  # <4>
+        try:
+            res = download_one(cc, base_url, verbose)  # <5>
+        except requests.exceptions.HTTPError as exc:  # <6>
+            error_msg = 'HTTP error {res.status_code} - {res.reason}'
+            error_msg = error_msg.format(res=exc.response)
+        except requests.exceptions.ConnectionError:  # <7>
+            error_msg = 'Connection error'
+        else:  # <8>
+            error_msg = ''
+            status = res.status
+
+        if error_msg:
+            status = HTTPStatus.error  # <9>
+        counter[status] += 1           # <10>
+        if verbose and error_msg:      # <11>
+            print(f'*** Error for {cc}: {error_msg}')
+
+    return counter  # <12>
+# end::FLAGS2_DOWNLOAD_MANY_SEQUENTIAL[]
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)

20-futures/getflags/flags2_threadpool.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+ThreadPool version
+
+Sample run::
+
+    $ python3 flags2_threadpool.py -s ERROR -e
+    ERROR site: http://localhost:8003/flags
+    Searching for 676 flags: from AA to ZZ
+    30 concurrent connections will be used.
+    --------------------
+    150 flags downloaded.
+    361 not found.
+    165 errors.
+    Elapsed time: 7.46s
+
+"""
+
+# tag::FLAGS2_THREADPOOL[]
+from collections import Counter
+from concurrent import futures
+
+import requests
+import tqdm  # type: ignore  # <1>
+
+from flags2_common import main, HTTPStatus  # <2>
+from flags2_sequential import download_one  # <3>
+
+DEFAULT_CONCUR_REQ = 30  # <4>
+MAX_CONCUR_REQ = 1000  # <5>
+
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[int]:
+    counter: Counter[int] = Counter()
+    with futures.ThreadPoolExecutor(max_workers=concur_req) as executor:  # <6>
+        to_do_map = {}  # <7>
+        for cc in sorted(cc_list):  # <8>
+            future = executor.submit(download_one, cc,
+                                     base_url, verbose)  # <9>
+            to_do_map[future] = cc  # <10>
+        done_iter = futures.as_completed(to_do_map)  # <11>
+        if not verbose:
+            done_iter = tqdm.tqdm(done_iter, total=len(cc_list))  # <12>
+        for future in done_iter:  # <13>
+            try:
+                res = future.result()  # <14>
+            except requests.exceptions.HTTPError as exc:  # <15>
+                error_fmt = 'HTTP {res.status_code} - {res.reason}'
+                error_msg = error_fmt.format(res=exc.response)
+            except requests.exceptions.ConnectionError:
+                error_msg = 'Connection error'
+            else:
+                error_msg = ''
+                status = res.status
+
+            if error_msg:
+                status = HTTPStatus.error
+            counter[status] += 1
+            if verbose and error_msg:
+                cc = to_do_map[future]  # <16>
+                print(f'*** Error for {cc}: {error_msg}')
+
+    return counter
+
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# end::FLAGS2_THREADPOOL[]

20-futures/getflags/flags3_asyncio.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+asyncio async/await version using run_in_executor for save_flag.
+
+"""
+
+import asyncio
+from collections import Counter
+
+import aiohttp
+import tqdm  # type: ignore
+
+from flags2_common import main, HTTPStatus, Result, save_flag
+
+# default set low to avoid errors from remote site, such as
+# 503 - Service Temporarily Unavailable
+DEFAULT_CONCUR_REQ = 5
+MAX_CONCUR_REQ = 1000
+
+
+class FetchError(Exception):
+    def __init__(self, country_code: str):
+        self.country_code = country_code
+
+async def get_flag(session: aiohttp.ClientSession,
+                   base_url: str,
+                   cc: str) -> bytes:
+    url = f'{base_url}/{cc}/{cc}.gif'
+    async with session.get(url) as resp:
+        if resp.status == 200:
+            return await resp.read()
+        else:
+            resp.raise_for_status()
+            return bytes()
+
+# tag::FLAGS3_ASYNCIO_GET_COUNTRY[]
+async def get_country(session: aiohttp.ClientSession,  # <1>
+                      base_url: str,
+                      cc: str) -> str:
+    url = f'{base_url}/{cc}/metadata.json'
+    async with session.get(url) as resp:
+        if resp.status == 200:
+            metadata = await resp.json()  # <2>
+            return metadata.get('country', 'no name')  # <3>
+        else:
+            resp.raise_for_status()
+            return ''
+# end::FLAGS3_ASYNCIO_GET_COUNTRY[]
+
+# tag::FLAGS3_ASYNCIO_DOWNLOAD_ONE[]
+async def download_one(session: aiohttp.ClientSession,
+                       cc: str,
+                       base_url: str,
+                       semaphore: asyncio.Semaphore,
+                       verbose: bool) -> Result:
+    try:
+        async with semaphore:
+            image = await get_flag(session, base_url, cc)  # <1>
+        async with semaphore:
+            country = await get_country(session, base_url, cc)  # <2>
+    except aiohttp.ClientResponseError as exc:
+        if exc.status == 404:
+            status = HTTPStatus.not_found
+            msg = 'not found'
+        else:
+            raise FetchError(cc) from exc
+    else:
+        filename = country.replace(' ', '_')  # <3>
+        filename = f'{filename}.gif'
+        loop = asyncio.get_running_loop()
+        loop.run_in_executor(None,
+                             save_flag, image, filename)
+        status = HTTPStatus.ok
+        msg = 'OK'
+    if verbose and msg:
+        print(cc, msg)
+    return Result(status, cc)
+# end::FLAGS3_ASYNCIO_DOWNLOAD_ONE[]
+
+async def supervisor(cc_list: list[str],
+                     base_url: str,
+                     verbose: bool,
+                     concur_req: int) -> Counter[HTTPStatus]:
+    counter: Counter[HTTPStatus] = Counter()
+    semaphore = asyncio.Semaphore(concur_req)
+    async with aiohttp.ClientSession() as session:
+        to_do = [download_one(session, cc, base_url,
+                              semaphore, verbose)
+                 for cc in sorted(cc_list)]
+
+        to_do_iter = asyncio.as_completed(to_do)
+        if not verbose:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
+        for coro in to_do_iter:
+            try:
+                res = await coro
+            except FetchError as exc:
+                country_code = exc.country_code
+                try:
+                    error_msg = exc.__cause__.message  # type: ignore
+                except AttributeError:
+                    error_msg = 'Unknown cause'
+                if verbose and error_msg:
+                    print(f'*** Error for {country_code}: {error_msg}')
+                status = HTTPStatus.error
+            else:
+                status = res.status
+
+            counter[status] += 1
+
+    return counter
+
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[HTTPStatus]:
+    coro = supervisor(cc_list, base_url, verbose, concur_req)
+    counts = asyncio.run(coro)  # <14>
+
+    return counts
+
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)

20-futures/getflags/flags_asyncio.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+"""Download flags of top 20 countries by population
+
+asyncio + aiottp version
+
+Sample run::
+
+    $ python3 flags_asyncio.py
+    EG VN IN TR RU ID US DE CN MX JP BD NG ET FR BR PH PK CD IR
+    20 flags downloaded in 1.07s
+
+"""
+# tag::FLAGS_ASYNCIO_TOP[]
+import asyncio
+
+from aiohttp import ClientSession  # <1>
+
+from flags import BASE_URL, save_flag, main  # <2>
+
+async def download_one(session: ClientSession, cc: str):  # <3>
+    image = await get_flag(session, cc)
+    save_flag(image, f'{cc}.gif')
+    print(cc, end=' ', flush=True)
+    return cc
+
+async def get_flag(session: ClientSession, cc: str) -> bytes:  # <4>
+    url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
+    async with session.get(url) as resp:  # <5>
+        return await resp.read()          # <6>
+# end::FLAGS_ASYNCIO_TOP[]
+
+# tag::FLAGS_ASYNCIO_START[]
+def download_many(cc_list: list[str]) -> int:     # <1>
+    return asyncio.run(supervisor(cc_list))       # <2>
+
+async def supervisor(cc_list: list[str]) -> int:
+    async with ClientSession() as session:        # <3>
+        to_do = [download_one(session, cc)        # <4>
+                 for cc in sorted(cc_list)]
+        res = await asyncio.gather(*to_do)        # <5>
+
+    return len(res)                               # <6>
+
+if __name__ == '__main__':
+    main(download_many)
+# end::FLAGS_ASYNCIO_START[]

20-futures/getflags/flags_threadpool.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+"""Download flags of top 20 countries by population
+
+ThreadPoolExecutor version
+
+Sample run::
+
+    $ python3 flags_threadpool.py
+    DE FR BD CN EG RU IN TR VN ID JP BR NG MX PK ET PH CD US IR
+    20 downloads in 0.35s
+
+"""
+
+# tag::FLAGS_THREADPOOL[]
+from concurrent import futures
+
+from flags import save_flag, get_flag, main  # <1>
+
+def download_one(cc: str):  # <2>
+    image = get_flag(cc)
+    save_flag(image, f'{cc}.gif')
+    print(cc, end=' ', flush=True)
+    return cc
+
+def download_many(cc_list: list[str]) -> int:
+    with futures.ThreadPoolExecutor() as executor:         # <3>
+        res = executor.map(download_one, sorted(cc_list))  # <4>
+
+    return len(list(res))                                  # <5>
+
+if __name__ == '__main__':
+    main(download_many)  # <6>
+# end::FLAGS_THREADPOOL[]

20-futures/getflags/flags_threadpool_futures.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+"""Download flags of top 20 countries by population
+
+ThreadPoolExecutor example with ``as_completed``.
+"""
+from concurrent import futures
+
+from flags import main
+from flags_threadpool import download_one
+
+
+# tag::FLAGS_THREADPOOL_AS_COMPLETED[]
+def download_many(cc_list: list[str]) -> int:
+    cc_list = cc_list[:5]  # <1>
+    with futures.ThreadPoolExecutor(max_workers=3) as executor:  # <2>
+        to_do: list[futures.Future] = []
+        for cc in sorted(cc_list):  # <3>
+            future = executor.submit(download_one, cc)  # <4>
+            to_do.append(future)  # <5>
+            print(f'Scheduled for {cc}: {future}')  # <6>
+
+        for count, future in enumerate(futures.as_completed(to_do), 1):  # <7>
+            res: str = future.result()  # <8>
+            print(f'{future} result: {res!r}')  # <9>
+
+    return count
+# end::FLAGS_THREADPOOL_AS_COMPLETED[]
+
+if __name__ == '__main__':
+    main(download_many)

20-futures/getflags/requirements.txt

@@ -0,0 +1,11 @@
+aiohttp==3.7.4
+async-timeout==3.0.1
+attrs==20.3.0
+certifi==2020.12.5
+chardet==4.0.0
+idna==2.10
+requests==2.25.1
+urllib3==1.26.5
+tqdm==4.56.2
+multidict==5.1.0
+yarl==1.6.3

20-futures/getflags/slow_server.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+"""Slow HTTP server class.
+
+This module implements a ThreadingHTTPServer using a custom
+SimpleHTTPRequestHandler subclass that introduces delays to all
+GET responses, and optionally returns errors to a fraction of
+the requests if given the --error_rate command-line argument.
+"""
+
+import contextlib
+import os
+import socket
+import time
+from functools import partial
+from http import server, HTTPStatus
+from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
+from random import random
+
+
+class SlowHTTPRequestHandler(SimpleHTTPRequestHandler):
+    """SlowHTTPRequestHandler adds delays and errors to test HTTP clients.
+
+    The optional error_rate argument determines how often GET requests
+    receive a 418 status code, "I'm a teapot".
+    If error_rate is .15, there's a 15% probability of each GET request
+    getting that error.
+    When the server believes it is a teapot, it refuses requests to serve files.
+
+    See: https://tools.ietf.org/html/rfc2324#section-2.3.2
+    """
+
+    def __init__(self, *args, error_rate=0.0, **kwargs):
+        self.error_rate = error_rate
+        super().__init__(*args, **kwargs)
+
+    def do_GET(self):
+        """Serve a GET request."""
+        time.sleep(.5)
+        if random() < self.error_rate:
+            self.send_error(HTTPStatus.IM_A_TEAPOT, "I'm a Teapot")
+        else:
+            f = self.send_head()
+            if f:
+                try:
+                    self.copyfile(f, self.wfile)
+                finally:
+                    f.close()
+
+# The code in the `if` block below, including comments, was copied
+# and adapted from the `http.server` module of Python 3.9
+# https://github.com/python/cpython/blob/master/Lib/http/server.py
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--bind', '-b', metavar='ADDRESS',
+                        help='Specify alternate bind address '
+                             '[default: all interfaces]')
+    parser.add_argument('--directory', '-d', default=os.getcwd(),
+                        help='Specify alternative directory '
+                             '[default:current directory]')
+    parser.add_argument('--error-rate', '-e', metavar='PROBABILITY',
+                        default=0.0, type=float,
+                        help='Error rate; e.g. use .25 for 25%% probability '
+                             '[default:0.0]')
+    parser.add_argument('port', action='store',
+                        default=8000, type=int,
+                        nargs='?',
+                        help='Specify alternate port [default: 8000]')
+    args = parser.parse_args()
+    handler_class = partial(SlowHTTPRequestHandler,
+                            directory=args.directory,
+                            error_rate=args.error_rate)
+
+    # ensure dual-stack is not disabled; ref #38907
+    class DualStackServer(ThreadingHTTPServer):
+        def server_bind(self):
+            # suppress exception when protocol is IPv4
+            with contextlib.suppress(Exception):
+                self.socket.setsockopt(
+                    socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+            return super().server_bind()
+
+    # test is a top-level function in http.server omitted from __all__
+    server.test(  # type: ignore
+        HandlerClass=handler_class,
+        ServerClass=DualStackServer,
+        port=args.port,
+        bind=args.bind,
+    )

20-futures/primes/primes.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import math
+
+
+PRIME_FIXTURE = [
+    (2, True),
+    (142702110479723, True),
+    (299593572317531, True),
+    (3333333333333301, True),
+    (3333333333333333, False),
+    (3333335652092209, False),
+    (4444444444444423, True),
+    (4444444444444444, False),
+    (4444444488888889, False),
+    (5555553133149889, False),
+    (5555555555555503, True),
+    (5555555555555555, False),
+    (6666666666666666, False),
+    (6666666666666719, True),
+    (6666667141414921, False),
+    (7777777536340681, False),
+    (7777777777777753, True),
+    (7777777777777777, False),
+    (9999999999999917, True),
+    (9999999999999999, False),
+]
+
+NUMBERS = [n for n, _ in PRIME_FIXTURE]
+
+# tag::IS_PRIME[]
+def is_prime(n: int) -> bool:
+    if n < 2:
+        return False
+    if n == 2:
+        return True
+    if n % 2 == 0:
+        return False
+
+    root = math.isqrt(n)
+    for i in range(3, root + 1, 2):
+        if n % i == 0:
+            return False
+    return True
+# end::IS_PRIME[]
+
+if __name__ == '__main__':
+
+    for n, prime in PRIME_FIXTURE:
+        prime_res = is_prime(n)
+        assert prime_res == prime
+        print(n, prime)

20-futures/primes/proc_pool.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+"""
+proc_pool.py: a version of the proc.py example from chapter 20,
+but using `concurrent.futures.ProcessPoolExecutor`.
+"""
+
+# tag::PRIMES_POOL[]
+import sys
+from concurrent import futures  # <1>
+from time import perf_counter
+from typing import NamedTuple
+
+from primes import is_prime, NUMBERS
+
+class PrimeResult(NamedTuple):  # <2>
+    n: int
+    flag: bool
+    elapsed: float
+
+def check(n: int) -> PrimeResult:
+    t0 = perf_counter()
+    res = is_prime(n)
+    return PrimeResult(n, res, perf_counter() - t0)
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        workers = None      # <3>
+    else:
+        workers = int(sys.argv[1])
+
+    executor = futures.ProcessPoolExecutor(workers)  # <4>
+    actual_workers = executor._max_workers  # type: ignore  # <5>
+
+    print(f'Checking {len(NUMBERS)} numbers with {actual_workers} processes:')
+
+    t0 = perf_counter()
+
+    numbers = sorted(NUMBERS, reverse=True)  # <6>
+    with executor:  # <7>
+        for n, prime, elapsed in executor.map(check, numbers):  # <8>
+            label = 'P' if prime else ' '
+            print(f'{n:16}  {label} {elapsed:9.6f}s')
+
+    time = perf_counter() - t0
+    print(f'Total time: {time:.2f}s')
+
+if __name__ == '__main__':
+    main()
+# end::PRIMES_POOL[]