ch21 examples

21-futures/getflags/flags2_threadpool.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+ThreadPool version
+
+Sample run::
+
+    $ python3 flags2_threadpool.py -s ERROR -e
+    ERROR site: http://localhost:8003/flags
+    Searching for 676 flags: from AA to ZZ
+    30 concurrent connections will be used.
+    --------------------
+    150 flags downloaded.
+    361 not found.
+    165 errors.
+    Elapsed time: 7.46s
+
+"""
+
+# tag::FLAGS2_THREADPOOL[]
+from collections import Counter
+from concurrent import futures
+
+import requests
+import tqdm  # type: ignore  # <1>
+
+from flags2_common import main, HTTPStatus  # <2>
+from flags2_sequential import download_one  # <3>
+
+DEFAULT_CONCUR_REQ = 30  # <4>
+MAX_CONCUR_REQ = 1000  # <5>
+
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[int]:
+    counter: Counter[int] = Counter()
+    with futures.ThreadPoolExecutor(max_workers=concur_req) as executor:  # <6>
+        to_do_map = {}  # <7>
+        for cc in sorted(cc_list):  # <8>
+            future = executor.submit(download_one,
+                                     cc, base_url, verbose)  # <9>
+            to_do_map[future] = cc  # <10>
+        done_iter = futures.as_completed(to_do_map)  # <11>
+        if not verbose:
+            done_iter = tqdm.tqdm(done_iter, total=len(cc_list))  # <12>
+        for future in done_iter:  # <13>
+            try:
+                res = future.result()  # <14>
+            except requests.exceptions.HTTPError as exc:  # <15>
+                error_msg = 'HTTP {res.status_code} - {res.reason}'
+                error_msg = error_msg.format(res=exc.response)
+            except requests.exceptions.ConnectionError:
+                error_msg = 'Connection error'
+            else:
+                error_msg = ''
+                status = res.status
+
+            if error_msg:
+                status = HTTPStatus.error
+            counter[status] += 1
+            if verbose and error_msg:
+                cc = to_do_map[future]  # <16>
+                print('*** Error for {}: {}'.format(cc, error_msg))
+
+    return counter
+
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# end::FLAGS2_THREADPOOL[]