final concurrency examples

futures/countries/flags2_threadpool.py

@@ -1,42 +1,68 @@
-"""Download flags of top 10 countries by population
+"""Download flags of countries (with error handling).
 
 ThreadPool version
 
 Sample run::
 
-    $
+    $ python3 flags2_threadpool.py -s ERROR -e
+    ERROR site: http://localhost:8003/flags
+    Searching for 676 flags: from AA to ZZ
+    30 concurrent connections will be used.
+    --------------------
+    150 flags downloaded.
+    361 not found.
+    165 errors.
+    Elapsed time: 7.46s
 
 """
 
+# BEGIN FLAGS2_THREADPOOL
+import collections
 from concurrent import futures
 
-import tqdm
+import requests
+import tqdm  # <1>
 
-from flag_utils import main, Counts
-from flags2_sequential import get_flag, download_one
+from flags2_common import main, HTTPStatus  # <2>
+from flags2_sequential import download_one  # <3>
 
-DEFAULT_CONCUR_REQ = 30
-MAX_CONCUR_REQ = 1000
+DEFAULT_CONCUR_REQ = 30  # <4>
+MAX_CONCUR_REQ = 1000  # <5>
 
 
 def download_many(cc_list, base_url, verbose, concur_req):
-    with futures.ThreadPoolExecutor(concur_req) as executor:
-        to_do = [executor.submit(download_one, cc, base_url, verbose)
-                 for cc in sorted(cc_list)]
-        counts = [0, 0, 0]
-        to_do_iter = futures.as_completed(to_do)
+    counter = collections.Counter()
+    with futures.ThreadPoolExecutor(concur_req) as executor:  # <6>
+        to_do_map = {}  # <7>
+        for cc in sorted(cc_list):  # <8>
+            future = executor.submit(download_one,
+                            cc, base_url, verbose)  # <9>
+            to_do_map[future] = cc  # <10>
+        to_do_iter = futures.as_completed(to_do_map)  # <11>
         if not verbose:
-            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
-        for future in to_do_iter:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <12>
+        for future in to_do_iter:  # <13>
             try:
-                res = future.result()
-            except Exception as exc:
-                print('*** Unexpected exception:', exc)
+                res = future.result()  # <14>
+            except requests.exceptions.HTTPError as exc:  # <15>
+                error_msg = 'HTTP {res.status_code} - {res.reason}'
+                error_msg = error_msg.format(res=exc.response)
+            except requests.exceptions.ConnectionError as exc:
+                error_msg = 'Connection error'
             else:
-                counts[res.status.value-1] += 1
+                error_msg = ''
+                status = res.status
 
-    return Counts(*counts)
+            if error_msg:
+                status = HTTPStatus.error
+            counter[status] += 1
+            if verbose and error_msg:
+                cc = to_do_map[future]  # <16>
+                print('*** Error for {}: {}'.format(cc, error_msg))
+
+    return counter
 
 
 if __name__ == '__main__':
     main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# END FLAGS2_THREADPOOL