sequential, threaded & async HTTP clients using HTTPX

20-futures/getflags/flags2_asyncio.py

@@ -8,65 +8,60 @@ asyncio async/await version
 # tag::FLAGS2_ASYNCIO_TOP[]
 import asyncio
 from collections import Counter
+from http import HTTPStatus
+from pathlib import Path
 
-import aiohttp
+import httpx
 import tqdm  # type: ignore
 
-from flags2_common import main, HTTPStatus, Result, save_flag
+from flags2_common import main, DownloadStatus, save_flag
 
 # default set low to avoid errors from remote site, such as
 # 503 - Service Temporarily Unavailable
 DEFAULT_CONCUR_REQ = 5
 MAX_CONCUR_REQ = 1000
 
-
-class FetchError(Exception):  # <1>
-    def __init__(self, country_code: str):
-        self.country_code = country_code
-
-
-async def get_flag(session: aiohttp.ClientSession,  # <2>
+async def get_flag(session: httpx.AsyncClient,  # <2>
                    base_url: str,
                    cc: str) -> bytes:
     url = f'{base_url}/{cc}/{cc}.gif'.lower()
-    async with session.get(url) as resp:
-        if resp.status == 200:
-            return await resp.read()
-        else:
-            resp.raise_for_status()  # <3>
-            return bytes()
+    resp = await session.get(url, timeout=3.1, follow_redirects=True)   # <3>
+    resp.raise_for_status()
+    return resp.content
 
-async def download_one(session: aiohttp.ClientSession,
+async def download_one(session: httpx.AsyncClient,
                        cc: str,
                        base_url: str,
                        semaphore: asyncio.Semaphore,  # <4>
-                       verbose: bool) -> Result:
+                       verbose: bool) -> DownloadStatus:
     try:
         async with semaphore:  # <5>
             image = await get_flag(session, base_url, cc)
-    except aiohttp.ClientResponseError as exc:
-        if exc.status == 404:               # <6>
-            status = HTTPStatus.not_found
-            msg = 'not found'
+    except httpx.HTTPStatusError as exc:  # <4>
+        res = exc.response
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            status = DownloadStatus.NOT_FOUND  # <5>
+            msg = f'not found: {res.url}'
         else:
-            raise FetchError(cc) from exc  # <7>
+            raise
+
     else:
-        save_flag(image, f'{cc}.gif')
-        status = HTTPStatus.ok
+        await asyncio.to_thread(save_flag, image, f'{cc}.gif')
+        status = DownloadStatus.OK
         msg = 'OK'
     if verbose and msg:
         print(cc, msg)
-    return Result(status, cc)
+    return status
 # end::FLAGS2_ASYNCIO_TOP[]
 
 # tag::FLAGS2_ASYNCIO_START[]
 async def supervisor(cc_list: list[str],
                      base_url: str,
                      verbose: bool,
-                     concur_req: int) -> Counter[HTTPStatus]:  # <1>
-    counter: Counter[HTTPStatus] = Counter()
+                     concur_req: int) -> Counter[DownloadStatus]:  # <1>
+    counter: Counter[DownloadStatus] = Counter()
     semaphore = asyncio.Semaphore(concur_req)  # <2>
-    async with aiohttp.ClientSession() as session:
+    async with httpx.AsyncClient() as session:
         to_do = [download_one(session, cc, base_url, semaphore, verbose)
                  for cc in sorted(cc_list)]  # <3>
         to_do_iter = asyncio.as_completed(to_do)  # <4>
@@ -74,25 +69,33 @@ async def supervisor(cc_list: list[str],
             to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <5>
         for coro in to_do_iter:  # <6>
             try:
-                res = await coro  # <7>
-            except FetchError as exc:  # <8>
-                country_code = exc.country_code  # <9>
-                try:
-                    error_msg = exc.__cause__.message  # type: ignore  # <10>
-                except AttributeError:
-                    error_msg = 'Unknown cause'  # <11>
-                if verbose and error_msg:
-                    print(f'*** Error for {country_code}: {error_msg}')
-                status = HTTPStatus.error
-            else:
-                status = res.status
-            counter[status] += 1  # <12>
-    return counter  # <13>
+                status = await coro  # <7>
+            except httpx.HTTPStatusError as exc:  # <8>
+                error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
+                error_msg = error_msg.format(resp=exc.response)
+                error = exc
+            except httpx.RequestError as exc:  # <9>
+                error_msg = f'{exc} {type(exc)}'.strip()
+                error = exc
+            except KeyboardInterrupt:  # <10>
+                break
+            else:  # <11>
+                error = None
+
+            if error:
+                status = DownloadStatus.ERROR  # <12>
+                if verbose:
+                    url = str(error.request.url)  # <13>
+                    cc = Path(url).stem.upper()   # <14>
+                    print(f'{cc} error: {error_msg}')
+            counter[status] += 1
+
+    return counter
 
 def download_many(cc_list: list[str],
                   base_url: str,
                   verbose: bool,
-                  concur_req: int) -> Counter[HTTPStatus]:
+                  concur_req: int) -> Counter[DownloadStatus]:
     coro = supervisor(cc_list, base_url, verbose, concur_req)
     counts = asyncio.run(coro)  # <14>