daviddoji/example-code-2e
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ch22: examples

Browse Source
This commit is contained in:
Luciano Ramalho 2021-02-28 21:19:40 -03:00
parent d2bcf655d7
commit a751c86836
10 changed files with 107 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
21-futures/getflags/flags.py
View File

@ -33,16 +33,15 @@ def save_flag(img: bytes, filename: str) -> None: # <5>
(DEST_DIR / filename).write_bytes(img)
def get_flag(cc: str) -> bytes: # <6>
cc = cc.lower()
url = f'{BASE_URL}/{cc}/{cc}.gif'
url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
resp = requests.get(url)
return resp.content
def download_many(cc_list: list[str]) -> int: # <7>
for cc in sorted(cc_list): # <8>
image = get_flag(cc)
save_flag(image, f'{cc}.gif')
print(cc, end=' ', flush=True) # <9>
save_flag(image, cc.lower() + '.gif')
return len(cc_list)
def main(downloader: Callable[[list[str]], int]) -> None: # <10>

82
21-futures/getflags/flags2_asyncio.py
View File

@ -5,13 +5,11 @@
asyncio async/await version
"""
# BEGIN FLAGS2_ASYNCIO_TOP
# tag::FLAGS2_ASYNCIO_TOP[]
import asyncio
from collections import Counter
import aiohttp
from aiohttp import web
from aiohttp.http_exceptions import HttpProcessingError
import tqdm # type: ignore
from flags2_common import main, HTTPStatus, Result, save_flag
@ -23,91 +21,83 @@ MAX_CONCUR_REQ = 1000
class FetchError(Exception): # <1>
def __init__(self, country_code):
def __init__(self, country_code: str):
self.country_code = country_code
async def get_flag(session, base_url, cc): # <2>
cc = cc.lower()
url = f'{base_url}/{cc}/{cc}.gif'
async def get_flag(session: aiohttp.ClientSession, # <2>
base_url: str,
cc: str) -> bytes:
url = f'{base_url}/{cc}/{cc}.gif'.lower()
async with session.get(url) as resp:
if resp.status == 200:
return await resp.read()
elif resp.status == 404:
raise web.HTTPNotFound()
else:
raise HttpProcessingError(
code=resp.status, message=resp.reason,
headers=resp.headers)
resp.raise_for_status() # <3>
return bytes()
async def download_one(session, cc, base_url, semaphore, verbose): # <3>
async def download_one(session: aiohttp.ClientSession, # <4>
cc: str,
base_url: str,
semaphore: asyncio.Semaphore,
verbose: bool) -> Result:
try:
async with semaphore: # <4>
image = await get_flag(session, base_url, cc) # <5>
except web.HTTPNotFound: # <6>
status = HTTPStatus.not_found
msg = 'not found'
except Exception as exc:
raise FetchError(cc) from exc # <7>
async with semaphore: # <5>
image = await get_flag(session, base_url, cc)
except aiohttp.ClientResponseError as exc:
if exc.status == 404: # <6>
status = HTTPStatus.not_found
msg = 'not found'
else:
raise FetchError(cc) from exc # <7>
else:
save_flag(image, cc.lower() + '.gif') # <8>
save_flag(image, f'{cc}.gif')
status = HTTPStatus.ok
msg = 'OK'
if verbose and msg:
print(cc, msg)
return Result(status, cc)
# END FLAGS2_ASYNCIO_TOP
# end::FLAGS2_ASYNCIO_TOP[]
# BEGIN FLAGS2_ASYNCIO_DOWNLOAD_MANY
async def downloader_coro(cc_list: list[str],
base_url: str,
verbose: bool,
concur_req: int) -> Counter[HTTPStatus]: # <1>
# tag::FLAGS2_ASYNCIO_START[]
async def supervisor(cc_list: list[str],
base_url: str,
verbose: bool,
concur_req: int) -> Counter[HTTPStatus]: # <1>
counter: Counter[HTTPStatus] = Counter()
semaphore = asyncio.Semaphore(concur_req) # <2>
async with aiohttp.ClientSession() as session: # <8>
async with aiohttp.ClientSession() as session:
to_do = [download_one(session, cc, base_url, semaphore, verbose)
for cc in sorted(cc_list)] # <3>
to_do_iter = asyncio.as_completed(to_do) # <4>
if not verbose:
to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5>
for future in to_do_iter: # <6>
for coro in to_do_iter: # <6>
try:
res = await future # <7>
res = await coro # <7>
except FetchError as exc: # <8>
country_code = exc.country_code # <9>
try:
if exc.__cause__ is None:
error_msg = 'Unknown cause'
else:
error_msg = exc.__cause__.args[0] # <10>
except IndexError:
error_msg = exc.__cause__.__class__.__name__ # <11>
error_msg = exc.__cause__.message # type: ignore # <10>
except AttributeError:
error_msg = 'Unknown cause' # <11>
if verbose and error_msg:
print(f'*** Error for {country_code}: {error_msg}')
status = HTTPStatus.error
else:
status = res.status
counter[status] += 1 # <12>
return counter # <13>
def download_many(cc_list: list[str],
base_url: str,
verbose: bool,
concur_req: int) -> Counter[HTTPStatus]:
coro = downloader_coro(cc_list, base_url, verbose, concur_req)
coro = supervisor(cc_list, base_url, verbose, concur_req)
counts = asyncio.run(coro) # <14>
return counts
if __name__ == '__main__':
main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
# END FLAGS2_ASYNCIO_DOWNLOAD_MANY
# end::FLAGS2_ASYNCIO_START[]

8
21-futures/getflags/flags2_common.py
View File

@ -45,8 +45,10 @@ def initial_report(cc_list: list[str],
print(f'{server_label} site: {SERVERS[server_label]}')
plural = 's' if len(cc_list) != 1 else ''
print(f'Searching for {len(cc_list)} flag{plural}: {cc_msg}')
plural = 's' if actual_req != 1 else ''
print(f'{actual_req} concurrent connection{plural} will be used.')
if actual_req == 1:
print('1 connection will be used.')
else:
print(f'{actual_req} concurrent connections will be used.')
def final_report(cc_list: list[str],
@ -60,7 +62,7 @@ def final_report(cc_list: list[str],
print(f'{counter[HTTPStatus.not_found]} not found.')
if counter[HTTPStatus.error]:
plural = 's' if counter[HTTPStatus.error] != 1 else ''
print(f'{counter[HTTPStatus.error]} error{plural}')
print(f'{counter[HTTPStatus.error]} error{plural}.')
print(f'Elapsed time: {elapsed:.2f}s')

5
21-futures/getflags/flags2_sequential.py
View File

@ -29,8 +29,7 @@ MAX_CONCUR_REQ = 1
# tag::FLAGS2_BASIC_HTTP_FUNCTIONS[]
def get_flag(base_url: str, cc: str) -> bytes:
cc = cc.lower()
url = f'{base_url}/{cc}/{cc}.gif'
url = f'{base_url}/{cc}/{cc}.gif'.lower()
resp = requests.get(url)
if resp.status_code != 200: # <1>
resp.raise_for_status()
@ -47,7 +46,7 @@ def download_one(cc: str, base_url: str, verbose: bool = False):
else: # <4>
raise
else:
save_flag(image, cc.lower() + '.gif')
save_flag(image, f'{cc}.gif')
status = HTTPStatus.ok
msg = 'OK'

4
21-futures/getflags/flags2_threadpool.py
View File

@ -40,8 +40,8 @@ def download_many(cc_list: list[str],
with futures.ThreadPoolExecutor(max_workers=concur_req) as executor: # <6>
to_do_map = {} # <7>
for cc in sorted(cc_list): # <8>
future = executor.submit(download_one,
cc, base_url, verbose) # <9>
future = executor.submit(download_one, cc,
base_url, verbose) # <9>
to_do_map[future] = cc # <10>
done_iter = futures.as_completed(to_do_map) # <11>
if not verbose:

26
21-futures/getflags/flags_asyncio.py
View File

@ -20,31 +20,27 @@ from flags import BASE_URL, save_flag, main # <2>
async def download_one(session: ClientSession, cc: str): # <3>
image = await get_flag(session, cc)
save_flag(image, f'{cc}.gif')
print(cc, end=' ', flush=True)
save_flag(image, cc.lower() + '.gif')
return cc
async def get_flag(session: ClientSession, cc: str) -> bytes: # <4>
cc = cc.lower()
url = f'{BASE_URL}/{cc}/{cc}.gif'
url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
async with session.get(url) as resp: # <5>
print(resp)
return await resp.read() # <6>
# end::FLAGS_ASYNCIO_TOP[]
# end::FLAGS_ASYNCIO_START[]
def download_many(cc_list): # <1>
return asyncio.run(supervisor(cc_list)) # <2>
# tag::FLAGS_ASYNCIO_START[]
def download_many(cc_list: list[str]) -> int: # <1>
return asyncio.run(supervisor(cc_list)) # <2>
async def supervisor(cc_list):
async with ClientSession() as session: # <3>
to_do = [download_one(session, cc)
for cc in sorted(cc_list)] # <4>
res = await asyncio.gather(*to_do) # <5>
async def supervisor(cc_list: list[str]) -> int:
async with ClientSession() as session: # <3>
to_do = [download_one(session, cc) # <4>
for cc in sorted(cc_list)]
res = await asyncio.gather(*to_do) # <5>
return len(res) # <6>
return len(res) # <6>
if __name__ == '__main__':
main(download_many)

2
21-futures/getflags/flags_threadpool.py
View File

@ -19,8 +19,8 @@ from flags import save_flag, get_flag, main # <1>
def download_one(cc: str): # <2>
image = get_flag(cc)
save_flag(image, f'{cc}.gif')
print(cc, end=' ', flush=True)
save_flag(image, cc.lower() + '.gif')
return cc
def download_many(cc_list: list[str]) -> int:

3
21-futures/getflags/slow_server.py
View File

@ -83,7 +83,8 @@ if __name__ == '__main__':
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
return super().server_bind()
server.test(
# test is a top-level function in http.server omitted from __all__
server.test( # type: ignore
HandlerClass=handler_class,
ServerClass=DualStackServer,
port=args.port,

25
22-asyncio/domains/netaddr.py Normal file
View File

@ -0,0 +1,25 @@
import asyncio
import socket
from collections.abc import Iterable, AsyncIterator
from typing import NamedTuple
class Result(NamedTuple):
name: str
found: bool
async def probe(loop: asyncio.AbstractEventLoop, name: str) -> Result:
try:
await loop.getaddrinfo(name, None)
except socket.gaierror:
return Result(name, False)
return Result(name, True)
async def multi_probe(names: Iterable[str]) -> AsyncIterator[Result]:
loop = asyncio.get_running_loop()
coros = [probe(loop, name) for name in names]
for coro in asyncio.as_completed(coros):
result = await coro
yield result

21
22-asyncio/domains/probe.py Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python3
import asyncio
import sys
from keyword import kwlist
from netaddr import multi_probe
async def main(tld: str) -> None:
names = (f'{w}.{tld}'.lower() for w in kwlist if len(w) <= 4)
async for name, found in multi_probe(sorted(names)):
mark = '.' if found else '?\t\t'
print(f'{mark} {name}')
if __name__ == '__main__':
if len(sys.argv) == 2:
asyncio.run(main(sys.argv[1]))
else:
print('Please provide a TLD.', f'Example: {sys.argv[0]} COM.BR')