sync from Atlas

This commit is contained in:
Luciano Ramalho 2021-10-12 22:37:28 -03:00
parent 43f1bf23b3
commit 6fb0832c40
7 changed files with 31 additions and 34 deletions

View File

@ -147,7 +147,7 @@ def test_factorial():
gcd_src = """ gcd_src = """
(define (mod m n) (define (mod m n)
(- m (* n (// m n)))) (- m (* n (quotient m n))))
(define (gcd m n) (define (gcd m n)
(if (= n 0) (if (= n 0)
m m

View File

@ -80,7 +80,7 @@ def standard_env() -> Environment:
'-': op.sub, '-': op.sub,
'*': op.mul, '*': op.mul,
'/': op.truediv, '/': op.truediv,
'//': op.floordiv, 'quotient': op.floordiv,
'>': op.gt, '>': op.gt,
'<': op.lt, '<': op.lt,
'>=': op.ge, '>=': op.ge,

View File

@ -146,7 +146,7 @@ def test_factorial():
gcd_src = """ gcd_src = """
(define (mod m n) (define (mod m n)
(- m (* n (// m n)))) (- m (* n (quotient m n))))
(define (gcd m n) (define (gcd m n)
(if (= n 0) (if (= n 0)
m m

View File

@ -83,7 +83,7 @@ def standard_env() -> Environment:
'-': op.sub, '-': op.sub,
'*': op.mul, '*': op.mul,
'/': op.truediv, '/': op.truediv,
'//': op.floordiv, 'quotient': op.floordiv,
'>': op.gt, '>': op.gt,
'<': op.lt, '<': op.lt,
'>=': op.ge, '>=': op.ge,

View File

@ -147,7 +147,7 @@ def test_factorial():
gcd_src = """ gcd_src = """
(define (mod m n) (define (mod m n)
(- m (* n (// m n)))) (- m (* n (quotient m n))))
(define (gcd m n) (define (gcd m n)
(if (= n 0) (if (= n 0)
m m
@ -255,4 +255,4 @@ closure_averager_src = """
def test_closure_averager(): def test_closure_averager():
got = run(closure_averager_src) got = run(closure_averager_src)
assert got == 12.0 assert got == 12.0
# end::RUN_AVERAGER[] # end::RUN_AVERAGER[]

View File

@ -80,7 +80,7 @@ def standard_env() -> Environment:
'-': op.sub, '-': op.sub,
'*': op.mul, '*': op.mul,
'/': op.truediv, '/': op.truediv,
'//': op.floordiv, 'quotient': op.floordiv,
'>': op.gt, '>': op.gt,
'<': op.lt, '<': op.lt,
'>=': op.ge, '>=': op.ge,

View File

@ -16,37 +16,36 @@ import tqdm # type: ignore
from flags2_common import main, DownloadStatus, save_flag from flags2_common import main, DownloadStatus, save_flag
# default set low to avoid errors from remote site, such as # low concurrency default to avoid errors from remote site,
# 503 - Service Temporarily Unavailable # such as 503 - Service Temporarily Unavailable
DEFAULT_CONCUR_REQ = 5 DEFAULT_CONCUR_REQ = 5
MAX_CONCUR_REQ = 1000 MAX_CONCUR_REQ = 1000
async def get_flag(session: httpx.AsyncClient, # <2> async def get_flag(client: httpx.AsyncClient, # <1>
base_url: str, base_url: str,
cc: str) -> bytes: cc: str) -> bytes:
url = f'{base_url}/{cc}/{cc}.gif'.lower() url = f'{base_url}/{cc}/{cc}.gif'.lower()
resp = await session.get(url, timeout=3.1, follow_redirects=True) # <3> resp = await client.get(url, timeout=3.1, follow_redirects=True) # <2>
resp.raise_for_status() resp.raise_for_status()
return resp.content return resp.content
async def download_one(session: httpx.AsyncClient, async def download_one(client: httpx.AsyncClient,
cc: str, cc: str,
base_url: str, base_url: str,
semaphore: asyncio.Semaphore, # <4> semaphore: asyncio.Semaphore,
verbose: bool) -> DownloadStatus: verbose: bool) -> DownloadStatus:
try: try:
async with semaphore: # <5> async with semaphore: # <3>
image = await get_flag(session, base_url, cc) image = await get_flag(client, base_url, cc)
except httpx.HTTPStatusError as exc: # <4> except httpx.HTTPStatusError as exc: # <5>
res = exc.response res = exc.response
if res.status_code == HTTPStatus.NOT_FOUND: if res.status_code == HTTPStatus.NOT_FOUND:
status = DownloadStatus.NOT_FOUND # <5> status = DownloadStatus.NOT_FOUND
msg = f'not found: {res.url}' msg = f'not found: {res.url}'
else: else:
raise raise
else: else:
await asyncio.to_thread(save_flag, image, f'{cc}.gif') await asyncio.to_thread(save_flag, image, f'{cc}.gif') # <6>
status = DownloadStatus.OK status = DownloadStatus.OK
msg = 'OK' msg = 'OK'
if verbose and msg: if verbose and msg:
@ -61,33 +60,31 @@ async def supervisor(cc_list: list[str],
concur_req: int) -> Counter[DownloadStatus]: # <1> concur_req: int) -> Counter[DownloadStatus]: # <1>
counter: Counter[DownloadStatus] = Counter() counter: Counter[DownloadStatus] = Counter()
semaphore = asyncio.Semaphore(concur_req) # <2> semaphore = asyncio.Semaphore(concur_req) # <2>
async with httpx.AsyncClient() as session: async with httpx.AsyncClient() as client:
to_do = [download_one(session, cc, base_url, semaphore, verbose) to_do = [download_one(client, cc, base_url, semaphore, verbose)
for cc in sorted(cc_list)] # <3> for cc in sorted(cc_list)] # <3>
to_do_iter = asyncio.as_completed(to_do) # <4> to_do_iter = asyncio.as_completed(to_do) # <4>
if not verbose: if not verbose:
to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5>
error: httpx.HTTPError | None = None error: httpx.HTTPError | None = None # <6>
for coro in to_do_iter: # <6> for coro in to_do_iter: # <7>
try: try:
status = await coro # <7> status = await coro # <8>
except httpx.HTTPStatusError as exc: # <8> except httpx.HTTPStatusError as exc:
error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}' error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
error_msg = error_msg.format(resp=exc.response) error_msg = error_msg.format(resp=exc.response)
error = exc error = exc # <9>
except httpx.RequestError as exc: # <9> except httpx.RequestError as exc:
error_msg = f'{exc} {type(exc)}'.strip() error_msg = f'{exc} {type(exc)}'.strip()
error = exc error = exc # <10>
except KeyboardInterrupt: # <10> except KeyboardInterrupt:
break break
else: # <11>
error = None
if error: if error:
status = DownloadStatus.ERROR # <12> status = DownloadStatus.ERROR # <11>
if verbose: if verbose:
url = str(error.request.url) # <13> url = str(error.request.url) # <12>
cc = Path(url).stem.upper() # <14> cc = Path(url).stem.upper() # <13>
print(f'{cc} error: {error_msg}') print(f'{cc} error: {error_msg}')
counter[status] += 1 counter[status] += 1