daviddoji/example-code-2e
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sync from Atlas

Browse Source
This commit is contained in:
Luciano Ramalho
2021-10-05 09:52:43 -03:00
parent 5d6b156047
commit 43f1bf23b3
5 changed files with 118 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
20-executors/getflags/.gitignore vendored
View File

@@ -1 +1,2 @@
flags/
flags/
downloaded/

10
20-executors/getflags/flags2_threadpool.py
View File

@@ -20,7 +20,7 @@ Sample run::
# tag::FLAGS2_THREADPOOL[]
from collections import Counter
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor, as_completed
import httpx
import tqdm # type: ignore
@@ -37,13 +37,13 @@ def download_many(cc_list: list[str],
verbose: bool,
concur_req: int) -> Counter[DownloadStatus]:
counter: Counter[DownloadStatus] = Counter()
with futures.ThreadPoolExecutor(max_workers=concur_req) as executor: # <4>
with ThreadPoolExecutor(max_workers=concur_req) as executor: # <4>
to_do_map = {} # <5>
for cc in sorted(cc_list): # <6>
future = executor.submit(download_one, cc,
base_url, verbose) # <7>
to_do_map[future] = cc # <8>
done_iter = futures.as_completed(to_do_map) # <9>
done_iter = as_completed(to_do_map) # <9>
if not verbose:
done_iter = tqdm.tqdm(done_iter, total=len(cc_list)) # <10>
for future in done_iter: # <11>
@@ -52,7 +52,7 @@ def download_many(cc_list: list[str],
except httpx.HTTPStatusError as exc: # <13>
error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
error_msg = error_msg.format(resp=exc.response)
except httpx.RequestError as exc: # <15>
except httpx.RequestError as exc:
error_msg = f'{exc} {type(exc)}'.strip()
except KeyboardInterrupt:
break
@@ -63,7 +63,7 @@ def download_many(cc_list: list[str],
status = DownloadStatus.ERROR
counter[status] += 1
if verbose and error_msg:
cc = to_do_map[future] # <16>
cc = to_do_map[future] # <14>
print(f'{cc} error: {error_msg}')
return counter