sync from Atlas repo

concurrency/flags/country-codes.tab

@@ -0,0 +1,195 @@
+# ISO-3166-1	US-GEC	name
+AF	AF	Afghanistan
+AL	AL	Albania
+DZ	AG	Algeria
+AD	AN	Andorra
+AO	AO	Angola
+AG	AC	Antigua and Barbuda
+AR	AR	Argentina
+AM	AM	Armenia
+AU	AS	Australia
+AT	AU	Austria
+AZ	AJ	Azerbaijan
+BS	BF	Bahamas
+BH	BA	Bahrain
+BD	BG	Bangladesh
+BB	BB	Barbados
+BY	BO	Belarus
+BE	BE	Belgium
+BZ	BH	Belize
+BJ	BN	Benin
+BT	BT	Bhutan
+BO	BL	Bolivia
+BA	BK	Bosnia and Herzegovina
+BW	BC	Botswana
+BR	BR	Brazil
+BN	BX	Brunei Darussalam
+BG	BU	Bulgaria
+BF	UV	Burkina Faso
+BI	BY	Burundi
+KH	CB	Cambodia
+CM	CM	Cameroon
+CA	CA	Canada
+CV	CV	Cape Verde
+CF	CT	Central African Republic
+TD	CD	Chad
+CL	CI	Chile
+CN	CH	China
+CO	CO	Colombia
+KM	CN	Comoros
+CG	CF	Congo (Brazzaville)
+CD	CG	Congo (Kinshasa)
+CR	CS	Costa Rica
+CI	IV	Côte d'Ivoire
+HR	HR	Croatia
+CU	CU	Cuba
+CY	CY	Cyprus
+CZ	EZ	Czech Republic
+DK	DA	Denmark
+DJ	DJ	Djibouti
+DM	DO	Dominica
+EC	EC	Ecuador
+EG	EG	Egypt
+SV	ES	El Salvador
+GQ	EK	Equatorial Guinea
+ER	ER	Eritrea
+EE	EN	Estonia
+ET	ET	Ethiopia
+FJ	FJ	Fiji
+FI	FI	Finland
+FR	FR	France
+GA	GB	Gabon
+GM	GA	Gambia
+GE	GG	Georgia
+DE	GM	Germany
+GH	GH	Ghana
+GR	GR	Greece
+GD	GJ	Grenada
+GT	GT	Guatemala
+GN	GV	Guinea
+GW	PU	Guinea-Bissau
+GY	GY	Guyana
+HT	HA	Haiti
+HN	HO	Honduras
+HU	HU	Hungary
+IS	IC	Iceland
+IN	IN	India
+ID	ID	Indonesia
+IR	IR	Iran
+IQ	IZ	Iraq
+IE	EI	Ireland
+IL	IS	Israel
+IT	IT	Italy
+JM	JM	Jamaica
+JP	JA	Japan
+JO	JO	Jordan
+KZ	KZ	Kazakhstan
+KE	KE	Kenya
+KI	KR	Kiribati
+KP	KN	Korea, North
+KR	KS	Korea, South
+KW	KU	Kuwait
+KG	KG	Kyrgyzstan
+LA	LA	Laos
+LV	LG	Latvia
+LB	LE	Lebanon
+LS	LT	Lesotho
+LR	LI	Liberia
+LY	LY	Libya
+LI	LS	Liechtenstein
+LT	LH	Lithuania
+LU	LU	Luxembourg
+MK	MK	Macedonia
+MG	MA	Madagascar
+MW	MI	Malawi
+MY	MY	Malaysia
+MV	MV	Maldives
+ML	ML	Mali
+MT	MT	Malta
+MH	RM	Marshall Islands
+MR	MR	Mauritania
+MU	MP	Mauritius
+MX	MX	Mexico
+FM	FM	Micronesia
+MD	MD	Moldova
+MC	MN	Monaco
+MN	MG	Mongolia
+ME	MJ	Montenegro
+MA	MO	Morocco
+MZ	MZ	Mozambique
+MM	BM	Myanmar
+NA	WA	Namibia
+NR	NR	Nauru
+NP	NP	Nepal
+NL	NL	Netherlands
+NZ	NZ	New Zealand
+NI	NU	Nicaragua
+NE	NG	Niger
+NG	NI	Nigeria
+NO	NO	Norway
+OM	MU	Oman
+PK	PK	Pakistan
+PW	PS	Palau
+PA	PM	Panama
+PG	PP	Papua New Guinea
+PY	PA	Paraguay
+PE	PE	Peru
+PH	RP	Philippines
+PL	PL	Poland
+PT	PO	Portugal
+QA	QA	Qatar
+RO	RO	Romania
+RU	RS	Russian Federation
+RW	RW	Rwanda
+KN	SC	Saint Kitts and Nevis
+LC	ST	Saint Lucia
+VC	VC	Grenadines
+WS	WS	Samoa
+SM	SM	San Marino
+ST	TP	Sao Tome and Principe
+SA	SA	Saudi Arabia
+SN	SG	Senegal
+RS	RI	Serbia
+SC	SE	Seychelles
+SL	SL	Sierra Leone
+SG	SN	Singapore
+SK	LO	Slovakia
+SI	SI	Slovenia
+SB	BP	Solomon Islands
+SO	SO	Somalia
+ZA	SF	South Africa
+SS	OD	South Sudan
+ES	SP	Spain
+LK	CE	Sri Lanka
+SD	SU	Sudan
+SR	NS	Suriname
+SZ	WZ	Swaziland
+SE	SW	Sweden
+CH	SZ	Switzerland
+SY	SY	Syria
+TW	TW	Taiwan
+TJ	TI	Tajikistan
+TZ	TZ	Tanzania
+TH	TH	Thailand
+TL	TT	Timor-Leste
+TG	TO	Togo
+TO	TN	Tonga
+TT	TD	Trinidad and Tobago
+TN	TS	Tunisia
+TR	TU	Turkey
+TM	TX	Turkmenistan
+TV	TV	Tuvalu
+UG	UG	Uganda
+UA	UP	Ukraine
+AE	AE	United Arab Emirates
+GB	UK	United Kingdom
+US	US	United States of America
+UY	UY	Uruguay
+UZ	UZ	Uzbekistan
+VU	NH	Vanuatu
+VA	VT	Vatican City
+VE	VE	Venezuela
+VN	VM	Vietnam
+YE	YM	Yemen
+ZM	ZA	Zambia
+ZW	ZI	Zimbabwe

concurrency/flags/countryflags.py

@@ -0,0 +1,46 @@
+"""
+Mappings of ISO-3166-1 alpha-2 country codes to names, to GEC
+(Geopolitical Entities and Codes used by the US government)
+and utility functions for flag download examples
+"""
+
+DATA_FILE = 'country-codes.tab'
+
+# original source
+CIA_URL = ('https://www.cia.gov/library/publications/'
+            'the-world-factbook/graphics/flags/large/{gec}-lgflag.gif')
+
+# local nginx web server
+NGINX_URL = 'http://localhost:8080/ciaflags/{gec}.gif'
+
+# Vaurien
+VAURIEN_URL = 'http://localhost:8000/ciaflags/{gec}.gif'
+
+BASE_URL = VAURIEN_URL
+
+DEST_PATH_NAME = 'img/{cc}.gif'
+
+cc2name = {}  # ISO-3166-1 to name
+cc2gec = {}   # ISO-3166-1 to GEC
+
+def _load():
+    with open(DATA_FILE, encoding='utf-8') as cc_txt:
+        for line in cc_txt:
+            line = line.rstrip()
+            if line.startswith('#'):
+                continue
+            iso_cc, gec, name = line.split('\t')
+            cc2name[iso_cc] = name
+            cc2gec[iso_cc] = gec
+
+
+def flag_url(iso_cc):
+    return BASE_URL.format(gec=cc2gec[iso_cc].lower())
+
+def iso_file_name(iso_cc):
+    return DEST_PATH_NAME.format(cc=iso_cc.lower())
+
+def gec_file_name(iso_cc):
+    return DEST_PATH_NAME.format(cc=cc2gec[iso_cc].lower())
+
+_load()

concurrency/flags/getsequential.py

@@ -0,0 +1,56 @@
+import requests
+import countryflags as cf
+import time
+
+
+times = {}
+
+def fetch(iso_cc):
+    resp = requests.get(cf.flag_url(iso_cc))
+    if resp.status_code != 200:
+        resp.raise_for_status()
+    file_name = cf.iso_file_name(iso_cc)
+    with open(file_name, 'wb') as img:
+        written = img.write(resp.content)
+    return written, file_name
+
+def main():
+    pending = sorted(cf.cc2name)
+    to_download = len(pending)
+    downloaded = 0
+    t0 = time.time()
+    for iso_cc in pending:
+        print('get:', iso_cc)
+        try:
+            times[iso_cc] = [time.time() - t0]
+            octets, file_name = fetch(iso_cc)
+            times[iso_cc].append(time.time() - t0)
+            downloaded += 1
+            print('\t--> {}: {:5d} bytes'.format(file_name, octets))
+        except Exception as exc:
+            print('\t***', iso_cc, 'generated an exception:', exc)
+    ratio = downloaded / to_download
+    print('{} of {} downloaded ({:.1%})'.format(downloaded, to_download, ratio))
+    for iso_cc in sorted(times):
+        start, end = times[iso_cc]
+        print('{}\t{:.6g}\t{:.6g}'.format(iso_cc, start, end))
+
+if __name__ == '__main__':
+    main()
+
+"""
+From cia.gov:
+real    3m26.679s
+user    0m5.212s
+sys     0m0.383s
+
+From localhost nginx:
+real    0m1.193s
+user    0m0.858s
+sys     0m0.179s
+
+From localhost nginx via Vaurien with .5s delay
+real    1m40.519s
+user    0m1.103s
+sys     0m0.243s
+"""

concurrency/flags/getthreadpool.py

@@ -0,0 +1,56 @@
+from concurrent import futures
+import sys
+import requests
+import countryflags as cf
+import time
+
+from getsequential import fetch
+
+DEFAULT_NUM_THREADS = 100
+GLOBAL_TIMEOUT = 300  # seconds
+
+times = {}
+
+def main(num_threads):
+    pool = futures.ThreadPoolExecutor(num_threads)
+    pending = {}
+    t0 = time.time()
+    # submit all jobs
+    for iso_cc in sorted(cf.cc2name):
+        print('get:', iso_cc)
+        times[iso_cc] = [time.time() - t0]
+        job = pool.submit(fetch, iso_cc)
+        pending[job] = iso_cc
+    to_download = len(pending)
+    downloaded = 0
+    # get results as jobs are done
+    for job in futures.as_completed(pending, timeout=GLOBAL_TIMEOUT):
+        try:
+            octets, file_name = job.result()
+            times[pending[job]].append(time.time() - t0)
+            downloaded += 1
+            print('\t--> {}: {:5d} bytes'.format(file_name, octets))
+        except Exception as exc:
+            print('\t***', pending[job], 'generated an exception:', exc)
+    ratio = downloaded / to_download
+    print('{} of {} downloaded ({:.1%})'.format(downloaded, to_download, ratio))
+    for iso_cc in sorted(times):
+        start, end = times[iso_cc]
+        print('{}\t{:.6g}\t{:.6g}'.format(iso_cc, start, end))
+
+if __name__ == '__main__':
+    if len(sys.argv) == 2:
+        num_threads = int(sys.argv[1])
+    else:
+        num_threads = DEFAULT_NUM_THREADS
+    main(num_threads)
+
+"""
+From localhost nginx:
+real    0m1.163s
+user    0m1.001s
+sys     0m0.289s
+
+
+
+"""

concurrency/flags/vaurien_delay.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+vaurien --protocol http --proxy localhost:8000 --backend localhost:8080 \
+        --behavior 100:delay --behavior-delay-sleep 1

concurrency/flags/vaurien_error_delay.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+vaurien --protocol http --proxy localhost:8000 --backend localhost:8080 \
+        --behavior 50:error,50:delay --behavior-delay-sleep .5