diff --git a/concurrency/flags/build_fixture.py b/concurrency/flags/build_fixture.py new file mode 100644 index 0000000..ea7a127 --- /dev/null +++ b/concurrency/flags/build_fixture.py @@ -0,0 +1,30 @@ +""" +Build flags fixture +""" + +import shutil +import os +import json + +SRC = 'img/' +DEST = 'fixture/' + +with open('country-codes.tab') as cc_fp: + for line in cc_fp: + if line.startswith('#'): + continue + iso_cc, gec_cc, name = line.strip().split('\t') + print(iso_cc, name) + cc = iso_cc.lower() + img_name = cc + '.gif' + from_file = os.path.join(SRC, img_name) + to_path = os.path.join(DEST, cc) + os.mkdir(to_path) + to_file = os.path.join(to_path, img_name) + shutil.copyfile(from_file, to_file) + tld_cc = 'uk' if cc == 'gb' else cc + metadata = {'country': name, 'iso_cc': iso_cc, + 'tld_cc': '.'+tld_cc, 'gec_cc': gec_cc} + + with open(os.path.join(to_path, 'metadata.json'), 'wt') as json_fp: + json.dump(metadata, json_fp, ensure_ascii=True) diff --git a/concurrency/flags/cc_count.py b/concurrency/flags/cc_count.py new file mode 100644 index 0000000..af1046e --- /dev/null +++ b/concurrency/flags/cc_count.py @@ -0,0 +1,19 @@ + +from collections import Counter +from operator import itemgetter +from string import ascii_uppercase + +with open('country-codes.tab') as fp: + ct = Counter() + for line in fp: + if line.startswith('#'): + continue + cc, _, _ = line.split('\t') + ct[cc[0]] += 1 + print(cc, end=' ') + +for key, value in sorted(ct.items(), key=itemgetter(1), reverse=True): + print(key, value) + +print('Total:', sum(ct.values())) +print('Missing:', ', '.join(set(ascii_uppercase) - ct.keys())) diff --git a/concurrency/flags/cc_tlds.py b/concurrency/flags/cc_tlds.py new file mode 100644 index 0000000..2c96dab --- /dev/null +++ b/concurrency/flags/cc_tlds.py @@ -0,0 +1,36 @@ +""" +Check country code TLDs +""" + +import shutil +import os +import json + +iso_cc_db = {} + +with open('country-codes.tab') as cc_fp: + for line in cc_fp: + if line.startswith('#'): + continue + iso_cc, gec_cc, name = line.strip().split('\t') + iso_cc_db[iso_cc.lower()] = name + +tld_cc_db = {} + +with open('tlds.tab') as cc_fp: + for line in cc_fp: + if line.startswith('#'): + continue + tld_cc, category, entity = line.strip().split('\t') + if category.strip() != 'country-code': + continue + if ascii(tld_cc) != repr(tld_cc): + continue + tld_cc_db[tld_cc[1:].strip()] = entity + +not_tld = iso_cc_db.keys() - tld_cc_db.keys() +print(sorted(not_tld)) + +for iso_cc, name in sorted(iso_cc_db.items()): + entity = tld_cc_db[iso_cc] + print('{}\t{}\t{}'.format(iso_cc, name, entity)) diff --git a/concurrency/flags/tlds.tab b/concurrency/flags/tlds.tab new file mode 100644 index 0000000..dcf1a70 --- /dev/null +++ b/concurrency/flags/tlds.tab @@ -0,0 +1,848 @@ +# https://www.iana.org/domains/root/db +.abogado generic Top Level Domain Holdings Limited +.ac country-code Network Information Center (AC Domain Registry) c/o Cable and Wireless (Ascension Island) +.academy generic Half Oaks, LLC +.accountants generic Knob Town, LLC +.active generic The Active Network, Inc +.actor generic United TLD Holdco Ltd. +.ad country-code Andorra Telecom +.adult generic ICM Registry AD LLC +.ae country-code Telecommunication Regulatory Authority (TRA) +.aero sponsored Societe Internationale de Telecommunications Aeronautique (SITA INC USA) +.af country-code Ministry of Communications and IT +.ag country-code UHSA School of Medicine +.agency generic Steel Falls, LLC +.ai country-code Government of Anguilla +.airforce generic United TLD Holdco Ltd. +.al country-code Electronic and Postal Communications Authority - AKEP +.allfinanz generic Allfinanz Deutsche Vermögensberatung Aktiengesellschaft +.alsace generic REGION D ALSACE +.am country-code Internet Society +.amsterdam generic Gemeente Amsterdam +.an country-code University of Curacao +.android generic Charleston Road Registry Inc. +.ao country-code Faculdade de Engenharia da Universidade Agostinho Neto +.apartments generic June Maple, LLC +.aq country-code Antarctica Network Information Centre Limited +.aquarelle generic Aquarelle.com +.ar country-code Presidencia de la Nación – Secretaría Legal y Técnica +.archi generic STARTING DOT LIMITED +.army generic United TLD Holdco Ltd. +.arpa infrastructure Internet Architecture Board (IAB) +.as country-code AS Domain Registry +.asia sponsored DotAsia Organisation Ltd. +.associates generic Baxter Hill, LLC +.at country-code nic.at GmbH +.attorney generic United TLD Holdco, Ltd +.au country-code .au Domain Administration (auDA) +.auction generic United TLD HoldCo, Ltd. +.audio generic Uniregistry, Corp. +.autos generic DERAutos, LLC +.aw country-code SETAR +.ax country-code Ålands landskapsregering +.axa generic AXA SA +.az country-code IntraNS +.ba country-code Universtiy Telinformatic Centre (UTIC) +.band generic United TLD Holdco, Ltd +.bank generic fTLD Registry Services, LLC +.bar generic Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable +.barclaycard generic Barclays Bank PLC +.barclays generic Barclays Bank PLC +.bargains generic Half Hallow, LLC +.bayern generic Bayern Connect GmbH +.bb country-code Government of Barbados Ministry of Economic Affairs and Development Telecommunications Unit +.bd country-code Ministry of Post & Telecommunications Bangladesh Secretariat +.be country-code DNS Belgium vzw/asbl +.beer generic Top Level Domain Holdings Limited +.berlin generic dotBERLIN GmbH & Co. KG +.best generic BestTLD Pty Ltd +.bf country-code ARCE-AutoritÈ de RÈgulation des Communications Electroniques +.bg country-code Register.BG +.bh country-code Telecommunications Regulatory Authority (TRA) +.bi country-code Centre National de l'Informatique +.bid generic dot Bid Limited +.bike generic Grand Hollow, LLC +.bingo generic Sand Cedar, LLC +.bio generic STARTING DOT LIMITED +.biz generic-restricted NeuStar, Inc. +.bj country-code Benin Telecoms S.A. +.bl country-code Not assigned +.black generic Afilias Limited +.blackfriday generic Uniregistry, Corp. +.bloomberg generic Bloomberg IP Holdings LLC +.blue generic Afilias Limited +.bm country-code Registry General Ministry of Labour and Immigration +.bmw generic Bayerische Motoren Werke Aktiengesellschaft +.bn country-code Telekom Brunei Berhad +.bnpparibas generic BNP Paribas +.bo country-code Agencia para el Desarrollo de la Información de la Sociedad en Bolivia +.boo generic Charleston Road Registry Inc. +.boutique generic Over Galley, LLC +.bq country-code Not assigned +.br country-code Comite Gestor da Internet no Brasil +.brussels generic DNS.be vzw +.bs country-code The College of the Bahamas +.bt country-code Ministry of Information and Communications +.budapest generic Top Level Domain Holdings Limited +.build generic Plan Bee LLC +.builders generic Atomic Madison, LLC +.business generic Spring Cross, LLC +.buzz generic DOTSTRATEGY CO. +.bv country-code UNINETT Norid A/S +.bw country-code Botswana Communications Regulatory Authority (BOCRA) +.by country-code Reliable Software Inc. +.bz country-code University of Belize +.bzh generic Association www.bzh +.ca country-code Canadian Internet Registration Authority (CIRA) Autorite Canadienne pour les Enregistrements Internet (ACEI) +.cab generic Half Sunset, LLC +.cal generic Charleston Road Registry Inc. +.camera generic Atomic Maple, LLC +.camp generic Delta Dynamite, LLC +.cancerresearch generic Australian Cancer Research Foundation +.canon generic Canon Inc. +.capetown generic ZA Central Registry NPC trading as ZA Central Registry +.capital generic Delta Mill, LLC +.caravan generic Caravan International, Inc. +.cards generic Foggy Hollow, LLC +.care generic Goose Cross, LLC +.career generic dotCareer LLC +.careers generic Wild Corner, LLC +.cartier generic Richemont DNS Inc. +.casa generic Top Level Domain Holdings Limited +.cash generic Delta Lake, LLC +.cat sponsored Fundacio puntCAT +.catering generic New Falls. LLC +.cbn generic The Christian Broadcasting Network, Inc. +.cc country-code eNIC Cocos (Keeling) Islands Pty. Ltd. d/b/a Island Internet Services +.cd country-code Office Congolais des Postes et Télécommunications - OCPT +.center generic Tin Mill, LLC +.ceo generic CEOTLD Pty Ltd +.cern generic European Organization for Nuclear Research ("CERN") +.cf country-code Societe Centrafricaine de Telecommunications (SOCATEL) +.cg country-code ONPT Congo and Interpoint Switzerland +.ch country-code SWITCH The Swiss Education & Research Network +.channel generic Charleston Road Registry Inc. +.chat generic Sand Fields, LLC +.cheap generic Sand Cover, LLC +.christmas generic Uniregistry, Corp. +.chrome generic Charleston Road Registry Inc. +.church generic Holly Fileds, LLC +.ci country-code INP-HB Institut National Polytechnique Felix Houphouet Boigny +.citic generic CITIC Group Corporation +.city generic Snow Sky, LLC +.ck country-code Telecom Cook Islands Ltd. +.cl country-code NIC Chile (University of Chile) +.claims generic Black Corner, LLC +.cleaning generic Fox Shadow, LLC +.click generic Uniregistry, Corp. +.clinic generic Goose Park, LLC +.clothing generic Steel Lake, LLC +.club generic .CLUB DOMAINS, LLC +.cm country-code Cameroon Telecommunications (CAMTEL) +.cn country-code Computer Network Information Center, Chinese Academy of Sciences +.co country-code .CO Internet S.A.S. +.coach generic Koko Island, LLC +.codes generic Puff Willow, LLC +.coffee generic Trixy Cover, LLC +.college generic XYZ.COM LLC +.cologne generic NetCologne Gesellschaft für Telekommunikation mbH +.com generic VeriSign Global Registry Services +.community generic Fox Orchard, LLC +.company generic Silver Avenue, LLC +.computer generic Pine Mill, LLC +.condos generic Pine House, LLC +.construction generic Fox Dynamite, LLC +.consulting generic United TLD Holdco, LTD. +.contractors generic Magic Woods, LLC +.cooking generic Top Level Domain Holdings Limited +.cool generic Koko Lake, LLC +.coop sponsored DotCooperation LLC +.country generic Top Level Domain Holdings Limited +.cr country-code National Academy of Sciences Academia Nacional de Ciencias +.credit generic Snow Shadow, LLC +.creditcard generic Binky Frostbite, LLC +.cricket generic dot Cricket Limited +.crs generic Federated Co-operatives Limited +.cruises generic Spring Way, LLC +.cu country-code CENIAInternet Industria y San Jose Capitolio Nacional +.cuisinella generic SALM S.A.S. +.cv country-code Agência Nacional das Comunicações (ANAC) +.cw country-code University of Curacao +.cx country-code Christmas Island Internet Administration Limited +.cy country-code University of Cyprus +.cymru generic Nominet UK +.cz country-code CZ.NIC, z.s.p.o +.dabur generic Dabur India Limited +.dad generic Charleston Road Registry Inc. +.dance generic United TLD Holdco Ltd. +.dating generic Pine Fest, LLC +.day generic Charleston Road Registry Inc. +.dclk generic Charleston Road Registry Inc. +.de country-code DENIC eG +.deals generic Sand Sunset, LLC +.degree generic United TLD Holdco, Ltd +.delivery generic Steel Station, LLC +.democrat generic United TLD Holdco Ltd. +.dental generic Tin Birch, LLC +.dentist generic United TLD Holdco, Ltd +.desi generic Desi Networks LLC +.design generic Top Level Design, LLC +.dev generic Charleston Road Registry Inc. +.diamonds generic John Edge, LLC +.diet generic Uniregistry, Corp. +.digital generic Dash Park, LLC +.direct generic Half Trail, LLC +.directory generic Extra Madison, LLC +.discount generic Holly Hill, LLC +.dj country-code Djibouti Telecom S.A +.dk country-code Dansk Internet Forum +.dm country-code DotDM Corporation +.dnp generic Dai Nippon Printing Co., Ltd. +.do country-code Pontificia Universidad Catolica Madre y Maestra Recinto Santo Tomas de Aquino +.docs generic Charleston Road Registry Inc. +.domains generic Sugar Cross, LLC +.doosan generic Doosan Corporation +.durban generic ZA Central Registry NPC trading as ZA Central Registry +.dvag generic Deutsche Vermögensberatung Aktiengesellschaft DVAG +.dz country-code CERIST +.eat generic Charleston Road Registry Inc. +.ec country-code NIC.EC (NICEC) S.A. +.edu sponsored EDUCAUSE +.education generic Brice Way, LLC +.ee country-code Eesti Interneti Sihtasutus (EIS) +.eg country-code Egyptian Universities Network (EUN) Supreme Council of Universities +.eh country-code Not assigned +.email generic Spring Madison, LLC +.emerck generic Merck KGaA +.energy generic Binky Birch, LLC +.engineer generic United TLD Holdco Ltd. +.engineering generic Romeo Canyon +.enterprises generic Snow Oaks, LLC +.equipment generic Corn Station, LLC +.er country-code Eritrea Telecommunication Services Corporation (EriTel) +.es country-code Red.es +.esq generic Charleston Road Registry Inc. +.estate generic Trixy Park, LLC +.et country-code Ethio telecom +.eu country-code EURid vzw/asbl +.eurovision generic European Broadcasting Union (EBU) +.eus generic Puntueus Fundazioa +.events generic Pioneer Maple, LLC +.everbank generic EverBank +.exchange generic Spring Falls, LLC +.expert generic Magic Pass, LLC +.exposed generic Victor Beach, LLC +.fail generic Atomic Pipe, LLC +.fans generic Asiamix Digital Limited +.farm generic Just Maple, LLC +.fashion generic Top Level Domain Holdings Limited +.feedback generic Top Level Spectrum, Inc. +.fi country-code Finnish Communications Regulatory Authority +.finance generic Cotton Cypress, LLC +.financial generic Just Cover, LLC +.firmdale generic Firmdale Holdings Limited +.fish generic Fox Woods, LLC +.fishing generic Top Level Domain Holdings Limited +.fit generic Minds + Machines Group Limited +.fitness generic Brice Orchard, LLC +.fj country-code The University of the South Pacific IT Services +.fk country-code Falkland Islands Government +.flights generic Fox Station, LLC +.florist generic Half Cypress, LLC +.flowers generic Uniregistry, Corp. +.flsmidth generic FLSmidth A/S +.fly generic Charleston Road Registry Inc. +.fm country-code FSM Telecommunications Corporation +.fo country-code FO Council +.foo generic Charleston Road Registry Inc. +.forsale generic United TLD Holdco, LLC +.foundation generic John Dale, LLC +.fr country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.frl generic FRLregistry B.V. +.frogans generic OP3FT +.fund generic John Castle, LLC +.furniture generic Lone Fields, LLC +.futbol generic United TLD Holdco, Ltd. +.ga country-code Agence Nationale des Infrastructures Numériques et des Fréquences (ANINF) +.gal generic Asociación puntoGAL +.gallery generic Sugar House, LLC +.garden generic Top Level Domain Holdings Limited +.gb country-code Reserved Domain - IANA +.gbiz generic Charleston Road Registry Inc. +.gd country-code The National Telecommunications Regulatory Commission (NTRC) +.gdn generic Joint Stock Company "Navigation-information systems" +.ge country-code Caucasus Online +.gent generic COMBELL GROUP NV/SA +.gf country-code Net Plus +.gg country-code Island Networks Ltd. +.ggee generic GMO Internet, Inc. +.gh country-code Network Computer Systems Limited +.gi country-code Sapphire Networks +.gift generic Uniregistry, Corp. +.gifts generic Goose Sky, LLC +.gives generic United TLD Holdco Ltd. +.gl country-code TELE Greenland A/S +.glass generic Black Cover, LLC +.gle generic Charleston Road Registry Inc. +.global generic Dot Global Domain Registry Limited +.globo generic Globo Comunicação e Participações S.A +.gm country-code GM-NIC +.gmail generic Charleston Road Registry Inc. +.gmo generic GMO Internet, Inc. +.gmx generic 1&1 Mail & Media GmbH +.gn country-code Centre National des Sciences Halieutiques de Boussoura +.goldpoint generic YODOBASHI CAMERA CO.,LTD. +.goog generic Charleston Road Registry Inc. +.google generic Charleston Road Registry Inc. +.gop generic Republican State Leadership Committee, Inc. +.gov sponsored General Services Administration Attn: QTDC, 2E08 (.gov Domain Registration) +.gp country-code Networking Technologies Group +.gq country-code GETESA +.gr country-code ICS-FORTH GR +.graphics generic Over Madison, LLC +.gratis generic Pioneer Tigers, LLC +.green generic Afilias Limited +.gripe generic Corn Sunset, LLC +.gs country-code Government of South Georgia and South Sandwich Islands (GSGSSI) +.gt country-code Universidad del Valle de Guatemala +.gu country-code University of Guam Computer Center +.guide generic Snow Moon, LLC +.guitars generic Uniregistry, Corp. +.guru generic Pioneer Cypress, LLC +.gw country-code Autoridade Reguladora Nacional - Tecnologias de Informação e Comunicação da Guiné-Bissau +.gy country-code University of Guyana +.hamburg generic Hamburg Top-Level-Domain GmbH +.hangout generic Charleston Road Registry Inc. +.haus generic United TLD Holdco, LTD. +.healthcare generic Silver Glen, LLC +.help generic Uniregistry, Corp. +.here generic Charleston Road Registry Inc. +.hermes generic Hermes International +.hiphop generic Uniregistry, Corp. +.hiv generic dotHIV gemeinnuetziger e.V. +.hk country-code Hong Kong Internet Registration Corporation Ltd. +.hm country-code HM Domain Registry +.hn country-code Red de Desarrollo Sostenible Honduras +.holdings generic John Madison, LLC +.holiday generic Goose Woods, LLC +.homes generic DERHomes, LLC +.horse generic Top Level Domain Holdings Limited +.host generic DotHost Inc. +.hosting generic Uniregistry, Corp. +.house generic Sugar Park, LLC +.how generic Charleston Road Registry Inc. +.hr country-code CARNet - Croatian Academic and Research Network +.ht country-code Consortium FDS/RDDH +.hu country-code Council of Hungarian Internet Providers (CHIP) +.ibm generic International Business Machines Corporation +.id country-code Perkumpulan Pengelola Nama Domain Internet Indonesia (PANDI) +.ie country-code University College Dublin Computing Services Computer Centre +.ifm generic ifm electronic gmbh +.il country-code Internet Society of Israel +.im country-code Isle of Man Government +.immo generic Auburn Bloom, LLC +.immobilien generic United TLD Holdco Ltd. +.in country-code National Internet Exchange of India +.industries generic Outer House, LLC +.info generic Afilias Limited +.ing generic Charleston Road Registry Inc. +.ink generic Top Level Design, LLC +.institute generic Outer Maple, LLC +.insure generic Pioneer Willow, LLC +.int sponsored Internet Assigned Numbers Authority +.international generic Wild Way, LLC +.investments generic Holly Glen, LLC +.io country-code IO Top Level Domain Registry Cable and Wireless +.iq country-code Communications and Media Commission (CMC) +.ir country-code Institute for Research in Fundamental Sciences +.irish generic Dot-Irish LLC +.is country-code ISNIC - Internet Iceland ltd. +.it country-code IIT - CNR +.iwc generic Richemont DNS Inc. +.jcb generic JCB Co., Ltd. +.je country-code Island Networks (Jersey) Ltd. +.jetzt generic New TLD Company AB +.jm country-code University of West Indies +.jo country-code National Information Technology Center (NITC) +.jobs sponsored Employ Media LLC +.joburg generic ZA Central Registry NPC trading as ZA Central Registry +.jp country-code Japan Registry Services Co., Ltd. +.juegos generic Uniregistry, Corp. +.kaufen generic United TLD Holdco Ltd. +.kddi generic KDDI CORPORATION +.ke country-code Kenya Network Information Center (KeNIC) +.kg country-code AsiaInfo Telecommunication Enterprise +.kh country-code Ministry of Post and Telecommunications +.ki country-code Ministry of Communications, Transport, and Tourism Development +.kim generic Afilias Limited +.kitchen generic Just Goodbye, LLC +.kiwi generic DOT KIWI LIMITED +.km country-code Comores Telecom +.kn country-code Ministry of Finance, Sustainable Development Information & Technology +.koeln generic NetCologne Gesellschaft für Telekommunikation mbH +.kp country-code Star Joint Venture Company +.kr country-code Korea Internet & Security Agency (KISA) +.krd generic KRG Department of Information Technology +.kred generic KredTLD Pty Ltd +.kw country-code Ministry of Communications +.ky country-code The Information and Communications Technology Authority +.kyoto generic Academic Institution: Kyoto Jyoho Gakuen +.kz country-code Association of IT Companies of Kazakhstan +.la country-code Lao National Internet Committee (LANIC), Ministry of Posts and Telecommunications +.lacaixa generic CAIXA D'ESTALVIS I PENSIONS DE BARCELONA +.land generic Pine Moon, LLC +.lat generic ECOM-LAC Federación de Latinoamérica y el Caribe para Internet y el Comercio Electrónico +.latrobe generic La Trobe University +.lawyer generic United TLD Holdco, Ltd +.lb country-code American University of Beirut Computing and Networking Services +.lc country-code University of Puerto Rico +.lds generic IRI Domain Management, LLC +.lease generic Victor Trail, LLC +.legal generic Blue Falls, LLC +.lgbt generic Afilias Limited +.li country-code Universitaet Liechtenstein +.lidl generic Schwarz Domains und Services GmbH & Co. KG +.life generic Trixy Oaks, LLC +.lighting generic John McCook, LLC +.limited generic Big Fest, LLC +.limo generic Hidden Frostbite, LLC +.link generic Uniregistry, Corp. +.lk country-code Council for Information Technology LK Domain Registrar +.loans generic June Woods, LLC +.london generic Dot London Domains Limited +.lotte generic Lotte Holdings Co., Ltd. +.lotto generic Afilias Limited +.lr country-code Data Technology Solutions, Inc. +.ls country-code National University of Lesotho +.lt country-code Kaunas University of Technology +.ltda generic InterNetX Corp. +.lu country-code RESTENA +.luxe generic Top Level Domain Holdings Limited +.luxury generic Luxury Partners LLC +.lv country-code University of Latvia Institute of Mathematics and Computer Science Department of Network Solutions (DNS) +.ly country-code General Post and Telecommunication Company +.ma country-code Agence Nationale de Réglementation des Télécommunications (ANRT) +.madrid generic Comunidad de Madrid +.maison generic Victor Frostbite, LLC +.management generic John Goodbye, LLC +.mango generic PUNTO FA S.L. +.market generic Unitied TLD Holdco, Ltd +.marketing generic Fern Pass, LLC +.marriott generic Marriott Worldwide Corporation +.mc country-code Gouvernement de Monaco Direction des Communications Electroniques +.md country-code MoldData S.E. +.me country-code Government of Montenegro +.media generic Grand Glen, LLC +.meet generic Afilias Limited +.melbourne generic The Crown in right of the State of Victoria, represented by its Department of State Development, Business and Innovation +.meme generic Charleston Road Registry Inc. +.memorial generic Dog Beach, LLC +.menu generic Wedding TLD2, LLC +.mf country-code Not assigned +.mg country-code NIC-MG (Network Information Center Madagascar) +.mh country-code Office of the Cabinet +.miami generic Top Level Domain Holdings Limited +.mil sponsored DoD Network Information Center +.mini generic Bayerische Motoren Werke Aktiengesellschaft +.mk country-code Macedonian Academic Research Network Skopje +.ml country-code Agence des Technologies de l’Information et de la Communication +.mm country-code Ministry of Communications, Posts & Telegraphs +.mn country-code Datacom Co., Ltd. +.mo country-code Bureau of Telecommunications Regulation (DSRT) +.mobi sponsored Afilias Technologies Limited dba dotMobi +.moda generic United TLD Holdco Ltd. +.moe generic Interlink Co., Ltd. +.monash generic Monash University +.money generic Outer McCook, LLC +.mormon generic IRI Domain Management, LLC ("Applicant") +.mortgage generic United TLD Holdco, Ltd +.moscow generic Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID) +.motorcycles generic DERMotorcycles, LLC +.mov generic Charleston Road Registry Inc. +.mp country-code Saipan Datacom, Inc. +.mq country-code MEDIASERV +.mr country-code Université des Sciences, de Technologie et de Médecine +.ms country-code MNI Networks Ltd. +.mt country-code NIC (Malta) +.mu country-code Internet Direct Ltd +.museum sponsored Museum Domain Management Association +.mv country-code Dhiraagu Pvt. Ltd. (DHIVEHINET) +.mw country-code Malawi Sustainable Development Network Programme (Malawi SDNP) +.mx country-code NIC-Mexico ITESM - Campus Monterrey +.my country-code MYNIC Berhad +.mz country-code Centro de Informatica de Universidade Eduardo Mondlane +.na country-code Namibian Network Information Center +.nagoya generic GMO Registry, Inc. +.name generic-restricted VeriSign Information Services, Inc. +.navy generic United TLD Holdco Ltd. +.nc country-code Office des Postes et Telecommunications +.ne country-code SONITEL +.net generic VeriSign Global Registry Services +.network generic Trixy Manor, LLC +.neustar generic NeuStar, Inc. +.new generic Charleston Road Registry Inc. +.nexus generic Charleston Road Registry Inc. +.nf country-code Norfolk Island Data Services +.ng country-code Nigeria Internet Registration Association +.ngo generic Public Interest Registry +.nhk generic Japan Broadcasting Corporation (NHK) +.ni country-code Universidad Nacional del Ingernieria Centro de Computo +.nico generic DWANGO Co., Ltd. +.ninja generic United TLD Holdco Ltd. +.nl country-code SIDN (Stichting Internet Domeinregistratie Nederland) +.no country-code UNINETT Norid A/S +.np country-code Mercantile Communications Pvt. Ltd. +.nr country-code CENPAC NET +.nra generic NRA Holdings Company, INC. +.nrw generic Minds + Machines GmbH +.ntt generic NIPPON TELEGRAPH AND TELEPHONE CORPORATION +.nu country-code The IUSN Foundation +.nyc generic The City of New York by and through the New York City Department of Information Technology & Telecommunications +.nz country-code InternetNZ +.okinawa generic BusinessRalliart inc. +.om country-code Telecommunications Regulatory Authority (TRA) +.one generic One.com A/S +.ong generic Public Interest Registry +.onl generic I-REGISTRY Ltd., Niederlassung Deutschland +.ooo generic INFIBEAM INCORPORATION LIMITED +.org generic Public Interest Registry (PIR) +.organic generic Afilias Limited +.osaka generic Interlink Co., Ltd. +.otsuka generic Otsuka Holdings Co., Ltd. +.ovh generic OVH SAS +.pa country-code Universidad Tecnologica de Panama +.paris generic City of Paris +.partners generic Magic Glen, LLC +.parts generic Sea Goodbye, LLC +.party generic Blue Sky Registry Limited +.pe country-code Red Cientifica Peruana +.pf country-code Gouvernement de la Polynésie française +.pg country-code PNG DNS Administration Vice Chancellors Office The Papua New Guinea University of Technology +.ph country-code PH Domain Foundation +.pharmacy generic National Association of Boards of Pharmacy +.photo generic Uniregistry, Corp. +.photography generic Sugar Glen, LLC +.photos generic Sea Corner, LLC +.physio generic PhysBiz Pty Ltd +.pics generic Uniregistry, Corp. +.pictures generic Foggy Sky, LLC +.pink generic Afilias Limited +.pizza generic Foggy Moon, LLC +.pk country-code PKNIC +.pl country-code Research and Academic Computer Network +.place generic Snow Galley, LLC +.plumbing generic Spring Tigers, LLC +.pm country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.pn country-code Pitcairn Island Administration +.pohl generic Deutsche Vermögensberatung Aktiengesellschaft DVAG +.poker generic Afilias Domains No. 5 Limited +.porn generic ICM Registry PN LLC +.post sponsored Universal Postal Union +.pr country-code Gauss Research Laboratory Inc. +.praxi generic Praxi S.p.A. +.press generic DotPress Inc. +.pro generic-restricted Registry Services Corporation dba RegistryPro +.prod generic Charleston Road Registry Inc. +.productions generic Magic Birch, LLC +.prof generic Charleston Road Registry Inc. +.properties generic Big Pass, LLC +.property generic Uniregistry, Corp. +.ps country-code Ministry Of Telecommunications & Information Technology, Government Computer Center. +.pt country-code Associação DNS.PT +.pub generic United TLD Holdco Ltd. +.pw country-code Micronesia Investment and Development Corporation +.py country-code NIC-PY +.qa country-code Communications Regulatory Authority +.qpon generic dotCOOL, Inc. +.quebec generic PointQuébec Inc +.re country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.realtor generic Real Estate Domains LLC +.recipes generic Grand Island, LLC +.red generic Afilias Limited +.rehab generic United TLD Holdco Ltd. +.reise generic dotreise GmbH +.reisen generic New Cypress, LLC +.reit generic National Association of Real Estate Investment Trusts, Inc. +.ren generic Beijing Qianxiang Wangjing Technology Development Co., Ltd. +.rentals generic Big Hollow,LLC +.repair generic Lone Sunset, LLC +.report generic Binky Glen, LLC +.republican generic United TLD Holdco Ltd. +.rest generic Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable +.restaurant generic Snow Avenue, LLC +.reviews generic United TLD Holdco, Ltd. +.rich generic I-REGISTRY Ltd., Niederlassung Deutschland +.rio generic Empresa Municipal de Informática SA - IPLANRIO +.rip generic United TLD Holdco Ltd. +.ro country-code National Institute for R&D in Informatics +.rocks generic United TLD Holdco, LTD. +.rodeo generic Top Level Domain Holdings Limited +.rs country-code Serbian National Internet Domain Registry (RNIDS) +.rsvp generic Charleston Road Registry Inc. +.ru country-code Coordination Center for TLD RU +.ruhr generic regiodot GmbH & Co. KG +.rw country-code Rwanda Information Communication and Technology Association (RICTA) +.ryukyu generic BusinessRalliart inc. +.sa country-code Communications and Information Technology Commission +.saarland generic dotSaarland GmbH +.sale generic United TLD Holdco, Ltd +.samsung generic SAMSUNG SDS CO., LTD +.sarl generic Delta Orchard, LLC +.saxo generic Saxo Bank A/S +.sb country-code Solomon Telekom Company Limited +.sc country-code VCS Pty Ltd +.sca generic SVENSKA CELLULOSA AKTIEBOLAGET SCA (publ) +.scb generic The Siam Commercial Bank Public Company Limited ("SCB") +.schmidt generic SALM S.A.S. +.schule generic Outer Moon, LLC +.schwarz generic Schwarz Domains und Services GmbH & Co. KG +.science generic dot Science Limited +.scot generic Dot Scot Registry Limited +.sd country-code Sudan Internet Society +.se country-code The Internet Infrastructure Foundation +.services generic Fox Castle, LLC +.sew generic SEW-EURODRIVE GmbH & Co KG +.sexy generic Uniregistry, Corp. +.sg country-code Singapore Network Information Centre (SGNIC) Pte Ltd +.sh country-code Government of St. Helena +.shiksha generic Afilias Limited +.shoes generic Binky Galley, LLC +.shriram generic Shriram Capital Ltd. +.si country-code Academic and Research Network of Slovenia (ARNES) +.singles generic Fern Madison, LLC +.sj country-code UNINETT Norid A/S +.sk country-code SK-NIC, a.s. +.sky generic Sky IP International Ltd, a company incorporated in England and Wales, operating via its registered Swiss branch +.sl country-code Sierratel +.sm country-code Telecom Italia San Marino S.p.A. +.sn country-code Universite Cheikh Anta Diop NIC Senegal +.so country-code Ministry of Post and Telecommunications +.social generic United TLD Holdco Ltd. +.software generic United TLD Holdco, Ltd +.sohu generic Sohu.com Limited +.solar generic Ruby Town, LLC +.solutions generic Silver Cover, LLC +.soy generic Charleston Road Registry Inc. +.space generic DotSpace Inc. +.spiegel generic SPIEGEL-Verlag Rudolf Augstein GmbH & Co. KG +.sr country-code Telesur +.ss country-code Not assigned +.st country-code Tecnisys +.style generic Binky Moon, LLC +.su country-code Russian Institute for Development of Public Networks (ROSNIIROS) +.supplies generic Atomic Fields, LLC +.supply generic Half Falls, LLC +.support generic Grand Orchard, LLC +.surf generic Top Level Domain Holdings Limited +.surgery generic Tin Avenue, LLC +.suzuki generic SUZUKI MOTOR CORPORATION +.sv country-code SVNet +.sx country-code SX Registry SA B.V. +.sy country-code National Agency for Network Services (NANS) +.sydney generic State of New South Wales, Department of Premier and Cabinet +.systems generic Dash Cypress, LLC +.sz country-code University of Swaziland Department of Computer Science +.taipei generic Taipei City Government +.tatar generic Limited Liability Company "Coordination Center of Regional Domain of Tatarstan Republic" +.tattoo generic Uniregistry, Corp. +.tax generic Storm Orchard, LLC +.tc country-code Melrex TC +.td country-code Société des télécommunications du Tchad (SOTEL TCHAD) +.technology generic Auburn Falls, LLC +.tel sponsored Telnic Ltd. +.temasek generic Temasek Holdings (Private) Limited +.tennis generic Cotton Bloom, LLC +.tf country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.tg country-code Cafe Informatique et Telecommunications +.th country-code Thai Network Information Center Foundation +.tienda generic Victor Manor, LLC +.tips generic Corn Willow, LLC +.tires generic Dog Edge, LLC +.tirol generic punkt Tirol GmbH +.tj country-code Information Technology Center +.tk country-code Telecommunication Tokelau Corporation (Teletok) +.tl country-code Ministry of Transport and Communications; National Division of Information and Technology +.tm country-code TM Domain Registry Ltd +.tn country-code Agence Tunisienne d'Internet +.to country-code Government of the Kingdom of Tonga H.R.H. Crown Prince Tupouto'a c/o Consulate of Tonga +.today generic Pearl Woods, LLC +.tokyo generic GMO Registry, Inc. +.tools generic Pioneer North, LLC +.top generic Jiangsu Bangning Science & Technology Co.,Ltd. +.toshiba generic TOSHIBA Corporation +.town generic Koko Moon, LLC +.toys generic Pioneer Orchard, LLC +.tp country-code - +.tr country-code Middle East Technical University Department of Computer Engineering +.trade generic Elite Registry Limited +.training generic Wild Willow, LLC +.travel sponsored Tralliance Registry Management Company, LLC. +.trust generic Artemis Internet Inc +.tt country-code University of the West Indies Faculty of Engineering +.tui generic TUI AG +.tv country-code Ministry of Finance and Tourism +.tw country-code Taiwan Network Information Center (TWNIC) +.tz country-code Tanzania Network Information Centre (tzNIC) +.ua country-code Hostmaster Ltd. +.ug country-code Uganda Online Ltd. +.uk country-code Nominet UK +.um country-code Not assigned +.university generic Little Station, LLC +.uno generic Dot Latin LLC +.uol generic UBN INTERNET LTDA. +.us country-code NeuStar, Inc. +.uy country-code SeCIU - Universidad de la Republica +.uz country-code Computerization and Information Technologies Developing Center UZINFOCOM +.va country-code Holy See Secretariat of State Department of Telecommunications +.vacations generic Atomic Tigers, LLC +.vc country-code Ministry of Telecommunications, Science, Technology and Industry +.ve country-code Comisión Nacional de Telecomunicaciones (CONATEL) +.vegas generic Dot Vegas, Inc. +.ventures generic Binky Lake, LLC +.versicherung generic dotversicherung-registry GmbH +.vet generic United TLD Holdco, Ltd +.vg country-code Telecommunications Regulatory Commission of the Virgin Islands +.vi country-code Virgin Islands Public Telcommunications System c/o COBEX Internet Services +.viajes generic Black Madison, LLC +.video generic United TLD Holdco, Ltd +.villas generic New Sky, LLC +.vision generic Koko Station, LLC +.vlaanderen generic DNS.be vzw +.vn country-code Ministry of Information and Communications of Socialist Republic of Viet Nam +.vodka generic Top Level Domain Holdings Limited +.vote generic Monolith Registry LLC +.voting generic Valuetainment Corp. +.voto generic Monolith Registry LLC +.voyage generic Ruby House, LLC +.vu country-code Telecom Vanuatu Limited +.wales generic Nominet UK +.wang generic Zodiac Registry Limited +.watch generic Sand Shadow, LLC +.webcam generic dot Webcam Limited +.website generic DotWebsite Inc. +.wed generic Atgron, Inc. +.wedding generic Top Level Domain Holdings Limited +.wf country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.whoswho generic Who's Who Registry +.wien generic punkt.wien GmbH +.wiki generic Top Level Design, LLC +.williamhill generic William Hill Organization Limited +.wme generic William Morris Endeavor Entertainment, LLC +.work generic Top Level Domain Holdings Limited +.works generic Little Dynamite, LLC +.world generic Bitter Fields, LLC +.ws country-code Government of Samoa Ministry of Foreign Affairs & Trade +.wtc generic World Trade Centers Association, Inc. +.wtf generic Hidden Way, LLC +.测试 test Internet Assigned Numbers Authority +.परीक्षा test Internet Assigned Numbers Authority +.佛山 generic Guangzhou YU Wei Information Technology Co., Ltd. +.集团 generic Eagle Horizon Limited +.在线 generic TLD REGISTRY LIMITED +.한국 country-code KISA (Korea Internet & Security Agency) +.ভারত country-code National Internet Exchange of India +.八卦 generic Zodiac Scorpio Limited +.موقع generic Suhub Electronic Establishment +.বাংলা country-code Not assigned +.公益 generic China Organizational Name Administration Center +.公司 generic Computer Network Information Center of Chinese Academy of Sciences （China Internet Network Information Center） +.移动 generic Afilias Limited +.我爱你 generic Tycoon Treasure Limited +.москва generic Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID) +.испытание test Internet Assigned Numbers Authority +.қаз country-code Association of IT Companies of Kazakhstan +.онлайн generic CORE Association +.сайт generic CORE Association +.срб country-code Serbian National Internet Domain Registry (RNIDS) +.бел country-code Reliable Software Inc. +.테스트 test Internet Assigned Numbers Authority +.淡马锡 generic Temasek Holdings (Private) Limited +.орг generic Public Interest Registry +.삼성 generic SAMSUNG SDS CO., LTD +.சிங்கப்பூர் country-code Singapore Network Information Centre (SGNIC) Pte Ltd +.商标 generic HU YI GLOBAL INFORMATION RESOURCES(HOLDING) COMPANY.HONGKONG LIMITED +.商店 generic Wild Island, LLC +.商城 generic Zodiac Aquarius Limited +.дети generic The Foundation for Network Initiatives “The Smart Internet” +.мкд country-code Macedonian Academic Research Network Skopje +.טעסט test Internet Assigned Numbers Authority +.中文网 generic TLD REGISTRY LIMITED +.中信 generic CITIC Group Corporation +.中国 country-code China Internet Network Information Center +.中國 country-code China Internet Network Information Center +.谷歌 generic Charleston Road Registry Inc. +.భారత్ country-code National Internet Exchange of India +.ලංකා country-code LK Domain Registry +.測試 test Internet Assigned Numbers Authority +.ભારત country-code National Internet Exchange of India +.भारत country-code National Internet Exchange of India +.آزمایشی test Internet Assigned Numbers Authority +.பரிட்சை test Internet Assigned Numbers Authority +.网店 generic Zodiac Libra Limited +.संगठन generic Public Interest Registry +.网络 generic Computer Network Information Center of Chinese Academy of Sciences （China Internet Network Information Center） +.укр country-code Ukrainian Network Information Centre (UANIC), Inc. +.香港 country-code Hong Kong Internet Registration Corporation Ltd. +.δοκιμή test Internet Assigned Numbers Authority +.إختبار test Internet Assigned Numbers Authority +.台湾 country-code Taiwan Network Information Center (TWNIC) +.台灣 country-code Taiwan Network Information Center (TWNIC) +.手机 generic Beijing RITT-Net Technology Development Co., Ltd +.мон country-code Datacom Co.,Ltd +.الجزائر country-code CERIST +.عمان country-code Telecommunications Regulatory Authority (TRA) +.ایران country-code Institute for Research in Fundamental Sciences (IPM) +.امارات country-code Telecommunications Regulatory Authority (TRA) +.بازار generic CORE Association +.پاکستان country-code Not assigned +.الاردن country-code National Information Technology Center (NITC) +.بھارت country-code National Internet Exchange of India +.المغرب country-code Agence Nationale de Réglementation des Télécommunications (ANRT) +.السعودية country-code Communications and Information Technology Commission +.سودان country-code Not assigned +.عراق country-code Not assigned +.مليسيا country-code MYNIC Berhad +.شبكة generic International Domain Registry Pty. Ltd. +.გე country-code Information Technologies Development Center (ITDC) +.机构 generic Public Interest Registry +.组织机构 generic Public Interest Registry +.ไทย country-code Thai Network Information Center Foundation +.سورية country-code National Agency for Network Services (NANS) +.рус generic Rusnames Limited +.рф country-code Coordination Center for TLD RU +.تونس country-code Agence Tunisienne d'Internet +.みんな generic Charleston Road Registry Inc. +.グーグル generic Charleston Road Registry Inc. +.世界 generic Stable Tone Limited +.ਭਾਰਤ country-code National Internet Exchange of India +.网址 generic KNET Co., Ltd +.游戏 generic Spring Fields, LLC +.vermögensberater generic Deutsche Vermögensberatung Aktiengesellschaft DVAG +.vermögensberatung generic Deutsche Vermögensberatung Aktiengesellschaft DVAG +.企业 generic Dash McCook, LLC +.مصر country-code National Telecommunication Regulatory Authority - NTRA +.قطر country-code Communications Regulatory Authority +.广东 generic Guangzhou YU Wei Information Technology Co., Ltd. +.இலங்கை country-code LK Domain Registry +.இந்தியா country-code National Internet Exchange of India +.հայ country-code Not assigned +.新加坡 country-code Singapore Network Information Centre (SGNIC) Pte Ltd +.فلسطين country-code Ministry of Telecom & Information Technology (MTIT) +.テスト test Internet Assigned Numbers Authority +.政务 generic China Organizational Name Administration Center +.xxx sponsored ICM Registry LLC +.xyz generic XYZ.COM LLC +.yachts generic DERYachts, LLC +.yandex generic YANDEX, LLC +.ye country-code TeleYemen +.yodobashi generic YODOBASHI CAMERA CO.,LTD. +.yoga generic Top Level Domain Holdings Limited +.yokohama generic GMO Registry, Inc. +.youtube generic Charleston Road Registry Inc. +.yt country-code Association Française pour le Nommage Internet en Coopération (A.F.N.I.C.) +.za country-code ZA Domain Name Authority +.zip generic Charleston Road Registry Inc. +.zm country-code Zambia Information and Communications Technology Authority (ZICTA) +.zone generic Outer Falls, LLC +.zuerich generic Kanton Zürich (Canton of Zurich) +.zw country-code Postal and Telecommunications Regulatory Authority of Zimbabwe (POTRAZ) \ No newline at end of file diff --git a/control/coroaverager3.py b/control/coroaverager3.py new file mode 100644 index 0000000..f2b419b --- /dev/null +++ b/control/coroaverager3.py @@ -0,0 +1,107 @@ +""" +A coroutine to compute a running average. + +Testing ``averager`` by itself:: + + >>> coro_avg = averager() + >>> next(coro_avg) + >>> coro_avg.send(10) + >>> coro_avg.send(30) + >>> coro_avg.send(6.5) + >>> coro_avg.send(None) + Traceback (most recent call last): + ... + StopIteration: Result(count=3, average=15.5) + + +Driving it with ``yield from``:: + + >>> def summarize(results): + ... while True: + ... result = yield from averager() + ... results.append(result) + ... + >>> results = [] + >>> summary = summarize(results) + >>> next(summary) + >>> for height in data['girls;m']: + ... summary.send(height) + ... + >>> summary.send(None) + >>> for height in data['boys;m']: + ... summary.send(height) + ... + >>> summary.send(None) + >>> results == [ + ... Result(count=10, average=1.4279999999999997), + ... Result(count=9, average=1.3888888888888888) + ... ] + True + +""" + +# BEGIN YIELD_FROM_AVERAGER +from collections import namedtuple + +Result = namedtuple('Result', 'count average') + + +# the subgenerator +def averager(): # <1> + total = 0.0 + count = 0 + average = None + while True: + term = yield # <2> + if term is None: # <3> + break + total += term + count += 1 + average = total/count + return Result(count, average) # <4> + + +# the delegating generator +def grouper(results, key): # <5> + while True: # <6> + results[key] = yield from averager() # <7> + + +# the client code, a.k.a. the caller +def main(data): # <8> + results = {} + for key, values in data.items(): + group = grouper(results, key) # <9> + next(group) # <10> + for value in values: + group.send(value) # <11> + group.send(None) # <12> + + # print(results) # uncomment to debug + report(results) + + +# output report +def report(results): + for key, result in sorted(results.items()): + group, unit = key.split(';') + print('{:2} {:5} averaging {:.2f}{}'.format( + result.count, group, result.average, unit)) + + +data = { + 'girls;kg': + [40.9, 38.5, 44.3, 42.2, 45.2, 41.7, 44.5, 38.0, 40.6, 44.5], + 'girls;m': + [1.6, 1.51, 1.4, 1.3, 1.41, 1.39, 1.33, 1.46, 1.45, 1.43], + 'boys;kg': + [39.0, 40.8, 43.2, 40.8, 43.1, 38.6, 41.4, 40.6, 36.3], + 'boys;m': + [1.38, 1.5, 1.32, 1.25, 1.37, 1.48, 1.25, 1.49, 1.46], +} + + +if __name__ == '__main__': + main(data) + +# END YIELD_FROM_AVERAGER diff --git a/control/kwcombos.py b/control/kwcombos.py new file mode 100644 index 0000000..8dffe33 --- /dev/null +++ b/control/kwcombos.py @@ -0,0 +1,5 @@ +from keyword import kwlist +from itertools import combinations + +for combo in combinations(kwlist, 2): + print(*combo) diff --git a/control/simulation/taxi_sim.py b/control/taxi_sim.py similarity index 71% rename from control/simulation/taxi_sim.py rename to control/taxi_sim.py index f58cf89..4078fdd 100644 --- a/control/simulation/taxi_sim.py +++ b/control/taxi_sim.py @@ -2,25 +2,25 @@ """ Taxi simulator -Sample run with two cars, random seed = 4. This is a valid doctest. +Sample run with two cars, random seed 10. This is a valid doctest. >>> main(num_taxis=2, seed=10) taxi: 0 Event(time=0, proc=0, action='leave garage') taxi: 0 Event(time=4, proc=0, action='pick up passenger') + taxi: 1 Event(time=5, proc=1, action='leave garage') + taxi: 1 Event(time=9, proc=1, action='pick up passenger') taxi: 0 Event(time=10, proc=0, action='drop off passenger') - taxi: 1 Event(time=10, proc=1, action='leave garage') - taxi: 1 Event(time=11, proc=1, action='pick up passenger') - taxi: 0 Event(time=14, proc=0, action='pick up passenger') + taxi: 1 Event(time=12, proc=1, action='drop off passenger') + taxi: 0 Event(time=17, proc=0, action='pick up passenger') + taxi: 1 Event(time=19, proc=1, action='pick up passenger') + taxi: 1 Event(time=21, proc=1, action='drop off passenger') + taxi: 1 Event(time=24, proc=1, action='pick up passenger') + taxi: 0 Event(time=28, proc=0, action='drop off passenger') taxi: 1 Event(time=28, proc=1, action='drop off passenger') - taxi: 0 Event(time=32, proc=0, action='drop off passenger') - taxi: 0 Event(time=33, proc=0, action='going home') - taxi: 1 Event(time=33, proc=1, action='pick up passenger') - taxi: 1 Event(time=35, proc=1, action='drop off passenger') - taxi: 1 Event(time=38, proc=1, action='pick up passenger') - taxi: 1 Event(time=42, proc=1, action='drop off passenger') - taxi: 1 Event(time=44, proc=1, action='pick up passenger') - taxi: 1 Event(time=75, proc=1, action='drop off passenger') - taxi: 1 Event(time=76, proc=1, action='going home') + taxi: 0 Event(time=29, proc=0, action='going home') + taxi: 1 Event(time=30, proc=1, action='pick up passenger') + taxi: 1 Event(time=61, proc=1, action='drop off passenger') + taxi: 1 Event(time=62, proc=1, action='going home') *** end of events *** See explanation and longer sample run at the end of this module. @@ -35,8 +35,9 @@ import argparse DEFAULT_NUMBER_OF_TAXIS = 3 DEFAULT_END_TIME = 80 -SEARCH_INTERVAL = 4 +SEARCH_DURATION = 4 TRIP_DURATION = 10 +DEPARTURE_INTERVAL = 5 Event = collections.namedtuple('Event', 'time proc action') @@ -50,7 +51,7 @@ def taxi_process(ident, trips, start_time=0): # <1> """Yield to simulator issuing event at each state change""" time = yield Event(start_time, ident, 'leave garage') # <2> for i in range(trips): # <3> - prowling_ends = time + compute_delay(SEARCH_INTERVAL) # <4> + prowling_ends = time + compute_delay(SEARCH_DURATION) # <4> time = yield Event(prowling_ends, ident, 'pick up passenger') # <5> trip_ends = time + compute_delay(TRIP_DURATION) # <6> @@ -107,7 +108,7 @@ def main(end_time=DEFAULT_END_TIME, num_taxis=DEFAULT_NUMBER_OF_TAXIS, if seed is not None: random.seed(seed) # get reproducible results - taxis = {i: taxi_process(i, (i+1)*2, i*10) + taxis = {i: taxi_process(i, (i+1)*2, i*DEPARTURE_INTERVAL) for i in range(num_taxis)} sim = Simulator(taxis) sim.run(end_time) @@ -216,40 +217,40 @@ Notes for the ``Simulator.run`` method:: sometimes). -Sample run from the command line:: +Sample run from the command line, seed=24, total elapsed time=160:: # BEGIN TAXI_SAMPLE_RUN -$ clear; python3 taxi_sim.py -t 3 -s 19 +$ python3 taxi_sim.py -s 24 -e 160 taxi: 0 Event(time=0, proc=0, action='leave garage') taxi: 0 Event(time=5, proc=0, action='pick up passenger') -taxi: 1 Event(time=10, proc=1, action='leave garage') -taxi: 1 Event(time=13, proc=1, action='pick up passenger') -taxi: 2 Event(time=20, proc=2, action='leave garage') -taxi: 0 Event(time=21, proc=0, action='drop off passenger') -taxi: 1 Event(time=21, proc=1, action='drop off passenger') -taxi: 1 Event(time=23, proc=1, action='pick up passenger') -taxi: 2 Event(time=23, proc=2, action='pick up passenger') -taxi: 1 Event(time=25, proc=1, action='drop off passenger') -taxi: 1 Event(time=27, proc=1, action='pick up passenger') -taxi: 2 Event(time=27, proc=2, action='drop off passenger') -taxi: 2 Event(time=29, proc=2, action='pick up passenger') -taxi: 1 Event(time=31, proc=1, action='drop off passenger') -taxi: 2 Event(time=31, proc=2, action='drop off passenger') -taxi: 1 Event(time=33, proc=1, action='pick up passenger') -taxi: 2 Event(time=33, proc=2, action='pick up passenger') -taxi: 2 Event(time=36, proc=2, action='drop off passenger') -taxi: 2 Event(time=37, proc=2, action='pick up passenger') -taxi: 2 Event(time=40, proc=2, action='drop off passenger') -taxi: 1 Event(time=42, proc=1, action='drop off passenger') -taxi: 1 Event(time=43, proc=1, action='going home') -taxi: 0 Event(time=44, proc=0, action='pick up passenger') -taxi: 2 Event(time=44, proc=2, action='pick up passenger') -taxi: 0 Event(time=49, proc=0, action='drop off passenger') -taxi: 0 Event(time=50, proc=0, action='going home') -taxi: 2 Event(time=58, proc=2, action='drop off passenger') -taxi: 2 Event(time=65, proc=2, action='pick up passenger') -taxi: 2 Event(time=71, proc=2, action='drop off passenger') -taxi: 2 Event(time=72, proc=2, action='going home') +taxi: 1 Event(time=5, proc=1, action='leave garage') +taxi: 1 Event(time=6, proc=1, action='pick up passenger') +taxi: 2 Event(time=10, proc=2, action='leave garage') +taxi: 2 Event(time=11, proc=2, action='pick up passenger') +taxi: 2 Event(time=23, proc=2, action='drop off passenger') +taxi: 0 Event(time=24, proc=0, action='drop off passenger') +taxi: 2 Event(time=24, proc=2, action='pick up passenger') +taxi: 2 Event(time=26, proc=2, action='drop off passenger') +taxi: 0 Event(time=30, proc=0, action='pick up passenger') +taxi: 2 Event(time=31, proc=2, action='pick up passenger') +taxi: 0 Event(time=43, proc=0, action='drop off passenger') +taxi: 0 Event(time=44, proc=0, action='going home') +taxi: 2 Event(time=46, proc=2, action='drop off passenger') +taxi: 2 Event(time=49, proc=2, action='pick up passenger') +taxi: 1 Event(time=70, proc=1, action='drop off passenger') +taxi: 2 Event(time=70, proc=2, action='drop off passenger') +taxi: 2 Event(time=71, proc=2, action='pick up passenger') +taxi: 2 Event(time=79, proc=2, action='drop off passenger') +taxi: 1 Event(time=88, proc=1, action='pick up passenger') +taxi: 2 Event(time=92, proc=2, action='pick up passenger') +taxi: 2 Event(time=98, proc=2, action='drop off passenger') +taxi: 2 Event(time=99, proc=2, action='going home') +taxi: 1 Event(time=102, proc=1, action='drop off passenger') +taxi: 1 Event(time=104, proc=1, action='pick up passenger') +taxi: 1 Event(time=135, proc=1, action='drop off passenger') +taxi: 1 Event(time=136, proc=1, action='pick up passenger') +taxi: 1 Event(time=151, proc=1, action='drop off passenger') +taxi: 1 Event(time=152, proc=1, action='going home') *** end of events *** # END TAXI_SAMPLE_RUN diff --git a/control/yield_from_expansion.py b/control/yield_from_expansion.py new file mode 100644 index 0000000..2c3ba5e --- /dev/null +++ b/control/yield_from_expansion.py @@ -0,0 +1,52 @@ +# Code below is the expansion of the statement: +# +# RESULT = yield from EXPR +# +# Copied verbatim from the Formal Semantics section of +# PEP 380 -- Syntax for Delegating to a Subgenerator +# +# https://www.python.org/dev/peps/pep-0380/#formal-semantics + + +# BEGIN YIELD_FROM_EXPANSION +_i = iter(EXPR) # <1> +try: + _y = next(_i) # <2> +except StopIteration as _e: + _r = _e.value # <3> +else: + while 1: # <4> + try: + _s = yield _y # <5> + except GeneratorExit as _e: # <6> + try: + _m = _i.close + except AttributeError: + pass + else: + _m() + raise _e + except BaseException as _e: # <7> + _x = sys.exc_info() + try: + _m = _i.throw + except AttributeError: + raise _e + else: # <8> + try: + _y = _m(*_x) + except StopIteration as _e: + _r = _e.value + break + else: # <9> + try: # <10> + if _s is None: # <11> + _y = next(_i) + else: + _y = _i.send(_s) + except StopIteration as _e: # <12> + _r = _e.value + break + +RESULT = _r # <13> +# END YIELD_FROM_EXPANSION diff --git a/futures/countries/.gitignore b/futures/countries/.gitignore new file mode 100644 index 0000000..8484300 --- /dev/null +++ b/futures/countries/.gitignore @@ -0,0 +1 @@ +flags/ \ No newline at end of file diff --git a/futures/countries/country_codes.txt b/futures/countries/country_codes.txt new file mode 100644 index 0000000..72c37f0 --- /dev/null +++ b/futures/countries/country_codes.txt @@ -0,0 +1,8 @@ +AD AE AF AG AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BI BJ BN BO BR BS BT +BW BY BZ CA CD CF CG CH CI CL CM CN CO CR CU CV CY CZ DE DJ DK DM DZ EC EE +EG ER ES ET FI FJ FM FR GA GB GD GE GH GM GN GQ GR GT GW GY HN HR HT HU ID +IE IL IN IQ IR IS IT JM JO JP KE KG KH KI KM KN KP KR KW KZ LA LB LC LI LK +LR LS LT LU LV LY MA MC MD ME MG MH MK ML MM MN MR MT MU MV MW MX MY MZ NA +NE NG NI NL NO NP NR NZ OM PA PE PG PH PK PL PT PW PY QA RO RS RU RW SA SB +SC SD SE SG SI SK SL SM SN SO SR SS ST SV SY SZ TD TG TH TJ TL TM TN TO TR +TT TV TW TZ UA UG US UY UZ VA VC VE VN VU WS YE ZA ZM ZW diff --git a/futures/countries/flag_utils.py b/futures/countries/flag_utils.py new file mode 100644 index 0000000..c16a72e --- /dev/null +++ b/futures/countries/flag_utils.py @@ -0,0 +1,148 @@ +"""Utilities for second set of flag examples. +""" + +import os +import time +import sys +import string +import argparse +from collections import namedtuple +from enum import Enum + + +Result = namedtuple('Result', 'status data') +Counts = namedtuple('Counts', 'ok not_found error') + +Status = Enum('Status', 'ok not_found error') + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() + +DEFAULT_CONCUR_REQ = 1 +MAX_CONCUR_REQ = 1 + +SERVERS = { + 'REMOTE': 'http://python.pro.br/fluent/data/flags', + 'LOCAL': 'http://localhost:8001/flags', + 'DELAY': 'http://localhost:8002/flags', + 'ERROR': 'http://localhost:8003/flags', +} +DEFAULT_SERVER = 'LOCAL' + +DEST_DIR = 'downloads/' +COUNTRY_CODES_FILE = 'country_codes.txt' + + +def save_flag(img, filename): + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +def initial_report(cc_list, actual_req, server_label): + if len(cc_list) <= 10: + cc_msg = ', '.join(cc_list) + else: + cc_msg = 'from {} to {}'.format(cc_list[0], cc_list[-1]) + print('{} site: {}'.format(server_label, SERVERS[server_label])) + msg = 'Searching for {} flag{}: {}' + plural = 's' if len(cc_list) != 1 else '' + print(msg.format(len(cc_list), plural, cc_msg)) + plural = 's' if actual_req != 1 else '' + msg = '{} concurrent connection{} will be used.' + print(msg.format(actual_req, plural)) + + +def final_report(cc_list, counts, start_time): + elapsed = time.time() - start_time + print('-' * 20) + msg = '{} flag{} downloaded.' + plural = 's' if counts.ok != 1 else '' + print(msg.format(counts.ok, plural)) + if counts.not_found: + print(counts.not_found, 'not found.') + if counts.error: + plural = 's' if counts.error != 1 else '' + print('{} error{}.'.format(counts.error, plural)) + print('Elapsed time: {:.2f}s'.format(elapsed)) + + +def expand_cc_args(every_cc, all_cc, cc_args, limit): + codes = set() + A_Z = string.ascii_uppercase + if every_cc: + codes.update(a+b for a in A_Z for b in A_Z) + elif all_cc: + with open(COUNTRY_CODES_FILE) as fp: + text = fp.read() + codes.update(text.split()) + else: + for cc in (c.upper() for c in cc_args): + if len(cc) == 1 and cc in A_Z: + codes.update(cc+c for c in A_Z) + elif len(cc) == 2 and all(c in A_Z for c in cc): + codes.add(cc) + else: + msg = 'each CC argument must be A to Z or AA to ZZ.' + raise ValueError('*** Usage error: '+msg) + return sorted(codes)[:limit] + + +def process_args(default_concur_req): + server_options = ', '.join(sorted(SERVERS)) + parser = argparse.ArgumentParser( + description='Download flags for country codes. ' + 'Default: top 20 countries by population.') + parser.add_argument('cc', metavar='CC', nargs='*', + help='country code or 1st letter (eg. B for BA...BZ)') + parser.add_argument('-a', '--all', action='store_true', + help='get all available flags (AD to ZW)') + parser.add_argument('-e', '--every', action='store_true', + help='get flags for every possible code (AA...ZZ)') + parser.add_argument('-l', '--limit', metavar='N', type=int, + help='limit to N first codes', default=sys.maxsize) + parser.add_argument('-m', '--max_req', metavar='CONCURRENT', type=int, + default=default_concur_req, + help='maximum concurrent requests (default={})' + .format(default_concur_req)) + parser.add_argument('-s', '--server', metavar='LABEL', + default=DEFAULT_SERVER, + help='Server to hit; one of {} (default={})' + .format(server_options, DEFAULT_SERVER)) + parser.add_argument('-v', '--verbose', action='store_true', + help='output detailed progress info') + args = parser.parse_args() + if args.max_req < 1: + print('*** Usage error: --max_req CONCURRENT must be >= 1') + parser.print_usage() + sys.exit(1) + if args.limit < 1: + print('*** Usage error: --limit N must be >= 1') + parser.print_usage() + sys.exit(1) + args.server = args.server.upper() + if args.server not in SERVERS: + print('*** Usage error: --server LABEL must be one of', + server_options) + parser.print_usage() + sys.exit(1) + try: + cc_list = expand_cc_args(args.every, args.all, args.cc, args.limit) + except ValueError as exc: + print(exc.args[0]) + parser.print_usage() + sys.exit(1) + + if not cc_list: + cc_list = sorted(POP20_CC) + return args, cc_list + + +def main(download_many, default_concur_req, max_concur_req): + args, cc_list = process_args(default_concur_req) + actual_req = min(args.max_req, max_concur_req, len(cc_list)) + initial_report(cc_list, actual_req, args.server) + base_url = SERVERS[args.server] + t0 = time.time() + counts = download_many(cc_list, base_url, args.verbose, actual_req) + final_report(cc_list, counts, t0) diff --git a/futures/countries/flags.py b/futures/countries/flags.py new file mode 100644 index 0000000..75ea66b --- /dev/null +++ b/futures/countries/flags.py @@ -0,0 +1,62 @@ +"""Download flags of top 20 countries by population + +Sequential version + +Sample run:: + + $ python3 flags.py + BD retrieved. + BR retrieved. + CD retrieved. + ... + TR retrieved. + US retrieved. + VN retrieved. + 20 flags downloaded in 10.16s + +""" + +import os +import time + +import requests + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() + +BASE_URL = 'http://python.pro.br/fluent/data/flags' + +DEST_DIR = 'downloads/' + + +def save_flag(img, filename): + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +def get_flag(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + res = requests.get(url) + return res.content + + +def download_many(cc_list): + for cc in sorted(cc_list): + image = get_flag(cc) + print('{} retrieved.'.format(cc)) + save_flag(image, cc.lower() + '.gif') + + return len(cc_list) + + +def main(download_many): + t0 = time.time() + count = download_many(POP20_CC) + elapsed = time.time() - t0 + msg = '{} flags downloaded in {:.2f}s' + print(msg.format(count, elapsed)) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags.zip b/futures/countries/flags.zip new file mode 100644 index 0000000..671b5a0 Binary files /dev/null and b/futures/countries/flags.zip differ diff --git a/futures/countries/flags2_asyncio.py b/futures/countries/flags2_asyncio.py new file mode 100644 index 0000000..98236a1 --- /dev/null +++ b/futures/countries/flags2_asyncio.py @@ -0,0 +1,112 @@ +"""Download flags of top 10 countries by population + +asyncio version + +Sample run:: + + $ + +""" + +import asyncio +from collections import namedtuple +from enum import Enum + +import aiohttp +from aiohttp import web +import tqdm + +from flag_utils import main, save_flag, Counts + +# default set low to avoid errors from remote site: +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + +TIMEOUT = 120 # seconds + +Status = Enum('Status', 'ok not_found error') +Result = namedtuple('Result', 'status data') + + +@asyncio.coroutine +def get_flag(base_url, cc): + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + res = yield from aiohttp.request('GET', url) + if res.status == 200: + image = yield from res.read() + return image + elif res.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.errors.HttpProcessingError( + code=res.status, message=res.reason, headers=res.headers) + + +@asyncio.coroutine +def download_one(cc, base_url, semaphore, verbose): + try: + with (yield from semaphore): + image = yield from get_flag(base_url, cc) + except web.HTTPNotFound: + status = Status.not_found + msg = '' + except aiohttp.errors.HttpProcessingError as exc: + status = Status.error + msg = '{} failed: {exc.code} - {exc.message}' + msg = msg.format(cc, exc=exc) + except aiohttp.errors.ClientError as exc: + try: + context = exc.__context__.__class__.__name__ + except AttributeError: + # we chain all exceptions, you should get original exception from __cause__ + context = '(unknown context)' + msg = '{} failed: {}'.format(cc, context) + status = Status.error + else: + save_flag(image, cc.lower() + '.gif') + status = Status.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) + + +@asyncio.coroutine +def downloader_coro(cc_list, base_url, verbose, max_req): + semaphore = asyncio.Semaphore(max_req) + to_do = [download_one(cc, base_url, semaphore, verbose) for cc in cc_list] + results = [] + to_do_iter = asyncio.as_completed(to_do) + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) + for future in to_do_iter: + result = yield from future + results.append(result) + return results + + +def download_many(cc_list, base_url, verbose, max_req): + loop = asyncio.get_event_loop() + #loop.set_debug(True) + try: + coro = downloader_coro(cc_list, base_url, verbose, max_req) + done = loop.run_until_complete(coro) + except Exception as exc: + print('*' * 60) + print(exc) + print(vars(exc)) + print('*' * 60) + counts = [] + for status in Status: + counts.append(len([res for res in done + if res.status == status])) + loop.close() + + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) diff --git a/futures/countries/flags2_asyncio_NOTQDM.py b/futures/countries/flags2_asyncio_NOTQDM.py new file mode 100644 index 0000000..0e9d175 --- /dev/null +++ b/futures/countries/flags2_asyncio_NOTQDM.py @@ -0,0 +1,100 @@ +"""Download flags of top 10 countries by population + +asyncio version + +Sample run:: + + $ + +""" + +import asyncio +from collections import namedtuple +from enum import Enum + +import aiohttp +from aiohttp import web + +from flag_utils import main, save_flag, Counts + +# default set low to avoid errors from remote site: +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + +TIMEOUT = 120 # seconds + +Status = Enum('Status', 'ok not_found error') +Result = namedtuple('Result', 'status data') + + +@asyncio.coroutine +def get_flag(base_url, cc): + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + res = yield from aiohttp.request('GET', url) + if res.status == 200: + image = yield from res.read() + return image + elif res.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.errors.HttpProcessingError( + code=res.status, message=res.reason, headers=res.headers) + + +@asyncio.coroutine +def download_one(cc, base_url, semaphore, verbose): + try: + with (yield from semaphore): + image = yield from get_flag(base_url, cc) + except web.HTTPNotFound: + status = Status.not_found + msg = '' + except aiohttp.errors.HttpProcessingError as exc: + status = Status.error + msg = '{} failed: {exc.code} - {exc.message}' + msg = msg.format(cc, exc=exc) + except aiohttp.errors.ClientError as exc: + try: + context = exc.__context__.__class__.__name__ + except AttributeError: + # we chain all exceptions, you should get original exception from __cause__ + context = '(unknown context)' + msg = '{} failed: {}'.format(cc, context) + status = Status.error + else: + save_flag(image, cc.lower() + '.gif') + status = Status.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) + + +def download_many(cc_list, base_url, verbose, max_req): + semaphore = asyncio.Semaphore(max_req) + to_do = [download_one(cc, base_url, semaphore, verbose) for cc in cc_list] + loop = asyncio.get_event_loop() + #loop.set_debug(True) + try: + done, pending = loop.run_until_complete(asyncio.wait(to_do, timeout=TIMEOUT)) + except Exception as exc: + print('*' * 60) + print(exc) + print(vars(exc)) + print('*' * 60) + counts = [] + for status in Status: + counts.append(len([task for task in done + if task.result().status == status])) + for task in pending: + task.cancel() + loop.close() + + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) diff --git a/futures/countries/flags2_sequential.py b/futures/countries/flags2_sequential.py new file mode 100644 index 0000000..e31bed8 --- /dev/null +++ b/futures/countries/flags2_sequential.py @@ -0,0 +1,74 @@ +"""Download flags of countries (with error handling). + +Sequential version + +Sample run:: + + $ + +""" + +import requests +import tqdm + +from flag_utils import main, save_flag, Counts, Status, Result + + +DEFAULT_CONCUR_REQ = 1 +MAX_CONCUR_REQ = 1 + + +def get_flag(base_url, cc): + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + res = requests.get(url) + if res.status_code != 200: + res.raise_for_status() + return res.content + + +def download_one(cc, base_url, verbose=False): + try: + image = get_flag(base_url, cc) + except requests.exceptions.HTTPError as exc: + res = exc.response + if res.status_code == 404: + status = Status.not_found + msg = '' + else: + status = Status.error + msg = 'error {res.status_code} - {res.reason}' + msg = msg.format(res=exc.response) + except requests.exceptions.ConnectionError as exc: + status = Status.error + msg = 'failed: {}'.format(cc, exc.args) + else: + save_flag(image, cc.lower() + '.gif') + status = Status.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) + + +def download_many(cc_list, base_url, verbose, max_req): + counts = [0, 0, 0] + if not verbose: + cc_iter = tqdm.tqdm(sorted(cc_list)) + else: + cc_iter = sorted(cc_list) + for cc in cc_iter: + try: + res = download_one(cc, base_url, verbose) + except Exception as exc: + msg = 'Unexpected exception for {}: {!r}' + print(msg.format(cc, exc)) + else: + counts[res.status.value-1] += 1 + + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) diff --git a/futures/countries/flags2_threadpool.py b/futures/countries/flags2_threadpool.py new file mode 100644 index 0000000..41927e0 --- /dev/null +++ b/futures/countries/flags2_threadpool.py @@ -0,0 +1,42 @@ +"""Download flags of top 10 countries by population + +ThreadPool version + +Sample run:: + + $ + +""" + +from concurrent import futures + +import tqdm + +from flag_utils import main, Counts +from flags2_sequential import get_flag, download_one + +DEFAULT_CONCUR_REQ = 30 +MAX_CONCUR_REQ = 1000 + + +def download_many(cc_list, base_url, verbose, concur_req): + with futures.ThreadPoolExecutor(concur_req) as executor: + to_do = [executor.submit(download_one, cc, base_url, verbose) + for cc in sorted(cc_list)] + counts = [0, 0, 0] + to_do_iter = futures.as_completed(to_do) + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) + for future in to_do_iter: + try: + res = future.result() + except Exception as exc: + print('*** Unexpected exception:', exc) + else: + counts[res.status.value-1] += 1 + + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) diff --git a/futures/countries/flags3_asyncio.py b/futures/countries/flags3_asyncio.py new file mode 100644 index 0000000..e318cf5 --- /dev/null +++ b/futures/countries/flags3_asyncio.py @@ -0,0 +1,122 @@ +"""Download flags of top 10 countries by population + +asyncio version + +Sample run:: + + $ python3 pop10_asyncio1.py + CN retrieved. + US retrieved. + BR retrieved. + NG retrieved. + PK retrieved. + RU retrieved. + ID retrieved. + IN retrieved. + BD retrieved. + JP retrieved. + 10 flags downloaded in 0.45s + +""" + +import asyncio +from collections import namedtuple +from enum import Enum + +import aiohttp +from aiohttp import web + +from flags_sequential2 import BASE_URL +from flags_sequential2 import save_flag, main, Counts + +MAX_TASKS = 100 if 'localhost' in BASE_URL else 5 +TIMEOUT = 120 # seconds + +Status = Enum('Status', 'ok not_found error') +Result = namedtuple('Result', 'status data') + + +@asyncio.coroutine +def http_get(url): + res = yield from aiohttp.request('GET', url) + if res.status == 200: + ctype = res.headers.get('Content-type', '').lower() + + if 'json' in ctype or url.endswith('json'): + data = yield from res.json() + else: + data = yield from res.read() + return data + elif res.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.errors.HttpProcessingError( + code=res.status, message=res.reason, headers=res.headers) + + +@asyncio.coroutine +def get_flag(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + return (yield from http_get(url)) + + +@asyncio.coroutine +def get_country(cc): + url = '{}/{cc}/metadata.json'.format(BASE_URL, cc=cc.lower()) + metadata = yield from http_get(url) + return metadata['country'] + +@asyncio.coroutine +def download_one(cc, semaphore): + try: + with (yield from semaphore): + image = yield from get_flag(cc) + with (yield from semaphore): + country = yield from get_country(cc) + except web.HTTPNotFound: + status = Status.not_found + except aiohttp.errors.HttpProcessingError as exc: + msg = '{} failed: {exc.code} - {exc.message}' + print(msg.format(cc, exc=exc)) + status = Status.error + except aiohttp.errors.ClientResponseError as exc: + try: + context = exc.__context__.__class__.__name__ + except AttributeError: + context = '(unknown context)' + msg = '{} failed: {}' + print(msg.format(cc, context)) + status = Status.error + else: + print('{} retrieved.'.format(cc.upper())) + country = country.replace(' ', '_') + save_flag(image, '{}-{}.gif'.format(country, cc)) + status = Status.ok + return Result(status, cc) + + +def download_many(cc_list): + semaphore = asyncio.Semaphore(MAX_TASKS) + to_do = [download_one(cc, semaphore) for cc in cc_list] + loop = asyncio.get_event_loop() + #loop.set_debug(True) + try: + done, pending = loop.run_until_complete(asyncio.wait(to_do, timeout=TIMEOUT)) + except Exception as exc: + print('*' * 60) + print(exc) + print(vars(exc)) + print('*' * 60) + counts = [] + for status in Status: + counts.append(len([task for task in done + if task.result().status == status])) + for task in pending: + task.cancel() + loop.close() + + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags3_threadpool.py b/futures/countries/flags3_threadpool.py new file mode 100644 index 0000000..140aafc --- /dev/null +++ b/futures/countries/flags3_threadpool.py @@ -0,0 +1,77 @@ +"""Download flags of top 10 countries by population + +ThreadPool version + +Sample run:: + + $ python3 pop10_threadpool1.py + BR retrieved. + PK retrieved. + BD retrieved. + JP retrieved. + CN retrieved. + IN retrieved. + RU retrieved. + NG retrieved. + US retrieved. + ID retrieved. + 10 flags downloaded in 0.63s + +""" + +from concurrent import futures +from collections import namedtuple +from enum import Enum + +import requests + +from flags_sequential2 import BASE_URL +from flags_sequential2 import save_flag, get_flag, main, Counts + +MAX_WORKERS = 200 + +Status = Enum('Status', 'ok not_found error') +Result = namedtuple('Result', 'status data') + + +def get_country(cc): + url = '{}/{cc}/metadata.json'.format(BASE_URL, cc=cc.lower()) + res = requests.get(url) + if res.status_code != 200: + res.raise_for_status() + return res.json()['country'] + + +def download_one(cc): + try: + image = get_flag(cc) + country = get_country(cc) + except requests.exceptions.HTTPError as exc: + res = exc.response + if res.status_code == 404: + status = Status.not_found + else: + msg = '{} failed: {res.status_code} - {res.reason}' + print(msg.format(cc, res=exc.response)) + status = Status.error + else: + print('{} retrieved.'.format(cc)) + country = country.replace(' ', '_') + save_flag(image, '{}-{}.gif'.format(country, cc)) + status = Status.ok + return Result(status, cc) + + +def download_many(cc_list): + workers = min(len(cc_list), MAX_WORKERS) + with futures.ThreadPoolExecutor(workers) as executor: + res = executor.map(download_one, sorted(cc_list)) + res = list(res) + counts = [] + for status in Status: + counts.append(len([r for r in res if r.status == status])) + return Counts(*counts) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags_asyncio0.py b/futures/countries/flags_asyncio0.py new file mode 100644 index 0000000..b991c4f --- /dev/null +++ b/futures/countries/flags_asyncio0.py @@ -0,0 +1,45 @@ +"""Download flags of top 20 countries by population + +asyncio+aiottp version + +Sample run:: + + $ python3 flags_asyncio0.py + EG retrieved. + BD retrieved. + JP retrieved. + ... + CD retrieved. + PH retrieved. + ET retrieved. + 20 flags downloaded in 1.05s + +""" + +import asyncio + +import aiohttp + +from flags import BASE_URL, save_flag, main + + +@asyncio.coroutine +def download_one(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + res = yield from aiohttp.request('GET', url) + image = yield from res.read() + print('{} retrieved.'.format(cc)) + save_flag(image, cc.lower() + '.gif') + return cc + + +def download_many(cc_list): + loop = asyncio.get_event_loop() + to_do = [download_one(cc) for cc in cc_list] + res, _ = loop.run_until_complete(asyncio.wait(to_do)) + loop.close() + return len(res) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags_asyncio1.py b/futures/countries/flags_asyncio1.py new file mode 100644 index 0000000..0b78841 --- /dev/null +++ b/futures/countries/flags_asyncio1.py @@ -0,0 +1,51 @@ +"""Download flags of top 20 countries by population + +asyncio+aiottp version + +Sample run:: + + $ python3 flags_asyncio.py + NG retrieved. + FR retrieved. + IN retrieved. + ... + EG retrieved. + DE retrieved. + IR retrieved. + 20 flags downloaded in 1.08s + +""" + +import asyncio + +import aiohttp + +from flags import BASE_URL, save_flag, main + + +@asyncio.coroutine +def get_flag(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + res = yield from aiohttp.request('GET', url) + image = yield from res.read() + return image + + +@asyncio.coroutine +def download_one(cc): + image = yield from get_flag(cc) + print('{} retrieved.'.format(cc)) + save_flag(image, cc.lower() + '.gif') + return cc + + +def download_many(cc_list): + loop = asyncio.get_event_loop() + to_do = [download_one(cc) for cc in cc_list] + res, _ = loop.run_until_complete(asyncio.wait(to_do)) + loop.close() + return len(res) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags_asyncio2.py b/futures/countries/flags_asyncio2.py new file mode 100644 index 0000000..2bcf0f8 --- /dev/null +++ b/futures/countries/flags_asyncio2.py @@ -0,0 +1,61 @@ +"""Download flags of top 20 countries by population + +asyncio+aiottp version + +Sample run:: + + $ python3 flags_asyncio.py + NG retrieved. + FR retrieved. + IN retrieved. + ... + EG retrieved. + DE retrieved. + IR retrieved. + 20 flags downloaded in 1.08s + +""" + +import asyncio + +import aiohttp + +from flags import BASE_URL, save_flag, main + + +@asyncio.coroutine +def get_flag(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + res = yield from aiohttp.request('GET', url) + image = yield from res.read() + return image + + +@asyncio.coroutine +def download_one(cc): + image = yield from get_flag(cc) + print('{} retrieved.'.format(cc)) + save_flag(image, cc.lower() + '.gif') + return cc + + +@asyncio.coroutine +def downloader_coro(cc_list): + to_do = [download_one(cc) for cc in cc_list] + results = [] + for future in asyncio.as_completed(to_do): + print(future) + result = yield from future + results.append(result) + return results + + +def download_many(cc_list): + loop = asyncio.get_event_loop() + results = loop.run_until_complete(downloader_coro(cc_list)) + loop.close() + return len(results) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/flags_threadpool.py b/futures/countries/flags_threadpool.py new file mode 100644 index 0000000..300d5c2 --- /dev/null +++ b/futures/countries/flags_threadpool.py @@ -0,0 +1,42 @@ +"""Download flags of top 20 countries by population + +ThreadPool version + +Sample run:: + + $ python3 flags_threadpool.py + BD retrieved. + EG retrieved. + CN retrieved. + ... + PH retrieved. + US retrieved. + IR retrieved. + 20 flags downloaded in 0.93s + +""" + +from concurrent import futures + +from flags import save_flag, get_flag, main + +MAX_WORKERS = 100 + + +def download_one(cc): + image = get_flag(cc) + print('{} retrieved.'.format(cc.upper())) + save_flag(image, cc.lower() + '.gif') + return cc + + +def download_many(cc_list): + workers = min(len(cc_list), MAX_WORKERS) + with futures.ThreadPoolExecutor(workers) as executor: + res = executor.map(download_one, sorted(cc_list)) + + return len(list(res)) + + +if __name__ == '__main__': + main(download_many) diff --git a/futures/countries/vaurien_delay.sh b/futures/countries/vaurien_delay.sh new file mode 100755 index 0000000..f5a7ffb --- /dev/null +++ b/futures/countries/vaurien_delay.sh @@ -0,0 +1,4 @@ +#!/bin/bash +vaurien --protocol http --backend localhost:8001 \ + --proxy localhost:8002 \ + --behavior 100:delay --behavior-delay-sleep .5 diff --git a/futures/countries/vaurien_error_delay.sh b/futures/countries/vaurien_error_delay.sh new file mode 100755 index 0000000..f13d02e --- /dev/null +++ b/futures/countries/vaurien_error_delay.sh @@ -0,0 +1,4 @@ +#!/bin/bash +vaurien --protocol http --backend localhost:8001 \ + --proxy localhost:8003 \ + --behavior 25:error,50:delay --behavior-delay-sleep .5 \ No newline at end of file