updated wikipedia examples

concurrency/wikipedia/build_fixture.py

@@ -1,18 +1,24 @@
 import sys
 import argparse
 import os
+import urllib
 
-from daypicts import get_picture_url, validate_date, gen_dates
+import requests
+
+from daypicts import get_picture_url, get_picture_urls
+from daypicts import validate_date, gen_dates, picture_type
 from daypicts import NoPictureForDate
-from daypicts import POTD_PATH
-
-FIXTURE_DIR = 'fixture/'
+from daypicts import REMOTE_PICT_BASE_URL, PICT_EXCEPTIONS
 
+FIXTURE_DOC_DIR = 'fixture/docroot/'
+FIXTURE_TEMPLATE_POTD_DIR = FIXTURE_DOC_DIR + 'Template-POTD/'
 
 def parse_args(argv):
     parser = argparse.ArgumentParser(description=main.__doc__)
     date_help = 'YYYY-MM-DD or YYYY-MM or YYYY: year, month and day'
     parser.add_argument('date', help=date_help)
+    parser.add_argument('-u', '--url_only', action='store_true',
+                        help='get picture URLS only')
 
     args = parser.parse_args(argv)
 
@@ -47,18 +53,45 @@ def save_picture_urls(dates, save_path):
             fp.write(snippet)
 
 
+def save_pictures(dates, save_path, verbose=False):
+    urls_ok = []
+    for date, url in get_picture_urls(dates, verbose):
+        response = requests.get(url)
+        file_path = os.path.join(save_path,
+                                 url.replace(REMOTE_PICT_BASE_URL, ''))
+        file_path = urllib.parse.unquote(file_path)
+        octets = response.content
+        # http://en.wikipedia.org/wiki/Template:POTD/2013-06-15
+
+        if date not in PICT_EXCEPTIONS:
+            assert picture_type(octets) is not None, url
+
+        try:
+            os.makedirs(os.path.dirname(file_path))
+        except FileExistsError:
+            pass
+        with open(file_path, 'wb') as fp:
+            fp.write(octets)
+
+        print(file_path)
+    return urls_ok
+
+
 def main(argv):
     """Build test fixture from Wikipedia "POTD" data"""
 
-    save_path = os.path.join(FIXTURE_DIR,POTD_PATH)
     try:
-        os.makedirs(save_path)
+        os.makedirs(FIXTURE_TEMPLATE_POTD_DIR)
     except FileExistsError:
         pass
 
     dates, args = parse_args(argv)
 
-    save_picture_urls(dates, save_path)
+    if args.url_only:
+        save_picture_urls(dates, FIXTURE_TEMPLATE_POTD_DIR)
+    else:
+        save_pictures(dates, FIXTURE_DOC_DIR)
+
 
 if __name__ == '__main__':
     main(sys.argv[1:])