#!/usr/bin/env python # -*- encoding: utf-8 -*- # isis2json.py: convert ISIS and ISO-2709 files to JSON # # Copyright (C) 2010 BIREME/PAHO/WHO # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published # by the Free Software Foundation, either version 2.1 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . ############################ # BEGIN ISIS2JSON # this script works with Python or Jython (versions >=2.5 and <3) import sys import argparse from uuid import uuid4 import os try: import json except ImportError: if os.name == 'java': # running Jython from com.xhaus.jyson import JysonCodec as json else: import simplejson as json SKIP_INACTIVE = True DEFAULT_QTY = 2**31 ISIS_MFN_KEY = 'mfn' ISIS_ACTIVE_KEY = 'active' SUBFIELD_DELIMITER = '^' INPUT_ENCODING = 'cp1252' def iter_iso_records(iso_file_name, isis_json_type): # <1> from iso2709 import IsoFile from subfield import expand iso = IsoFile(iso_file_name) for record in iso: fields = {} for field in record.directory: field_key = str(int(field.tag)) # remove leading zeroes field_occurrences = fields.setdefault(field_key, []) content = field.value.decode(INPUT_ENCODING, 'replace') if isis_json_type == 1: field_occurrences.append(content) elif isis_json_type == 2: field_occurrences.append(expand(content)) elif isis_json_type == 3: field_occurrences.append(dict(expand(content))) else: raise NotImplementedError('ISIS-JSON type %s conversion ' 'not yet implemented for .iso input' % isis_json_type) yield fields iso.close() def iter_mst_records(master_file_name, isis_json_type): # <2> try: from bruma.master import MasterFactory, Record except ImportError: print('IMPORT ERROR: Jython 2.5 and Bruma.jar ' 'are required to read .mst files') raise SystemExit mst = MasterFactory.getInstance(master_file_name).open() for record in mst: fields = {} if SKIP_INACTIVE: if record.getStatus() != Record.Status.ACTIVE: continue else: # save status only there are non-active records fields[ISIS_ACTIVE_KEY] = (record.getStatus() == Record.Status.ACTIVE) fields[ISIS_MFN_KEY] = record.getMfn() for field in record.getFields(): field_key = str(field.getId()) field_occurrences = fields.setdefault(field_key, []) if isis_json_type == 3: content = {} for subfield in field.getSubfields(): subfield_key = subfield.getId() if subfield_key == '*': content['_'] = subfield.getContent() else: subfield_occurrences = content.setdefault(subfield_key, []) subfield_occurrences.append(subfield.getContent()) field_occurrences.append(content) elif isis_json_type == 1: content = [] for subfield in field.getSubfields(): subfield_key = subfield.getId() if subfield_key == '*': content.insert(0, subfield.getContent()) else: content.append(SUBFIELD_DELIMITER + subfield_key + subfield.getContent()) field_occurrences.append(''.join(content)) else: raise NotImplementedError('ISIS-JSON type %s conversion ' 'not yet implemented for .mst input' % isis_json_type) yield fields mst.close() def write_json(input_gen, file_name, output, qty, skip, id_tag, # <3> gen_uuid, mongo, mfn, isis_json_type, prefix, constant): start = skip end = start + qty if id_tag: id_tag = str(id_tag) ids = set() else: id_tag = '' for i, record in enumerate(input_gen): if i >= end: break if not mongo: if i == 0: output.write('[') elif i > start: output.write(',') if start <= i < end: if id_tag: occurrences = record.get(id_tag, None) if occurrences is None: msg = 'id tag #%s not found in record %s' if ISIS_MFN_KEY in record: msg = msg + (' (mfn=%s)' % record[ISIS_MFN_KEY]) raise KeyError(msg % (id_tag, i)) if len(occurrences) > 1: msg = 'multiple id tags #%s found in record %s' if ISIS_MFN_KEY in record: msg = msg + (' (mfn=%s)' % record[ISIS_MFN_KEY]) raise TypeError(msg % (id_tag, i)) else: # ok, we have one and only one id field if isis_json_type == 1: id = occurrences[0] elif isis_json_type == 2: id = occurrences[0][0][1] elif isis_json_type == 3: id = occurrences[0]['_'] if id in ids: msg = 'duplicate id %s in tag #%s, record %s' if ISIS_MFN_KEY in record: msg = msg + (' (mfn=%s)' % record[ISIS_MFN_KEY]) raise TypeError(msg % (id, id_tag, i)) record['_id'] = id ids.add(id) elif gen_uuid: record['_id'] = unicode(uuid4()) elif mfn: record['_id'] = record[ISIS_MFN_KEY] if prefix: # iterate over a fixed sequence of tags for tag in tuple(record): if str(tag).isdigit(): record[prefix+tag] = record[tag] del record[tag] # this is why we iterate over a tuple # with the tags, and not directly on the record dict if constant: constant_key, constant_value = constant.split(':') record[constant_key] = constant_value output.write(json.dumps(record).encode('utf-8')) output.write('\n') if not mongo: output.write(']\n') def main(): # <4> # create the parser parser = argparse.ArgumentParser( description='Convert an ISIS .mst or .iso file to a JSON array') # add the arguments parser.add_argument( 'file_name', metavar='INPUT.(mst|iso)', help='.mst or .iso file to read') parser.add_argument( '-o', '--out', type=argparse.FileType('w'), default=sys.stdout, metavar='OUTPUT.json', help='the file where the JSON output should be written' ' (default: write to stdout)') parser.add_argument( '-c', '--couch', action='store_true', help='output array within a "docs" item in a JSON document' ' for bulk insert to CouchDB via POST to db/_bulk_docs') parser.add_argument( '-m', '--mongo', action='store_true', help='output individual records as separate JSON dictionaries, one' ' per line for bulk insert to MongoDB via mongoimport utility') parser.add_argument( '-t', '--type', type=int, metavar='ISIS_JSON_TYPE', default=1, help='ISIS-JSON type, sets field structure: 1=string, 2=alist,' ' 3=dict (default=1)') parser.add_argument( '-q', '--qty', type=int, default=DEFAULT_QTY, help='maximum quantity of records to read (default=ALL)') parser.add_argument( '-s', '--skip', type=int, default=0, help='records to skip from start of .mst (default=0)') parser.add_argument( '-i', '--id', type=int, metavar='TAG_NUMBER', default=0, help='generate an "_id" from the given unique TAG field number' ' for each record') parser.add_argument( '-u', '--uuid', action='store_true', help='generate an "_id" with a random UUID for each record') parser.add_argument( '-p', '--prefix', type=str, metavar='PREFIX', default='', help='concatenate prefix to every numeric field tag' ' (ex. 99 becomes "v99")') parser.add_argument( '-n', '--mfn', action='store_true', help='generate an "_id" from the MFN of each record' ' (available only for .mst input)') parser.add_argument( '-k', '--constant', type=str, metavar='TAG:VALUE', default='', help='Include a constant tag:value in every record (ex. -k type:AS)') ''' # TODO: implement this to export large quantities of records to CouchDB parser.add_argument( '-r', '--repeat', type=int, default=1, help='repeat operation, saving multiple JSON files' ' (default=1, use -r 0 to repeat until end of input)') ''' # parse the command line args = parser.parse_args() if args.file_name.lower().endswith('.mst'): input_gen_func = iter_mst_records # <5> else: if args.mfn: print('UNSUPORTED: -n/--mfn option only available for .mst input.') raise SystemExit input_gen_func = iter_iso_records # <6> input_gen = input_gen_func(args.file_name, args.type) # <7> if args.couch: args.out.write('{ "docs" : ') write_json(input_gen, args.file_name, args.out, args.qty, # <8> args.skip, args.id, args.uuid, args.mongo, args.mfn, args.type, args.prefix, args.constant) if args.couch: args.out.write('}\n') args.out.close() if __name__ == '__main__': main() # END ISIS2JSON