| #!/usr/bin/env python |
| # -*- coding: utf-8 -*- |
| from __future__ import print_function |
| import csv |
| import json |
| import re |
| import sys |
| import logging |
| import argparse |
| import difflib |
| import multiprocessing.dummy |
| import multiprocessing |
| col_module = 'Module' |
| col_cls = 'Legacy Trace Class' |
| col_debug_level = 'Debug Level' |
| col_sensitive = 'Sensitive' |
| col_tag = 'Tag' |
| def init_logger(log_filename): |
| logging.basicConfig(level=logging.WARNING, |
| format='[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s', |
| datefmt='%m-%d %H:%M:%S', |
| filename=log_filename) |
| console = logging.StreamHandler() |
| console.setLevel(logging.ERROR) |
| formatter = logging.Formatter('[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s') |
| console.setFormatter(formatter) |
| logging.getLogger('').addHandler(console) |
| |
| def traceApiTraceType(dhl_api_name): |
| if dhl_api_name == 'dhl_internal_trace': |
| return 'CoreDesign' |
| else: |
| return 'Public' |
| |
| def traceApiType(dhl_api_name): |
| if dhl_api_name == 'dhl_peer_trace': |
| return 'ota' |
| else: |
| return 'index' |
| |
| def similar(a, b): |
| return difflib.SequenceMatcher(None, a, b).quick_ratio() |
| |
| def getMsgClass(utmd, traceName, traceDef): |
| for cls in utmd['traceClassDefs']: |
| try: |
| if cls.get(traceDef['traceClass']): |
| return cls |
| except: |
| return None |
| |
| def updateMsgTraceType(utmd, traceName, traceDef, apiName): |
| traceClass = getMsgClass(utmd, traceName, traceDef) |
| if traceClass: |
| if traceClass.keys()[0] == traceDef['traceClass']: |
| #update trace type by api |
| logging.info('Original traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName))) |
| traceClass.get(traceDef['traceClass'])['traceType'] = traceApiTraceType(apiName) |
| logging.warning('Changed traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName))) |
| |
| def updateMsgCls(utmd, traceName, traceDef, userCls): |
| g_similarity = 0 |
| g_traceClassName = '' |
| for traceClass in utmd['traceClassDefs']: |
| for traceClassName, tracClassDef in traceClass.iteritems(): |
| similarity = similar(userCls, traceClassName) |
| if similarity > g_similarity: |
| g_similarity = similarity |
| g_traceClassName = traceClassName |
| logging.warning('Choose {} closest to {}'.format(g_traceClassName, userCls)) |
| |
| if len(g_traceClassName) > 0: |
| logging.info('Original traceClass: {}-{}'.format(traceName, traceDef['traceClass'])) |
| traceDef['traceClass'] = g_traceClassName |
| logging.warning('Changed traceClass: {}-{}'.format(traceName, traceDef['traceClass'])) |
| |
| def readUtmdToJson(utmd_filename): |
| with open(utmd_filename, 'r') as utmdfile: |
| j = json.load(utmdfile) |
| utmdfile.close() |
| return j |
| |
| def writeJsonToUtmd(j, utmd_filename): |
| with open(utmd_filename, 'w') as utmdfile: |
| utmdfile.write(json.dumps(j, |
| sort_keys=True, |
| ensure_ascii=True, |
| indent=2)) |
| utmdfile.close() |
| |
| def updateUtmdTraceDefs(utmd, trace, ps_trc_listing_filename): |
| ps_trc_listing = open(ps_trc_listing_filename, 'r') |
| fieldnames = ('filename', 'LN', 'API', 'CLS', 'MSG') |
| for traceName, traceDef in trace.iteritems(): |
| ps_trcs = csv.DictReader( ps_trc_listing, fieldnames) |
| #find match row (with trace API reference) |
| #match_rows = filter(lambda row: traceName == row['MSG'], ps_trcs) |
| match_rows = filter(lambda row: similar(traceName, row['MSG']) > 0.9, ps_trcs) |
| if len(match_rows) == 0: |
| logging.warning('Trace Usage not found {}'.format(traceName)) |
| traceDef['_comment'] = 'Trace reference not found' |
| else: |
| #update new cls def |
| #update trace type by api |
| updateMsgTraceType(utmd, traceName, traceDef, match_rows[0]['API']) |
| #update trace class |
| updateMsgCls(utmd, traceName, traceDef, match_rows[0]['CLS']) |
| #update API type |
| if 'apiType' in traceDef: |
| logging.info('Original apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| traceDef['apiType'] = traceApiType(match_rows[0]['API']) |
| logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| else: |
| traceDef['apiType'] = traceApiType(match_rows[0]['API']) |
| logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| return trace |
| |
| def updateUtmd(utmd_filename, csv_filename, ps_trc_listing_filename, jobs): |
| logging.error('Process {}'.format(utmd_filename)) |
| if utmd_filename.startswith('./'): |
| utmd_filename = utmd_filename[2::] |
| csvfile = open(csv_filename, 'r') |
| utmd = readUtmdToJson(utmd_filename) |
| if utmd is None: |
| logging.error('read UTMD failed') |
| sys.exit() |
| if utmd['traceFamily'] != 'PS': |
| logging.error('{}-{} not PS UTMD'.format(utmd['module'], utmd['traceFamily'])) |
| sys.exit() |
| fieldnames = ('Category', 'Task Index', col_module, col_cls, col_debug_level, col_sensitive, col_tag) |
| #update debug level / tag / sensitive tag from survey table |
| survey_table = csv.DictReader( csvfile, fieldnames) |
| logging.info('Module: {} - {}'.format(utmd['module'], utmd_filename)) |
| for row in filter(lambda row: utmd['module'] == row[col_module], survey_table): |
| logging.info(row) |
| for traceClass in utmd['traceClassDefs']: |
| for traceClassName, traceClassDef in traceClass.iteritems(): |
| if(traceClassName == row[col_cls]): |
| #debug level |
| logging.info('Original dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel'])) |
| if row[col_debug_level] == 'N/A': |
| traceClassDef['debugLevel'] = 'Ultra-Low' |
| else: |
| traceClassDef['debugLevel'] = row[col_debug_level] |
| logging.warning('Changed dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel'])) |
| #tag |
| if row[col_debug_level] not in traceClassDef['tag']: |
| logging.info('Original tag: {}-{}'.format(traceClassName, traceClassDef['tag'])) |
| traceClassDef['tag'] = [ row[col_tag], traceClassName ] |
| logging.warning('Changed tag: {}-{}'.format(traceClassName, traceClassDef['tag'])) |
| #sensitive |
| if row[col_sensitive] == 'Y': |
| logging.info('mark sensitive: {}'.format(traceClassName)) |
| traceClassDef['tag'].append('Sensitive') |
| #update trace class name |
| for traceClass in utmd['traceClassDefs']: |
| for traceClassName, tracClassDef in traceClass.iteritems(): |
| if traceClassName == 'TRACE_GROUP1': |
| traceClass['TRACE_GROUP_1'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP2': |
| traceClass['TRACE_GROUP_2'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP3': |
| traceClass['TRACE_GROUP_3'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP4': |
| traceClass['TRACE_GROUP_4'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP5': |
| traceClass['TRACE_GROUP_5'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP6': |
| traceClass['TRACE_GROUP_6'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP7': |
| traceClass['TRACE_GROUP_7'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP8': |
| traceClass['TRACE_GROUP_8'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP9': |
| traceClass['TRACE_GROUP_9'] = traceClass.pop(traceClassName) |
| elif traceClassName == 'TRACE_GROUP10': |
| traceClass['TRACE_GROUP_10'] = traceClass.pop(traceClassName) |
| #update trace Class of msg and API type for msg |
| pool = multiprocessing.Pool(jobs) |
| results = [pool.apply_async(updateUtmdTraceDefs, args=(utmd, trace, ps_trc_listing_filename,)) for trace in utmd['traceDefs']] |
| pool.close() |
| pool.join() |
| utmd['traceDefs'] = [r.get() for r in results] |
| writeJsonToUtmd(utmd, utmd_filename) |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='update-ps-utmd:\n\ |
| Update tag of trace class', |
| #formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| formatter_class=argparse.RawDescriptionHelpFormatter) |
| parser.add_argument("-v", action="version", version='1.0.0') |
| parser.add_argument("utmd_file", |
| help="input PS UTMD file") |
| parser.add_argument("csv_file", |
| help="input CSV file(survey table)") |
| parser.add_argument("ps_trc_listing", |
| help="input PS trace listing") |
| parser.add_argument("-j", dest="jobs", |
| help="number of parallel jobs", |
| default=1, |
| type=int, |
| action="store") |
| parser.add_argument("-l", dest="log_file", |
| help="log file", |
| default='update-ps-utmd.log', |
| action="store") |
| args = parser.parse_args() |
| if args.utmd_file is None: |
| parser.print_help() |
| quit() |
| init_logger(args.log_file) |
| updateUtmd(args.utmd_file, args.csv_file, args.ps_trc_listing, args.jobs) |
| if __name__ == '__main__': |
| main() |