yu.dong | c33b307 | 2024-08-21 23:14:49 -0700 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | # -*- coding: utf-8 -*- |
| 3 | from __future__ import print_function |
| 4 | import csv |
| 5 | import json |
| 6 | import re |
| 7 | import sys |
| 8 | import logging |
| 9 | import argparse |
| 10 | import difflib |
| 11 | import multiprocessing.dummy |
| 12 | import multiprocessing |
| 13 | col_module = 'Module' |
| 14 | col_cls = 'Legacy Trace Class' |
| 15 | col_debug_level = 'Debug Level' |
| 16 | col_sensitive = 'Sensitive' |
| 17 | col_tag = 'Tag' |
| 18 | def init_logger(log_filename): |
| 19 | logging.basicConfig(level=logging.WARNING, |
| 20 | format='[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s', |
| 21 | datefmt='%m-%d %H:%M:%S', |
| 22 | filename=log_filename) |
| 23 | console = logging.StreamHandler() |
| 24 | console.setLevel(logging.ERROR) |
| 25 | formatter = logging.Formatter('[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s') |
| 26 | console.setFormatter(formatter) |
| 27 | logging.getLogger('').addHandler(console) |
| 28 | |
| 29 | def traceApiTraceType(dhl_api_name): |
| 30 | if dhl_api_name == 'dhl_internal_trace': |
| 31 | return 'CoreDesign' |
| 32 | else: |
| 33 | return 'Public' |
| 34 | |
| 35 | def traceApiType(dhl_api_name): |
| 36 | if dhl_api_name == 'dhl_peer_trace': |
| 37 | return 'ota' |
| 38 | else: |
| 39 | return 'index' |
| 40 | |
| 41 | def similar(a, b): |
| 42 | return difflib.SequenceMatcher(None, a, b).quick_ratio() |
| 43 | |
| 44 | def getMsgClass(utmd, traceName, traceDef): |
| 45 | for cls in utmd['traceClassDefs']: |
| 46 | try: |
| 47 | if cls.get(traceDef['traceClass']): |
| 48 | return cls |
| 49 | except: |
| 50 | return None |
| 51 | |
| 52 | def updateMsgTraceType(utmd, traceName, traceDef, apiName): |
| 53 | traceClass = getMsgClass(utmd, traceName, traceDef) |
| 54 | if traceClass: |
| 55 | if traceClass.keys()[0] == traceDef['traceClass']: |
| 56 | #update trace type by api |
| 57 | logging.info('Original traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName))) |
| 58 | traceClass.get(traceDef['traceClass'])['traceType'] = traceApiTraceType(apiName) |
| 59 | logging.warning('Changed traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName))) |
| 60 | |
| 61 | def updateMsgCls(utmd, traceName, traceDef, userCls): |
| 62 | g_similarity = 0 |
| 63 | g_traceClassName = '' |
| 64 | for traceClass in utmd['traceClassDefs']: |
| 65 | for traceClassName, tracClassDef in traceClass.iteritems(): |
| 66 | similarity = similar(userCls, traceClassName) |
| 67 | if similarity > g_similarity: |
| 68 | g_similarity = similarity |
| 69 | g_traceClassName = traceClassName |
| 70 | logging.warning('Choose {} closest to {}'.format(g_traceClassName, userCls)) |
| 71 | |
| 72 | if len(g_traceClassName) > 0: |
| 73 | logging.info('Original traceClass: {}-{}'.format(traceName, traceDef['traceClass'])) |
| 74 | traceDef['traceClass'] = g_traceClassName |
| 75 | logging.warning('Changed traceClass: {}-{}'.format(traceName, traceDef['traceClass'])) |
| 76 | |
| 77 | def readUtmdToJson(utmd_filename): |
| 78 | with open(utmd_filename, 'r') as utmdfile: |
| 79 | j = json.load(utmdfile) |
| 80 | utmdfile.close() |
| 81 | return j |
| 82 | |
| 83 | def writeJsonToUtmd(j, utmd_filename): |
| 84 | with open(utmd_filename, 'w') as utmdfile: |
| 85 | utmdfile.write(json.dumps(j, |
| 86 | sort_keys=True, |
| 87 | ensure_ascii=True, |
| 88 | indent=2)) |
| 89 | utmdfile.close() |
| 90 | |
| 91 | def updateUtmdTraceDefs(utmd, trace, ps_trc_listing_filename): |
| 92 | ps_trc_listing = open(ps_trc_listing_filename, 'r') |
| 93 | fieldnames = ('filename', 'LN', 'API', 'CLS', 'MSG') |
| 94 | for traceName, traceDef in trace.iteritems(): |
| 95 | ps_trcs = csv.DictReader( ps_trc_listing, fieldnames) |
| 96 | #find match row (with trace API reference) |
| 97 | #match_rows = filter(lambda row: traceName == row['MSG'], ps_trcs) |
| 98 | match_rows = filter(lambda row: similar(traceName, row['MSG']) > 0.9, ps_trcs) |
| 99 | if len(match_rows) == 0: |
| 100 | logging.warning('Trace Usage not found {}'.format(traceName)) |
| 101 | traceDef['_comment'] = 'Trace reference not found' |
| 102 | else: |
| 103 | #update new cls def |
| 104 | #update trace type by api |
| 105 | updateMsgTraceType(utmd, traceName, traceDef, match_rows[0]['API']) |
| 106 | #update trace class |
| 107 | updateMsgCls(utmd, traceName, traceDef, match_rows[0]['CLS']) |
| 108 | #update API type |
| 109 | if 'apiType' in traceDef: |
| 110 | logging.info('Original apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| 111 | traceDef['apiType'] = traceApiType(match_rows[0]['API']) |
| 112 | logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| 113 | else: |
| 114 | traceDef['apiType'] = traceApiType(match_rows[0]['API']) |
| 115 | logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType'])) |
| 116 | return trace |
| 117 | |
| 118 | def updateUtmd(utmd_filename, csv_filename, ps_trc_listing_filename, jobs): |
| 119 | logging.error('Process {}'.format(utmd_filename)) |
| 120 | if utmd_filename.startswith('./'): |
| 121 | utmd_filename = utmd_filename[2::] |
| 122 | csvfile = open(csv_filename, 'r') |
| 123 | utmd = readUtmdToJson(utmd_filename) |
| 124 | if utmd is None: |
| 125 | logging.error('read UTMD failed') |
| 126 | sys.exit() |
| 127 | if utmd['traceFamily'] != 'PS': |
| 128 | logging.error('{}-{} not PS UTMD'.format(utmd['module'], utmd['traceFamily'])) |
| 129 | sys.exit() |
| 130 | fieldnames = ('Category', 'Task Index', col_module, col_cls, col_debug_level, col_sensitive, col_tag) |
| 131 | #update debug level / tag / sensitive tag from survey table |
| 132 | survey_table = csv.DictReader( csvfile, fieldnames) |
| 133 | logging.info('Module: {} - {}'.format(utmd['module'], utmd_filename)) |
| 134 | for row in filter(lambda row: utmd['module'] == row[col_module], survey_table): |
| 135 | logging.info(row) |
| 136 | for traceClass in utmd['traceClassDefs']: |
| 137 | for traceClassName, traceClassDef in traceClass.iteritems(): |
| 138 | if(traceClassName == row[col_cls]): |
| 139 | #debug level |
| 140 | logging.info('Original dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel'])) |
| 141 | if row[col_debug_level] == 'N/A': |
| 142 | traceClassDef['debugLevel'] = 'Ultra-Low' |
| 143 | else: |
| 144 | traceClassDef['debugLevel'] = row[col_debug_level] |
| 145 | logging.warning('Changed dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel'])) |
| 146 | #tag |
| 147 | if row[col_debug_level] not in traceClassDef['tag']: |
| 148 | logging.info('Original tag: {}-{}'.format(traceClassName, traceClassDef['tag'])) |
| 149 | traceClassDef['tag'] = [ row[col_tag], traceClassName ] |
| 150 | logging.warning('Changed tag: {}-{}'.format(traceClassName, traceClassDef['tag'])) |
| 151 | #sensitive |
| 152 | if row[col_sensitive] == 'Y': |
| 153 | logging.info('mark sensitive: {}'.format(traceClassName)) |
| 154 | traceClassDef['tag'].append('Sensitive') |
| 155 | #update trace class name |
| 156 | for traceClass in utmd['traceClassDefs']: |
| 157 | for traceClassName, tracClassDef in traceClass.iteritems(): |
| 158 | if traceClassName == 'TRACE_GROUP1': |
| 159 | traceClass['TRACE_GROUP_1'] = traceClass.pop(traceClassName) |
| 160 | elif traceClassName == 'TRACE_GROUP2': |
| 161 | traceClass['TRACE_GROUP_2'] = traceClass.pop(traceClassName) |
| 162 | elif traceClassName == 'TRACE_GROUP3': |
| 163 | traceClass['TRACE_GROUP_3'] = traceClass.pop(traceClassName) |
| 164 | elif traceClassName == 'TRACE_GROUP4': |
| 165 | traceClass['TRACE_GROUP_4'] = traceClass.pop(traceClassName) |
| 166 | elif traceClassName == 'TRACE_GROUP5': |
| 167 | traceClass['TRACE_GROUP_5'] = traceClass.pop(traceClassName) |
| 168 | elif traceClassName == 'TRACE_GROUP6': |
| 169 | traceClass['TRACE_GROUP_6'] = traceClass.pop(traceClassName) |
| 170 | elif traceClassName == 'TRACE_GROUP7': |
| 171 | traceClass['TRACE_GROUP_7'] = traceClass.pop(traceClassName) |
| 172 | elif traceClassName == 'TRACE_GROUP8': |
| 173 | traceClass['TRACE_GROUP_8'] = traceClass.pop(traceClassName) |
| 174 | elif traceClassName == 'TRACE_GROUP9': |
| 175 | traceClass['TRACE_GROUP_9'] = traceClass.pop(traceClassName) |
| 176 | elif traceClassName == 'TRACE_GROUP10': |
| 177 | traceClass['TRACE_GROUP_10'] = traceClass.pop(traceClassName) |
| 178 | #update trace Class of msg and API type for msg |
| 179 | pool = multiprocessing.Pool(jobs) |
| 180 | results = [pool.apply_async(updateUtmdTraceDefs, args=(utmd, trace, ps_trc_listing_filename,)) for trace in utmd['traceDefs']] |
| 181 | pool.close() |
| 182 | pool.join() |
| 183 | utmd['traceDefs'] = [r.get() for r in results] |
| 184 | writeJsonToUtmd(utmd, utmd_filename) |
| 185 | |
| 186 | def main(): |
| 187 | parser = argparse.ArgumentParser(description='update-ps-utmd:\n\ |
| 188 | Update tag of trace class', |
| 189 | #formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| 190 | formatter_class=argparse.RawDescriptionHelpFormatter) |
| 191 | parser.add_argument("-v", action="version", version='1.0.0') |
| 192 | parser.add_argument("utmd_file", |
| 193 | help="input PS UTMD file") |
| 194 | parser.add_argument("csv_file", |
| 195 | help="input CSV file(survey table)") |
| 196 | parser.add_argument("ps_trc_listing", |
| 197 | help="input PS trace listing") |
| 198 | parser.add_argument("-j", dest="jobs", |
| 199 | help="number of parallel jobs", |
| 200 | default=1, |
| 201 | type=int, |
| 202 | action="store") |
| 203 | parser.add_argument("-l", dest="log_file", |
| 204 | help="log file", |
| 205 | default='update-ps-utmd.log', |
| 206 | action="store") |
| 207 | args = parser.parse_args() |
| 208 | if args.utmd_file is None: |
| 209 | parser.print_help() |
| 210 | quit() |
| 211 | init_logger(args.log_file) |
| 212 | updateUtmd(args.utmd_file, args.csv_file, args.ps_trc_listing, args.jobs) |
| 213 | if __name__ == '__main__': |
| 214 | main() |