blob: 63c7d423f685b0d88765bac4ab2953220f886821 [file] [log] [blame]
yu.dongc33b3072024-08-21 23:14:49 -07001#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3from __future__ import print_function
4import csv
5import json
6import re
7import sys
8import logging
9import argparse
10import difflib
11import multiprocessing.dummy
12import multiprocessing
13col_module = 'Module'
14col_cls = 'Legacy Trace Class'
15col_debug_level = 'Debug Level'
16col_sensitive = 'Sensitive'
17col_tag = 'Tag'
18def init_logger(log_filename):
19 logging.basicConfig(level=logging.WARNING,
20 format='[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s',
21 datefmt='%m-%d %H:%M:%S',
22 filename=log_filename)
23 console = logging.StreamHandler()
24 console.setLevel(logging.ERROR)
25 formatter = logging.Formatter('[%(process)d] %(asctime)s %(name)-12s %(levelname)-8s %(message)s')
26 console.setFormatter(formatter)
27 logging.getLogger('').addHandler(console)
28
29def traceApiTraceType(dhl_api_name):
30 if dhl_api_name == 'dhl_internal_trace':
31 return 'CoreDesign'
32 else:
33 return 'Public'
34
35def traceApiType(dhl_api_name):
36 if dhl_api_name == 'dhl_peer_trace':
37 return 'ota'
38 else:
39 return 'index'
40
41def similar(a, b):
42 return difflib.SequenceMatcher(None, a, b).quick_ratio()
43
44def getMsgClass(utmd, traceName, traceDef):
45 for cls in utmd['traceClassDefs']:
46 try:
47 if cls.get(traceDef['traceClass']):
48 return cls
49 except:
50 return None
51
52def updateMsgTraceType(utmd, traceName, traceDef, apiName):
53 traceClass = getMsgClass(utmd, traceName, traceDef)
54 if traceClass:
55 if traceClass.keys()[0] == traceDef['traceClass']:
56 #update trace type by api
57 logging.info('Original traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName)))
58 traceClass.get(traceDef['traceClass'])['traceType'] = traceApiTraceType(apiName)
59 logging.warning('Changed traceType: {}-{}'.format(traceClass.get(traceDef['traceClass'])['traceType'], traceApiTraceType(apiName)))
60
61def updateMsgCls(utmd, traceName, traceDef, userCls):
62 g_similarity = 0
63 g_traceClassName = ''
64 for traceClass in utmd['traceClassDefs']:
65 for traceClassName, tracClassDef in traceClass.iteritems():
66 similarity = similar(userCls, traceClassName)
67 if similarity > g_similarity:
68 g_similarity = similarity
69 g_traceClassName = traceClassName
70 logging.warning('Choose {} closest to {}'.format(g_traceClassName, userCls))
71
72 if len(g_traceClassName) > 0:
73 logging.info('Original traceClass: {}-{}'.format(traceName, traceDef['traceClass']))
74 traceDef['traceClass'] = g_traceClassName
75 logging.warning('Changed traceClass: {}-{}'.format(traceName, traceDef['traceClass']))
76
77def readUtmdToJson(utmd_filename):
78 with open(utmd_filename, 'r') as utmdfile:
79 j = json.load(utmdfile)
80 utmdfile.close()
81 return j
82
83def writeJsonToUtmd(j, utmd_filename):
84 with open(utmd_filename, 'w') as utmdfile:
85 utmdfile.write(json.dumps(j,
86 sort_keys=True,
87 ensure_ascii=True,
88 indent=2))
89 utmdfile.close()
90
91def updateUtmdTraceDefs(utmd, trace, ps_trc_listing_filename):
92 ps_trc_listing = open(ps_trc_listing_filename, 'r')
93 fieldnames = ('filename', 'LN', 'API', 'CLS', 'MSG')
94 for traceName, traceDef in trace.iteritems():
95 ps_trcs = csv.DictReader( ps_trc_listing, fieldnames)
96 #find match row (with trace API reference)
97 #match_rows = filter(lambda row: traceName == row['MSG'], ps_trcs)
98 match_rows = filter(lambda row: similar(traceName, row['MSG']) > 0.9, ps_trcs)
99 if len(match_rows) == 0:
100 logging.warning('Trace Usage not found {}'.format(traceName))
101 traceDef['_comment'] = 'Trace reference not found'
102 else:
103 #update new cls def
104 #update trace type by api
105 updateMsgTraceType(utmd, traceName, traceDef, match_rows[0]['API'])
106 #update trace class
107 updateMsgCls(utmd, traceName, traceDef, match_rows[0]['CLS'])
108 #update API type
109 if 'apiType' in traceDef:
110 logging.info('Original apiType: {}-{}'.format(traceName, traceDef['apiType']))
111 traceDef['apiType'] = traceApiType(match_rows[0]['API'])
112 logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType']))
113 else:
114 traceDef['apiType'] = traceApiType(match_rows[0]['API'])
115 logging.warning('Changed apiType: {}-{}'.format(traceName, traceDef['apiType']))
116 return trace
117
118def updateUtmd(utmd_filename, csv_filename, ps_trc_listing_filename, jobs):
119 logging.error('Process {}'.format(utmd_filename))
120 if utmd_filename.startswith('./'):
121 utmd_filename = utmd_filename[2::]
122 csvfile = open(csv_filename, 'r')
123 utmd = readUtmdToJson(utmd_filename)
124 if utmd is None:
125 logging.error('read UTMD failed')
126 sys.exit()
127 if utmd['traceFamily'] != 'PS':
128 logging.error('{}-{} not PS UTMD'.format(utmd['module'], utmd['traceFamily']))
129 sys.exit()
130 fieldnames = ('Category', 'Task Index', col_module, col_cls, col_debug_level, col_sensitive, col_tag)
131 #update debug level / tag / sensitive tag from survey table
132 survey_table = csv.DictReader( csvfile, fieldnames)
133 logging.info('Module: {} - {}'.format(utmd['module'], utmd_filename))
134 for row in filter(lambda row: utmd['module'] == row[col_module], survey_table):
135 logging.info(row)
136 for traceClass in utmd['traceClassDefs']:
137 for traceClassName, traceClassDef in traceClass.iteritems():
138 if(traceClassName == row[col_cls]):
139 #debug level
140 logging.info('Original dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel']))
141 if row[col_debug_level] == 'N/A':
142 traceClassDef['debugLevel'] = 'Ultra-Low'
143 else:
144 traceClassDef['debugLevel'] = row[col_debug_level]
145 logging.warning('Changed dbg lvl: {}-{}'.format(traceClassName, traceClassDef['debugLevel']))
146 #tag
147 if row[col_debug_level] not in traceClassDef['tag']:
148 logging.info('Original tag: {}-{}'.format(traceClassName, traceClassDef['tag']))
149 traceClassDef['tag'] = [ row[col_tag], traceClassName ]
150 logging.warning('Changed tag: {}-{}'.format(traceClassName, traceClassDef['tag']))
151 #sensitive
152 if row[col_sensitive] == 'Y':
153 logging.info('mark sensitive: {}'.format(traceClassName))
154 traceClassDef['tag'].append('Sensitive')
155 #update trace class name
156 for traceClass in utmd['traceClassDefs']:
157 for traceClassName, tracClassDef in traceClass.iteritems():
158 if traceClassName == 'TRACE_GROUP1':
159 traceClass['TRACE_GROUP_1'] = traceClass.pop(traceClassName)
160 elif traceClassName == 'TRACE_GROUP2':
161 traceClass['TRACE_GROUP_2'] = traceClass.pop(traceClassName)
162 elif traceClassName == 'TRACE_GROUP3':
163 traceClass['TRACE_GROUP_3'] = traceClass.pop(traceClassName)
164 elif traceClassName == 'TRACE_GROUP4':
165 traceClass['TRACE_GROUP_4'] = traceClass.pop(traceClassName)
166 elif traceClassName == 'TRACE_GROUP5':
167 traceClass['TRACE_GROUP_5'] = traceClass.pop(traceClassName)
168 elif traceClassName == 'TRACE_GROUP6':
169 traceClass['TRACE_GROUP_6'] = traceClass.pop(traceClassName)
170 elif traceClassName == 'TRACE_GROUP7':
171 traceClass['TRACE_GROUP_7'] = traceClass.pop(traceClassName)
172 elif traceClassName == 'TRACE_GROUP8':
173 traceClass['TRACE_GROUP_8'] = traceClass.pop(traceClassName)
174 elif traceClassName == 'TRACE_GROUP9':
175 traceClass['TRACE_GROUP_9'] = traceClass.pop(traceClassName)
176 elif traceClassName == 'TRACE_GROUP10':
177 traceClass['TRACE_GROUP_10'] = traceClass.pop(traceClassName)
178 #update trace Class of msg and API type for msg
179 pool = multiprocessing.Pool(jobs)
180 results = [pool.apply_async(updateUtmdTraceDefs, args=(utmd, trace, ps_trc_listing_filename,)) for trace in utmd['traceDefs']]
181 pool.close()
182 pool.join()
183 utmd['traceDefs'] = [r.get() for r in results]
184 writeJsonToUtmd(utmd, utmd_filename)
185
186def main():
187 parser = argparse.ArgumentParser(description='update-ps-utmd:\n\
188 Update tag of trace class',
189 #formatter_class=argparse.ArgumentDefaultsHelpFormatter)
190 formatter_class=argparse.RawDescriptionHelpFormatter)
191 parser.add_argument("-v", action="version", version='1.0.0')
192 parser.add_argument("utmd_file",
193 help="input PS UTMD file")
194 parser.add_argument("csv_file",
195 help="input CSV file(survey table)")
196 parser.add_argument("ps_trc_listing",
197 help="input PS trace listing")
198 parser.add_argument("-j", dest="jobs",
199 help="number of parallel jobs",
200 default=1,
201 type=int,
202 action="store")
203 parser.add_argument("-l", dest="log_file",
204 help="log file",
205 default='update-ps-utmd.log',
206 action="store")
207 args = parser.parse_args()
208 if args.utmd_file is None:
209 parser.print_help()
210 quit()
211 init_logger(args.log_file)
212 updateUtmd(args.utmd_file, args.csv_file, args.ps_trc_listing, args.jobs)
213if __name__ == '__main__':
214 main()