[Feature]Upload Modem source code
Change-Id: Id4294f30faced84d3e6fd6d5e61e1111bf287a37
diff --git a/mcu/tools/expand-pstrc-to-utmd.py b/mcu/tools/expand-pstrc-to-utmd.py
new file mode 100755
index 0000000..d8791b4
--- /dev/null
+++ b/mcu/tools/expand-pstrc-to-utmd.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import re
+import os
+import json
+import chardet
+import logging
+import argparse
+from tempfile import mkstemp
+from shutil import move, copy
+class ParserState():
+ EXPECT_BEGIN='(BEGIN_TRACE_MAP_WITH_RANGE|BEGIN_TRACE_MAP)\(\s*(\w+)\s*(,*\s*(\d+)\s*)?\s*(,*\s*(\d+)\s*)?\)'
+ TRC_MSG= r'TRC_MSG\s*\(\s*([_a-zA-Z][_a-zA-Z0-9]+)\s*,\s*\"((\\.|[^"\\])*)\"(\w+\"(\\.|[^"\\]*)\")*\s*\)'
+ END_TRACE_MAP='(END_TRACE_MAP_WITH_RANGE|END_TRACE_MAP)\(\s*(\w+)\s*(,*\s*(\d+)\s*)?\s*(,*\s*(\d+)\s*)?\)'
+args = None
+module = 'DUMMY'
+legacyParams = {}
+msgs = []
+moduleList = []
+classes = [
+ { 'TRACE_INFO': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_WARNING': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_ERROR': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_FUNC': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_STATE': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP1': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP2': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP3': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP4': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP5': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP6': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP7': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP8': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP9': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
+ { 'TRACE_GROUP10': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } }
+]
+
+def init_logger(log_filename):
+ logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+ datefmt='%m-%d %H:%M:%S',
+ filename=log_filename)
+ console = logging.StreamHandler()
+ console.setLevel(logging.DEBUG)
+ formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
+ console.setFormatter(formatter)
+ logging.getLogger('').addHandler(console)
+def parsePsTrcPreprocessed(pstrc_file, inc_path):
+ global moduleList
+ tmp_h, tmp_pstrc_file = mkstemp()
+ with os.fdopen(tmp_h, 'w') as new_file:
+ prep_pstrc_file = pstrc_file+'.tmp'
+ logging.info('PreProcessing {}'.format(prep_pstrc_file))
+ os.system(r'gcc -fdirectives-only -P -E -I{} -Iinterface/service/kal/ -Iinterface/service/config/ -Iinterface/sap -I interface/service/dhl/v2 -I interface/service/dhl -I interface/service/config/kal_config/ -I interface/driver/regbase -I interface/service/event_info -D __VGTCM__ -D __LPP_CP_SUPPORT__ -D __MD97__ -D MT6297 -D __TST_MODULE__ -D GEN_FOR_PC {} | gcc -fpreprocessed -dD -E -P - > {}'.format(inc_path, pstrc_file, prep_pstrc_file))
+ parsePsTrc(prep_pstrc_file, new_file)
+ logging.info('Processing {}'.format(pstrc_file))
+ parserState = ParserState.EXPECT_BEGIN
+ tmp_h, tmp_pstrc_file = mkstemp()
+ with os.fdopen(tmp_h, 'w') as new_file:
+ file = open(pstrc_file)
+ line = file.readline()
+ while line:
+ stripped= line.rstrip().lstrip()
+ if stripped == '':
+ line = file.readline()
+ continue
+ logging.debug('[IO] State {}: line: {}'.format(parserState, stripped))
+ if parserState == ParserState.EXPECT_BEGIN:
+ if re.match(ParserState.EXPECT_BEGIN, stripped, re.IGNORECASE):
+ match = re.search(ParserState.EXPECT_BEGIN, stripped)
+ if match:
+ parserState = ParserState.TRC_MSG
+ else:
+ new_file.write(line)
+ else:
+ new_file.write(line)
+ elif parserState == ParserState.TRC_MSG:
+ if re.match(ParserState.END_TRACE_MAP, stripped, re.IGNORECASE):
+ _m = moduleList.pop(0)
+ j = json.dumps(_m,
+ sort_keys=True,
+ # escape utf-8 to ascii
+ ensure_ascii=True,
+ indent=2
+ )
+ filename = os.path.splitext(pstrc_file)[0] +'_'+ _m['module'].lower() + '_utmd.json'
+ new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
+ new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
+ new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
+ parserState = ParserState.EXPECT_BEGIN
+ else:
+ new_file.write(line)
+ line = file.readline()
+ #force output for lpp_trc.h (no END_TRACE_MAP in file...)
+ for _m in moduleList:
+ filename = os.path.splitext(pstrc_file)[0] +'_'+ _m['module'].lower() + '_utmd.json'
+ new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
+ new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
+ new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
+ if pstrc_file.endswith('lpp_trc.h'):
+ new_file.write('#endif /* _LPP_TRC_H */\n')
+ move(tmp_pstrc_file, pstrc_file)
+def parsePsTrc(pstrc_file, new_file):
+ global module
+ global legacyParams
+ global classes
+ global msgs
+ global moduleList
+ logging.info('Processing {}'.format(pstrc_file))
+ os.system(r"sed -i -E ':a;N;$!ba;s/(TRC_MSG)\s*\(\s*(\w+)\s*,\n/\1\(\2, /g;:a:N;$!ba;s/\\\n//g' {}".format(pstrc_file))
+ parserState = ParserState.EXPECT_BEGIN
+ with open(pstrc_file) as file:
+ line = file.readline()
+ while line:
+ stripped= line.rstrip().lstrip()
+ if stripped == '':
+ line = file.readline()
+ continue
+ logging.debug('[IO] State {}: line: {}'.format(parserState, stripped))
+ if parserState == ParserState.EXPECT_BEGIN:
+ if re.match(ParserState.EXPECT_BEGIN, stripped, re.IGNORECASE):
+ match = re.search(ParserState.EXPECT_BEGIN, stripped)
+ if match:
+ global module
+ module = match.group(2)
+ if match.group(4) and match.group(6):
+ legacyParams.update( {'predefinedMessageRange': { 'min': int(match.group(4)), 'max': int(match.group(6)) } } )
+ else:
+ logging.warning("4th group / 6th group not found")
+ parserState = ParserState.TRC_MSG
+ else:
+ new_file.write(line)
+ else:
+ new_file.write(line)
+ elif parserState == ParserState.TRC_MSG:
+ if stripped.startswith('/*'):
+ while not stripped.endswith('*/'):
+ c = file.read(1)
+ stripped = stripped + c
+ line = file.readline()
+ continue
+ if stripped.startswith('TRC_MSG'):
+ while not re.match(ParserState.TRC_MSG, stripped, re.IGNORECASE) and line:
+ if stripped.endswith('\\'):
+ stripped = stripped[:-1:]
+ line = file.readline().lstrip().rstrip()
+ stripped = stripped + line
+ logging.error('strip anti-slash {}'.format(stripped))
+ else:
+ line = file.readline().lstrip().rstrip()
+ stripped = stripped + line
+ logging.error('strip anti-slash {}'.format(stripped))
+ # detect character encoding and transform to utf8
+ if chardet.detect(stripped)['confidence'] > 0.75:
+ logging.debug('decode by {}'.format(chardet.detect(stripped)['encoding']))
+ stripped = stripped.decode(chardet.detect(stripped)['encoding']).encode('utf8')
+ match = re.search(ParserState.TRC_MSG, stripped)
+ trc = {}
+ trc['traceClass'] = 'TRACE_INFO'
+ if match.group(4):
+ trc['format'] = match.group(2) + re.sub('"', '', match.group(4))
+ else:
+ trc['format'] = match.group(2)
+ if not filter( lambda x: match.group(1) in x, msgs):
+ msgs.append( { match.group(1): trc } )
+ else:
+ logging.error('Msg redefined {}'.format(match.group(1)))
+ logging.debug(match)
+ elif re.match(ParserState.END_TRACE_MAP, stripped, re.IGNORECASE):
+ moduleList.append({'module':module,
+ 'legacyParameters': legacyParams,
+ 'traceClassDefs': classes,
+ 'traceDefs': msgs,
+ 'traceFamily': 'PS'
+ })
+ j = json.dumps({'module':module,
+ 'legacyParameters': legacyParams,
+ 'traceClassDefs': classes,
+ 'traceDefs': msgs,
+ 'traceFamily': 'PS'
+ },
+ sort_keys=True,
+ # escape utf-8 to ascii
+ ensure_ascii=True,
+ indent=2
+ )
+ filename = os.path.splitext(os.path.splitext(pstrc_file)[0])[0] +'_'+ module.lower() + '_utmd.json'
+ logging.warning('write to {}'.format(filename))
+ utmd_file = open(filename, 'w')
+ utmd_file.write(j)
+ new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
+ new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
+ new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
+ parserState = ParserState.EXPECT_BEGIN
+ module = 'DUMMY'
+ legacyParams = {}
+ msgs = []
+ else:
+ new_file.write(line)
+ line = file.readline()
+def main():
+ global args
+ parser = argparse.ArgumentParser(description='PSTRC-UTMD: UTMD file will be automatically generated aside pstrc_file\n\
+ xxx_trc.h -> xxx_trc_utmd.json for single trace_map header\n\
+ xxx_trc.h -> xxx_trc_mod_ooo_utmd.json, xxx_trc_mod_abc_utmd.json for multiple trace_map header\n\
+ in this case the there are trace_map for MOD_OOO and MOD_ABC',
+ #formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("-v", action="version", version='1.0.0')
+ parser.add_argument("pstrc_file",
+ help="input PS TRC file")
+ parser.add_argument("-l", dest="log_file",
+ help="log file",
+ default='legacy-ps-utmd.log',
+ action="store")
+ args = parser.parse_args()
+ if args.pstrc_file is None:
+ parser.print_help()
+ quit()
+ init_logger(args.log_file)
+ copy(args.pstrc_file, args.pstrc_file+'.bak')
+ expand_file_mapping = {'lpp_trc': 'protocol/agps/lpp/include/',
+ 'mod_dast_trc': 'protocol/ddm/mod_dast/include/',
+ 'vgsm_trc': 'protocol/interface/vgnas/trace/',
+ 'ltecsr_trc': 'protocol/lte_csr/csr/inc/',
+ 'SST_trc': 'service/sst/include/',
+ 'tcm_trc': 'protocol/layer4/l4/tcm/common/'
+ }
+ if(args.pstrc_file.endswith('lpp_trc.h') or
+ args.pstrc_file.endswith('mod_dast_trc.h') or
+ args.pstrc_file.endswith('vgsm_trc.h') or
+ args.pstrc_file.endswith('ltecsr_trc.h') or
+ args.pstrc_file.endswith('SST_trc.h') or
+ os.path.basename(args.pstrc_file) == 'tcm_trc.h'
+ ):
+ logging.info('Inc path: {}'.format(expand_file_mapping[ os.path.splitext(os.path.basename(args.pstrc_file))[0] ]) )
+ parsePsTrcPreprocessed(args.pstrc_file, expand_file_mapping[ os.path.splitext(os.path.basename(args.pstrc_file))[0] ])
+ else:
+ tmp_h, tmp_pstrc_file = mkstemp()
+ with os.fdopen(tmp_h, 'w') as new_file:
+ parsePsTrc(args.pstrc_file, new_file)
+ move(tmp_pstrc_file, args.pstrc_file)
+if __name__ == '__main__':
+ main()
+