mcu/tools/expand-pstrc-to-utmd.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import os
import json
import chardet
import logging
import argparse
from tempfile import mkstemp
from shutil import move, copy
class ParserState():
    EXPECT_BEGIN='(BEGIN_TRACE_MAP_WITH_RANGE|BEGIN_TRACE_MAP)\(\s*(\w+)\s*(,*\s*(\d+)\s*)?\s*(,*\s*(\d+)\s*)?\)'
    TRC_MSG= r'TRC_MSG\s*\(\s*([_a-zA-Z][_a-zA-Z0-9]+)\s*,\s*\"((\\.|[^"\\])*)\"(\w+\"(\\.|[^"\\]*)\")*\s*\)'
    END_TRACE_MAP='(END_TRACE_MAP_WITH_RANGE|END_TRACE_MAP)\(\s*(\w+)\s*(,*\s*(\d+)\s*)?\s*(,*\s*(\d+)\s*)?\)'
args = None
module = 'DUMMY'
legacyParams = {}
msgs = []
moduleList = []
classes = [
    { 'TRACE_INFO': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_WARNING': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_ERROR': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_FUNC': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_STATE': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP1': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP2': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP3': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP4': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP5': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP6': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP7': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP8': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP9': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } },
    { 'TRACE_GROUP10': { 'tag': ['Baseline'], 'traceType': 'Public', 'debugLevel': 'Ultra-Low' } }
]

def init_logger(log_filename):
    logging.basicConfig(level=logging.DEBUG,
            format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
            datefmt='%m-%d %H:%M:%S',
            filename=log_filename)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
def parsePsTrcPreprocessed(pstrc_file, inc_path):
    global moduleList
    tmp_h, tmp_pstrc_file = mkstemp()
    with os.fdopen(tmp_h, 'w') as new_file:
        prep_pstrc_file = pstrc_file+'.tmp'
        logging.info('PreProcessing {}'.format(prep_pstrc_file))
        os.system(r'gcc -fdirectives-only -P -E -I{} -Iinterface/service/kal/ -Iinterface/service/config/ -Iinterface/sap -I interface/service/dhl/v2 -I interface/service/dhl -I interface/service/config/kal_config/ -I interface/driver/regbase -I interface/service/event_info -D __VGTCM__ -D __LPP_CP_SUPPORT__ -D __MD97__ -D MT6297 -D __TST_MODULE__ -D GEN_FOR_PC {} | gcc -fpreprocessed -dD -E -P - > {}'.format(inc_path, pstrc_file, prep_pstrc_file))
        parsePsTrc(prep_pstrc_file, new_file)
    logging.info('Processing {}'.format(pstrc_file))
    parserState = ParserState.EXPECT_BEGIN
    tmp_h, tmp_pstrc_file = mkstemp()
    with os.fdopen(tmp_h, 'w') as new_file:
        file = open(pstrc_file)
        line = file.readline()
        while line:
            stripped= line.rstrip().lstrip()
            if stripped == '':
                line = file.readline()
                continue
            logging.debug('[IO] State {}: line: {}'.format(parserState, stripped))
            if parserState == ParserState.EXPECT_BEGIN:
                if re.match(ParserState.EXPECT_BEGIN, stripped, re.IGNORECASE):
                    match = re.search(ParserState.EXPECT_BEGIN, stripped)
                    if match:
                        parserState = ParserState.TRC_MSG
                    else:
                        new_file.write(line)
                else:
                    new_file.write(line)
            elif parserState == ParserState.TRC_MSG:
                if re.match(ParserState.END_TRACE_MAP, stripped, re.IGNORECASE):
                    _m = moduleList.pop(0)
                    j = json.dumps(_m,
                        sort_keys=True,
                        # escape utf-8 to ascii
                        ensure_ascii=True,
                        indent=2
                        )
                    filename = os.path.splitext(pstrc_file)[0] +'_'+ _m['module'].lower() + '_utmd.json'
                    new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
                    new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
                    new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
                    parserState = ParserState.EXPECT_BEGIN
            else:
                new_file.write(line)
            line = file.readline()
        #force output for lpp_trc.h (no END_TRACE_MAP in file...)
        for _m in moduleList:
            filename = os.path.splitext(pstrc_file)[0] +'_'+ _m['module'].lower() + '_utmd.json'
            new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
            new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
            new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
        if pstrc_file.endswith('lpp_trc.h'):
            new_file.write('#endif  /* _LPP_TRC_H */\n')
    move(tmp_pstrc_file, pstrc_file)
def parsePsTrc(pstrc_file, new_file):
    global module
    global legacyParams
    global classes
    global msgs
    global moduleList
    logging.info('Processing {}'.format(pstrc_file))
    os.system(r"sed -i -E ':a;N;$!ba;s/(TRC_MSG)\s*\(\s*(\w+)\s*,\n/\1\(\2, /g;:a:N;$!ba;s/\\\n//g' {}".format(pstrc_file))
    parserState = ParserState.EXPECT_BEGIN
    with open(pstrc_file) as file:
        line = file.readline()
        while line:
            stripped= line.rstrip().lstrip()
            if stripped == '':
                line = file.readline()
                continue
            logging.debug('[IO] State {}: line: {}'.format(parserState, stripped))
            if parserState == ParserState.EXPECT_BEGIN:
                if re.match(ParserState.EXPECT_BEGIN, stripped, re.IGNORECASE):
                    match = re.search(ParserState.EXPECT_BEGIN, stripped)
                    if match:
                        global module
                        module = match.group(2)
                        if match.group(4) and match.group(6):
                            legacyParams.update( {'predefinedMessageRange': { 'min': int(match.group(4)), 'max': int(match.group(6)) } } )
                        else:
                            logging.warning("4th group / 6th group not found")
                        parserState = ParserState.TRC_MSG
                    else:
                        new_file.write(line)
                else:
                    new_file.write(line)
            elif parserState == ParserState.TRC_MSG:
                if stripped.startswith('/*'):
                    while not stripped.endswith('*/'):
                        c = file.read(1)
                        stripped = stripped + c
                    line = file.readline()
                    continue
                if stripped.startswith('TRC_MSG'):
                    while not re.match(ParserState.TRC_MSG, stripped, re.IGNORECASE) and line:
                        if stripped.endswith('\\'):
                            stripped = stripped[:-1:]
                            line = file.readline().lstrip().rstrip()
                            stripped = stripped + line
                            logging.error('strip anti-slash {}'.format(stripped))
                        else:
                            line = file.readline().lstrip().rstrip()
                            stripped = stripped + line
                            logging.error('strip anti-slash {}'.format(stripped))
                    # detect character encoding and transform to utf8
                    if chardet.detect(stripped)['confidence'] > 0.75:
                        logging.debug('decode by {}'.format(chardet.detect(stripped)['encoding']))
                        stripped = stripped.decode(chardet.detect(stripped)['encoding']).encode('utf8')
                    match = re.search(ParserState.TRC_MSG, stripped)
                    trc = {}
                    trc['traceClass'] = 'TRACE_INFO'
                    if match.group(4):
                        trc['format'] = match.group(2) + re.sub('"', '', match.group(4))
                    else:
                        trc['format'] = match.group(2)
                    if not filter( lambda x: match.group(1) in x, msgs):
                        msgs.append( { match.group(1): trc } )
                    else:
                        logging.error('Msg redefined {}'.format(match.group(1)))
                    logging.debug(match)
                elif re.match(ParserState.END_TRACE_MAP, stripped, re.IGNORECASE):
                    moduleList.append({'module':module,
                        'legacyParameters': legacyParams,
                        'traceClassDefs': classes,
                        'traceDefs': msgs,
                        'traceFamily': 'PS'
                        })
                    j = json.dumps({'module':module,
                        'legacyParameters': legacyParams,
                        'traceClassDefs': classes,
                        'traceDefs': msgs,
                        'traceFamily': 'PS'
                        },
                        sort_keys=True,
                        # escape utf-8 to ascii
                        ensure_ascii=True,
                        indent=2
                        )
                    filename = os.path.splitext(os.path.splitext(pstrc_file)[0])[0] +'_'+ module.lower() + '_utmd.json'
                    logging.warning('write to {}'.format(filename))
                    utmd_file = open(filename, 'w')
                    utmd_file.write(j)
                    new_file.write('// The trace map of {} is converted to UTMD file.\n'.format(module))
                    new_file.write('// If you wished to modify your trace, please refer to {}\n'.format(os.path.basename(filename)))
                    new_file.write('#include"{}_precodegen.h"\n'.format(os.path.splitext(os.path.basename(filename))[0]))
                    parserState = ParserState.EXPECT_BEGIN
                    module = 'DUMMY'
                    legacyParams = {}
                    msgs = []
            else:
                new_file.write(line)
            line = file.readline()
def main():
    global args
    parser = argparse.ArgumentParser(description='PSTRC-UTMD: UTMD file will be automatically generated aside pstrc_file\n\
            xxx_trc.h -> xxx_trc_utmd.json for single trace_map header\n\
            xxx_trc.h -> xxx_trc_mod_ooo_utmd.json, xxx_trc_mod_abc_utmd.json for multiple trace_map header\n\
            in this case the there are trace_map for MOD_OOO and MOD_ABC',
            #formatter_class=argparse.ArgumentDefaultsHelpFormatter)
            formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-v", action="version", version='1.0.0')
    parser.add_argument("pstrc_file",
            help="input PS TRC file")
    parser.add_argument("-l", dest="log_file",
            help="log file",
            default='legacy-ps-utmd.log',
            action="store")
    args = parser.parse_args()
    if args.pstrc_file is None:
        parser.print_help()
        quit()
    init_logger(args.log_file)
    copy(args.pstrc_file, args.pstrc_file+'.bak')
    expand_file_mapping  =  {'lpp_trc': 'protocol/agps/lpp/include/',
            'mod_dast_trc': 'protocol/ddm/mod_dast/include/',
            'vgsm_trc': 'protocol/interface/vgnas/trace/',
            'ltecsr_trc': 'protocol/lte_csr/csr/inc/',
            'SST_trc': 'service/sst/include/',
            'tcm_trc': 'protocol/layer4/l4/tcm/common/'
            }
    if(args.pstrc_file.endswith('lpp_trc.h') or
            args.pstrc_file.endswith('mod_dast_trc.h') or
            args.pstrc_file.endswith('vgsm_trc.h') or
            args.pstrc_file.endswith('ltecsr_trc.h') or
            args.pstrc_file.endswith('SST_trc.h') or
            os.path.basename(args.pstrc_file) == 'tcm_trc.h'
            ):
        logging.info('Inc path: {}'.format(expand_file_mapping[ os.path.splitext(os.path.basename(args.pstrc_file))[0] ]) )
        parsePsTrcPreprocessed(args.pstrc_file, expand_file_mapping[ os.path.splitext(os.path.basename(args.pstrc_file))[0] ])
    else:
        tmp_h, tmp_pstrc_file = mkstemp()
        with os.fdopen(tmp_h, 'w') as new_file:
            parsePsTrc(args.pstrc_file, new_file)
        move(tmp_pstrc_file, args.pstrc_file)
if __name__ == '__main__':
    main()