[Feature]add MT2731_MP2_MR2_SVN388 baseline version
Change-Id: Ief04314834b31e27effab435d3ca8ba33b499059
diff --git a/meta/poky/scripts/buildstats-diff b/meta/poky/scripts/buildstats-diff
new file mode 100755
index 0000000..a128dd3
--- /dev/null
+++ b/meta/poky/scripts/buildstats-diff
@@ -0,0 +1,306 @@
+#!/usr/bin/python3
+#
+# Script for comparing buildstats from two different builds
+#
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+import argparse
+import glob
+import logging
+import math
+import os
+import sys
+from operator import attrgetter
+
+# Import oe libs
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(scripts_path, 'lib'))
+from buildstats import BuildStats, diff_buildstats, taskdiff_fields, BSVerDiff
+
+
+# Setup logging
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+log = logging.getLogger()
+
+
+class ScriptError(Exception):
+ """Exception for internal error handling of this script"""
+ pass
+
+
+def read_buildstats(path, multi):
+ """Read buildstats"""
+ if not os.path.exists(path):
+ raise ScriptError("No such file or directory: {}".format(path))
+
+ if os.path.isfile(path):
+ return BuildStats.from_file_json(path)
+
+ if os.path.isfile(os.path.join(path, 'build_stats')):
+ return BuildStats.from_dir(path)
+
+ # Handle a non-buildstat directory
+ subpaths = sorted(glob.glob(path + '/*'))
+ if len(subpaths) > 1:
+ if multi:
+ log.info("Averaging over {} buildstats from {}".format(
+ len(subpaths), path))
+ else:
+ raise ScriptError("Multiple buildstats found in '{}'. Please give "
+ "a single buildstat directory of use the --multi "
+ "option".format(path))
+ bs = None
+ for subpath in subpaths:
+ if os.path.isfile(subpath):
+ _bs = BuildStats.from_file_json(subpath)
+ else:
+ _bs = BuildStats.from_dir(subpath)
+ if bs is None:
+ bs = _bs
+ else:
+ bs.aggregate(_bs)
+ if not bs:
+ raise ScriptError("No buildstats found under {}".format(path))
+
+ return bs
+
+
+def print_ver_diff(bs1, bs2):
+ """Print package version differences"""
+
+ diff = BSVerDiff(bs1, bs2)
+
+ maxlen = max([len(r) for r in set(bs1.keys()).union(set(bs2.keys()))])
+ fmt_str = " {:{maxlen}} ({})"
+
+ if diff.new:
+ print("\nNEW RECIPES:")
+ print("------------")
+ for name, val in sorted(diff.new.items()):
+ print(fmt_str.format(name, val.nevr, maxlen=maxlen))
+
+ if diff.dropped:
+ print("\nDROPPED RECIPES:")
+ print("----------------")
+ for name, val in sorted(diff.dropped.items()):
+ print(fmt_str.format(name, val.nevr, maxlen=maxlen))
+
+ fmt_str = " {0:{maxlen}} {1:<20} ({2})"
+ if diff.rchanged:
+ print("\nREVISION CHANGED:")
+ print("-----------------")
+ for name, val in sorted(diff.rchanged.items()):
+ field1 = "{} -> {}".format(val.left.revision, val.right.revision)
+ field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
+ print(fmt_str.format(name, field1, field2, maxlen=maxlen))
+
+ if diff.vchanged:
+ print("\nVERSION CHANGED:")
+ print("----------------")
+ for name, val in sorted(diff.vchanged.items()):
+ field1 = "{} -> {}".format(val.left.version, val.right.version)
+ field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
+ print(fmt_str.format(name, field1, field2, maxlen=maxlen))
+
+ if diff.echanged:
+ print("\nEPOCH CHANGED:")
+ print("--------------")
+ for name, val in sorted(diff.echanged.items()):
+ field1 = "{} -> {}".format(val.left.epoch, val.right.epoch)
+ field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
+ print(fmt_str.format(name, field1, field2, maxlen=maxlen))
+
+
+def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
+ """Diff task execution times"""
+ def val_to_str(val, human_readable=False):
+ """Convert raw value to printable string"""
+ def hms_time(secs):
+ """Get time in human-readable HH:MM:SS format"""
+ h = int(secs / 3600)
+ m = int((secs % 3600) / 60)
+ s = secs % 60
+ if h == 0:
+ return "{:02d}:{:04.1f}".format(m, s)
+ else:
+ return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
+
+ if 'time' in val_type:
+ if human_readable:
+ return hms_time(val)
+ else:
+ return "{:.1f}s".format(val)
+ elif 'bytes' in val_type and human_readable:
+ prefix = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi']
+ dec = int(math.log(val, 2) / 10)
+ prec = 1 if dec > 0 else 0
+ return "{:.{prec}f}{}B".format(val / (2 ** (10 * dec)),
+ prefix[dec], prec=prec)
+ elif 'ops' in val_type and human_readable:
+ prefix = ['', 'k', 'M', 'G', 'T', 'P']
+ dec = int(math.log(val, 1000))
+ prec = 1 if dec > 0 else 0
+ return "{:.{prec}f}{}ops".format(val / (1000 ** dec),
+ prefix[dec], prec=prec)
+ return str(int(val))
+
+ def sum_vals(buildstats):
+ """Get cumulative sum of all tasks"""
+ total = 0.0
+ for recipe_data in buildstats.values():
+ for bs_task in recipe_data.tasks.values():
+ total += getattr(bs_task, val_type)
+ return total
+
+ if min_val:
+ print("Ignoring tasks less than {} ({})".format(
+ val_to_str(min_val, True), val_to_str(min_val)))
+ if min_absdiff:
+ print("Ignoring differences less than {} ({})".format(
+ val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
+
+ # Prepare the data
+ tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff)
+
+ # Sort our list
+ for field in reversed(sort_by):
+ if field.startswith('-'):
+ field = field[1:]
+ reverse = True
+ else:
+ reverse = False
+ tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse)
+
+ linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF',
+ val_type.upper() + '1', val_type.upper() + '2')]
+ field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])])
+
+ # Prepare fields in string format and measure field lengths
+ for diff in tasks_diff:
+ task_prefix = diff.task_op if diff.pkg_op == ' ' else ' '
+ linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task,
+ val_to_str(diff.absdiff),
+ '{:+.1f}%'.format(diff.reldiff),
+ val_to_str(diff.value1),
+ val_to_str(diff.value2)))
+ for i, field in enumerate(linedata[-1]):
+ key = 'len_{}'.format(i)
+ if len(field) > field_lens[key]:
+ field_lens[key] = len(field)
+
+ # Print data
+ print()
+ for fields in linedata:
+ print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format(
+ *fields, **field_lens))
+
+ # Print summary of the diffs
+ total1 = sum_vals(bs1)
+ total2 = sum_vals(bs2)
+ print("\nCumulative {}:".format(val_type))
+ print (" {} {:+.1f}% {} ({}) -> {} ({})".format(
+ val_to_str(total2 - total1), 100 * (total2-total1) / total1,
+ val_to_str(total1, True), val_to_str(total1),
+ val_to_str(total2, True), val_to_str(total2)))
+
+
+def parse_args(argv):
+ """Parse cmdline arguments"""
+ description="""
+Script for comparing buildstats of two separate builds."""
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ description=description)
+
+ min_val_defaults = {'cputime': 3.0,
+ 'read_bytes': 524288,
+ 'write_bytes': 524288,
+ 'read_ops': 500,
+ 'write_ops': 500,
+ 'walltime': 5}
+ min_absdiff_defaults = {'cputime': 1.0,
+ 'read_bytes': 131072,
+ 'write_bytes': 131072,
+ 'read_ops': 50,
+ 'write_ops': 50,
+ 'walltime': 2}
+
+ parser.add_argument('--debug', '-d', action='store_true',
+ help="Verbose logging")
+ parser.add_argument('--ver-diff', action='store_true',
+ help="Show package version differences and exit")
+ parser.add_argument('--diff-attr', default='cputime',
+ choices=min_val_defaults.keys(),
+ help="Buildstat attribute which to compare")
+ parser.add_argument('--min-val', default=min_val_defaults, type=float,
+ help="Filter out tasks less than MIN_VAL. "
+ "Default depends on --diff-attr.")
+ parser.add_argument('--min-absdiff', default=min_absdiff_defaults, type=float,
+ help="Filter out tasks whose difference is less than "
+ "MIN_ABSDIFF, Default depends on --diff-attr.")
+ parser.add_argument('--sort-by', default='absdiff',
+ help="Comma-separated list of field sort order. "
+ "Prepend the field name with '-' for reversed sort. "
+ "Available fields are: {}".format(', '.join(taskdiff_fields)))
+ parser.add_argument('--multi', action='store_true',
+ help="Read all buildstats from the given paths and "
+ "average over them")
+ parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat")
+ parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat")
+
+ args = parser.parse_args(argv)
+
+ # We do not nedd/want to read all buildstats if we just want to look at the
+ # package versions
+ if args.ver_diff:
+ args.multi = False
+
+ # Handle defaults for the filter arguments
+ if args.min_val is min_val_defaults:
+ args.min_val = min_val_defaults[args.diff_attr]
+ if args.min_absdiff is min_absdiff_defaults:
+ args.min_absdiff = min_absdiff_defaults[args.diff_attr]
+
+ return args
+
+
+def main(argv=None):
+ """Script entry point"""
+ args = parse_args(argv)
+ if args.debug:
+ log.setLevel(logging.DEBUG)
+
+ # Validate sort fields
+ sort_by = []
+ for field in args.sort_by.split(','):
+ if field.lstrip('-') not in taskdiff_fields:
+ log.error("Invalid sort field '%s' (must be one of: %s)" %
+ (field, ', '.join(taskdiff_fields)))
+ sys.exit(1)
+ sort_by.append(field)
+
+ try:
+ bs1 = read_buildstats(args.buildstats1, args.multi)
+ bs2 = read_buildstats(args.buildstats2, args.multi)
+
+ if args.ver_diff:
+ print_ver_diff(bs1, bs2)
+ else:
+ print_task_diff(bs1, bs2, args.diff_attr, args.min_val,
+ args.min_absdiff, sort_by)
+ except ScriptError as err:
+ log.error(str(err))
+ return 1
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())