graph-build-time: generate graphs based on timing data

This script generates graphs of packages build time, from the timing
data generated by Buildroot in the $(O)/build-time.log file.

Example usage:

  ./support/scripts/graph-build-time \
      --type=histogram --input=$(O)/build-time.log --output=foobar.pdf

Three graph types are available :

  * histogram, which creates an histogram of the build time for each
    package, decomposed by each step (extract, patch, configure,
    etc.). The order in which the packages are shown is
    configurable: by package name, by build order, or by duration
    order. See the --order option.

  * pie-packages, which creates a pie chart of the build time of
    each package (without decomposition in steps). Packages that
    contributed to less than 1% of the overall build time are all
    grouped together in an "Other" entry.

  * pie-steps, which creates a pie chart of the time spent globally
    on each step (extract, patch, configure, etc...)

The default is to generate an histogram ordered by package name.

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
[yann.morin.1998@free.fr: adapt to the format of the step-hooks build-time.log,
    add sort order by name, default to name-ordered histogram, use our colours
    for pie-charts, add alternate color-scheme, add short-options, add
    --input/-i]
Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>

1 files changed, 291 insertions, 0 deletions

diff --git a/support/scripts/graph-build-time b/support/scripts/graph-build-time
new file mode 100755
index 000000000..2216db2b6
--- /dev/null
+++ b/support/scripts/graph-build-time
@@ -0,0 +1,291 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2011 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+# Copyright (C) 2013 by Yann E. MORIN <yann.morin.1998@free.fr>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This script generates graphs of packages build time, from the timing
+# data generated by Buildroot in the $(O)/build-time.log file.
+#
+# Example usage:
+#
+#   cat $(O)/build-time.log | ./support/scripts/graph-build-time --type=histogram --output=foobar.pdf
+#
+# Three graph types are available :
+#
+#   * histogram, which creates an histogram of the build time for each
+#     package, decomposed by each step (extract, patch, configure,
+#     etc.). The order in which the packages are shown is
+#     configurable: by package name, by build order, or by duration
+#     order. See the --order option.
+#
+#   * pie-packages, which creates a pie chart of the build time of
+#     each package (without decomposition in steps). Packages that
+#     contributed to less than 1% of the overall build time are all
+#     grouped together in an "Other" entry.
+#
+#   * pie-steps, which creates a pie chart of the time spent globally
+#     on each step (extract, patch, configure, etc...)
+#
+# The default is to generate an histogram ordered by package name.
+#
+# Requirements:
+#
+#   * matplotlib (python-matplotlib on Debian/Ubuntu systems)
+#   * numpy (python-numpy on Debian/Ubuntu systems)
+#   * argparse (by default in Python 2.7, requires python-argparse if
+#     Python 2.6 is used)
+
+import matplotlib
+import numpy
+matplotlib.use('PDF')
+
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import csv
+import argparse
+import sys
+
+steps = [ 'extract', 'patch', 'configure', 'build',
+          'install-target', 'install-staging', 'install-images',
+          'install-host']
+
+default_colors = ['#e60004', '#009836', '#2e1d86', '#ffed00',
+                  '#0068b5', '#f28e00', '#940084', '#97c000']
+
+alternate_colors = ['#00e0e0', '#3f7f7f', '#ff0000', '#00c000',
+                    '#0080ff', '#c000ff', '#00eeee', '#e0e000']
+
+class Package:
+    def __init__(self, name):
+        self.name = name
+        self.steps_duration = {}
+        self.steps_start = {}
+        self.steps_end = {}
+
+    def add_step(self, step, state, time):
+        if state == "start":
+            self.steps_start[step] = time
+        else:
+            self.steps_end[step] = time
+        if self.steps_start.has_key(step) and self.steps_end.has_key(step):
+            self.steps_duration[step] = self.steps_end[step] - self.steps_start[step]
+
+    def get_duration(self, step=None):
+        if step == None:
+            duration = 0
+            for step in self.steps_duration.keys():
+                duration += self.steps_duration[step]
+            return duration
+        if self.steps_duration.has_key(step):
+            return self.steps_duration[step]
+        return 0
+
+# Generate an histogram of the time spent in each step of each
+# package.
+def pkg_histogram(data, output, order="build"):
+    n_pkgs = len(data)
+    ind = numpy.arange(n_pkgs)
+
+    if order == "duration":
+        data = sorted(data, key=lambda p: p.get_duration(), reverse=True)
+    elif order == "name":
+        data = sorted(data, key=lambda p: p.name, reverse=False)
+
+    # Prepare the vals array, containing one entry for each step
+    vals = []
+    for step in steps:
+        val = []
+        for p in data:
+            val.append(p.get_duration(step))
+        vals.append(val)
+
+    bottom = [0] * n_pkgs
+    legenditems = []
+
+    plt.figure()
+
+    # Draw the bars, step by step
+    for i in range(0, len(vals)):
+        b = plt.bar(ind+0.1, vals[i], width=0.8, color=colors[i], bottom=bottom, linewidth=0.25)
+        legenditems.append(b[0])
+        bottom = [ bottom[j] + vals[i][j] for j in range(0, len(vals[i])) ]
+
+    # Draw the package names
+    plt.xticks(ind + .6, [ p.name for p in data ], rotation=-60, rotation_mode="anchor", fontsize=8, ha='left')
+
+    # Adjust size of graph (double the width)
+    sz = plt.gcf().get_size_inches()
+    plt.gcf().set_size_inches(sz[0] * 2, sz[1])
+
+    # Add more space for the package names at the bottom
+    plt.gcf().subplots_adjust(bottom=0.2)
+
+    # Remove ticks in the graph for each package
+    axes = plt.gcf().gca()
+    for line in axes.get_xticklines():
+        line.set_markersize(0)
+
+    axes.set_ylabel('Time (seconds)')
+
+    # Reduce size of legend text
+    leg_prop = fm.FontProperties(size=6)
+
+    # Draw legend
+    plt.legend(legenditems, steps, prop=leg_prop)
+
+    if order == "name":
+        plt.title('Build time of packages\n')
+    elif order == "build":
+        plt.title('Build time of packages, by build order\n')
+    elif order == "duration":
+        plt.title('Build time of packages, by duration order\n')
+
+    # Save graph
+    plt.savefig(output)
+
+# Generate a pie chart with the time spent building each package.
+def pkg_pie_time_per_package(data, output):
+    # Compute total build duration
+    total = 0
+    for p in data:
+        total += p.get_duration()
+
+    # Build the list of labels and values, and filter the packages
+    # that account for less than 1% of the build time.
+    labels = []
+    values = []
+    other_value = 0
+    for p in data:
+        if p.get_duration() < (total * 0.01):
+            other_value += p.get_duration()
+        else:
+            labels.append(p.name)
+            values.append(p.get_duration())
+
+    labels.append('Other')
+    values.append(other_value)
+
+    plt.figure()
+
+    # Draw pie graph
+    patches, texts, autotexts = plt.pie(values, labels=labels,
+                                        autopct='%1.1f%%', shadow=True,
+                                        colors=colors)
+
+    # Reduce text size
+    proptease = fm.FontProperties()
+    proptease.set_size('xx-small')
+    plt.setp(autotexts, fontproperties=proptease)
+    plt.setp(texts, fontproperties=proptease)
+
+    plt.title('Build time per package')
+    plt.savefig(output)
+
+# Generate a pie chart with a portion for the overall time spent in
+# each step for all packages.
+def pkg_pie_time_per_step(data, output):
+    steps_values = []
+    for step in steps:
+        val = 0
+        for p in data:
+            val += p.get_duration(step)
+        steps_values.append(val)
+
+    plt.figure()
+
+    # Draw pie graph
+    patches, texts, autotexts = plt.pie(steps_values, labels=steps,
+                                        autopct='%1.1f%%', shadow=True,
+                                        colors=colors)
+
+    # Reduce text size
+    proptease = fm.FontProperties()
+    proptease.set_size('xx-small')
+    plt.setp(autotexts, fontproperties=proptease)
+    plt.setp(texts, fontproperties=proptease)
+
+    plt.title('Build time per step')
+    plt.savefig(output)
+
+# Parses the csv file passed on standard input and returns a list of
+# Package objects, filed with the duration of each step and the total
+# duration of the package.
+def read_data(input_file):
+    if input_file is None:
+        input_file = sys.stdin
+    else:
+        input_file = open(input_file)
+    reader = csv.reader(input_file, delimiter=':')
+    pkgs = []
+
+    # Auxilliary function to find a package by name in the list.
+    def getpkg(name):
+        for p in pkgs:
+            if p.name == name:
+                return p
+        return None
+
+    for row in reader:
+        time = int(row[0].strip())
+        state = row[1].strip()
+        step = row[2].strip()
+        pkg = row[3].strip()
+
+        p = getpkg(pkg)
+        if p is None:
+            p = Package(pkg)
+            pkgs.append(p)
+
+        p.add_step(step, state, time)
+
+    return pkgs
+
+parser = argparse.ArgumentParser(description='Draw build time graphs')
+parser.add_argument("--type", '-t', metavar="GRAPH_TYPE",
+                    help="Type of graph (histogram, pie-packages, pie-steps)")
+parser.add_argument("--order", '-O', metavar="GRAPH_ORDER",
+                    help="Ordering of packages: build or duration (for histogram only)")
+parser.add_argument("--alternate-colors", '-c', action="store_true",
+                    help="Use alternate colour-scheme")
+parser.add_argument("--input", '-i', metavar="OUTPUT",
+                    help="Input file (usually $(O)/build/build-time.log)")
+parser.add_argument("--output", '-o', metavar="OUTPUT", required=True,
+                    help="Output file (PDF extension)")
+args = parser.parse_args()
+
+d = read_data(args.input)
+
+if args.alternate_colors:
+    colors = alternate_colors
+else:
+    colors = default_colors
+
+if args.type == "histogram" or args.type == None:
+    if args.order == "build" or args.order == "duration" or args.order == "name":
+        pkg_histogram(d, args.output, args.order)
+    elif args.order == None:
+        pkg_histogram(d, args.output, "name")
+    else:
+        sys.stderr.write("Unknown ordering: %s\n" % args.order)
+        exit(1)
+elif args.type == "pie-packages":
+    pkg_pie_time_per_package(d, args.output)
+elif args.type == "pie-steps":
+    pkg_pie_time_per_step(d, args.output)
+else:
+    sys.stderr.write("Unknown type: %s\n" % args.type)
+    exit(1)