U-Boot

# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2014 Google, Inc
#

"""Implementation the bulider threads

This module provides the BuilderThread class, which handles calling the builder
based on the jobs provided.
"""

from collections import namedtuple
import errno
import glob
import io
import os
import shutil
import threading

from buildman import cfgutil
from u_boot_pylib import command
from u_boot_pylib import gitutil
from u_boot_pylib import tools

# Named tuple for run_commit() options that don't change during a job
#
# Members:
#     brd (Board): Board to build
#     work_dir (str): Directory to which the source will be checked out
#     work_in_output (bool): Use the output directory as the work directory and
#         don't write to a separate output directory
#     adjust_cfg (list of str): List of changes to make to .config file before
#         building. Each is one of (where C is either CONFIG_xxx or just xxx):
#             C to enable C
#             ~C to disable C
#             C=val to set the value of C (val must have quotes if C is a
#                 string Kconfig
#     fragments (str): Config fragments added to defconfig
RunRequest = namedtuple('RunRequest', ['brd', 'work_dir', 'work_in_output',
                                       'adjust_cfg', 'fragments'])

# Named tuple for _setup_build() return value
#
# Members:
#     env (dict): Environment variables for the build
#     args (list of str): Arguments to pass to make
#     config_args (list of str): Arguments for configuration
#     cwd (str): Current working directory for the build
#     src_dir (str): Source directory path
BuildSetup = namedtuple('BuildSetup', ['env', 'args', 'config_args', 'cwd',
                                       'src_dir'])

RETURN_CODE_RETRY = -1
BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl']

# Per-commit cache for the kconfig_changed_since() result.  The answer only
# depends on the commit (did any Kconfig file change since the previous
# checkout?), so one thread can do the walk and every other thread building
# the same commit reuses the boolean.
_kconfig_cache = {}
_kconfig_cache_lock = threading.Lock()


def _kconfig_changed_uncached(fname, srcdir, target):
    """Check if any Kconfig or defconfig files are newer than fname.

    This does the real work — an os.walk() of srcdir. It should only be
    called once per commit; the result is cached by kconfig_changed_since().
    """
    if not os.path.exists(fname):
        return False
    ref_time = os.path.getmtime(fname)

    # Check the board's specific defconfig if target is provided
    if target:
        defconfig = os.path.join(srcdir, 'configs', f'{target}_defconfig')
        if os.path.exists(defconfig):
            if os.path.getmtime(defconfig) > ref_time:
                return True

    for dirpath, dirnames, filenames in os.walk(srcdir):
        # Prune in-place so os.walk() skips dotdirs and build dirs
        dirnames[:] = [d for d in dirnames if d[0] != '.' and d != 'build']
        for filename in filenames:
            if filename.startswith('Kconfig'):
                filepath = os.path.join(dirpath, filename)
                if os.path.getmtime(filepath) > ref_time:
                    return True
    return False


def reset_kconfig_cache():
    """Reset the cached kconfig results, for testing"""
    _kconfig_cache.clear()


def kconfig_changed_since(fname, srcdir='.', target=None, commit_upto=None):
    """Check if any Kconfig or defconfig files are newer than the given file.

    Args:
        fname (str): Path to file to compare against (typically '.config')
        srcdir (str): Source directory to search for Kconfig/defconfig files
        target (str): Board target name; if provided, only check that board's
            defconfig file (e.g. 'sandbox' checks 'configs/sandbox_defconfig')
        commit_upto (int or None): Commit index for caching. When set, only
            the first thread to check a given commit does the walk; all
            other threads reuse that result.

    Returns:
        bool: True if any Kconfig* file (or the board's defconfig) in srcdir
            is newer than fname, False otherwise. Also returns False if fname
            doesn't exist.
    """
    if commit_upto is None:
        return _kconfig_changed_uncached(fname, srcdir, target)

    if commit_upto in _kconfig_cache:
        return _kconfig_cache[commit_upto]

    with _kconfig_cache_lock:
        if commit_upto in _kconfig_cache:
            return _kconfig_cache[commit_upto]
        result = _kconfig_changed_uncached(fname, srcdir, target)
        _kconfig_cache[commit_upto] = result
        return result

# Common extensions for images
COMMON_EXTS = ['.bin', '.rom', '.itb', '.img']

def mkdir(dirname, parents=False):
    """Make a directory if it doesn't already exist.

    Args:
        dirname (str): Directory to create, or None to do nothing
        parents (bool): True to also make parent directories

    Raises:
        OSError: File already exists
        ValueError: Trying to create the current working directory
    """
    if not dirname or os.path.exists(dirname):
        return
    try:
        if parents:
            os.makedirs(dirname)
        else:
            os.mkdir(dirname)
    except OSError as err:
        if err.errno == errno.EEXIST:
            if os.path.realpath('.') == os.path.realpath(dirname):
                raise ValueError(
                    f"Cannot create the current working directory "
                    f"'{dirname}'!") from err
        else:
            raise


def _remove_old_outputs(out_dir):
    """Remove any old output-target files

    Args:
        out_dir (str): Output directory for the build, or None for current dir

    Since we use a build directory that was previously used by another
    board, it may have produced an SPL image. If we don't remove it (i.e.
    see do_config and self.mrproper below) then it will appear to be the
    output of this build, even if it does not produce SPL images.
    """
    for elf in BASE_ELF_FILENAMES:
        fname = os.path.join(out_dir or '', elf)
        if os.path.exists(fname):
            os.remove(fname)


def copy_files(out_dir, build_dir, dirname, patterns):
    """Copy files from the build directory to the output.

    Args:
        out_dir (str): Path to output directory containing the files
        build_dir (str): Place to copy the files
        dirname (str): Source directory, '' for normal U-Boot, 'spl' for SPL
        patterns (list of str): A list of filenames to copy, each relative
           to the build directory
    """
    for pattern in patterns:
        file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
        for fname in file_list:
            target = os.path.basename(fname)
            if dirname:
                base, ext = os.path.splitext(target)
                if ext:
                    target = f'{base}-{dirname}{ext}'
            shutil.copy(fname, os.path.join(build_dir, target))


# pylint: disable=R0903
class BuilderJob:
    """Holds information about a job to be performed by a thread

    Members:
        brd: Board object to build
        commits: List of Commit objects to build
        keep_outputs: True to save build output files
        step: 1 to process every commit, n to process every nth commit
        work_in_output: Use the output directory as the work directory and
            don't write to a separate output directory.
    """
    def __init__(self):
        self.brd = None
        self.commits = []
        self.keep_outputs = False
        self.step = 1
        self.work_in_output = False


class ResultThread(threading.Thread):
    """This thread processes results from builder threads.

    It simply passes the results on to the builder. There is only one
    result thread, and this helps to serialise the build output.
    """
    def __init__(self, builder):
        """Set up a new result thread

        Args:
            builder: Builder which will be sent each result
        """
        threading.Thread.__init__(self)
        self.builder = builder

    def run(self):
        """Called to start up the result thread.

        We collect the next result job and pass it on to the build.
        """
        while True:
            result = self.builder.out_queue.get()
            self.builder.process_result(result)
            self.builder.out_queue.task_done()


class BuilderThread(threading.Thread):
    """This thread builds U-Boot for a particular board.

    An input queue provides each new job. We run 'make' to build U-Boot
    and then pass the results on to the output queue.

    Members:
        builder: The builder which contains information we might need
        thread_num: Our thread number (0-n-1), used to decide on a
            temporary directory. If this is -1 then there are no threads
            and we are the (only) main process
        mrproper: Use 'make mrproper' before each reconfigure
        per_board_out_dir: True to build in a separate persistent directory per
            board rather than a thread-specific directory
        test_exception: Used for testing; True to raise an exception instead of
            reporting the build result
        toolchain: Toolchain object to use for building, or None if not yet
            selected
    """
    # pylint: disable=R0913
    def __init__(self, builder, thread_num, mrproper, per_board_out_dir,
                 test_exception=False):
        """Set up a new builder thread"""
        threading.Thread.__init__(self)
        self.builder = builder
        self.thread_num = thread_num
        self.mrproper = mrproper
        self.per_board_out_dir = per_board_out_dir
        self.test_exception = test_exception
        self.toolchain = None

    def make(self, commit, brd, stage, cwd, *args, **kwargs):
        """Run 'make' on a particular commit and board.

        The source code will already be checked out, so the 'commit'
        argument is only for information.

        Args:
            commit (Commit): Commit that is being built
            brd (Board): Board that is being built
            stage (str): Stage of the build. Valid stages are:
                        mrproper - can be called to clean source
                        config - called to configure for a board
                        build - the main make invocation - it does the build
            cwd (str): Working directory to set, or None to leave it alone
            *args (list of str): Arguments to pass to 'make'
            **kwargs (dict): A list of keyword arguments to pass to
                command.run_one()

        Returns:
            CommandResult: Result of the make operation
        """
        return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs)

    def _build_args(self, brd, out_dir, out_rel_dir, work_dir, commit_upto):
        """Set up arguments to the args list based on the settings

        Args:
            brd (Board): Board to create arguments for
            out_dir (str): Path to output directory containing the files, or
                or None to not use a separate output directory
            out_rel_dir (str): Output directory relative to the current dir
            work_dir (str): Directory to which the source will be checked out,
                or None to use current directory
            commit_upto (int): Commit number to build (0...n-1)

        Returns:
            tuple:
                list of str: Arguments to pass to make
                str: Current working directory, or None if no commit
                str: Source directory (typically the work directory)
        """
        args = []
        cwd = work_dir
        src_dir = os.path.realpath(work_dir) if work_dir else os.getcwd()
        if commit_upto is None:
            # In this case we are building in the original source directory
            # (i.e. the current directory where buildman is invoked. The
            # output directory is set to this thread's selected work
            # directory.
            #
            # Symlinks can confuse U-Boot's Makefile since we may use '..'
            # in our path, so remove them.
            if out_dir:
                real_dir = os.path.realpath(out_dir)
                args.append(f'O={real_dir}')
            cwd = None
            src_dir = os.getcwd()
        elif out_rel_dir:
            args.append(f'O={out_rel_dir}')
        if self.builder.verbose_build:
            args.append('V=1')
        else:
            args.append('-s')
        num_jobs = self.builder.num_jobs
        if num_jobs is None and self.builder.active_boards:
            active = self.builder.active_boards
            nthreads = self.builder.num_threads
            baseline = self.builder.max_boards or nthreads
            if active < baseline:
                # Tail: ramp up 2x to compensate for make overhead
                num_jobs = max(1, nthreads * 2 // active)
            else:
                num_jobs = max(1, nthreads // active)
        if num_jobs is not None:
            args.extend(['-j', str(num_jobs)])
            args.append(f'NPROC={num_jobs}')
        if self.builder.warnings_as_errors:
            args.append('KCFLAGS=-Werror')
            args.append('HOSTCFLAGS=-Werror')
        if self.builder.allow_missing:
            args.append('BINMAN_ALLOW_MISSING=1')
        if self.builder.no_lto:
            args.append('NO_LTO=1')
        if self.builder.reproducible_builds:
            args.append('SOURCE_DATE_EPOCH=0')
        args.extend(self.builder.toolchains.get_make_arguments(brd))
        args.extend(self.toolchain.make_args())
        return args, cwd, src_dir

    def _reconfigure(self, commit, brd, cwd, args, env, config_args, config_out,
                     cmd_list, mrproper):
        """Reconfigure the build

        Args:
            commit (Commit): Commit only being built
            brd (Board): Board being built
            cwd (str): Current working directory
            args (list of str): Arguments to pass to make
            env (dict): Environment strings
            config_args (list of str): defconfig arg for this board
            cmd_list (list of str): List to add the commands to, for logging
            mrproper (bool): True to run mrproper first

        Returns:
            CommandResult object
        """
        if mrproper:
            result = self.make(commit, brd, 'mrproper', cwd, 'mrproper', *args,
                               env=env)
            config_out.write(result.combined)
            cmd_list.append([self.builder.gnu_make, 'mrproper', *args])
        result = self.make(commit, brd, 'config', cwd, *(args + config_args),
                           env=env)
        cmd_list.append([self.builder.gnu_make] + args + config_args)
        config_out.write(result.combined)
        return result

    def _build(self, commit, brd, cwd, args, env, cmd_list, config_only):
        """Perform the build

        Args:
            commit (Commit): Commit only being built
            brd (Board): Board being built
            cwd (str): Current working directory
            args (list of str): Arguments to pass to make
            env (dict): Environment strings
            cmd_list (list of str): List to add the commands to, for logging
            config_only (bool): True if this is a config-only build (using the
                'make cfg' target)

        Returns:
            CommandResult object
        """
        if config_only:
            args.append('cfg')
        elif self.builder.build_target:
            args.append(self.builder.build_target)
        result = self.make(commit, brd, 'build', cwd, *args, env=env)
        cmd_list.append([self.builder.gnu_make] + args)
        if (result.return_code == 2 and
            ('Some images are invalid' in result.stderr)):
            # This is handled later by the check for output in stderr
            result.return_code = 0
        return result

    def _read_done_file(self, commit_upto, brd, force_build,
                        force_build_failures):
        """Check the 'done' file and see if this commit should be built

        Args:
            commit (Commit): Commit only being built
            brd (Board): Board being built
            force_build (bool): Force a build even if one was previously done
            force_build_failures (bool): Force a bulid if the previous result
                showed failure

        Returns:
            tuple:
                bool: True if build should be built
                CommandResult: if there was a previous run:
                    - already_done set to True
                    - return_code set to return code
                    - result.stderr set to 'bad' if stderr output was recorded
        """
        result = command.CommandResult()
        result.remote = None
        done_file = self.builder.get_done_file(commit_upto, brd.target)
        result.already_done = os.path.exists(done_file)
        result.kconfig_reconfig = False
        will_build = (force_build or force_build_failures or
            not result.already_done)
        if result.already_done:
            with open(done_file, 'r', encoding='utf-8') as outf:
                try:
                    result.return_code = int(outf.readline())
                except ValueError:
                    # The file may be empty due to running out of disk space.
                    # Try a rebuild
                    result.return_code = RETURN_CODE_RETRY

            # Check the signal that the build needs to be retried
            if result.return_code == RETURN_CODE_RETRY:
                will_build = True
            elif will_build:
                err_file = self.builder.get_err_file(commit_upto, brd.target)
                if os.path.exists(err_file) and os.stat(err_file).st_size:
                    result.stderr = 'bad'
                elif not force_build:
                    # The build passed, so no need to build it again
                    will_build = False

        return will_build, result

    def _decide_dirs(self, req):
        """Decide the output directory to use

        Args:
            req (RunRequest): Run request (see RunRequest for details)

        Returns:
            tuple:
                out_dir (str): Output directory for the build
                out_rel_dir (str): Output directory relative to the current dir
        """
        if req.work_in_output or self.builder.in_tree:
            out_rel_dir = None
            out_dir = req.work_dir
        else:
            if self.per_board_out_dir:
                out_rel_dir = os.path.join('..', req.brd.target)
            else:
                out_rel_dir = 'build'
            out_dir = os.path.join(req.work_dir, out_rel_dir)
        return out_dir, out_rel_dir

    def _checkout(self, commit_upto, work_dir):
        """Checkout the right commit

        Args:
            commit_upto (int): Commit number to build (0...n-1)
            work_dir (str): Directory to which the source will be checked out

        Returns:
            Commit: Commit being built, or 'current' for current source
        """
        if self.builder.commits:
            commit = self.builder.commits[commit_upto]
            if self.builder.checkout:
                git_dir = os.path.join(work_dir, '.git')
                gitutil.checkout(commit.hash, git_dir, work_dir, force=True)
        else:
            commit = 'current'
        return commit

    def _setup_build(self, req, commit_upto, out_dir, out_rel_dir):
        """Set up the build environment and arguments

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            commit_upto (int): Commit number to build (0...n-1)
            out_dir (str): Output directory for the build, or None to use
               current
            out_rel_dir (str): Output directory relative to the current dir

        Returns:
            BuildSetup: Build setup (see BuildSetup for details)
        """
        env = self.builder.make_environment(self.toolchain)
        if out_dir and not os.path.exists(out_dir):
            mkdir(out_dir)

        args, cwd, src_dir = self._build_args(req.brd, out_dir, out_rel_dir,
                                              req.work_dir, commit_upto)
        if req.brd.extended:
            config_args = [f'{req.brd.orig_target}_defconfig']
            for frag in req.brd.extended.fragments:
                fname = os.path.join(f'{frag}.config')
                config_args.append(fname)
        else:
            config_args = [f'{req.brd.target}_defconfig']
        if req.fragments is not None:
            config_args.extend(req.fragments.split(','))

        _remove_old_outputs(out_dir)

        return BuildSetup(env, args, config_args, cwd, src_dir)

    def _reconfig_if_needed(self, req, setup, commit, config_out, cmd_list,
                            out_dir, do_config, mrproper, result):
        """Reconfigure the build if needed

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            setup (BuildSetup): Build setup (see BuildSetup for details)
            commit (Commit): Commit being built
            config_out (StringIO): Buffer for configuration output
            cmd_list (list of str): List to add the commands to, for logging
            out_dir (str): Output directory for the build
            do_config (bool): True to run a make <board>_defconfig on the source
            mrproper (bool): True to run mrproper first
            result (CommandResult): Previous result

        Returns:
            tuple:
                result (CommandResult): Result of the reconfiguration
                do_config (bool): Whether config is still needed
                cfg_file (str): Path to the .config file
        """
        cfg_file = os.path.join(out_dir or '', '.config')
        if do_config or req.adjust_cfg:
            result = self._reconfigure(
                commit, req.brd, setup.cwd, setup.args, setup.env,
                setup.config_args, config_out, cmd_list, mrproper)
            do_config = False   # No need to configure next time
            if req.adjust_cfg:
                merge_result = cfgutil.run_merge_config(
                    setup.cwd, out_dir, cfg_file, req.adjust_cfg, setup.env)
                config_out.write(merge_result.combined)
                if merge_result.return_code:
                    result = merge_result
        return result, do_config, cfg_file

    def _build_and_get_result(self, req, setup, commit, cmd_list, config_out,
                              cfg_file, config_only, result):
        """Perform the build and finalise the result

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            setup (BuildSetup): Build setup (see BuildSetup for details)
            commit (Commit): Commit being built
            cmd_list (list of str): List to add the commands to, for logging
            config_out (StringIO): Buffer for configuration output
            cfg_file (str): Path to the .config file
            config_only (bool): Only configure the source, do not build it
            result (CommandResult): Previous result

        Returns:
            CommandResult: Result of the build
        """
        if result.return_code == 0:
            result = self._build(commit, req.brd, setup.cwd, setup.args,
                                 setup.env, cmd_list, config_only)
            if req.adjust_cfg:
                errs = cfgutil.check_cfg_file(cfg_file, req.adjust_cfg)
                if errs:
                    result.stderr += errs
                    result.return_code = 1
        result.stderr = result.stderr.replace(setup.src_dir + '/', '')
        if self.builder.verbose_build:
            result.stdout = config_out.getvalue() + result.stdout
        result.cmd_list = cmd_list
        return result

    def _config_and_build(self, req, commit_upto, do_config, mrproper,
                          config_only, commit, out_dir, out_rel_dir, result):
        """Do the build, configuring first if necessary

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            commit_upto (int): Commit number to build (0...n-1)
            do_config (bool): True to run a make <board>_defconfig on the source
            mrproper (bool): True to run mrproper first
            config_only (bool): Only configure the source, do not build it
            commit (Commit): Commit being built
            out_dir (str): Output directory for the build, or None to use
               current
            out_rel_dir (str): Output directory relative to the current dir
            result (CommandResult): Previous result

        Returns:
            tuple:
                result (CommandResult): Result of the build
                do_config (bool): indicates whether 'make config' is needed on
                    the next incremental build
        """
        setup = self._setup_build(req, commit_upto, out_dir, out_rel_dir)

        config_out = io.StringIO()
        cmd_list = []

        result, do_config, cfg_file = self._reconfig_if_needed(
            req, setup, commit, config_out, cmd_list, out_dir, do_config,
            mrproper, result)

        result = self._build_and_get_result(
            req, setup, commit, cmd_list, config_out, cfg_file, config_only,
            result)

        return result, do_config

    def _do_build(self, req, commit_upto, do_config, mrproper, config_only,
                  out_dir, out_rel_dir, result):
        """Perform a build if a toolchain can be obtained

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            commit_upto (int): Commit number to build (0...n-1)
            do_config (bool): True to run a make <board>_defconfig on the source
            mrproper (bool): True to run mrproper first
            config_only (bool): Only configure the source, do not build it
            out_dir (str): Output directory for the build
            out_rel_dir (str): Output directory relative to the current dir
            result (CommandResult): Previous result

        Returns:
            tuple:
                result (CommandResult): Result of the build
                do_config (bool): Whether config is needed next time
                kconfig_reconfig (bool): Whether Kconfig triggered a reconfig
        """
        kconfig_reconfig = False

        # We are going to have to build it. First, get a toolchain
        if not self.toolchain:
            try:
                self.toolchain = self.builder.toolchains.select(req.brd.arch)
            except ValueError as err:
                result.return_code = 10
                result.stdout = ''
                result.stderr = f'Tool chain error for {req.brd.arch}: {err}'

        if self.toolchain:
            commit = self._checkout(commit_upto, req.work_dir)

            # Check if Kconfig files have changed since last config. Skip
            # when do_config is already True (e.g. first commit) since
            # defconfig will run anyway. This avoids an expensive os.walk()
            # of the source tree that can be very slow when many threads
            # do it simultaneously.
            if self.builder.kconfig_check and not do_config:
                config_file = os.path.join(out_dir, '.config')
                if kconfig_changed_since(config_file, req.work_dir,
                                         req.brd.target, commit_upto):
                    kconfig_reconfig = True
                    do_config = True

            result, do_config = self._config_and_build(
                req, commit_upto, do_config, mrproper, config_only,
                commit, out_dir, out_rel_dir, result)

        result.already_done = False
        result.kconfig_reconfig = kconfig_reconfig
        return result, do_config, kconfig_reconfig

    def run_commit(self, req, commit_upto, do_config, mrproper, config_only,
                   force_build, force_build_failures):
        """Build a particular commit.

        If the build is already done, and we are not forcing a build, we skip
        the build and just return the previously-saved results.

        Args:
            req (RunRequest): Run request (see RunRequest for details)
            commit_upto (int): Commit number to build (0...n-1)
            do_config (bool): True to run a make <board>_defconfig on the source
            mrproper (bool): True to run mrproper first
            config_only (bool): Only configure the source, do not build it
            force_build (bool): Force a build even if one was previously done
            force_build_failures (bool): Force a build if the previous result
                showed failure

        Returns:
            tuple:
                CommandResult: Results of the build
                bool: Indicates whether 'make config' is still needed
        """
        # Create a default result - it will be overwritte by the call to
        # self.make() below, in the event that we do a build.
        out_dir, out_rel_dir = self._decide_dirs(req)

        # Check if the job was already completed last time
        will_build, result = self._read_done_file(commit_upto, req.brd,
                                                  force_build,
                                                  force_build_failures)

        if will_build:
            result, do_config, _ = self._do_build(
                req, commit_upto, do_config, mrproper, config_only,
                out_dir, out_rel_dir, result)

        result.remote = None
        result.toolchain = self.toolchain
        result.brd = req.brd
        result.commit_upto = commit_upto
        result.out_dir = out_dir
        return result, do_config

    def _process_elf_file(self, result, fname, env):
        """Process an ELF file to extract size information

        Runs nm, objdump, and size on the ELF file and writes the output
        to appropriate files.

        Args:
            result (CommandResult): Result containing build information
            fname (str): ELF filename to process
            env (dict): Environment variables for the commands

        Returns:
            str: Size line with rodata size appended, or None if size failed
        """
        cmd = [f'{self.toolchain.cross}nm', '--size-sort', fname]
        nm_result = command.run_one(*cmd, capture=True,
                                    capture_stderr=True,
                                    cwd=result.out_dir,
                                    raise_on_error=False, env=env)
        if nm_result.stdout:
            nm_fname = self.builder.get_func_sizes_file(
                result.commit_upto, result.brd.target, fname)
            with open(nm_fname, 'w', encoding='utf-8') as outf:
                print(nm_result.stdout, end=' ', file=outf)

        cmd = [f'{self.toolchain.cross}objdump', '-h', fname]
        dump_result = command.run_one(*cmd, capture=True,
                                      capture_stderr=True,
                                      cwd=result.out_dir,
                                      raise_on_error=False, env=env)
        rodata_size = ''
        if dump_result.stdout:
            objdump = self.builder.get_objdump_file(result.commit_upto,
                            result.brd.target, fname)
            with open(objdump, 'w', encoding='utf-8') as outf:
                print(dump_result.stdout, end=' ', file=outf)
            for line in dump_result.stdout.splitlines():
                fields = line.split()
                if len(fields) > 5 and fields[1] == '.rodata':
                    rodata_size = fields[2]

        cmd = [f'{self.toolchain.cross}size', fname]
        size_result = command.run_one(*cmd, capture=True,
                                      capture_stderr=True,
                                      cwd=result.out_dir,
                                      raise_on_error=False, env=env)
        if size_result.stdout:
            return size_result.stdout.splitlines()[1] + ' ' + rodata_size
        return None

    def _write_toolchain_result(self, result, done_file, build_dir,
                                maybe_aborted, work_in_output):
        """Write build result and toolchain information

        Args:
            result (CommandResult): Result to write
            done_file (str): Path to the 'done' file
            build_dir (str): Build directory path
            maybe_aborted (bool): True if the build may have been aborted
            work_in_output (bool): Use the output directory as the work
                directory and don't write to a separate output directory
        """
        # Write the build result and toolchain information.
        with open(done_file, 'w', encoding='utf-8') as outf:
            if maybe_aborted:
                # Special code to indicate we need to retry
                outf.write(f'{RETURN_CODE_RETRY}')
            else:
                outf.write(f'{result.return_code}')
        with open(os.path.join(build_dir, 'toolchain'), 'w',
                  encoding='utf-8') as outf:
            print('gcc', result.toolchain.gcc, file=outf)
            print('path', result.toolchain.path, file=outf)
            print('cross', result.toolchain.cross, file=outf)
            print('arch', result.toolchain.arch, file=outf)
            outf.write(f'{result.return_code}')

        # Write out the image and function size information and an objdump
        env = self.builder.make_environment(self.toolchain)
        with open(os.path.join(build_dir, 'out-env'), 'wb') as outf:
            for var in sorted(env.keys()):
                outf.write(b'%s="%s"' % (var, env[var]))

        with open(os.path.join(build_dir, 'out-cmd'), 'w',
                  encoding='utf-8') as outf:
            for cmd in result.cmd_list:
                print(' '.join(cmd), file=outf)

        lines = []
        for fname in BASE_ELF_FILENAMES:
            line = self._process_elf_file(result, fname, env)
            if line:
                lines.append(line)

        # Extract the environment from U-Boot and dump it out
        cmd = [f'{self.toolchain.cross}objcopy', '-O', 'binary',
               '-j', '.rodata.default_environment',
               'env/built-in.o', 'uboot.env']
        command.run_one(*cmd, capture=True, capture_stderr=True,
                        cwd=result.out_dir, raise_on_error=False, env=env)
        if not work_in_output:
            copy_files(result.out_dir, build_dir, '', ['uboot.env'])

        # Write out the image sizes file. This is similar to the output
        # of binutil's 'size' utility, but it omits the header line and
        # adds an additional hex value at the end of each line for the
        # rodata size
        if lines:
            sizes = self.builder.get_sizes_file(result.commit_upto,
                            result.brd.target)
            with open(sizes, 'w', encoding='utf-8') as outf:
                print('\n'.join(lines), file=outf)

    def _write_result(self, result, keep_outputs, work_in_output):
        """Write a built result to the output directory.

        Args:
            result (CommandResult): result to write
            keep_outputs (bool): True to store the output binaries, False
                to delete them
            work_in_output (bool): Use the output directory as the work
                directory and don't write to a separate output directory.
        """
        # If we think this might have been aborted with Ctrl-C, record the
        # failure but not that we are 'done' with this board. A retry may fix
        # it.
        maybe_aborted = result.stderr and 'No child processes' in result.stderr

        if result.return_code >= 0 and result.already_done:
            return

        # Write the output and stderr
        output_dir = self.builder.get_output_dir(result.commit_upto)
        mkdir(output_dir)
        build_dir = self.builder.get_build_dir(result.commit_upto,
                result.brd.target)
        mkdir(build_dir)

        outfile = os.path.join(build_dir, 'log')
        with open(outfile, 'w', encoding='utf-8') as outf:
            if result.stdout:
                outf.write(result.stdout)

        errfile = self.builder.get_err_file(result.commit_upto,
                result.brd.target)
        if result.stderr:
            with open(errfile, 'w', encoding='utf-8') as outf:
                outf.write(result.stderr)
        elif os.path.exists(errfile):
            os.remove(errfile)

        # Fatal error
        if result.return_code < 0:
            return

        done_file = self.builder.get_done_file(result.commit_upto,
                result.brd.target)
        if result.toolchain:
            self._write_toolchain_result(result, done_file, build_dir,
                                         maybe_aborted, work_in_output)
        else:
            # Indicate that the build failure due to lack of toolchain
            tools.write_file(done_file, '2\n', binary=False)

        if not work_in_output:
            # Write out the configuration files, with a special case for SPL
            for dirname in ['', 'spl', 'tpl']:
                copy_files(
                    result.out_dir, build_dir, dirname,
                    ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg',
                     '.config', 'include/autoconf.mk',
                     'include/generated/autoconf.h'])

            # Now write the actual build output
            if keep_outputs:
                to_copy = ['u-boot*', '*.map', 'MLO', 'SPL',
                           'include/autoconf.mk', 'spl/u-boot-spl*',
                           'tpl/u-boot-tpl*', 'vpl/u-boot-vpl*']
                to_copy += [f'*{ext}' for ext in COMMON_EXTS]
                copy_files(result.out_dir, build_dir, '', to_copy)

    def _send_result(self, result):
        """Send a result to the builder for processing

        Args:
            result (CommandResult): results of the build

        Raises:
            ValueError: self.test_exception is true (for testing)
        """
        if self.test_exception:
            raise ValueError('test exception')
        if self.thread_num != -1:
            self.builder.out_queue.put(result)
        else:
            self.builder.process_result(result)

    def run_job(self, job):
        """Run a single job

        A job consists of a building a list of commits for a particular board.

        Args:
            job (Job): Job to build

        Raises:
            ValueError: Thread was interrupted
        """
        brd = job.brd
        work_dir = self.builder.get_thread_dir(self.thread_num)
        self.toolchain = None
        req = RunRequest(brd, work_dir, job.work_in_output, job.adjust_cfg,
                         job.fragments)
        if job.commits:
            # Run 'make board_defconfig' on the first commit
            do_config = True
            commit_upto  = 0
            force_build = False
            for commit_upto in range(0, len(job.commits), job.step):
                result, request_config = self.run_commit(
                        req, commit_upto, do_config, self.mrproper,
                        self.builder.config_only,
                        force_build or self.builder.force_build,
                        self.builder.force_build_failures)
                failed = result.return_code
                did_config = do_config
                if failed and not do_config and not self.mrproper:
                    # If our incremental build failed, try building again
                    # with a reconfig.
                    if self.builder.force_config_on_failure:
                        result, request_config = self.run_commit(
                            req, commit_upto, True,
                            self.mrproper or self.builder.fallback_mrproper,
                            False, True, False)
                        did_config = True
                if not self.builder.force_reconfig:
                    do_config = request_config

                # If we built that commit, then config is done. But if we got
                # an warning, reconfig next time to force it to build the same
                # files that created warnings this time. Otherwise an
                # incremental build may not build the same file, and we will
                # think that the warning has gone away.
                # We could avoid this by using -Werror everywhere...
                # For errors, the problem doesn't happen, since presumably
                # the build stopped and didn't generate output, so will retry
                # that file next time. So we could detect warnings and deal
                # with them specially here. For now, we just reconfigure if
                # anything goes work.
                # Of course this is substantially slower if there are build
                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
                # have problems).
                if (failed and not result.already_done and not did_config and
                        self.builder.force_config_on_failure):
                    # If this build failed, try the next one with a
                    # reconfigure.
                    # Sometimes if the board_config.h file changes it can mess
                    # with dependencies, and we get:
                    # make: *** No rule to make target `include/autoconf.mk',
                    #     needed by `depend'.
                    do_config = True
                    force_build = True
                else:
                    force_build = False
                    if self.builder.force_config_on_failure:
                        if failed:
                            do_config = True
                    result.commit_upto = commit_upto
                    if result.return_code < 0:
                        raise ValueError('Interrupt')

                # We have the build results, so output the result
                self._write_result(result, job.keep_outputs, job.work_in_output)
                self._send_result(result)
        else:
            # Just build the currently checked-out build
            result, request_config = self.run_commit(
                        req, None, True, self.mrproper,
                        self.builder.config_only, True,
                        self.builder.force_build_failures)
            failed = result.return_code
            if failed and not self.mrproper:
                result, request_config = self.run_commit(
                            req, None, True, self.builder.fallback_mrproper,
                            self.builder.config_only, True,
                            self.builder.force_build_failures)

            result.commit_upto = 0
            self._write_result(result, job.keep_outputs, job.work_in_output)
            self._send_result(result)

    def run(self):
        """Our thread's run function

        This thread picks a job from the queue, runs it, and then goes to the
        next job.
        """
        while True:
            job = self.builder.queue.get()
            with self.builder.active_lock:
                self.builder.active_boards += 1
            try:
                self.run_job(job)
            except Exception as exc:  # pylint: disable=W0718
                print('Thread exception (use -T0 to run without threads):',
                      exc)
                self.builder.thread_exceptions.append(exc)
            finally:
                with self.builder.active_lock:
                    self.builder.active_boards -= 1
            self.builder.queue.task_done()