U-Boot

# SPDX-License-Identifier: GPL-2.0+
# Copyright (c) 2012 The Chromium OS Authors.

"""Handles finding and using toolchains for building U-Boot."""

import re
import glob
from html.parser import HTMLParser
import os
import sys
import urllib.error
import urllib.parse
import urllib.request

from buildman import bsettings
from u_boot_pylib import command
from u_boot_pylib import terminal
from u_boot_pylib import tools

# Toolchain priority levels (lower number = higher priority):
#   PRIORITY_FULL_PREFIX: Explicit [toolchain-prefix] path exists as a file
#   PRIORITY_PREFIX_GCC: [toolchain-prefix] path + 'gcc' exists as a file
#   PRIORITY_PREFIX_GCC_PATH: [toolchain-prefix] path + 'gcc' found in PATH
#   PRIORITY_DOWNLOADED: Toolchain downloaded via --fetch-arch
#   PRIORITY_CALC: Toolchain found by scanning [toolchain] paths; actual
#       priority is PRIORITY_CALC + offset based on toolchain name
(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
    PRIORITY_DOWNLOADED, PRIORITY_CALC) = list(range(5))

# Environment variable / argument types for get_env_args()
(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)

# Matches a repeated prefix, e.g. 'aarch64-linux-aarch64-linux-gcc'
RE_DOUBLED_PREFIX = re.compile(r'^(.+)\1gcc$')


class MyHTMLParser(HTMLParser):
    """Simple class to collect links from a page

    Public members:
        arch_link (str): URL of the toolchain for the desired architecture,
            or None if not found
        links (list of str): List of URLs for all .xz archives found
    """

    def __init__(self, arch):
        """Create a new parser

        After the parser runs, self.links will be set to a list of the links
        to .xz archives found in the page, and self.arch_link will be set to
        the one for the given architecture (or None if not found).

        Args:
            arch (str): Architecture to search for
        """
        HTMLParser.__init__(self)
        self.arch_link = None
        self.links = []
        self._re_arch = re.compile(f'[-_]{arch}-')

    def handle_starttag(self, tag, attrs):
        """Handle a start tag in the HTML being parsed"""
        if tag == 'a':
            for attr, value in attrs:
                if attr == 'href':
                    if value and value.endswith('.xz'):
                        self.links.append(value)
                        if self._re_arch.search(value):
                            self.arch_link = value


class Toolchain:
    """A single toolchain

    Public members:
        gcc (str): Full path to C compiler
        path (str): Directory path containing C compiler
        cross (str): Cross compile string, e.g. 'arm-linux-'
        arch (str): Architecture of toolchain as determined from the first
            component of the filename. E.g. arm-linux-gcc becomes arm
        priority (int): Toolchain priority (0=highest, 20=lowest)
        override_toolchain (str): Toolchain to use for sandbox, overriding the
            normal one
        ok (bool): True if the toolchain works, False otherwise
    """
    # pylint: disable=too-many-arguments,too-many-positional-arguments
    def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
                 arch=None, override_toolchain=None):
        """Create a new toolchain object.

        Args:
            fname (str): Filename of the gcc component, possibly with ~ or
                $HOME in it
            test (bool): True to run the toolchain to test it
            verbose (bool): True to print out the information
            priority (int): Priority to use for this toolchain, or
                PRIORITY_CALC to calculate it
            arch (str): Architecture of toolchain, or None to detect from
                filename
            override_toolchain (str): Toolchain to use for sandbox, or None
        """
        fname = os.path.expanduser(fname)
        self.gcc = fname
        self.path = os.path.dirname(fname)
        self.override_toolchain = override_toolchain

        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
        basename = os.path.basename(fname)
        pos = basename.rfind('-')
        self.cross = basename[:pos + 1] if pos != -1 else ''

        # The architecture is the first part of the name
        pos = self.cross.find('-')
        if arch:
            self.arch = arch
        else:
            self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
        if self.arch == 'sandbox' and override_toolchain:
            self.gcc = override_toolchain

        env = self.make_environment(False)

        # As a basic sanity check, run the C compiler with --version
        cmd = [fname, '--version']
        if priority == PRIORITY_CALC:
            self.priority = self.get_priority(fname)
        else:
            self.priority = priority
        if test:
            result = command.run_one(*cmd, capture=True, env=env,
                                     raise_on_error=False)
            self.ok = result.return_code == 0
            if verbose:
                print('Tool chain test: ', end=' ')
                if self.ok:
                    print(f"OK, arch='{self.arch}', priority {self.priority}")
                else:
                    print('BAD')
                    print(f"Command: {' '.join(cmd)}")
                    print(result.stdout)
                    print(result.stderr)
        else:
            self.ok = True

    def get_priority(self, fname):
        """Return the priority of the toolchain.

        Toolchains are ranked according to their suitability by their
        filename prefix.

        Args:
            fname (str): Filename of toolchain

        Returns:
            int: Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
        """
        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
            '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
            '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
            '-linux-gnueabihf', '-le-linux', '-uclinux']
        for prio, item in enumerate(priority_list):
            if item in fname:
                return PRIORITY_CALC + prio
        return PRIORITY_CALC + prio

    def get_wrapper(self):
        """Get toolchain wrapper from the setting file.
        """
        value = ''
        for _, value in bsettings.get_items('toolchain-wrapper'):
            if not value:
                print("Warning: Wrapper not found")
        if value:
            value = value + ' '

        return value

    def get_env_args(self, which):
        """Get an environment variable/args value based on the the toolchain

        Args:
            which (int): VAR_... value to get

        Returns:
            str: Value of that environment variable or arguments
        """
        if which == VAR_CROSS_COMPILE:
            wrapper = self.get_wrapper()
            base = '' if self.arch == 'sandbox' else self.path
            if (base == '' and self.cross == ''):
                return ''
            return wrapper + os.path.join(base, self.cross)
        if which == VAR_PATH:
            return self.path
        if which == VAR_ARCH:
            return self.arch
        if which == VAR_MAKE_ARGS:
            args = self.make_args()
            if args:
                return ' '.join(args)
            return ''
        raise ValueError(f'Unknown arg to GetEnvArgs ({which})')

    def make_environment(self, full_path, env=None):
        """Returns an environment for using the toolchain.

        This takes the current environment and adds CROSS_COMPILE so that
        the tool chain will operate correctly. This also disables localized
        output and possibly Unicode encoded output of all build tools by
        adding LC_ALL=C. For the case where full_path is False, it prepends
        the toolchain to PATH

        Note that os.environb is used to obtain the environment, since in some
        cases the environment many contain non-ASCII characters and we see
        errors like:

          UnicodeEncodeError: 'utf-8' codec can't encode characters in position
             569-570: surrogates not allowed

        When running inside a Python venv, care is taken not to put the
        toolchain path before the venv path, so that builds initiated by
        buildman will still respect the venv.

        Args:
            full_path (bool): Return the full path in CROSS_COMPILE and don't
                set PATH
            env (dict of bytes): Original environment, used for testing

        Returns:
            dict of bytes: Environment to use. This is based on the current
                environment, with changes as needed to CROSS_COMPILE, PATH
                and LC_ALL.
        """
        env = dict(env or os.environb)

        wrapper = self.get_wrapper()

        if self.override_toolchain:
            # We'll use MakeArgs() to provide this
            pass
        elif full_path and self.cross:
            env[b'CROSS_COMPILE'] = tools.to_bytes(
                wrapper + os.path.join(self.path, self.cross))
        elif self.cross:
            env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross)

            # Detect a Python sandbox and avoid defeating it
            if sys.prefix != sys.base_prefix:
                paths = env[b'PATH'].split(b':')
                new_paths = []
                to_insert = tools.to_bytes(self.path)
                insert_after = tools.to_bytes(sys.prefix)
                for path in paths:
                    new_paths.append(path)
                    if to_insert and path.startswith(insert_after):
                        new_paths.append(to_insert)
                        to_insert = None
                if to_insert:
                    new_paths.append(to_insert)
                env[b'PATH'] = b':'.join(new_paths)
            else:
                env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH']

        env[b'LC_ALL'] = b'C'

        return env

    def make_args(self):
        """Create the 'make' arguments for a toolchain

        This is only used when the toolchain is being overridden. Since the
        U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
        environment (and MakeEnvironment()) to override these values. This
        function returns the arguments to accomplish this.

        Returns:
            List of arguments to pass to 'make'
        """
        if self.override_toolchain:
            return [f'HOSTCC={self.override_toolchain}',
                    f'CC={self.override_toolchain}']
        return []


class Toolchains:
    """Manage a list of toolchains for building U-Boot

    We select one toolchain for each architecture type

    Public members:
        toolchains: Dict of Toolchain objects, keyed by architecture name
        prefixes: Dict of prefixes to check, keyed by architecture. This can
            be a full path and toolchain prefix, for example
            {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
            something on the search path, for example
            {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
        paths: List of paths to check for toolchains (may contain wildcards)
    """

    def __init__(self, override_toolchain=None):
        self.toolchains = {}
        self.prefixes = {}
        self.paths = []
        self.download_paths = set()
        self.override_toolchain = override_toolchain
        self._make_flags = dict(bsettings.get_items('make-flags'))

    def get_path_list(self, show_warning=True):
        """Get a list of available toolchain paths

        Args:
            show_warning (bool): True to show a warning if there are no tool
                chains.

        Returns:
            list of str: List of paths to toolchains mentioned in the
                [toolchain] section of the settings file.
        """
        toolchains = bsettings.get_items('toolchain')
        if show_warning and not toolchains:
            print(f"Warning: No tool chains. Please run 'buildman "
                  f"--fetch-arch all' to download all available toolchains, or "
                  f"add a [toolchain] section to your buildman config file "
                  f"{bsettings.config_fname}. See buildman.rst for details")

        paths = []
        for _, value in toolchains:
            fname = os.path.expanduser(value)
            if '*' in value:
                paths += glob.glob(fname)
            else:
                paths.append(fname)
        return paths

    def get_settings(self, show_warning=True):
        """Get toolchain settings from the settings file.

        Args:
            show_warning (bool): True to show a warning if there are no tool
                chains.
        """
        self.prefixes = bsettings.get_items('toolchain-prefix')
        self.paths += self.get_path_list(show_warning)

        # Track which paths are from downloaded toolchains
        for name, value in bsettings.get_items('toolchain'):
            if name == 'download':
                fname = os.path.expanduser(value)
                if '*' in value:
                    self.download_paths.update(glob.glob(fname))
                else:
                    self.download_paths.add(fname)

    # pylint: disable=too-many-arguments,too-many-positional-arguments
    def add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
            arch=None):
        """Add a toolchain to our list

        We select the given toolchain as our preferred one for its
        architecture if it is a higher priority than the others.

        Args:
            fname (str): Filename of toolchain's gcc driver
            test (bool): True to run the toolchain to test it
            verbose (bool): True to print out progress information
            priority (int): Priority to use for this toolchain
            arch (str): Toolchain architecture, or None if not known
        """
        toolchain = Toolchain(fname, test, verbose, priority, arch,
                              self.override_toolchain)
        add_it = toolchain.ok
        if add_it:
            if toolchain.arch in self.toolchains:
                add_it = (toolchain.priority <
                          self.toolchains[toolchain.arch].priority)
            if add_it:
                self.toolchains[toolchain.arch] = toolchain
            elif verbose:
                print(f"Toolchain '{toolchain.gcc}' at priority "
                      f"{toolchain.priority} will be ignored because another "
                      f"toolchain for arch '{toolchain.arch}' has priority "
                      f"{self.toolchains[toolchain.arch].priority}")

    @staticmethod
    def is_doubled_prefix(fname):
        """Check if a gcc filename has a doubled prefix

        Some toolchain tarballs contain symlinks with the cross-compile prefix
        repeated, e.g. 'x86_64-linux-x86_64-linux-gcc'. These are not valid
        toolchains and should be ignored.

        Args:
            fname (str): Filename to check (basename, not full path)

        Returns:
            bool: True if the prefix is doubled, False otherwise
        """
        return bool(RE_DOUBLED_PREFIX.match(fname))

    def scan_path(self, path, verbose):
        """Scan a path for a valid toolchain

        Args:
            path (str): Path to scan
            verbose (bool): True to print out progress information

        Returns:
            list of str: Filenames of C compilers found
        """
        fnames = []
        for subdir in ['.', 'bin', 'usr/bin']:
            dirname = os.path.join(path, subdir)
            if verbose:
                print(f"      - looking in '{dirname}'")
            for fname in glob.glob(dirname + '/*gcc'):
                basename = os.path.basename(fname)
                if self.is_doubled_prefix(basename):
                    if verbose:
                        print(f"         - ignoring '{fname}' (doubled prefix)")
                    continue
                if verbose:
                    print(f"         - found '{fname}'")
                fnames.append(fname)
        return fnames

    def scan_path_env(self, fname):
        """Scan the PATH environment variable for a given filename.

        Args:
            fname (str): Filename to scan for

        Returns:
            list of str: List of matching pathnames, or [] if none
        """
        pathname_list = []
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            pathname = os.path.join(path, fname)
            if os.path.exists(pathname):
                pathname_list.append(pathname)
        return pathname_list

    def scan(self, verbose, raise_on_error=True):
        """Scan for available toolchains and select the best for each arch.

        We look for all the toolchains we can file, figure out the
        architecture for each, and whether it works. Then we select the
        highest priority toolchain for each arch.

        Args:
            verbose (bool): True to print out progress information
            raise_on_error (bool): True to raise an error if a toolchain is
                not found
        """
        if verbose:
            print('Scanning for tool chains')
        for name, value in self.prefixes:
            fname = os.path.expanduser(value)
            if verbose:
                print(f"   - scanning prefix '{fname}'")
            if os.path.exists(fname):
                self.add(fname, True, verbose, PRIORITY_FULL_PREFIX, name)
                continue
            fname += 'gcc'
            if os.path.exists(fname):
                self.add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
                continue
            fname_list = self.scan_path_env(fname)
            for f in fname_list:
                self.add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
            if not fname_list:
                msg = f"No tool chain found for prefix '{fname}'"
                if raise_on_error:
                    raise ValueError(msg)
                print(f'Error: {msg}')
        for path in self.paths:
            if verbose:
                print(f"   - scanning path '{path}'")
            fnames = self.scan_path(path, verbose)
            priority = (PRIORITY_DOWNLOADED if path in self.download_paths
                        else PRIORITY_CALC)
            for fname in fnames:
                self.add(fname, True, verbose, priority)

    def list(self):
        """List out the selected toolchains for each architecture"""
        col = terminal.Color()
        print(col.build(
            col.BLUE,
            f'List of available toolchains ({len(self.toolchains)}):'))
        if len(self.toolchains):
            for key, value in sorted(self.toolchains.items()):
                print(f'{key:10}: {value.gcc}')
        else:
            print('None')

    def select(self, arch):
        """Returns the toolchain for a given architecture

        Args:
            arch (str): Name of architecture (e.g. 'arm', 'ppc_8xx')

        Returns:
            Toolchain: toolchain object, or None if none found
        """
        for tag, value in bsettings.get_items('toolchain-alias'):
            if arch == tag:
                for alias in value.split():
                    if alias in self.toolchains:
                        return self.toolchains[alias]

        if not arch in self.toolchains:
            raise ValueError(f"No tool chain found for arch '{arch}'")
        return self.toolchains[arch]

    def resolve_references(self, var_dict, args):
        """Resolve variable references in a string

        This converts ${blah} within the string to the value of blah.
        This function works recursively.

        Args:
            var_dict (dict): Dictionary containing variables and their values
            args (str): String containing make arguments

        Returns:
            str: Resolved string

        >>> bsettings.setup(None)
        >>> tcs = Toolchains()
        >>> tcs.add('fred', False)
        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
                        'second' : '2nd'}
        >>> tcs.resolve_references(var_dict, 'this=${oblique}_set')
        'this=OBLIQUE_set'
        >>> tcs.resolve_references(var_dict, 'this=${oblique}_set${first}nd')
        'this=OBLIQUE_setfi2ndrstnd'
        """
        re_var = re.compile(r'(\$\{[-_a-z0-9A-Z]{1,}\})')

        while True:
            m = re_var.search(args)
            if not m:
                break
            lookup = m.group(0)[2:-1]
            value = var_dict.get(lookup, '')
            args = args[:m.start(0)] + value + args[m.end(0):]
        return args

    def get_make_arguments(self, brd):
        """Returns 'make' arguments for a given board

        The flags are in a section called 'make-flags'. Flags are named
        after the target they represent, for example snapper9260=TESTING=1
        will pass TESTING=1 to make when building the snapper9260 board.

        References to other boards can be added in the string also. For
        example:

        [make-flags]
        at91-boards=ENABLE_AT91_TEST=1
        snapper9260=${at91-boards} BUILD_TAG=442
        snapper9g45=${at91-boards} BUILD_TAG=443

        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.

        A special 'target' variable is set to the board target.

        Args:
            brd (Board): Board object for the board to check.

        Returns:
            list of str: 'make' flags for that board, or [] if none
        """
        self._make_flags['target'] = brd.target
        arg_str = self.resolve_references(self._make_flags,
                           self._make_flags.get(brd.target, ''))
        args = re.findall(r"(?:\".*?\"|\S)+", arg_str)
        i = 0
        while i < len(args):
            args[i] = args[i].replace('"', '')
            if not args[i]:
                del args[i]
            else:
                i += 1
        return args

    def locate_arch_url(self, fetch_arch):
        """Find a toolchain available online

        Look in standard places for available toolchains. At present the
        only standard place is at kernel.org.

        Args:
            fetch_arch (str): Architecture to look for, or 'list' for all

        Returns:
            tuple or str or None: If fetch_arch is 'list', a tuple of
                (machine architecture, list of toolchains). Otherwise the
                URL containing this toolchain, if available, else None.
        """
        arch = command.output_one_line('uname', '-m')
        if arch == 'aarch64':
            arch = 'arm64'
        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
        versions = ['14.2.0', '13.2.0']
        links = []
        for version in versions:
            url = f'{base}/{arch}/{version}/'
            print(f'Checking: {url}')
            with urllib.request.urlopen(url) as response:
                html = tools.to_string(response.read())
                parser = MyHTMLParser(fetch_arch)
                parser.feed(html)
                if fetch_arch == 'list':
                    links += parser.links
                elif parser.arch_link:
                    return url + parser.arch_link
        if fetch_arch == 'list':
            return arch, links
        return None

    def unpack(self, fname, dest):
        """Unpack a tar file

        Args:
            fname (str): Filename to unpack
            dest (str): Destination directory

        Returns:
            str: Directory name of the first entry in the archive, without
                the trailing /
        """
        stdout = command.output('tar', 'xvfJ', fname, '-C', dest)
        dirs = stdout.splitlines()[1].split('/')[:2]
        return '/'.join(dirs)

    def test_settings_has_path(self, path):
        """Check if buildman will find this toolchain

        Returns:
            True if the path is in settings, False if not
        """
        paths = self.get_path_list(False)
        return path in paths

    def list_archs(self):
        """List architectures with available toolchains to download"""
        host_arch, archives = self.locate_arch_url('list')
        re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
        arch_set = set()
        for archive in archives:
            # Remove the host architecture from the start
            arch = re_arch.match(archive[len(host_arch):])
            if arch:
                if arch.group(1) != '2.0' and arch.group(1) != '64':
                    arch_set.add(arch.group(1))
        return sorted(arch_set)

    def fetch_and_install(self, arch):
        """Fetch and install a new toolchain

        arch:
            Architecture to fetch, or 'list' to list
        """
        # Fist get the URL for this architecture
        col = terminal.Color()
        print(col.build(col.BLUE, f"Downloading toolchain for arch '{arch}'"))
        url = self.locate_arch_url(arch)
        if not url:
            print(f"Cannot find toolchain for arch '{arch}' - "
                  "use 'list' to list")
            return 2
        home = os.environ['HOME']
        dest = os.path.join(home, '.buildman-toolchains')
        if not os.path.exists(dest):
            os.mkdir(dest)

        # Download the tar file for this toolchain and unpack it
        tarfile, tmpdir = tools.download(url, '.buildman')
        if not tarfile:
            return 1
        print(col.build(col.GREEN, f'Unpacking to: {dest}'), end=' ')
        sys.stdout.flush()
        path = self.unpack(tarfile, dest)
        os.remove(tarfile)
        os.rmdir(tmpdir)
        print()

        # Check that the toolchain works
        print(col.build(col.GREEN, 'Testing'))
        dirpath = os.path.join(dest, path)
        compiler_fname_list = self.scan_path(dirpath, True)
        if not compiler_fname_list:
            print('Could not locate C compiler - fetch failed.')
            return 1
        if len(compiler_fname_list) != 1:
            print(col.build(col.RED,
                            f"Warning, ambiguous toolchains: "
                            f"{', '.join(compiler_fname_list)}"))
        # Instantiate to verify the toolchain works
        Toolchain(compiler_fname_list[0], True, True)

        # Make sure that it will be found by buildman
        if not self.test_settings_has_path(dirpath):
            print(f"Adding 'download' to config file "
                  f"'{bsettings.config_fname}'")
            bsettings.set_item('toolchain', 'download', f'{dest}/*/*')
        return 0