chromeos/scripts/kernelconfig

#!/usr/bin/env python3
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# pylint: disable=missing-function-docstring

"""Chrome OS kernel configuration tool

Script to merge all configs and run 'make oldconfig' on it to wade out bad
juju, then split the configs into distro-commmon and flavour-specific parts

See this page for more details:
http://dev.chromium.org/chromium-os/how-tos-and-troubleshooting/kernel-configuration
"""

import argparse
import difflib
import fnmatch
import glob
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
from typing import Dict, List


MODES = ["oldconfig", "olddefconfig", "editconfig", "genconfig", "checkconfig"]
TOOLCHAIN_PREFIXES = {
    "x86_64": "x86_64-cros-linux-gnu",
    "armel": "armv7a-cros-linux-gnueabihf",
    "arm64": "aarch64-cros-linux-gnu",
}


def splitconfig(srcdir: str) -> Dict[str, str]:
    """Split and merge common configs

    Return dict of filename-to-contents.
    """
    allconfigs = {}

    # Parse config files.
    for config in os.listdir(srcdir):
        # Only config.*
        if not config.endswith(".config"):
            continue

        allconfigs[config] = set()

        for line in open(os.path.join(srcdir, config), encoding="utf-8"):
            m = re.match(r"#*\s*CONFIG_(\w+)[\s=](.*)$", line)
            if not m:
                continue
            option, value = m.groups()
            allconfigs[config].add((option, value))

    # Split out common config options.
    common = None
    for config in allconfigs:
        if common is None:
            common = allconfigs[config].copy()
        else:
            common &= allconfigs[config]
    for config in allconfigs:
        allconfigs[config] -= common
    allconfigs["common.config"] = common

    ret = {}
    # Generate new splitconfigs.
    for config in allconfigs:
        if allconfigs[config] is None:
            continue
        contents = "#\n# Config options generated by splitconfig\n#\n"
        for option, value in sorted(list(allconfigs[config])):
            if value == "is not set":
                contents += "# CONFIG_%s %s\n" % (option, value)
            else:
                contents += "CONFIG_%s=%s\n" % (option, value)
        ret[config] = contents
    return ret


def die(msg: str = "Aborting"):
    logging.error(msg)
    sys.exit(1)


def in_chroot() -> bool:
    return os.path.exists("/etc/cros_chroot_version")


def rerun_in_chroot():
    cmd = [
        "cros_sdk",
        "--working-dir",
        ".",
        "--",
        os.path.join(os.path.curdir, os.path.relpath(sys.argv[0])),
    ] + sys.argv[1:]
    logging.info("Re-running in chroot")
    logging.debug("Run: %s", cmd)
    sys.exit(subprocess.run(cmd, check=False).returncode)


def make_toolchain_args(arch: str) -> Dict[str, str]:
    ret = {
        "LD": "ld.lld",
    }

    if in_chroot():
        prefix = TOOLCHAIN_PREFIXES[arch]
        ccompiler = prefix + "-clang"
        # CrOS build tooling wants us to use these, and avoid a default of
        # unprefixed 'gcc' or 'clang'. Supply defaults here, but also consider
        # environment varibles below.
        ret["HOSTCC"] = "x86_64-pc-linux-gnu-clang"
        ret["HOSTCXX"] = "x86_64-pc-linux-gnu-clang++"
    else:
        prefix = "x86_64-linux-gnu"
        ccompiler = prefix + "-gcc"

    if not shutil.which(ccompiler):
        if in_chroot():
            die(
                '%s not found. Try running "`sudo cros_setup_toolchains -t %s`"'
                % (ccompiler, prefix)
            )
        else:
            die("%s not found. Try running inside chroot." % ccompiler)

    # Propagate a few environment variables as make args.
    for key in ("HOSTCC", "HOSTCXX"):
        val = os.getenv(key)
        if val:
            ret[key] = val

    ret.update(
        {
            "CC": ccompiler,
            "CXX": prefix + "-g++",
            "CROSS_COMPILE": prefix + "-",
        }
    )
    return ret


def getchar() -> str:
    # We don't want line-buffering, and shelling out to bash is much simpler
    # than messing with tty settings.
    return (
        subprocess.check_output(["bash", "-c", 'read -n 1 -s; echo "${REPLY}"'])
        .decode("utf-8")
        .strip()
    )


def editconfig_prompt(variant: str) -> bool:
    print("* %s: press <Enter> to edit, S to skip" % variant)
    return getchar() not in ("S", "s")


def filter_match(target: str, pattern: str) -> bool:
    return fnmatch.fnmatch(target, "*" + pattern + "*")


def is_editconfig_interactive(
    arch_flavour: str, prompt: bool, pattern: str
) -> bool:
    if filter_match(arch_flavour, pattern):
        if not prompt:
            return True
        return editconfig_prompt(arch_flavour)
    return False


def make_cmd(builddir: str, arch: str, target: str) -> List[str]:
    pairs = make_toolchain_args(arch)
    pairs["ARCH"] = "arm" if arch == "armel" else arch
    pairs["O"] = builddir

    make_args = [k + "=" + v for k, v in pairs.items()]

    return ["make", "-j"] + make_args + [target, "savedefconfig"]


def checkconfig(srcdir: str, outdir: str) -> int:
    error = 0
    for src_config in glob.glob(
        srcdir + "/chromeos/config/**/*.config", recursive=True
    ):
        relative = os.path.relpath(src_config, srcdir)
        out_config = os.path.join(outdir, relative)
        logging.debug("Compare %s to %s", src_config, out_config)
        before = open(src_config, "r", encoding="utf-8").readlines()
        after = open(out_config, "r", encoding="utf-8").readlines()

        if before == after:
            continue

        error = 1
        diff = difflib.context_diff(
            before, after, fromfile="a/" + relative, tofile="b/" + relative
        )
        sys.stdout.writelines(diff)
        logging.error("checkconfig failed for config %s", relative)

    if error:
        logging.error(
            """Consider running `%s olddefconfig` to normalize.

NOTE: chromeos/config changes should always be in a CL by themselves and never
squashed into the same patch as code changes. If code and config changes need
to land together, consider using Cq-Depend to make a circular dependency.""",
            sys.argv[0],
        )
    else:
        logging.info("All good!")
    return error


def build_one_arch(
    args, tmpdir: str, srcdir: str, family: str, arch: str
) -> List[subprocess.Popen]:
    procs = []
    config_base_dir = srcdir + "/chromeos/config/" + family
    config_arch_dir = config_base_dir + "/" + arch
    for flavourconfig in glob.glob(
        os.path.join(srcdir, config_arch_dir, "*.flavour.config")
    ):
        flavour = os.path.basename(flavourconfig)
        builddir = os.path.join(tmpdir, "build", family, arch, flavour)
        os.makedirs(builddir)

        # Merge base.config, common.config, <flavour>.flavour.config
        conf = "\n".join(
            open(os.path.join(srcdir, f), encoding="utf-8").read()
            for f in [
                config_base_dir + "/base.config",
                config_arch_dir + "/common.config",
                config_arch_dir + "/" + flavour,
            ]
        )
        open(os.path.join(builddir, ".config"), "w", encoding="utf-8").write(
            conf
        )

        interactive = False
        if args.mode == "genconfig":
            mode = "olddefconfig"
        elif args.mode == "editconfig":
            interactive = is_editconfig_interactive(
                family + "/" + arch + "/" + flavour, not args.yes, args.filter
            )
            mode = "menuconfig" if interactive else "olddefconfig"
        elif args.mode == "checkconfig":
            mode = "olddefconfig"
        else:
            if args.mode == "oldconfig":
                # 'oldconfig' may run into interactive prompts.
                interactive = True
            mode = args.mode

        cmd = make_cmd(builddir, arch, mode)
        if interactive:
            logging.info("Starting interactive cmd: %s", " ".join(cmd))
            ret = subprocess.run(cmd, check=False, cwd=srcdir)
            if ret.returncode:
                die('cmd "%s" failed' % " ".join(cmd))
        else:
            logging.info("Starting background cmd: %s", " ".join(cmd))
            procs += [
                subprocess.Popen(
                    cmd,
                    cwd=srcdir,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
            ]
    return procs


def do_splitconfig(
    tmpdir: str,
    srcdir: str,
    outdir: str,
    family: str,
    arches: List[str],
    save_configs: bool,
):
    family_cfg_dir = "chromeos/config/" + family

    if save_configs:
        keep_dir = "CONFIGS/" + family
        os.makedirs(keep_dir, exist_ok=True)

    for arch in arches:
        dest = os.path.join(tmpdir, family, arch)
        family_arch_dir = family_cfg_dir + "/" + arch
        os.makedirs(dest)
        for flavourconfig in glob.glob(
            os.path.join(srcdir, family_arch_dir, "*.flavour.config")
        ):
            flavour = os.path.basename(flavourconfig)
            builddir = os.path.join(tmpdir, "build", family, arch, flavour)
            shutil.copy(
                os.path.join(builddir, "defconfig"), os.path.join(dest, flavour)
            )
            if save_configs:
                shutil.copy(
                    os.path.join(builddir, ".config"),
                    os.path.join(keep_dir, arch + "-" + flavour),
                )
                shutil.copy(
                    os.path.join(builddir, "defconfig"),
                    os.path.join(keep_dir, arch + "-" + flavour + ".def"),
                )
        os.makedirs(os.path.join(outdir, family_arch_dir), exist_ok=True)
        # Find per-arch common items; flavour-unique options can be written out
        # immediately.
        for config, contents in splitconfig(dest).items():
            if config == "common.config":
                open(
                    os.path.join(tmpdir, family, arch + ".config"),
                    "w",
                    encoding="utf-8",
                ).write(contents)
            else:
                open(
                    os.path.join(outdir, family_arch_dir, config),
                    "w",
                    encoding="utf-8",
                ).write(contents)

    # Find cross-arch common items; common ones go to base.config, and unique
    # ones to <arch>/common.config.
    for config, contents in splitconfig(tmpdir + "/" + family).items():
        if config == "common.config":
            open(
                os.path.join(outdir, family_cfg_dir, "base.config"),
                "w",
                encoding="utf-8",
            ).write(contents)
        else:
            assert config.endswith(".config")
            # Python3.9: arch = config.removesuffix('.config')
            arch = config[: -len(".config")]
            open(
                os.path.join(outdir, family_cfg_dir, arch, "common.config"),
                "w",
                encoding="utf-8",
            ).write(contents)


# Arches relevant to this family.
def family_arches(srcdir: str, family: str) -> List[str]:
    return [
        os.path.basename(p)
        for p in glob.glob(srcdir + "/chromeos/config/" + family + "/*")
        if os.path.isdir(p)
    ]


def doit(args, tmpdir: str) -> int:
    for d in (
        os.getcwd(),
        os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "..")),
    ):
        if os.path.exists(os.path.join(d, "MAINTAINERS")) and os.path.exists(
            os.path.join(d, "Makefile")
        ):
            srcdir = d
            logging.info("Using top kernel dir: %s", srcdir)
            break
    else:
        die("This does not appear to be a kernel source directory.")

    if not in_chroot() and not args.force:
        rerun_in_chroot()
        assert False  # Should not reach this.

    # Most commands write back to source directory.
    if args.mode == "checkconfig":
        outdir = tmpdir
    else:
        outdir = srcdir

    save_configs = args.mode == "genconfig"

    procs = []

    families = [
        os.path.basename(p)
        for p in glob.glob(srcdir + "/chromeos/config/*")
        if os.path.isdir(p)
    ]
    for family in families:
        for arch in family_arches(srcdir, family):
            if not arch in TOOLCHAIN_PREFIXES:
                die("Unexpected arch: %s" % arch)
            procs += build_one_arch(args, tmpdir, srcdir, family, arch)

    for p in procs:
        ret = p.wait()
        if ret:
            assert p.stderr  # Remind pytype we captured stderr.
            logging.error(p.stderr.read().decode("utf-8"))
            die('cmd "%s" failed' % " ".join(p.args))

    logging.info("Generating splitconfigs")
    for family in families:
        do_splitconfig(
            tmpdir,
            srcdir,
            outdir,
            family,
            family_arches(srcdir, family),
            save_configs,
        )

    if args.mode == "checkconfig":
        return checkconfig(srcdir, outdir)

    return 0


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Chrome OS kernel configuration script",
        epilog="""
               Note that Kbuild will evaluate some features depending on the
               toolchain, so we try to enter the SDK chroot. This can be
               overridden, with a potentially degraded experience.
               """,
    )

    log_level_choices = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
    parser.add_argument(
        "--log_level", "-l", choices=log_level_choices, default="INFO"
    )
    parser.add_argument(
        "--force",
        "-F",
        action="store_true",
        help="Force allowing to run outside the chroot",
    )
    parser.add_argument(
        "--filter",
        "-f",
        default="",
        help="Only attempt to edit configs which match filter",
    )
    parser.add_argument(
        "--yes",
        "-y",
        action="store_true",
        help="Edit all configs which match unconditionally",
    )
    parser.add_argument("mode", choices=MODES, help="sub-command/mode")

    args = parser.parse_args()
    logging.basicConfig(
        level=args.log_level, format="%(levelname)s - %(message)s"
    )

    with tempfile.TemporaryDirectory() as tmpdir:
        return doit(args, tmpdir)


sys.exit(main())