kubernetes/hack/boilerplate/boilerplate.py

#!/usr/bin/env python3

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import difflib
import glob
import os
import re
import sys

parser = argparse.ArgumentParser()
parser.add_argument(
    "filenames", help="list of files to check, all files if unspecified", nargs="*"
)

rootdir = os.path.dirname(__file__) + "/../../"
rootdir = os.path.abspath(rootdir)
parser.add_argument("--rootdir", default=rootdir, help="root directory to examine")

default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir)

parser.add_argument(
    "-v",
    "--verbose",
    help="give verbose output regarding why a file does not pass",
    action="store_true",
)

args = parser.parse_args()

verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")


def get_refs():
    refs = {}

    for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
        extension = os.path.basename(path).split(".")[1]

        with open(path, "r") as ref_file:
            refs[extension] = ref_file.read().splitlines()

    return refs


def is_generated_file(data, regexs):
    return regexs["generated"].search(data)


def file_passes(filename, refs, regexs):
    try:
        with open(filename) as stream:
            data = stream.read()
    except OSError as exc:
        print(f"Unable to open {filename}: {exc}", file=verbose_out)
        return False

    # determine if the file is automatically generated
    generated = is_generated_file(data, regexs)

    basename = os.path.basename(filename)
    extension = file_extension(filename)
    if generated:
        if extension == "go":
            extension = "generatego"

    if extension != "":
        ref = refs[extension]
    else:
        ref = refs[basename]

    # remove extra content from the top of files
    if extension in ("go", "generatego"):
        data, found = regexs["go_build_constraints"].subn("", data, 1)
    elif extension in ["sh", "py"]:
        data, found = regexs["shebang"].subn("", data, 1)

    data = data.splitlines()

    # if our test file is smaller than the reference it surely fails!
    if len(ref) > len(data):
        print(
            f"File {filename} smaller than reference ({len(data)} < {len(ref)})",
            file=verbose_out,
        )
        return False

    # trim our file to the same number of lines as the reference file
    data = data[: len(ref)]

    pattern = regexs["year"]
    for line in data:
        if pattern.search(line):
            if generated:
                print(
                    f"File {filename} has the YEAR field, but it should not be in generated file",
                    file=verbose_out,
                )
            else:
                print(
                    "File {filename} has the YEAR field, but missing the year of date",
                    file=verbose_out,
                )
            return False

    if not generated:
        # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR"
        pattern = regexs["date"]
        for i, line in enumerate(data):
            data[i], found = pattern.subn("YEAR", line)
            if found != 0:
                break

    # if we don't match the reference at this point, fail
    if ref != data:
        print(f"Header in {filename} does not match reference, diff:", file=verbose_out)
        if args.verbose:
            print(file=verbose_out)
            for line in difflib.unified_diff(
                ref, data, "reference", filename, lineterm=""
            ):
                print(line, file=verbose_out)
            print(file=verbose_out)
        return False

    return True


def file_extension(filename):
    return os.path.splitext(filename)[1].split(".")[-1].lower()


skipped_names = [
    "third_party",
    "_output",
    ".git",
    "cluster/env.sh",
    "vendor",
    "testdata",
    "test/e2e/generated/bindata.go",
    "hack/boilerplate/test",
    "staging/src/k8s.io/kubectl/pkg/generated/bindata.go",
]


def normalize_files(files):
    newfiles = []
    for pathname in files:
        if any(x in pathname for x in skipped_names):
            continue
        newfiles.append(pathname)
    for i, pathname in enumerate(newfiles):
        if not os.path.isabs(pathname):
            newfiles[i] = os.path.join(args.rootdir, pathname)
    return newfiles


def get_files(extensions):
    files = []
    if len(args.filenames) > 0:
        files = args.filenames
    else:
        for root, dirs, walkfiles in os.walk(args.rootdir):
            # don't visit certain dirs. This is just a performance improvement
            # as we would prune these later in normalize_files(). But doing it
            # cuts down the amount of filesystem walking we do and cuts down
            # the size of the file list
            for dname in skipped_names:
                if dname in dirs:
                    dirs.remove(dname)
            for dname in dirs:
                # dirs that start with __ are ignored
                if dname.startswith("__"):
                    dirs.remove(dname)

            for name in walkfiles:
                pathname = os.path.join(root, name)
                files.append(pathname)

    files = normalize_files(files)
    outfiles = []
    for pathname in files:
        basename = os.path.basename(pathname)
        extension = file_extension(pathname)
        if extension in extensions or basename in extensions:
            outfiles.append(pathname)
    return outfiles


def get_dates():
    years = datetime.datetime.now().year
    return "(%s)" % "|".join(str(year) for year in range(2014, years + 1))


def get_regexs():
    regexs = {}
    # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
    regexs["year"] = re.compile("YEAR")
    # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year
    # as a regex like: "(2014|2015|2016|2017|2018)";
    # company holder names can be anything
    regexs["date"] = re.compile(get_dates())
    # strip the following build constraints/tags:
    # //go:build
    # // +build \n\n
    regexs["go_build_constraints"] = re.compile(
        r"^(//(go:build| \+build).*\n)+\n", re.MULTILINE
    )
    # strip #!.* from scripts
    regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
    # Search for generated files
    regexs["generated"] = re.compile(r"^[/*#]+ +.* DO NOT EDIT\.$", re.MULTILINE)
    return regexs


def main():
    regexs = get_regexs()
    refs = get_refs()
    filenames = get_files(refs)

    for filename in filenames:
        if not file_passes(filename, refs, regexs):
            print(filename)

    return 0


if __name__ == "__main__":
    sys.exit(main())