30d34d0e59
Check to make sure there is not an alphanumeric character immeditely before or after the 'flag'. It there is an alphanumeric character then this is obviously not actually the flag we care about. For example if the project declares a flag "valid-name" but the regex finds something like "invalid_name" we should not match. Clearly this "invalid_name" is not actually a wrong usage of the "valid-name" flag.
213 lines
7.2 KiB
Python
Executable File
213 lines
7.2 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright 2015 The Kubernetes Authors All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import json
|
|
import mmap
|
|
import os
|
|
import re
|
|
import sys
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*')
|
|
parser.add_argument("-e", "--skip-exceptions", help="ignore hack/verify-flags/exceptions.txt and print all output", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
|
|
dashRE = re.compile('[-_]')
|
|
|
|
# Cargo culted from http://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python
|
|
def is_binary(pathname):
|
|
"""Return true if the given filename is binary.
|
|
@raise EnvironmentError: if the file does not exist or cannot be accessed.
|
|
@attention: found @ http://bytes.com/topic/python/answers/21222-determine-file-type-binary-text on 6/08/2010
|
|
@author: Trent Mick <TrentM@ActiveState.com>
|
|
@author: Jorge Orpinel <jorge@orpinel.com>"""
|
|
try:
|
|
f = open(pathname, 'r')
|
|
CHUNKSIZE = 1024
|
|
while 1:
|
|
chunk = f.read(CHUNKSIZE)
|
|
if '\0' in chunk: # found null byte
|
|
return True
|
|
if len(chunk) < CHUNKSIZE:
|
|
break # done
|
|
except:
|
|
return True
|
|
finally:
|
|
f.close()
|
|
return False
|
|
|
|
def get_all_files(rootdir):
|
|
all_files = []
|
|
for root, dirs, files in os.walk(rootdir):
|
|
# don't visit certain dirs
|
|
if 'Godeps' in dirs:
|
|
dirs.remove('Godeps')
|
|
if 'third_party' in dirs:
|
|
dirs.remove('third_party')
|
|
if '.git' in dirs:
|
|
dirs.remove('.git')
|
|
if 'exceptions.txt' in files:
|
|
files.remove('exceptions.txt')
|
|
if 'known-flags.txt' in files:
|
|
files.remove('known-flags.txt')
|
|
|
|
for name in files:
|
|
if name.endswith(".svg"):
|
|
continue
|
|
if name.endswith(".gliffy"):
|
|
continue
|
|
pathname = os.path.join(root, name)
|
|
if is_binary(pathname):
|
|
continue
|
|
all_files.append(pathname)
|
|
return all_files
|
|
|
|
def normalize_files(rootdir, files):
|
|
newfiles = []
|
|
a = ['Godeps', 'third_party', 'exceptions.txt', 'known-flags.txt']
|
|
for f in files:
|
|
if any(x in f for x in a):
|
|
continue
|
|
if f.endswith(".svg"):
|
|
continue
|
|
if f.endswith(".gliffy"):
|
|
continue
|
|
newfiles.append(f)
|
|
for i, f in enumerate(newfiles):
|
|
if not os.path.isabs(f):
|
|
newfiles[i] = os.path.join(rootdir, f)
|
|
return newfiles
|
|
|
|
def line_has_bad_flag(line, flagre):
|
|
results = flagre.findall(line)
|
|
for result in results:
|
|
if "_" in result:
|
|
return True
|
|
return False
|
|
|
|
# The list of files might not be the whole repo. If someone only changed a
|
|
# couple of files we don't want to run all of the golang files looking for
|
|
# flags. Instead load the list of flags from hack/verify-flags/known-flags.txt
|
|
# If running the golang files finds a new flag not in that file, return an
|
|
# error and tell the user to add the flag to the flag list.
|
|
def get_flags(rootdir, files):
|
|
# use a set for uniqueness
|
|
flags = set()
|
|
|
|
# preload the 'known' flags
|
|
pathname = os.path.join(rootdir, "hack/verify-flags/known-flags.txt")
|
|
f = open(pathname, 'r')
|
|
for line in f.read().splitlines():
|
|
flags.add(line)
|
|
f.close()
|
|
|
|
regexs = [ re.compile('Var[P]?\([^,]*, "([^"]*)"'),
|
|
re.compile('.String[P]?\("([^"]*)",[^,]+,[^)]+\)'),
|
|
re.compile('.Int[P]?\("([^"]*)",[^,]+,[^)]+\)'),
|
|
re.compile('.Bool[P]?\("([^"]*)",[^,]+,[^)]+\)'),
|
|
re.compile('.Duration[P]?\("([^"]*)",[^,]+,[^)]+\)'),
|
|
re.compile('.StringSlice[P]?\("([^"]*)",[^,]+,[^)]+\)') ]
|
|
|
|
new_flags = set()
|
|
# walk all the files looking for any flags being declared
|
|
for pathname in files:
|
|
if not pathname.endswith(".go"):
|
|
continue
|
|
f = open(pathname, 'r')
|
|
data = f.read()
|
|
f.close()
|
|
matches = []
|
|
for regex in regexs:
|
|
matches = matches + regex.findall(data)
|
|
for flag in matches:
|
|
# if the flag doesn't have a - or _ it is not interesting
|
|
if not dashRE.search(flag):
|
|
continue
|
|
if flag not in flags:
|
|
new_flags.add(flag)
|
|
if len(new_flags) != 0:
|
|
print("Found flags in golang files not in the list of known flags. Please add these to hack/verify-flags/known-flags.txt")
|
|
print("%s" % "\n".join(new_flags))
|
|
sys.exit(1)
|
|
return list(flags)
|
|
|
|
def flags_to_re(flags):
|
|
"""turn the list of all flags we found into a regex find both - and _ version"""
|
|
flagREs = []
|
|
for flag in flags:
|
|
# turn all flag names into regexs which will find both types
|
|
newre = dashRE.sub('[-_]', flag)
|
|
flagREs.append("[^\w]" + newre + "[^\w]")
|
|
# turn that list of regex strings into a single large RE
|
|
flagRE = "|".join(flagREs)
|
|
flagRE = re.compile(flagRE)
|
|
return flagRE
|
|
|
|
def load_exceptions(rootdir):
|
|
exceptions = set()
|
|
if args.skip_exceptions:
|
|
return exceptions
|
|
exception_filename = os.path.join(rootdir, "hack/verify-flags/exceptions.txt")
|
|
exception_file = open(exception_filename, 'r')
|
|
for exception in exception_file.read().splitlines():
|
|
out = exception.split(":", 1)
|
|
if len(out) != 2:
|
|
printf("Invalid line in exceptions file: %s" % exception)
|
|
continue
|
|
filename = out[0]
|
|
line = out[1]
|
|
exceptions.add((filename, line))
|
|
return exceptions
|
|
|
|
def main():
|
|
rootdir = os.path.dirname(__file__) + "/../"
|
|
rootdir = os.path.abspath(rootdir)
|
|
|
|
exceptions = load_exceptions(rootdir)
|
|
|
|
if len(args.filenames) > 0:
|
|
files = args.filenames
|
|
else:
|
|
files = get_all_files(rootdir)
|
|
files = normalize_files(rootdir, files)
|
|
|
|
flags = get_flags(rootdir, files)
|
|
flagRE = flags_to_re(flags)
|
|
|
|
bad_lines = []
|
|
# walk all the file looking for any flag that was declared and now has an _
|
|
for pathname in files:
|
|
relname = os.path.relpath(pathname, rootdir)
|
|
f = open(pathname, 'r')
|
|
for line in f.read().splitlines():
|
|
if line_has_bad_flag(line, flagRE):
|
|
if (relname, line) not in exceptions:
|
|
bad_lines.append((relname, line))
|
|
f.close()
|
|
|
|
if len(bad_lines) != 0:
|
|
if not args.skip_exceptions:
|
|
print("Found illegal 'flag' usage. If this is a false positive add the following line(s) to hack/verify-flags/exceptions.txt:")
|
|
for (relname, line) in bad_lines:
|
|
print("%s:%s" % (relname, line))
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|