Code policy tool improvements

- Detect plain identifiers (e.g. 'fpclassify')
- Strip comments and literals for some checks
- Strip expect strings for API testcase checks

util/check_code_policy.py

@@ -2,19 +2,9 @@
 #
 #  Check various C source code policy rules and issue warnings for offenders
 #
-
-#  Check debug log macro calls in C code for consistency.  Allowed forms:
-#
-#    DUK_D(DUK_DPRINT(...))
-#    DUK_DD(DUK_DDPRINT(...))
-#    DUK_DDD(DUK_DDDPRINT(...))
-#
-#  The calls may span multiple lines, but the wrapper (DUK_D) and the log
-#  macro (DUK_DPRINT) must be on the same line.
-#
 #  Usage:
 #
-#    $ python check_debuglog_calls.py src/*.c
+#    $ python check_code_policy.py src/*.c
 #
 
 import os
@@ -38,6 +28,91 @@ class Problem:
 re_debuglog_callsite = re.compile(r'^.*?(DUK_D+PRINT).*?$')
 re_trailing_ws = re.compile(r'^.*?\s$')
 re_only_ws = re.compile(r'^\s*$')
+re_identifier = re.compile(r'[A-Za-z0-9_]+')
+
+# These identifiers are wrapped in duk_features.h.in, and should only be used
+# through the wrappers elsewhere.
+rejected_plain_identifiers_list = [
+	# math classification
+	'fpclassify',
+	'signbit',
+	'isfinite',
+	'isnan',
+	'isinf',
+	'FP_NAN',
+	'FP_INFINITE',
+	'FP_ZERO',
+	'FP_SUBNORMAL',
+	'FP_NORMAL',
+
+	# math functions
+	'fabs',
+	'fmin',
+	'fmax',
+	'floor',
+	'ceil',
+	'fmod',
+	'pow',
+	'acos',
+	'asin',
+	'atan',
+	'atan2',
+	'sin',
+	'cos',
+	'tan',
+	'exp',
+	'log',
+	'sqrt',
+
+	# memory functions
+	'malloc',
+	'realloc',
+	'calloc',
+	'free',
+	'memcpy',
+	'memmove',
+	'memcmp',
+	'memset',
+
+	# string functions
+	'strlen',
+	'strcmp',
+	'strncmp',
+	'printf',
+	'fprintf',
+	'sprintf',
+	'_snprintf',
+	'snprintf',
+	'vsprintf',
+	'_vsnprintf',
+	'vsnprintf',
+	'sscanf',
+	'vsscanf',
+
+	# streams
+	'stdout',
+	'stderr',
+	'stdin',
+
+	# file ops
+	'fopen',
+	'fclose',
+	'fread',
+	'fwrite',
+	'fseek',
+	'ftell',
+	'fflush',
+	'fputc',
+
+	# misc
+	'abort',
+	'exit',
+	'setjmp',
+	'longjmp'
+]
+rejected_plain_identifiers = {}
+for id in rejected_plain_identifiers_list:
+	rejected_plain_identifiers[id] = True
 
 debuglog_wrappers = {
 	'DUK_DPRINT': 'DUK_D',
@@ -47,7 +122,52 @@ debuglog_wrappers = {
 
 problems = []
 
-def checkDebugLogCalls(line):
+re_repl_c_comments = re.compile(r'/\*.*?\*/', re.DOTALL)
+re_repl_cpp_comments = re.compile(r'//.*?\n', re.DOTALL)
+re_repl_string_literals_dquot = re.compile(r'''\"(?:\\\"|[^\"])+\"''')
+re_repl_string_literals_squot = re.compile(r'''\'(?:\\\'|[^\'])+\'''')
+re_repl_expect_strings = re.compile(r'/\*===.*?===*?\*/', re.DOTALL)
+re_not_newline = re.compile(r'[^\n]+', re.DOTALL)
+
+def removeCommentsAndLiterals(data):
+	def repl_c(m):
+		tmp = re.sub(re_not_newline, '', m.group(0))
+		if tmp == '':
+			tmp = ' '  # avoid /**/
+		return '/*' + tmp + '*/'
+	def repl_cpp(m):
+		return '// removed\n'
+	def repl_dquot(m):
+		return '"' + ('.' * (len(m.group(0)) - 2)) + '"'
+	def repl_squot(m):
+		return "'" + ('.' * (len(m.group(0)) - 2)) + "'"
+
+	data = re.sub(re_repl_c_comments, repl_c, data)
+	data = re.sub(re_repl_cpp_comments, repl_cpp, data)
+	data = re.sub(re_repl_string_literals_dquot, repl_dquot, data)
+	data = re.sub(re_repl_string_literals_squot, repl_squot, data)
+	return data
+
+def removeExpectStrings(data):
+	def repl(m):
+		tmp = re.sub(re_not_newline, '', m.group(0))
+		if tmp == '':
+			tmp = ' '  # avoid /*======*/
+		return '/*===' + tmp + '===*/'
+
+	data = re.sub(re_repl_expect_strings, repl, data)
+	return data
+
+def checkDebugLogCalls(line, filename):
+	# Allowed debug log forms:
+	#
+	#     DUK_D(DUK_DPRINT(...))
+	#     DUK_DD(DUK_DDPRINT(...))
+	#     DUK_DDD(DUK_DDDPRINT(...))
+	#
+	# The calls may span multiple lines, but the wrapper (DUK_D)
+	# and the log macro (DUK_DPRINT) must be on the same line.
+
 	if 'DPRINT' not in line:
 		return
 
@@ -70,7 +190,7 @@ def checkDebugLogCalls(line):
 
 	raise Exception('invalid debug log call form')
 
-def checkTrailingWhitespace(line):
+def checkTrailingWhitespace(line, filename):
 	if len(line) > 0 and line[-1] == '\n':
 		line = line[:-1]
 
@@ -80,13 +200,13 @@ def checkTrailingWhitespace(line):
 
 	raise Exception('trailing whitespace')
 
-def checkCarriageReturns(line):
+def checkCarriageReturns(line, filename):
 	if not '\x0d' in line:
 		return
 
 	raise Exception('carriage return')
 
-def checkMixedIndent(line):
+def checkMixedIndent(line, filename):
 	if not '\x20\x09' in line:
 		return
 
@@ -99,39 +219,63 @@ def checkMixedIndent(line):
 
 	raise Exception('mixed space/tab indent (idx %d)' % idx)
 
-def checkFixme(line):
+def checkFixme(line, filename):
 	if not 'FIXME' in line:
 		return
 
 	raise Exception('FIXME on line')
 
-def processFile(filename, checkers):
-	f = open(filename, 'rb')
+def checkIdentifiers(line, filename):
+	bn = os.path.basename(filename)
+	excludePlain = (bn == 'duk_features.h.in' or \
+	                bn[0:5] == 'test-')
+
+	for m in re.finditer(re_identifier, line):
+		if rejected_plain_identifiers.has_key(m.group(0)):
+			if not excludePlain:
+				raise Exception('invalid identifier %r (perhaps plain)' % m.group(0))
 
-	linenumber = 0
-	for line in f:
-		linenumber += 1
-		for fun in checkers:
-			try:
-				fun(line)
-			except Exception as e:
-				problems.append(Problem(filename, linenumber, line, str(e)))
+def processFile(filename, checkersRaw, checkersNoComments, checkersNoExpectStrings):
+	f = open(filename, 'rb')
+	dataRaw = f.read()
 	f.close()
 
+	dataNoComments = removeCommentsAndLiterals(dataRaw)   # no c/javascript comments, literals removed
+	dataNoExpectStrings = removeExpectStrings(dataRaw)    # no testcase expect strings
+
+	def f(data, checkers):
+		linenumber = 0
+		for line in data.split('\n'):
+			linenumber += 1
+			for fun in checkers:
+				try:
+					fun(line, filename)
+				except Exception as e:
+					problems.append(Problem(filename, linenumber, line, str(e)))
+
+	f(dataRaw, checkersRaw)
+	f(dataNoComments, checkersNoComments)
+	f(dataNoExpectStrings, checkersNoExpectStrings)
+
 def main():
 	parser = optparse.OptionParser()
 	parser.add_option('--dump-vim-commands', dest='dump_vim_commands', default=False, help='Dump oneline vim command')
 	(opts, args) = parser.parse_args()
 
-	checkers = []
-	checkers.append(checkDebugLogCalls)
-	checkers.append(checkTrailingWhitespace)
-	checkers.append(checkCarriageReturns)
-	checkers.append(checkMixedIndent)
-	checkers.append(checkFixme)
+	checkersRaw = []
+	checkersRaw.append(checkDebugLogCalls)
+	checkersRaw.append(checkCarriageReturns)
+	checkersRaw.append(checkFixme)
+
+	checkersNoComments = []
+	checkersNoComments.append(checkIdentifiers)
+
+	checkersNoExpectStrings = []
+	checkersNoExpectStrings.append(checkTrailingWhitespace)
+	checkersNoExpectStrings.append(checkMixedIndent)
 
 	for filename in args:
-		processFile(filename, checkers)
+		processFile(filename, checkersRaw, checkersNoComments, checkersNoExpectStrings)
 
 	if len(problems) > 0:
 		for i in problems: