test: Extend white space check to verify files are valid utf-8

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

1 files changed, 8 insertions, 2 deletions

diff --git a/scripts/check_whitespace.py b/scripts/check_whitespace.py
index 6f32c24f..13c4a3ed 100755
--- a/scripts/check_whitespace.py
+++ b/scripts/check_whitespace.py
@@ -4,7 +4,7 @@
 # Copyright (C) 2018  Kevin O'Connor <kevin@koconnor.net>
 #
 # This file may be distributed under the terms of the GNU GPLv3 license.
-import sys, os.path
+import sys, os.path, unicodedata
 
 HaveError = False
 
@@ -28,10 +28,16 @@ def check_file(filename):
     # Do checks
     lineno = 0
     for lineno, line in enumerate(data.split('\n')):
+        # Verify line is valid utf-8
+        try:
+            line = line.decode('utf-8')
+        except UnicodeDecodeError:
+            report_error(filename, lineno, "Non utf-8 character")
+            continue
         # Check for control characters
         for c in line:
             oc = ord(c)
-            if oc < 32:
+            if oc < 32 or unicodedata.category(c).startswith('C'):
                 char_name = repr(c)
                 if oc == 9:
                     if os.path.basename(filename).lower() == 'makefile':