From bd02f998f2bf00fad5255f43ad0621a80d94ee83 Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 21 Nov 2018 11:32:28 +0700 Subject: [PATCH 01/31] remove last config access from main cycle --- src/compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compare.py b/src/compare.py index dca8a88..24c4a15 100644 --- a/src/compare.py +++ b/src/compare.py @@ -91,6 +91,7 @@ frames = 0 timeout = config.getint("video", "timout") dark_threshold = config.getfloat("video", "dark_threshold") end_report = config.getboolean("debug", "end_report") +video_certainty = config.getfloat("video", "certainty") / 10 while True: # Increment the frame count every loop frames += 1 @@ -145,7 +146,7 @@ while True: match_index += 1 # Try to find a match that's confident enough - if match * 10 < float(config.get("video", "certainty")) and match > 0: + if 0 < match < video_certainty: timings.append(time.time()) # If set to true in the config, print debug text From 9ca5be97fb470ab1b4b7a09a7173357ac27b622d Mon Sep 17 00:00:00 2001 From: Luya Tshimbalanga Date: Wed, 28 Nov 2018 18:18:12 -0800 Subject: [PATCH 02/31] Add Fedora --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4493f0c..d216bb8 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,14 @@ Install the `howdy` package from the AUR. For AUR installation instructions, tak You will need to do some additional configuration steps. Please read the [ArchWiki entry](https://wiki.archlinux.org/index.php/Howdy) for more information. +### Fedora +The `howdy` package is now available in a [Fedora COPR repository](https://copr.fedorainfracloud.org/coprs/luya/howdy/) by simply execute the following command from a terminal: + +``` +sudo dnf copr enable luya/howdy +sudo dnf install howdy +``` + ## Setup After installation, you need to let Howdy learn your face. Run `sudo howdy add` to add a face model. From 683837df91b008f03ddb16d4c8e37d9fcb09db53 Mon Sep 17 00:00:00 2001 From: dmig Date: Fri, 7 Dec 2018 14:26:01 +0700 Subject: [PATCH 03/31] new option for CNN --- src/config.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config.ini b/src/config.ini index 1f577e9..55e29d6 100644 --- a/src/config.ini +++ b/src/config.ini @@ -21,6 +21,11 @@ dismiss_lockscreen = false # The howdy command will still function disabled = false +# Use CNN instead of HOG +# CNN model is much more accurate than the HOG based model, but takes much more +# computational power to run, and is meant to be executed on a GPU to attain reasonable speed. +use_cnn = false + [video] # The certainty of the detected face belonging to the user of the account # On a scale from 1 to 10, values above 5 are not recommended From d7bc38ea167f54719de12373debff11761c6542e Mon Sep 17 00:00:00 2001 From: dmig Date: Fri, 7 Dec 2018 14:28:07 +0700 Subject: [PATCH 04/31] get rid of face_recognition module, support CNN --- src/cli/test.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/cli/test.py b/src/cli/test.py index cceb1a4..63af1d2 100644 --- a/src/cli/test.py +++ b/src/cli/test.py @@ -5,7 +5,7 @@ import configparser import os import time import cv2 -import face_recognition +import dlib # Get the absolute path to the current file path = os.path.dirname(os.path.abspath(__file__)) @@ -51,6 +51,14 @@ def print_text(line_number, text): """Print the status text by line number""" cv2.putText(overlay, text, (10, height - 10 - (10 * line_number)), cv2.FONT_HERSHEY_SIMPLEX, .3, (0, 255, 0), 0, cv2.LINE_AA) +use_cnn = config.getboolean('core', 'use_cnn') +if use_cnn: + face_detector = dlib.cnn_face_detection_model_v1( + path + '/../dlib-data/mmod_human_face_detector.dat' + ) +else: + face_detector = dlib.get_frontal_face_detector() + # Open the window and attach a a mouse listener cv2.namedWindow("Howdy Test") cv2.setMouseCallback("Howdy Test", mouse) @@ -86,7 +94,7 @@ try: # Grab a single frame of video - ret, frame = (video_capture.read()) + ret, frame = video_capture.read() # Make a frame to put overlays in overlay = frame.copy() @@ -100,7 +108,7 @@ try: # Fill with the overal containing percentage hist_perc = [] - # Loop though all values to calculate a pensentage and add it to the overlay + # Loop though all values to calculate a percentage and add it to the overlay for index, value in enumerate(hist): value_perc = float(value[0]) / hist_total * 100 hist_perc.append(value_perc) @@ -135,17 +143,20 @@ try: rec_tm = time.time() # Get the locations of all faces and their locations - face_locations = face_recognition.face_locations(frame) + face_locations = face_detector(frame, 1) # upsample 1 time rec_tm = time.time() - rec_tm # Loop though all faces and paint a circle around them for loc in face_locations: + if use_cnn: + loc = loc.rect + # Get the center X and Y from the rectangular points - x = int((loc[1] - loc[3]) / 2) + loc[3] - y = int((loc[2] - loc[0]) / 2) + loc[0] + x = int((loc.right() - loc.left()) / 2) + loc.left() + y = int((loc.bottom() - loc.top()) / 2) + loc.top() # Get the raduis from the with of the square - r = (loc[1] - loc[3]) / 2 + r = (loc.right() - loc.left()) / 2 # Add 20% padding r = int(r + (r * 0.2)) From d9e72c74dc6a4d93464992dcc59211b3f040d58b Mon Sep 17 00:00:00 2001 From: dmig Date: Sat, 8 Dec 2018 13:18:05 +0700 Subject: [PATCH 05/31] directory and instructions for dlib data files --- src/dlib-data/.gitignore | 2 ++ src/dlib-data/Readme.md | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 src/dlib-data/.gitignore create mode 100644 src/dlib-data/Readme.md diff --git a/src/dlib-data/.gitignore b/src/dlib-data/.gitignore new file mode 100644 index 0000000..e4d30eb --- /dev/null +++ b/src/dlib-data/.gitignore @@ -0,0 +1,2 @@ +*.dat +*.dat.bz2 diff --git a/src/dlib-data/Readme.md b/src/dlib-data/Readme.md new file mode 100644 index 0000000..a940a5f --- /dev/null +++ b/src/dlib-data/Readme.md @@ -0,0 +1,7 @@ +Download and unpack `dlib` data files from https://github.com/davisking/dlib-models repository: +```shell +wget https://github.com/davisking/dlib-models/raw/master/dlib_face_recognition_resnet_model_v1.dat.bz2 +wget https://github.com/davisking/dlib-models/raw/master/mmod_human_face_detector.dat.bz2 +wget https://github.com/davisking/dlib-models/raw/master/shape_predictor_5_face_landmarks.dat.bz2 +bunzip *bz2 +``` From 0c384baef2a7760e6da44b9fb83b9cdc39e675ed Mon Sep 17 00:00:00 2001 From: dmig Date: Sat, 8 Dec 2018 13:53:30 +0700 Subject: [PATCH 06/31] speedup and improve face recognition; stable slow mode fps --- src/cli/test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/cli/test.py b/src/cli/test.py index 63af1d2..8c8e034 100644 --- a/src/cli/test.py +++ b/src/cli/test.py @@ -1,3 +1,4 @@ +#! /usr/bin/python3 # Show a windows with the video stream and testing information # Import required modules @@ -59,6 +60,8 @@ if use_cnn: else: face_detector = dlib.get_frontal_face_detector() +clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + # Open the window and attach a a mouse listener cv2.namedWindow("Howdy Test") cv2.setMouseCallback("Howdy Test", mouse) @@ -79,22 +82,26 @@ rec_tm = 0 # Wrap everything in an keyboard interupt handler try: while True: + frame_tm = time.time() + # Increment the frames total_frames += 1 sec_frames += 1 # Id we've entered a new second - if sec != int(time.time()): + if sec != int(frame_tm): # Set the last seconds FPS fps = sec_frames # Set the new second and reset the counter - sec = int(time.time()) + sec = int(frame_tm) sec_frames = 0 # Grab a single frame of video ret, frame = video_capture.read() + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + frame = clahe.apply(frame) # Make a frame to put overlays in overlay = frame.copy() @@ -174,9 +181,11 @@ try: if cv2.waitKey(1) != -1: raise KeyboardInterrupt() + frame_time = time.time() - frame_tm + # Delay the frame if slowmode is on if slow_mode: - time.sleep(.55) + time.sleep(.5 - frame_time) # On ctrl+C except KeyboardInterrupt: From 31dab29aa7ce6dec24aaee374ede04be35095f83 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 12:28:45 +0700 Subject: [PATCH 07/31] use default values when reading from config --- src/cli/test.py | 8 ++++---- src/compare.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cli/test.py b/src/cli/test.py index 8c8e034..c65c92b 100644 --- a/src/cli/test.py +++ b/src/cli/test.py @@ -19,13 +19,13 @@ config.read(path + "/../config.ini") video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true -if config.getboolean("video", "force_mjpeg"): +if config.getboolean("video", "force_mjpeg", fallback=False): # Set a magic number, will enable MJPEG but is badly documented video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # Set the frame width and height if requested -fw = config.getint("video", "frame_width") -fh = config.getint("video", "frame_height") +fw = config.getint("video", "frame_width", fallback=-1) +fh = config.getint("video", "frame_height", fallback=-1) if fw != -1: video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) @@ -52,7 +52,7 @@ def print_text(line_number, text): """Print the status text by line number""" cv2.putText(overlay, text, (10, height - 10 - (10 * line_number)), cv2.FONT_HERSHEY_SIMPLEX, .3, (0, 255, 0), 0, cv2.LINE_AA) -use_cnn = config.getboolean('core', 'use_cnn') +use_cnn = config.getboolean('core', 'use_cnn', fallback=False) if use_cnn: face_detector = dlib.cnn_face_detection_model_v1( path + '/../dlib-data/mmod_human_face_detector.dat' diff --git a/src/compare.py b/src/compare.py index 24c4a15..dbfd5fb 100644 --- a/src/compare.py +++ b/src/compare.py @@ -59,13 +59,13 @@ timings.append(time.time()) video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true -if config.getboolean("video", "force_mjpeg"): +if config.getboolean("video", "force_mjpeg", fallback=False): # Set a magic number, will enable MJPEG but is badly documentated video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # Set the frame width and height if requested -fw = config.getint("video", "frame_width") -fh = config.getint("video", "frame_height") +fw = config.getint("video", "frame_width", fallback=-1) +fh = config.getint("video", "frame_height", fallback=-1) if fw != -1: video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) @@ -84,7 +84,7 @@ import face_recognition timings.append(time.time()) # Fetch the max frame height -max_height = int(config.get("video", "max_height")) +max_height = config.getfloat("video", "max_height", fallback=0.0) # Start the read loop frames = 0 From 29c3a31a5bbffa3503d3da664598677f226e0430 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:11:08 +0700 Subject: [PATCH 08/31] fix models loading problem --- src/compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compare.py b/src/compare.py index dbfd5fb..9fb19e5 100644 --- a/src/compare.py +++ b/src/compare.py @@ -44,7 +44,8 @@ try: models = json.load(open(os.path.dirname(os.path.abspath(__file__)) + "/models/" + user + ".dat")) # Put all models together into 1 array - encodings = [model["data"] for model in models] + for model in models: + encodings += model["data"] except FileNotFoundError: sys.exit(10) From 01e706fce6d371658006d4e1bb4f8d8d35d1c4a5 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:12:28 +0700 Subject: [PATCH 09/31] speedup startup a bit --- src/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compare.py b/src/compare.py index 9fb19e5..6505eec 100644 --- a/src/compare.py +++ b/src/compare.py @@ -75,7 +75,7 @@ if fh != -1: # Capture a single frame so the camera becomes active # This will let the camera adjust its light levels while we're importing for faster scanning -video_capture.read() +video_capture.grab() # Note the time it took to open the camera timings.append(time.time()) From c8c481aed25930209e5a97af2710a69c18ee863b Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:15:31 +0700 Subject: [PATCH 10/31] use PATH constant --- src/compare.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/compare.py b/src/compare.py index 6505eec..73c3d1b 100644 --- a/src/compare.py +++ b/src/compare.py @@ -14,9 +14,12 @@ import os import json import configparser +# Get the absolute path to the current directory +PATH = os.path.abspath(__file__ + '/..') + # Read config from disk config = configparser.ConfigParser() -config.read(os.path.dirname(os.path.abspath(__file__)) + "/config.ini") +config.read(PATH + "/config.ini") def stop(status): """Stop the execution and close video stream""" @@ -41,7 +44,7 @@ dark_tries = 0 # Try to load the face model from the models folder try: - models = json.load(open(os.path.dirname(os.path.abspath(__file__)) + "/models/" + user + ".dat")) + models = json.load(open(PATH + "/models/" + user + ".dat")) # Put all models together into 1 array for model in models: From 1b8a7dc449f739386ad7956dda4193985be70c4d Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:21:44 +0700 Subject: [PATCH 11/31] replace face_recognition with dlib+numpy; support for cnn detector; calculate scaling_factor outside of main cycle; use grayscale image for darkframe and face detection; fix winning model index detection; some minor code cleanups --- src/compare.py | 126 ++++++++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/src/compare.py b/src/compare.py index 73c3d1b..2d2f0c4 100644 --- a/src/compare.py +++ b/src/compare.py @@ -84,11 +84,33 @@ video_capture.grab() timings.append(time.time()) # Import face recognition, takes some time -import face_recognition +import dlib +import numpy as np + +use_cnn = config.getboolean('core', 'use_cnn', fallback=False) +if use_cnn: + face_detector = dlib.cnn_face_detection_model_v1( + PATH + '/dlib-data/mmod_human_face_detector.dat' + ) +else: + face_detector = dlib.get_frontal_face_detector() + +pose_predictor = dlib.shape_predictor( + PATH + '/dlib-data/shape_predictor_5_face_landmarks.dat' +) +face_encoder = dlib.face_recognition_model_v1( + PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' +) + timings.append(time.time()) # Fetch the max frame height max_height = config.getfloat("video", "max_height", fallback=0.0) +# Get the height of the image +height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 1 + +# Calculate the amount the image has to shrink +scaling_factor = (max_height / height) or 1 # Start the read loop frames = 0 @@ -105,80 +127,74 @@ while True: stop(11) # Grab a single frame of video - # Don't remove ret, it doesn't work without it - ret, frame = video_capture.read() + _, frame = video_capture.read() + gsframe = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Create a histogram of the image with 8 values - hist = cv2.calcHist([frame], [0], None, [8], [0, 256]) + hist = cv2.calcHist([gsframe], [0], None, [8], [0, 256]) # All values combined for percentage calculation - hist_total = int(sum(hist)[0]) + hist_total = np.sum(hist) - # If the image is fully black, skip to the next frame - if hist_total == 0: + # If the image is fully black or the frame exceeds threshold, + # skip to the next frame + if hist_total == 0 or (hist[0] / hist_total * 100 > dark_threshold): dark_tries += 1 continue - # Scrip the frame if it exceeds the threshold - if float(hist[0]) / hist_total * 100 > dark_threshold: - dark_tries += 1 - continue - - # Get the height and with of the image - height, width = frame.shape[:2] - # If the hight is too high - if max_height < height: - # Calculate the amount the image has to shrink - scaling_factor = max_height / float(height) + if scaling_factor != 1: # Apply that factor to the frame frame = cv2.resize(frame, None, fx=scaling_factor, fy=scaling_factor, interpolation=cv2.INTER_AREA) - - # Save the new size for diagnostics - scale_height, scale_width = frame.shape[:2] + gsframe = cv2.resize(gsframe, None, fx=scaling_factor, fy=scaling_factor, interpolation=cv2.INTER_AREA) # Get all faces from that frame as encodings - face_encodings = face_recognition.face_encodings(frame) + face_locations = face_detector(gsframe, 1) # upsample 1 time # Loop through each face - for face_encoding in face_encodings: + for fl in face_locations: + if use_cnn: + fl = fl.rect + + face_landmark = pose_predictor(frame, fl) + face_encoding = np.array( + face_encoder.compute_face_descriptor(frame, face_landmark, 1) # num_jitters=1 + ) # Match this found face against a known face - matches = face_recognition.face_distance(encodings, face_encoding) + matches = np.linalg.norm(encodings - face_encoding, axis=1) - # Check if any match is certain enough to be the user we're looking for - match_index = 0 - for match in matches: - match_index += 1 + # Get best match + match_index = np.argmin(matches) + match = matches[match_index] - # Try to find a match that's confident enough - if 0 < match < video_certainty: - timings.append(time.time()) + # Check if a match that's confident enough + if 0 < match < video_certainty: + timings.append(time.time()) - # If set to true in the config, print debug text - if end_report: - def print_timing(label, offset): - """Helper function to print a timing from the list""" - print(" %s: %dms" % (label, round((timings[1 + offset] - timings[offset]) * 1000))) + # If set to true in the config, print debug text + if end_report: + def print_timing(label, offset): + """Helper function to print a timing from the list""" + print(" %s: %dms" % (label, round((timings[1 + offset] - timings[offset]) * 1000))) - print("Time spent") - print_timing("Starting up", 0) - print_timing("Opening the camera", 1) - print_timing("Importing face_recognition", 2) - print_timing("Searching for known face", 3) + print("Time spent") + print_timing("Starting up", 0) + print_timing("Opening the camera", 1) + print_timing("Importing recognition libs", 2) + print_timing("Searching for known face", 3) - print("\nResolution") - print(" Native: %dx%d" % (height, width)) - print(" Used: %dx%d" % (scale_height, scale_width)) + print("\nResolution") + width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 1 + print(" Native: %dx%d" % (height, width)) + # Save the new size for diagnostics + scale_height, scale_width = frame.shape[:2] + print(" Used: %dx%d" % (scale_height, scale_width)) - # Show the total number of frames and calculate the FPS by deviding it by the total scan time - print("\nFrames searched: %d (%.2f fps)" % (frames, frames / (timings[4] - timings[3]))) - print("Dark frames ignored: %d " % (dark_tries, )) - print("Certainty of winning frame: %.3f" % (match * 10, )) + # Show the total number of frames and calculate the FPS by deviding it by the total scan time + print("\nFrames searched: %d (%.2f fps)" % (frames, frames / (timings[4] - timings[3]))) + print("Dark frames ignored: %d " % (dark_tries, )) + print("Certainty of winning frame: %.3f" % (match * 10, )) - # Catch older 3-encoding models - if not match_index in models: - match_index = 0 + print("Winning model: %d (\"%s\")" % (match_index, models[match_index]["label"])) - print("Winning model: %d (\"%s\")" % (match_index, models[match_index]["label"])) - - # End peacefully - stop(0) + # End peacefully + stop(0) From 82e4e9e10fd3b6f96f662f92744fcb3a309070f9 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:23:55 +0700 Subject: [PATCH 12/31] replace git dependency with wget|curl; fix opencv dependency; add recommended and suggested packages --- debian/control | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 8d2efa1..457d11d 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,9 @@ Vcs-Git: https://github.com/boltgolt/howdy Package: howdy Homepage: https://github.com/boltgolt/howdy Architecture: all -Depends: ${misc:Depends}, git, python3, python3-pip, python3-dev, python3-setuptools, libpam-python, fswebcam, libopencv-dev, python-opencv, cmake, streamer +Depends: ${misc:Depends}, curl|wget, python3-pip, python3-dev, python3-setuptools, libpam-python, fswebcam, libopencv-dev, python3-opencv, cmake, streamer +Recommends: libatlas-base-dev | libopenblas-dev | liblapack-dev +Suggests: nvidia-cuda-dev (>= 7.5) Description: Howdy: Windows Hello style authentication for Linux. Use your built-in IR emitters and camera in combination with face recognition to prove who you are. From 9c2507ed137c4aea8d0543eac55acfd8df3fcd6c Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:25:02 +0700 Subject: [PATCH 13/31] replace subprocess calls with native calls --- debian/prerm | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/debian/prerm b/debian/prerm index e0c5126..958c37e 100755 --- a/debian/prerm +++ b/debian/prerm @@ -13,6 +13,7 @@ def col(id): import subprocess import sys import os +from shutil import rmtree # Only run when we actually want to remove if "remove" not in sys.argv and "purge" not in sys.argv: @@ -24,21 +25,21 @@ if not os.path.exists("/lib/security/howdy/cli"): # Remove files and symlinks try: - subprocess.call(["rm /usr/local/bin/howdy"], shell=True) -except e: + os.unlink('/usr/local/bin/howdy') +except Exception: print("Can't remove executable") try: - subprocess.call(["rm /usr/share/bash-completion/completions/howdy"], shell=True) -except e: + os.unlink('/usr/share/bash-completion/completions/howdy') +except Exception: print("Can't remove autocompletion script") try: - subprocess.call(["rm -rf /lib/security/howdy"], shell=True) -except e: + rmtree('/lib/security/howdy') +except Exception: # This error is normal pass # Remove face_recognition and dlib -subprocess.call(["pip3 uninstall face_recognition face_recognition_models dlib -y --no-cache-dir"], shell=True) +subprocess.call(['pip3', 'uninstall', 'dlib', '-y', '--no-cache-dir']) # Print a tearbending message print(col(2) + """ From ff02a723d48a64b5e2863b0009f49042ca9106c6 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:30:25 +0700 Subject: [PATCH 14/31] replace most subprocess calls with native calls; download and install dlib from tarball instead of making git clone; configure dlib according to cpu flags; do not install opencv using pip; set use_cnn option automatically; minor code cleanup --- debian/postinst | 131 +++++++++++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 47 deletions(-) diff --git a/debian/postinst b/debian/postinst index dffcc5d..53f0afa 100755 --- a/debian/postinst +++ b/debian/postinst @@ -10,14 +10,13 @@ def col(id): return "\033[0m" # Import required modules +import fileinput import subprocess -import time import sys import os import re -import signal -import fileinput -import urllib.parse +import tarfile +from shutil import rmtree, which # Don't run unless we need to configure the install # Will also happen on upgrade but we will catch that later on @@ -35,6 +34,7 @@ def handleStatus(status): print(col(3) + "Error while running last command" + col(0)) sys.exit(1) +sc = subprocess.call # We're not in fresh configuration mode so don't continue the setup if not os.path.exists("/tmp/howdy_picked_device"): @@ -78,64 +78,108 @@ picked = in_file.read() in_file.close() # Remove the temporary file -subprocess.call(["rm /tmp/howdy_picked_device"], shell=True) +os.unlink("/tmp/howdy_picked_device") log("Upgrading pip to the latest version") # Update pip -handleStatus(subprocess.call(["pip3 install --upgrade pip"], shell=True)) +handleStatus(sc(["pip3", "install", "--upgrade", "pip"])) -log("Cloning dlib") +dlib_archive = '/tmp/dlib_latest.tar.gz' -# Clone the dlib git to /tmp, but only the last commit -handleStatus(subprocess.call(["git", "clone", "--depth", "1", "https://github.com/davisking/dlib.git", "/tmp/dlib_clone"])) +log('Downloading dlib') + +loader = which('curl') +LOADER_CMD = None +if loader: + LOADER_CMD = [loader, '--silent', '--retry', '5', '--location', '--output'] +else: + loader = which('wget') + LOADER_CMD = [loader, '--quiet', '--tries', '5', '--output-document'] + +cmd = LOADER_CMD + [dlib_archive, 'https://api.github.com/repos/davisking/dlib/tarball/latest'] + +handleStatus(sc(cmd)) + +DLIB_DIR = None + +excludes = re.compile( + 'davisking-dlib-\w+/(dlib/(http_client|java|matlab|test/)|' + '(docs|examples|python_examples)|' + 'tools/(archive|convert_dlib_nets_to_caffe|htmlify|imglab|python/test|visual_studio_natvis))' +) +with tarfile.open(dlib_archive) as tf: + for item in tf: + # tarball contains directory davisking-dlib-, so peek into archive for the name + if not DLIB_DIR: + DLIB_DIR = item.name + + # extract only files sufficent for building + if not excludes.match(item.name): + tf.extract(item, '/tmp') + +os.unlink(dlib_archive) log("Building dlib") -# Start the build without GPU -handleStatus(subprocess.call(["cd /tmp/dlib_clone/; python3 setup.py install --yes USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA"], shell=True)) +# Start the build +cmd = ["python3", "setup.py", "install"] + +flags = '' +with open('/proc/cpuinfo') as info: + for line in info: + if 'flags' in line: + flags = line + break + +if 'avx' in flags: + cmd += ["--yes", "USE_AVX_INSTRUCTIONS"] +elif 'sse4' in flags: + cmd += ["--yes", "USE_SSE4_INSTRUCTIONS"] +elif 'sse3' in flags: + cmd += ["--yes", "USE_SSE3_INSTRUCTIONS"] +elif 'sse2' in flags: + cmd += ["--yes", "USE_SSE2_INSTRUCTIONS"] + +sp = subprocess.run(cmd, cwd=DLIB_DIR, stderr=subprocess.STDOUT) +handleStatus(sp.returncode) + +# simple check for CUDA +cuda_used = 'DLIB WILL USE CUDA' in sp.stdout log("Cleaning up dlib") # Remove the no longer needed git clone -handleStatus(subprocess.call(["rm", "-rf", "/tmp/dlib_clone"])) -print("Temporary dlib files removed") +del sp +rmtree(DLIB_DIR) -log("Installing python dependencies") +log("Temporary dlib files removed") -# Install direct dependencies so pip does not freak out with the manual dlib install -handleStatus(subprocess.call(["pip3", "install", "--cache-dir", "/tmp/pip_howdy", "face_recognition_models==0.3.0", "Click>=6.0", "numpy", "Pillow"])) - -log("Installing face_recognition") - -# Install face_recognition though pip -handleStatus(subprocess.call(["pip3", "install", "--cache-dir", "/tmp/pip_howdy", "--no-deps", "face_recognition==1.2.2"])) - -try: - import cv2 -except Exception as e: - log("Reinstalling opencv2") - handleStatus(subprocess.call(["pip3", "install", "opencv-python"])) log("Configuring howdy") # Manually change the camera id to the one picked -for line in fileinput.input(["/lib/security/howdy/config.ini"], inplace = 1): - print(line.replace("device_path = none", "device_path = " + picked), end="") +for line in fileinput.input(["/lib/security/howdy/config.ini"], inplace=1): + print( + line + .replace("device_path = none", "device_path = " + picked) + .replace("use_cnn = false", "use_cnn = " + str(cuda_used).lower()), + end="" + ) print("Camera ID saved") # Secure the howdy folder -handleStatus(subprocess.call(["chmod 744 -R /lib/security/howdy/"], shell=True)) +handleStatus(sc(["chmod 744 -R /lib/security/howdy/"], shell=True)) # Allow anyone to execute the python CLI -handleStatus(subprocess.call(["chmod 755 /lib/security/howdy"], shell=True)) -handleStatus(subprocess.call(["chmod 755 /lib/security/howdy/cli.py"], shell=True)) -handleStatus(subprocess.call(["chmod 755 -R /lib/security/howdy/cli"], shell=True)) +os.chmod('/lib/security/howdy', 0o755) +os.chmod('/lib/security/howdy/cli.py', 0o755) +handleStatus(sc(["chmod 755 -R /lib/security/howdy/cli"], shell=True)) print("Permissions set") # Make the CLI executable as howdy -handleStatus(subprocess.call(["ln -s /lib/security/howdy/cli.py /usr/local/bin/howdy"], shell=True)) -handleStatus(subprocess.call(["chmod +x /usr/local/bin/howdy"], shell=True)) +os.symlink("/lib/security/howdy/cli.py", "/usr/local/bin/howdy") +os.chmod("/usr/local/bin/howdy", 0o755) print("Howdy command installed") log("Adding howdy as PAM module") @@ -149,10 +193,7 @@ inserted = False # Open the PAM config file with open("/etc/pam.d/common-auth") as fp: - # Read the first line - line = fp.readline() - - while line: + for line in fp.readline(): # Add the line to the output directly, we're not deleting anything outlines.append(line) @@ -179,9 +220,6 @@ with open("/etc/pam.d/common-auth") as fp: # Mark as inserted inserted = True - # Go to the next line - line = fp.readline() - # Print a file Header print("\033[33m" + ">>> START OF /etc/pam.d/common-auth" + "\033[0m") @@ -196,19 +234,18 @@ print("\033[33m" + ">>> END OF /etc/pam.d/common-auth" + "\033[0m" + "\n") if "HOWDY_NO_PROMPT" not in os.environ: # Ask the user if this change is okay print("Lines will be insterted in /etc/pam.d/common-auth as shown above") - ans = input("Apply this change? [y/N]: ") + ans = input("Apply this change? [y/N]: ").strip().lower() # Abort the whole thing if it's not - if ans.lower().strip() != "y" or ans.lower().strip() == "yes": + if ans not in ("y", "yes"): print("Interpreting as a \"NO\", aborting") sys.exit(1) print("Adding lines to PAM\n") # Write to PAM -common_auth = open("/etc/pam.d/common-auth", "w") -common_auth.write("".join(outlines)) -common_auth.close() +with open("/etc/pam.d/common-auth", "w") as common_auth: + common_auth.write("".join(outlines)) # Sign off print("Installation complete.") From 377241bd2ef0d685e6fb85a7ad1c1071d47d5e76 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 13:50:35 +0700 Subject: [PATCH 15/31] don't frighten user with slowdown message, it's not so significant; notify user about max name length --- src/cli/add.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/add.py b/src/cli/add.py index 4fe11d8..4283f1d 100644 --- a/src/cli/add.py +++ b/src/cli/add.py @@ -46,8 +46,8 @@ except FileNotFoundError: # Print a warning if too many encodings are being added if len(encodings) > 3: - print("WARNING: Every additional model slows down the face recognition engine") - print("Press ctrl+C to cancel\n") + print("NOTICE: Each additional model slows down the face recognition engine slightly") + print("Press Ctrl+C to cancel\n") print("Adding face model for the user " + user) @@ -63,7 +63,7 @@ if builtins.howdy_args.y: print('Using default label "%s" because of -y flag' % (label, )) else: # Ask the user for a custom label - label_in = input("Enter a label for this new model [" + label + "]: ") + label_in = input("Enter a label for this new model [" + label + "] (max 24 characters): ") # Set the custom label (if any) and limit it to 24 characters if label_in != "": From b465982092a00523ac1c28c533dc98550649d353 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 14:22:55 +0700 Subject: [PATCH 16/31] dark frame detection --- src/cli/add.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/cli/add.py b/src/cli/add.py index 4283f1d..4432cbc 100644 --- a/src/cli/add.py +++ b/src/cli/add.py @@ -106,14 +106,26 @@ time.sleep(2) enc = [] # Count the amount or read frames frames = 0 +dark_threshold = config.getfloat("video", "dark_threshold") # Loop through frames till we hit a timeout while frames < 60: - frames += 1 - # Grab a single frame of video # Don't remove ret, it doesn't work without it ret, frame = video_capture.read() + gsframe = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + + # Create a histogram of the image with 8 values + hist = cv2.calcHist([gsframe], [0], None, [8], [0, 256]) + # All values combined for percentage calculation + hist_total = np.sum(hist) + + # If the image is fully black or the frame exceeds threshold, + # skip to the next frame + if hist_total == 0 or (hist[0] / hist_total * 100 > dark_threshold): + continue + + frames += 1 # Get the encodings in the frame enc = face_recognition.face_encodings(frame) From 8671be425d2e880cb6249c70aafa1f8449e15f67 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 14:24:50 +0700 Subject: [PATCH 17/31] use defaults for config values; simplify path detection --- src/cli/add.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cli/add.py b/src/cli/add.py index 4432cbc..86b0ab8 100644 --- a/src/cli/add.py +++ b/src/cli/add.py @@ -20,8 +20,8 @@ except ImportError as err: print("pip3 show face_recognition") sys.exit(1) -# Get the absolute path to the current file -path = os.path.dirname(os.path.abspath(__file__)) +# Get the absolute path to the current directory +path = os.path.abspath(__file__ + '/..') # Read config from disk config = configparser.ConfigParser() @@ -81,13 +81,13 @@ insert_model = { video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true -if config.getboolean("video", "force_mjpeg"): +if config.getboolean("video", "force_mjpeg", fallback=False): # Set a magic number, will enable MJPEG but is badly documentated video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # Set the frame width and height if requested -fw = config.getint("video", "frame_width") -fh = config.getint("video", "frame_height") +fw = config.getint("video", "frame_width", fallback=-1) +fh = config.getint("video", "frame_height", fallback=-1) if fw != -1: video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) @@ -95,7 +95,7 @@ if fh != -1: video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh) # Request a frame to wake the camera up -video_capture.read() +video_capture.grab() print("\nPlease look straight into the camera") From 2c48350b232d2eb4068dcdddedff22d207dbc8fb Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 14:27:21 +0700 Subject: [PATCH 18/31] replace face_recognition with dlib+numpy; add video_capture.release(); minor code cleanup --- src/cli/add.py | 59 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/src/cli/add.py b/src/cli/add.py index 86b0ab8..28f6022 100644 --- a/src/cli/add.py +++ b/src/cli/add.py @@ -8,16 +8,17 @@ import json import configparser import builtins import cv2 +import numpy as np -# Try to import face_recognition and give a nice error if we can't +# Try to import dlib and give a nice error if we can't # Add should be the first point where import issues show up try: - import face_recognition + import dlib except ImportError as err: print(err) - print("\nCan't import the face_recognition module, check the output of") - print("pip3 show face_recognition") + print("\nCan't import the dlib module, check the output of") + print("pip3 show dlib") sys.exit(1) # Get the absolute path to the current directory @@ -27,6 +28,21 @@ path = os.path.abspath(__file__ + '/..') config = configparser.ConfigParser() config.read(path + "/../config.ini") +use_cnn = config.getboolean('core', 'use_cnn', fallback=False) +if use_cnn: + face_detector = dlib.cnn_face_detection_model_v1( + path + '/../dlib-data/mmod_human_face_detector.dat' + ) +else: + face_detector = dlib.get_frontal_face_detector() + +pose_predictor = dlib.shape_predictor( + path + '/../dlib-data/shape_predictor_5_face_landmarks.dat' +) +face_encoder = dlib.face_recognition_model_v1( + path + '/../dlib-data/dlib_face_recognition_resnet_model_v1.dat' +) + user = builtins.howdy_user # The permanent file to store the encoded model in enc_file = path + "/../models/" + user + ".dat" @@ -127,29 +143,34 @@ while frames < 60: frames += 1 - # Get the encodings in the frame - enc = face_recognition.face_encodings(frame) + # Get all faces from that frame as encodings + face_locations = face_detector(gsframe, 1) # upsample 1 time # If we've found at least one, we can continue - if enc: + if face_locations: break -else: + +video_capture.release() + +# If more than 1 faces are detected we can't know wich one belongs to the user +if len(face_locations) > 1: + print("Multiple faces detected, aborting") + sys.exit(1) +elif not face_locations: print("No face detected, aborting") sys.exit(1) -# If more than 1 faces are detected we can't know wich one belongs to the user -if len(enc) > 1: - print("Multiple faces detected, aborting") - sys.exit(1) +face_location = face_locations[0] +if use_cnn: + face_location = face_location.rect -# Totally clean array that can be exported as JSON -clean_enc = [] +# Get the encodings in the frame +face_landmark = pose_predictor(frame, face_location) +face_encoding = np.array( + face_encoder.compute_face_descriptor(frame, face_landmark, 1) # num_jitters=1 +) -# Copy the values into a clean array so we can export it as JSON later on -for point in enc[0]: - clean_enc.append(point) - -insert_model["data"].append(clean_enc) +insert_model["data"].append(face_encoding.tolist()) # Insert full object into the list encodings.append(insert_model) From d1d4d0af2314715d7ad2ef8a49a620e2c61ef1b3 Mon Sep 17 00:00:00 2001 From: dmig Date: Sun, 9 Dec 2018 14:28:16 +0700 Subject: [PATCH 19/31] remove last face_recognition references --- .travis.yml | 4 ++-- debian/prerm | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3eb3ce..531897f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,8 @@ script: # Confirm the cv2 module has been installed correctly - sudo /usr/bin/env python3 -c "import cv2; print(cv2.__version__);" - # Confirm the face_recognition module has been installed correctly - - sudo /usr/bin/env python3 -c "import face_recognition; print(face_recognition.__version__);" + # Confirm the dlib module has been installed correctly + - sudo /usr/bin/env python3 -c "import dlib; print(dlib.__version__);" # Check if the username passthough works correctly with sudo - 'howdy | ack-grep --passthru --color "current active user: travis"' diff --git a/debian/prerm b/debian/prerm index 958c37e..a4537f0 100755 --- a/debian/prerm +++ b/debian/prerm @@ -38,7 +38,7 @@ except Exception: # This error is normal pass -# Remove face_recognition and dlib +# Remove dlib subprocess.call(['pip3', 'uninstall', 'dlib', '-y', '--no-cache-dir']) # Print a tearbending message From 02a9aee63a97415a89a1664b5d2f8675e295fa41 Mon Sep 17 00:00:00 2001 From: dmig Date: Tue, 11 Dec 2018 16:24:35 +0700 Subject: [PATCH 20/31] convert timings from list to dict; move camera initialization to separate thread --- src/compare.py | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/compare.py b/src/compare.py index 2d2f0c4..d17b330 100644 --- a/src/compare.py +++ b/src/compare.py @@ -5,7 +5,9 @@ import time # Start timing -timings = [time.time()] +timings = { + 'st': time.time() +} # Import required modules import cv2 @@ -13,6 +15,7 @@ import sys import os import json import configparser +from threading import Thread # Get the absolute path to the current directory PATH = os.path.abspath(__file__ + '/..') @@ -56,8 +59,15 @@ except FileNotFoundError: if not encodings: sys.exit(10) +timings['st'] = time.time() - timings['st'] + +video_capture = None + +def initialize_cam(): + global video_capture, timings + # Add the time needed to start the script -timings.append(time.time()) + timings['ic'] = time.time() # Start video capture on the IR camera video_capture = cv2.VideoCapture(config.get("video", "device_path")) @@ -81,7 +91,12 @@ if fh != -1: video_capture.grab() # Note the time it took to open the camera -timings.append(time.time()) + timings['ic'] = time.time() - timings['ic'] + +init_thread = Thread(target=initialize_cam) +init_thread.start() + +timings['ll'] = time.time() # Import face recognition, takes some time import dlib @@ -102,7 +117,11 @@ face_encoder = dlib.face_recognition_model_v1( PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' ) -timings.append(time.time()) +timings['ll'] = time.time() - timings['ll'] + +# wait for camera initialization to finish +init_thread.join() +del init_thread # Fetch the max frame height max_height = config.getfloat("video", "max_height", fallback=0.0) @@ -113,6 +132,7 @@ height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 1 scaling_factor = (max_height / height) or 1 # Start the read loop +timings['fr'] = time.time() frames = 0 timeout = config.getint("video", "timout") dark_threshold = config.getfloat("video", "dark_threshold") @@ -123,7 +143,7 @@ while True: frames += 1 # Stop if we've exceded the time limit - if time.time() - timings[3] > timeout: + if time.time() - timings['fr'] > timeout: stop(11) # Grab a single frame of video @@ -168,19 +188,19 @@ while True: # Check if a match that's confident enough if 0 < match < video_certainty: - timings.append(time.time()) + timings['fr'] = time.time() - timings['fr'] # If set to true in the config, print debug text if end_report: - def print_timing(label, offset): + def print_timing(label, k): """Helper function to print a timing from the list""" - print(" %s: %dms" % (label, round((timings[1 + offset] - timings[offset]) * 1000))) + print(" %s: %dms" % (label, round(timings[k] * 1000))) print("Time spent") - print_timing("Starting up", 0) - print_timing("Opening the camera", 1) - print_timing("Importing recognition libs", 2) - print_timing("Searching for known face", 3) + print_timing("Starting up", 'st') + print_timing("Opening the camera", 'ic') + print_timing("Importing recognition libs", 'll') + print_timing("Searching for known face", 'fr') print("\nResolution") width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 1 @@ -190,7 +210,7 @@ while True: print(" Used: %dx%d" % (scale_height, scale_width)) # Show the total number of frames and calculate the FPS by deviding it by the total scan time - print("\nFrames searched: %d (%.2f fps)" % (frames, frames / (timings[4] - timings[3]))) + print("\nFrames searched: %d (%.2f fps)" % (frames, frames / timings['fr'])) print("Dark frames ignored: %d " % (dark_tries, )) print("Certainty of winning frame: %.3f" % (match * 10, )) From bedfa8020cf4557e8cc914f5cf357ef3127c7eb6 Mon Sep 17 00:00:00 2001 From: dmig Date: Tue, 11 Dec 2018 16:30:45 +0700 Subject: [PATCH 21/31] more informative debug --- src/compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/compare.py b/src/compare.py index d17b330..cafdb0d 100644 --- a/src/compare.py +++ b/src/compare.py @@ -59,14 +59,13 @@ except FileNotFoundError: if not encodings: sys.exit(10) +# Add the time needed to start the script timings['st'] = time.time() - timings['st'] video_capture = None def initialize_cam(): global video_capture, timings - -# Add the time needed to start the script timings['ic'] = time.time() # Start video capture on the IR camera @@ -198,8 +197,9 @@ while True: print("Time spent") print_timing("Starting up", 'st') - print_timing("Opening the camera", 'ic') - print_timing("Importing recognition libs", 'll') + print(" Open cam + load libs: %dms" % (round(max(timings['ll'], timings['ic']) * 1000, ))) + print_timing(" Opening the camera", 'ic') + print_timing(" Importing recognition libs", 'll') print_timing("Searching for known face", 'fr') print("\nResolution") From c90d977352d7e93221ca04db7f19f4d0aeac6518 Mon Sep 17 00:00:00 2001 From: dmig Date: Tue, 11 Dec 2018 16:40:06 +0700 Subject: [PATCH 22/31] fix broken previous commit --- src/compare.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/compare.py b/src/compare.py index cafdb0d..8460ec1 100644 --- a/src/compare.py +++ b/src/compare.py @@ -68,28 +68,28 @@ def initialize_cam(): global video_capture, timings timings['ic'] = time.time() -# Start video capture on the IR camera -video_capture = cv2.VideoCapture(config.get("video", "device_path")) + # Start video capture on the IR camera + video_capture = cv2.VideoCapture(config.get("video", "device_path")) -# Force MJPEG decoding if true -if config.getboolean("video", "force_mjpeg", fallback=False): - # Set a magic number, will enable MJPEG but is badly documentated - video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) + # Force MJPEG decoding if true + if config.getboolean("video", "force_mjpeg", fallback=False): + # Set a magic number, will enable MJPEG but is badly documentated + video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) -# Set the frame width and height if requested -fw = config.getint("video", "frame_width", fallback=-1) -fh = config.getint("video", "frame_height", fallback=-1) -if fw != -1: - video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) + # Set the frame width and height if requested + fw = config.getint("video", "frame_width", fallback=-1) + fh = config.getint("video", "frame_height", fallback=-1) + if fw != -1: + video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) -if fh != -1: - video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh) + if fh != -1: + video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh) -# Capture a single frame so the camera becomes active -# This will let the camera adjust its light levels while we're importing for faster scanning -video_capture.grab() + # Capture a single frame so the camera becomes active + # This will let the camera adjust its light levels while we're importing for faster scanning + video_capture.grab() -# Note the time it took to open the camera + # Note the time it took to open the camera timings['ic'] = time.time() - timings['ic'] init_thread = Thread(target=initialize_cam) From f4d231cca8d49e03861cb1ca3491825c38651f36 Mon Sep 17 00:00:00 2001 From: dmig Date: Tue, 11 Dec 2018 23:44:30 +0700 Subject: [PATCH 23/31] proper cam init time measuring --- src/compare.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/compare.py b/src/compare.py index 8460ec1..758cdd1 100644 --- a/src/compare.py +++ b/src/compare.py @@ -65,8 +65,7 @@ timings['st'] = time.time() - timings['st'] video_capture = None def initialize_cam(): - global video_capture, timings - timings['ic'] = time.time() + global video_capture # Start video capture on the IR camera video_capture = cv2.VideoCapture(config.get("video", "device_path")) @@ -89,9 +88,7 @@ def initialize_cam(): # This will let the camera adjust its light levels while we're importing for faster scanning video_capture.grab() - # Note the time it took to open the camera - timings['ic'] = time.time() - timings['ic'] - +timings['ic'] = time.time() init_thread = Thread(target=initialize_cam) init_thread.start() @@ -121,6 +118,8 @@ timings['ll'] = time.time() - timings['ll'] # wait for camera initialization to finish init_thread.join() del init_thread +# Note the time it took to open the camera +timings['ic'] = time.time() - timings['ic'] # Fetch the max frame height max_height = config.getfloat("video", "max_height", fallback=0.0) From 62aabdf480f80b3400278cb1fc922886da297bc3 Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 00:08:13 +0700 Subject: [PATCH 24/31] initialize recognizers in threads instead of camera init --- src/compare.py | 87 +++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/src/compare.py b/src/compare.py index 758cdd1..9069e96 100644 --- a/src/compare.py +++ b/src/compare.py @@ -62,35 +62,31 @@ if not encodings: # Add the time needed to start the script timings['st'] = time.time() - timings['st'] -video_capture = None - -def initialize_cam(): - global video_capture - - # Start video capture on the IR camera - video_capture = cv2.VideoCapture(config.get("video", "device_path")) - - # Force MJPEG decoding if true - if config.getboolean("video", "force_mjpeg", fallback=False): - # Set a magic number, will enable MJPEG but is badly documentated - video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) - - # Set the frame width and height if requested - fw = config.getint("video", "frame_width", fallback=-1) - fh = config.getint("video", "frame_height", fallback=-1) - if fw != -1: - video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) - - if fh != -1: - video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh) - - # Capture a single frame so the camera becomes active - # This will let the camera adjust its light levels while we're importing for faster scanning - video_capture.grab() - timings['ic'] = time.time() -init_thread = Thread(target=initialize_cam) -init_thread.start() + +# Start video capture on the IR camera +video_capture = cv2.VideoCapture(config.get("video", "device_path")) + +# Force MJPEG decoding if true +if config.getboolean("video", "force_mjpeg", fallback=False): + # Set a magic number, will enable MJPEG but is badly documentated + video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) + +# Set the frame width and height if requested +fw = config.getint("video", "frame_width", fallback=-1) +fh = config.getint("video", "frame_height", fallback=-1) +if fw != -1: + video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw) + +if fh != -1: + video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh) + +# Capture a single frame so the camera becomes active +# This will let the camera adjust its light levels while we're importing for faster scanning +video_capture.grab() + +# Note the time it took to open the camera +timings['ic'] = time.time() - timings['ic'] timings['ll'] = time.time() @@ -98,6 +94,9 @@ timings['ll'] = time.time() import dlib import numpy as np +pose_predictor = None +face_encoder = None + use_cnn = config.getboolean('core', 'use_cnn', fallback=False) if use_cnn: face_detector = dlib.cnn_face_detection_model_v1( @@ -106,21 +105,29 @@ if use_cnn: else: face_detector = dlib.get_frontal_face_detector() -pose_predictor = dlib.shape_predictor( - PATH + '/dlib-data/shape_predictor_5_face_landmarks.dat' -) -face_encoder = dlib.face_recognition_model_v1( - PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' -) +def init_predictor(): + global pose_predictor + pose_predictor = dlib.shape_predictor( + PATH + '/dlib-data/shape_predictor_5_face_landmarks.dat' + ) +def init_encoder(): + global face_encoder + face_encoder = dlib.face_recognition_model_v1( + PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' + ) + + +init_thread1 = Thread(target=init_encoder) +init_thread2 = Thread(target=init_predictor) +init_thread1.start() +init_thread2.start() + +init_thread2.join() +init_thread1.join() +del init_thread1, init_thread2 timings['ll'] = time.time() - timings['ll'] -# wait for camera initialization to finish -init_thread.join() -del init_thread -# Note the time it took to open the camera -timings['ic'] = time.time() - timings['ic'] - # Fetch the max frame height max_height = config.getfloat("video", "max_height", fallback=0.0) # Get the height of the image From 4f11da686f2d23c6f10637d53e997d713e3948c4 Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 00:15:35 +0700 Subject: [PATCH 25/31] one more detector init thread --- src/compare.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/compare.py b/src/compare.py index 9069e96..4ac8cb6 100644 --- a/src/compare.py +++ b/src/compare.py @@ -94,16 +94,19 @@ timings['ll'] = time.time() import dlib import numpy as np +face_detector = None pose_predictor = None face_encoder = None - use_cnn = config.getboolean('core', 'use_cnn', fallback=False) -if use_cnn: - face_detector = dlib.cnn_face_detection_model_v1( - PATH + '/dlib-data/mmod_human_face_detector.dat' - ) -else: - face_detector = dlib.get_frontal_face_detector() + +def init_detector(): + global face_detector + if use_cnn: + face_detector = dlib.cnn_face_detection_model_v1( + PATH + '/dlib-data/mmod_human_face_detector.dat' + ) + else: + face_detector = dlib.get_frontal_face_detector() def init_predictor(): global pose_predictor @@ -120,12 +123,15 @@ def init_encoder(): init_thread1 = Thread(target=init_encoder) init_thread2 = Thread(target=init_predictor) +init_thread3 = Thread(target=init_detector) +init_thread3.start() init_thread1.start() init_thread2.start() +init_thread3.join() init_thread2.join() init_thread1.join() -del init_thread1, init_thread2 +del init_thread1, init_thread2, init_thread3 timings['ll'] = time.time() - timings['ll'] # Fetch the max frame height From 7aac419bd7066e979fb6a9adc341e0655a756c5c Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 14:45:23 +0700 Subject: [PATCH 26/31] importing all modules at the beginning is ~40% faster --- src/compare.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/compare.py b/src/compare.py index 4ac8cb6..53ba3d4 100644 --- a/src/compare.py +++ b/src/compare.py @@ -10,12 +10,14 @@ timings = { } # Import required modules -import cv2 import sys import os import json import configparser from threading import Thread +import cv2 +import dlib +import numpy as np # Get the absolute path to the current directory PATH = os.path.abspath(__file__ + '/..') @@ -90,10 +92,6 @@ timings['ic'] = time.time() - timings['ic'] timings['ll'] = time.time() -# Import face recognition, takes some time -import dlib -import numpy as np - face_detector = None pose_predictor = None face_encoder = None From 01c6364e65c5017d8455028ca3de018db7d8454c Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 14:58:03 +0700 Subject: [PATCH 27/31] simplify argument check; comments typos --- src/compare.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/compare.py b/src/compare.py index 53ba3d4..b47d8a8 100644 --- a/src/compare.py +++ b/src/compare.py @@ -31,14 +31,12 @@ def stop(status): video_capture.release() sys.exit(status) -# Make sure we were given an username to tast against -try: - if not isinstance(sys.argv[1], str): - sys.exit(1) -except IndexError: + +# Make sure we were got an username to test against +if len(sys.argv) < 2: sys.exit(1) -# The username of the authenticating user +# The username of the user being authenticated user = sys.argv[1] # The model file contents models = [] @@ -61,7 +59,7 @@ except FileNotFoundError: if not encodings: sys.exit(10) -# Add the time needed to start the script +# Save the time needed to start the script timings['st'] = time.time() - timings['st'] timings['ic'] = time.time() @@ -71,8 +69,8 @@ video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true if config.getboolean("video", "force_mjpeg", fallback=False): - # Set a magic number, will enable MJPEG but is badly documentated - video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) + # Set a magic number, will enable MJPEG but is badly documented + video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # 1196444237 is 'MJPEG' in ASCII # Set the frame width and height if requested fw = config.getint("video", "frame_width", fallback=-1) From 5fcb7383ca8a74b0481ba26b709f22bd6320377f Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 15:00:59 +0700 Subject: [PATCH 28/31] reorder lines for readability --- src/compare.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/compare.py b/src/compare.py index b47d8a8..03f268b 100644 --- a/src/compare.py +++ b/src/compare.py @@ -19,8 +19,6 @@ import cv2 import dlib import numpy as np -# Get the absolute path to the current directory -PATH = os.path.abspath(__file__ + '/..') # Read config from disk config = configparser.ConfigParser() @@ -36,6 +34,9 @@ def stop(status): if len(sys.argv) < 2: sys.exit(1) +# Get the absolute path to the current directory +PATH = os.path.abspath(__file__ + '/..') + # The username of the user being authenticated user = sys.argv[1] # The model file contents @@ -44,6 +45,12 @@ models = [] encodings = [] # Amount of ingnored dark frames dark_tries = 0 +# Total amount of frames captured +frames = 0 +# face recognition/detection instances +face_detector = None +pose_predictor = None +face_encoder = None # Try to load the face model from the models folder try: @@ -59,6 +66,17 @@ except FileNotFoundError: if not encodings: sys.exit(10) +# Read config from disk +config = configparser.ConfigParser() +config.read(PATH + "/config.ini") + +# CNN usage flag +use_cnn = config.getboolean('core', 'use_cnn', fallback=False) +timeout = config.getint("video", "timout", fallback=5) +dark_threshold = config.getfloat("video", "dark_threshold", fallback=50.0) +video_certainty = config.getfloat("video", "certainty", fallback=3.5) / 10 +end_report = config.getboolean("debug", "end_report", fallback=False) + # Save the time needed to start the script timings['st'] = time.time() - timings['st'] @@ -90,11 +108,6 @@ timings['ic'] = time.time() - timings['ic'] timings['ll'] = time.time() -face_detector = None -pose_predictor = None -face_encoder = None -use_cnn = config.getboolean('core', 'use_cnn', fallback=False) - def init_detector(): global face_detector if use_cnn: @@ -128,6 +141,7 @@ init_thread3.join() init_thread2.join() init_thread1.join() del init_thread1, init_thread2, init_thread3 +# Note the time it took to initalize detectors timings['ll'] = time.time() - timings['ll'] # Fetch the max frame height @@ -140,11 +154,6 @@ scaling_factor = (max_height / height) or 1 # Start the read loop timings['fr'] = time.time() -frames = 0 -timeout = config.getint("video", "timout") -dark_threshold = config.getfloat("video", "dark_threshold") -end_report = config.getboolean("debug", "end_report") -video_certainty = config.getfloat("video", "certainty") / 10 while True: # Increment the frame count every loop frames += 1 From cc1b9785a83ec1f125558f9397848631dd3256a6 Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 15:04:11 +0700 Subject: [PATCH 29/31] replace threading module with low-level _thread; leave only one initialization helper thread --- src/compare.py | 67 +++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/src/compare.py b/src/compare.py index 03f268b..10b863c 100644 --- a/src/compare.py +++ b/src/compare.py @@ -14,15 +14,28 @@ import sys import os import json import configparser -from threading import Thread import cv2 import dlib import numpy as np +import _thread as thread +def init_detector(lock): + global face_detector, pose_predictor, face_encoder + if use_cnn: + face_detector = dlib.cnn_face_detection_model_v1( + PATH + '/dlib-data/mmod_human_face_detector.dat' + ) + else: + face_detector = dlib.get_frontal_face_detector() -# Read config from disk -config = configparser.ConfigParser() -config.read(PATH + "/config.ini") + pose_predictor = dlib.shape_predictor( + PATH + '/dlib-data/shape_predictor_5_face_landmarks.dat' + ) + + face_encoder = dlib.face_recognition_model_v1( + PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' + ) + lock.release() def stop(status): """Stop the execution and close video stream""" @@ -80,9 +93,15 @@ end_report = config.getboolean("debug", "end_report", fallback=False) # Save the time needed to start the script timings['st'] = time.time() - timings['st'] -timings['ic'] = time.time() +# Import face recognition, takes some time +timings['ll'] = time.time() + +lock = thread.allocate_lock() +lock.acquire() +thread.start_new_thread(init_detector, (lock, )) # Start video capture on the IR camera +timings['ic'] = time.time() video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true @@ -106,41 +125,11 @@ video_capture.grab() # Note the time it took to open the camera timings['ic'] = time.time() - timings['ic'] -timings['ll'] = time.time() +# wait for thread to finish +lock.acquire() +lock.release() +del lock -def init_detector(): - global face_detector - if use_cnn: - face_detector = dlib.cnn_face_detection_model_v1( - PATH + '/dlib-data/mmod_human_face_detector.dat' - ) - else: - face_detector = dlib.get_frontal_face_detector() - -def init_predictor(): - global pose_predictor - pose_predictor = dlib.shape_predictor( - PATH + '/dlib-data/shape_predictor_5_face_landmarks.dat' - ) - -def init_encoder(): - global face_encoder - face_encoder = dlib.face_recognition_model_v1( - PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' - ) - - -init_thread1 = Thread(target=init_encoder) -init_thread2 = Thread(target=init_predictor) -init_thread3 = Thread(target=init_detector) -init_thread3.start() -init_thread1.start() -init_thread2.start() - -init_thread3.join() -init_thread2.join() -init_thread1.join() -del init_thread1, init_thread2, init_thread3 # Note the time it took to initalize detectors timings['ll'] = time.time() - timings['ll'] From 443c5e70635db419ddd819ed4877ee051e034259 Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 16:30:52 +0700 Subject: [PATCH 30/31] proper library load timing; add total time reporting --- src/compare.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/compare.py b/src/compare.py index 10b863c..f6f21a5 100644 --- a/src/compare.py +++ b/src/compare.py @@ -35,6 +35,8 @@ def init_detector(lock): face_encoder = dlib.face_recognition_model_v1( PATH + '/dlib-data/dlib_face_recognition_resnet_model_v1.dat' ) + # Note the time it took to initalize detectors + timings['ll'] = time.time() - timings['ll'] lock.release() def stop(status): @@ -91,7 +93,7 @@ video_certainty = config.getfloat("video", "certainty", fallback=3.5) / 10 end_report = config.getboolean("debug", "end_report", fallback=False) # Save the time needed to start the script -timings['st'] = time.time() - timings['st'] +timings['in'] = time.time() - timings['st'] # Import face recognition, takes some time timings['ll'] = time.time() @@ -130,8 +132,6 @@ lock.acquire() lock.release() del lock -# Note the time it took to initalize detectors -timings['ll'] = time.time() - timings['ll'] # Fetch the max frame height max_height = config.getfloat("video", "max_height", fallback=0.0) @@ -193,6 +193,7 @@ while True: # Check if a match that's confident enough if 0 < match < video_certainty: + timings['tt'] = time.time() - timings['st'] timings['fr'] = time.time() - timings['fr'] # If set to true in the config, print debug text @@ -202,11 +203,12 @@ while True: print(" %s: %dms" % (label, round(timings[k] * 1000))) print("Time spent") - print_timing("Starting up", 'st') + print_timing("Starting up", 'in') print(" Open cam + load libs: %dms" % (round(max(timings['ll'], timings['ic']) * 1000, ))) print_timing(" Opening the camera", 'ic') print_timing(" Importing recognition libs", 'll') print_timing("Searching for known face", 'fr') + print_timing("Total time", 'tt') print("\nResolution") width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 1 From a8d68d945b51f871bc9876d1885fcb8b9fe3e14e Mon Sep 17 00:00:00 2001 From: dmig Date: Wed, 12 Dec 2018 16:43:27 +0700 Subject: [PATCH 31/31] no changes, comment --- src/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compare.py b/src/compare.py index f6f21a5..23fd486 100644 --- a/src/compare.py +++ b/src/compare.py @@ -109,7 +109,7 @@ video_capture = cv2.VideoCapture(config.get("video", "device_path")) # Force MJPEG decoding if true if config.getboolean("video", "force_mjpeg", fallback=False): # Set a magic number, will enable MJPEG but is badly documented - video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # 1196444237 is 'MJPEG' in ASCII + video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237) # 1196444237 is 'GPJM' in ASCII # Set the frame width and height if requested fw = config.getint("video", "frame_width", fallback=-1)