Merge pull request #118 from boltgolt/dev

Version 2.5.0

.travis.yml

@@ -8,19 +8,16 @@ script:
   # Install the binary, running the debian scripts in the process
   - sudo apt install ../*.deb -y
 
-  # Confirm the cv2 module has been installed correctly
+  # Go through function tests
-  - sudo /usr/bin/env python3 -c "import cv2; print(cv2.__version__);"
+  - ./tests/importing.sh
-  # Confirm the face_recognition module has been installed correctly
+  - ./tests/passthrough.sh
-  - sudo /usr/bin/env python3 -c "import face_recognition; print(face_recognition.__version__);"
+  # Skip PAM integration tests for now because of broken pamtester
-
+  # - ./tests/pam.sh
-  # Check if the username passthough works correctly with sudo
+  - ./tests/compare.sh
-  - 'howdy | ack-grep --passthru --color "current active user: travis"'
-  - 'sudo howdy | ack-grep --passthru --color "current active user: travis"'
 
   # Remove howdy from the installation
   - sudo apt purge howdy -y
 
-
 notifications:
   email:
     on_success: never
@@ -34,3 +31,4 @@ addons:
       - ack-grep
       - devscripts
       - fakeroot
+      - pamtester

README.md

@@ -28,6 +28,14 @@ Install the `howdy` package from the AUR. For AUR installation instructions, tak
 
 You will need to do some additional configuration steps. Please read the [ArchWiki entry](https://wiki.archlinux.org/index.php/Howdy) for more information.
 
+### Fedora
+The `howdy` package is now available in a [Fedora COPR repository](https://copr.fedorainfracloud.org/coprs/luya/howdy/) by simply execute the following command from a terminal:
+
+```
+sudo dnf copr enable luya/howdy
+sudo dnf install howdy
+```
+
 ## Setup
 
 After installation, you need to let Howdy learn your face. Run `sudo howdy add` to add a face model.

debian/changelog

@@ -1,3 +1,15 @@
+howdy (2.5.0) xenial; urgency=medium
+
+  * Added FFmpeg and v4l2 recorders (thanks @timwelch!)
+  * Added automatic PAM inclusion on installation
+  * Added optional notice on detection attempt (thanks @mrkmg!)
+  * Added support for grayscale frame encoding (thanks @dmig and @sapjunior!)
+  * Massively improved recognition speed (thanks @dmig!)
+  * Fixed typo in "timout" config value
+  * Removed unneeded dependencies (thanks @dmig!)
+
+ -- boltgolt <boltgolt@gmail.com>  Sun, 06 Jan 2019 14:37:41 +0100
+
 howdy (2.4.0) xenial; urgency=medium
 
   * Cameras are now selected by path instead of by video device number (thanks @Rhiyo!)

debian/control

@@ -9,7 +9,9 @@ Vcs-Git: https://github.com/boltgolt/howdy
 Package: howdy
 Homepage: https://github.com/boltgolt/howdy
 Architecture: all
-Depends: ${misc:Depends}, git, python3, python3-pip, python3-dev, python3-setuptools, libpam-python, fswebcam, libopencv-dev, python-opencv, cmake, streamer
+Depends: ${misc:Depends}, curl|wget, python3, python3-pip, python3-dev, python3-setuptools, libpam-python, fswebcam, libopencv-dev, cmake, streamer
+Recommends: libatlas-base-dev | libopenblas-dev | liblapack-dev
+Suggests: nvidia-cuda-dev (>= 7.5)
 Description: Howdy: Windows Hello style authentication for Linux.
  Use your built-in IR emitters and camera in combination with face recognition
  to prove who you are.

debian/install

@@ -1,2 +1,3 @@
 src/. lib/security/howdy
+src/pam-config/.  /usr/share/pam-configs
 autocomplete/. usr/share/bash-completion/completions

debian/postinst

@@ -2,6 +2,7 @@
 # Installation script to install howdy
 # Executed after primary apt install
 
+
 def col(id):
 	"""Add color escape sequences"""
 	if id == 1: return "\033[32m"
@@ -9,15 +10,15 @@ def col(id):
 	if id == 3: return "\033[31m"
 	return "\033[0m"
 
+
 # Import required modules
+import fileinput
 import subprocess
-import time
 import sys
 import os
 import re
-import signal
+import tarfile
-import fileinput
+from shutil import rmtree, which
-import urllib.parse
 
 # Don't run unless we need to configure the install
 # Will also happen on upgrade but we will catch that later on
@@ -29,6 +30,7 @@ def log(text):
 	"""Print a nicely formatted line to stdout"""
 	print("\n>>> " + col(1) + text + col(0) + "\n")
 
+
 def handleStatus(status):
 	"""Abort if a command fails"""
 	if (status != 0):
@@ -36,6 +38,9 @@ def handleStatus(status):
 		sys.exit(1)
 
 
+# Create shorthand for subprocess creation
+sc = subprocess.call
+
 # We're not in fresh configuration mode so don't continue the setup
 if not os.path.exists("/tmp/howdy_picked_device"):
 	# Check if we have an older config we can restore
@@ -53,15 +58,22 @@ if not os.path.exists("/tmp/howdy_picked_device"):
 			# Go through every setting in the old config and apply it to the new file
 			for section in oldConf.sections():
 				for (key, value) in oldConf.items(section):
+
+					# MIGRATION 2.3.1 -> 2.4.0
 					# If config is still using the old device_id parameter, convert it to a path
 					if key == "device_id":
 						key = "device_path"
 						value = "/dev/video" + value
 
+					# MIGRATION 2.4.0 -> 2.5.0
+					# Finally correct typo in "timout" config value
+					if key == "timout":
+						key = "timeout"
+
 					try:
 						newConf.set(section, key, value)
 					# Add a new section where needed
-					except configparser.NoSectionError as e:
+					except configparser.NoSectionError:
 						newConf.add_section(section)
 						newConf.set(section, key, value)
 
@@ -69,6 +81,11 @@ if not os.path.exists("/tmp/howdy_picked_device"):
 			with open("/lib/security/howdy/config.ini", "w") as configfile:
 				newConf.write(configfile)
 
+		# Install dlib data files if needed
+		if not os.path.exists("/lib/security/howdy/dlib-data/shape_predictor_5_face_landmarks.dat"):
+			print("Attempting installation of missing data files")
+			handleStatus(subprocess.call(["./install.sh"], shell=True, cwd="/lib/security/howdy/dlib-data"))
+
 	sys.exit(0)
 
 # Open the temporary file containing the device ID
@@ -78,137 +95,139 @@ picked = in_file.read()
 in_file.close()
 
 # Remove the temporary file
-subprocess.call(["rm /tmp/howdy_picked_device"], shell=True)
+os.unlink("/tmp/howdy_picked_device")
 
 log("Upgrading pip to the latest version")
 
 # Update pip
-handleStatus(subprocess.call(["pip3 install --upgrade pip"], shell=True))
+handleStatus(sc(["pip3", "install", "--upgrade", "pip"]))
 
-log("Cloning dlib")
+log("Downloading and unpacking data files")
 
-# Clone the dlib git to /tmp, but only the last commit
+# Run the bash script to download and unpack the .dat files needed
-handleStatus(subprocess.call(["git", "clone", "--depth", "1", "https://github.com/davisking/dlib.git", "/tmp/dlib_clone"]))
+handleStatus(subprocess.call(["./install.sh"], shell=True, cwd="/lib/security/howdy/dlib-data"))
+
+log("Downloading dlib")
+
+dlib_archive = "/tmp/v19.16.tar.gz"
+loader = which("wget")
+LOADER_CMD = None
+
+# If wget is installed, use that as the downloader
+if loader:
+	LOADER_CMD = [loader, "--tries", "5", "--output-document"]
+# Otherwise, fall back on curl
+else:
+	loader = which("curl")
+	LOADER_CMD = [loader, "--retry", "5", "--location", "--output"]
+
+# Assemble and execute the download command
+cmd = LOADER_CMD + [dlib_archive, "https://github.com/davisking/dlib/archive/v19.16.tar.gz"]
+handleStatus(sc(cmd))
+
+# The folder containing the dlib source
+DLIB_DIR = None
+# A regex of all files to ignore while unpacking the archive
+excludes = re.compile(
+	"davisking-dlib-\w+/(dlib/(http_client|java|matlab|test/)|"
+	"(docs|examples|python_examples)|"
+	"tools/(archive|convert_dlib_nets_to_caffe|htmlify|imglab|python/test|visual_studio_natvis))"
+)
+
+# Open the archive
+with tarfile.open(dlib_archive) as tf:
+	for item in tf:
+		# Set the destenation dir if unset
+		if not DLIB_DIR:
+			DLIB_DIR = "/tmp/" + item.name
+
+		# extract only files sufficient for building
+		if not excludes.match(item.name):
+			tf.extract(item, "/tmp")
+
+# Delete the downloaded archive
+os.unlink(dlib_archive)
 
 log("Building dlib")
 
-# Start the build without GPU
+cmd = ["sudo", "python3", "setup.py", "install"]
-handleStatus(subprocess.call(["cd /tmp/dlib_clone/; python3 setup.py install --yes USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA"], shell=True))
+cuda_used = False
+flags = ""
+
+# Get the CPU details
+with open("/proc/cpuinfo") as info:
+	for line in info:
+		if "flags" in line:
+			flags = line
+			break
+
+# Use the most efficient instruction set the CPU supports
+if "avx" in flags:
+	cmd += ["--yes", "USE_AVX_INSTRUCTIONS"]
+elif "sse4" in flags:
+	cmd += ["--yes", "USE_SSE4_INSTRUCTIONS"]
+elif "sse3" in flags:
+	cmd += ["--yes", "USE_SSE3_INSTRUCTIONS"]
+elif "sse2" in flags:
+	cmd += ["--yes", "USE_SSE2_INSTRUCTIONS"]
+
+# Compile and link dlib
+try:
+	sp = subprocess.Popen(cmd, cwd=DLIB_DIR, stdout=subprocess.PIPE)
+except subprocess.CalledProcessError:
+	print("Error while building dlib")
+	raise
+
+# Go through each line from stdout
+while sp.poll() is None:
+	line = sp.stdout.readline().decode("utf-8")
+
+	if "DLIB WILL USE CUDA" in line:
+		cuda_used = True
+
+	print(line, end="")
 
 log("Cleaning up dlib")
 
 # Remove the no longer needed git clone
-handleStatus(subprocess.call(["rm", "-rf", "/tmp/dlib_clone"]))
+del sp
+rmtree(DLIB_DIR)
+
 print("Temporary dlib files removed")
 
-log("Installing python dependencies")
+log("Installing OpenCV")
 
-# Install direct dependencies so pip does not freak out with the manual dlib install
+handleStatus(subprocess.call(["pip3", "install", "--no-cache-dir", "opencv-python"]))
-handleStatus(subprocess.call(["pip3", "install", "--cache-dir", "/tmp/pip_howdy", "face_recognition_models==0.3.0", "Click>=6.0", "numpy", "Pillow"]))
-
-log("Installing face_recognition")
-
-# Install face_recognition though pip
-handleStatus(subprocess.call(["pip3", "install", "--cache-dir", "/tmp/pip_howdy", "--no-deps", "face_recognition==1.2.2"]))
-
-try:
-	import cv2
-except Exception as e:
-	log("Reinstalling opencv2")
-	handleStatus(subprocess.call(["pip3", "install", "opencv-python"]))
 
 log("Configuring howdy")
 
 # Manually change the camera id to the one picked
-for line in fileinput.input(["/lib/security/howdy/config.ini"], inplace = 1):
+for line in fileinput.input(["/lib/security/howdy/config.ini"], inplace=1):
-	print(line.replace("device_path = none", "device_path = " + picked), end="")
+	line = line.replace("device_path = none", "device_path = " + picked)
+	line = line.replace("use_cnn = false", "use_cnn = " + str(cuda_used).lower())
+
+	print(line, end="")
+
 print("Camera ID saved")
 
 # Secure the howdy folder
-handleStatus(subprocess.call(["chmod 744 -R /lib/security/howdy/"], shell=True))
+handleStatus(sc(["chmod 744 -R /lib/security/howdy/"], shell=True))
 
 # Allow anyone to execute the python CLI
-handleStatus(subprocess.call(["chmod 755 /lib/security/howdy"], shell=True))
+os.chmod("/lib/security/howdy", 0o755)
-handleStatus(subprocess.call(["chmod 755 /lib/security/howdy/cli.py"], shell=True))
+os.chmod("/lib/security/howdy/cli.py", 0o755)
-handleStatus(subprocess.call(["chmod 755 -R /lib/security/howdy/cli"], shell=True))
+handleStatus(sc(["chmod 755 -R /lib/security/howdy/cli"], shell=True))
 print("Permissions set")
 
 # Make the CLI executable as howdy
-handleStatus(subprocess.call(["ln -s /lib/security/howdy/cli.py /usr/local/bin/howdy"], shell=True))
+os.symlink("/lib/security/howdy/cli.py", "/usr/local/bin/howdy")
-handleStatus(subprocess.call(["chmod +x /usr/local/bin/howdy"], shell=True))
+os.chmod("/usr/local/bin/howdy", 0o755)
 print("Howdy command installed")
 
 log("Adding howdy as PAM module")
 
-# Will be filled with the actual output lines
+# Activate the pam-config file
-outlines = []
+handleStatus(subprocess.call(["pam-auth-update --package"], shell=True))
-# Will be fillled with lines that contain coloring
-printlines = []
-# Track if the new lines have been insterted yet
-inserted = False
-
-# Open the PAM config file
-with open("/etc/pam.d/common-auth") as fp:
-	# Read the first line
-	line = fp.readline()
-
-	while line:
-		# Add the line to the output directly, we're not deleting anything
-		outlines.append(line)
-
-		# Print the comments in gray and don't insert into comments
-		if line[:1] == "#":
-			printlines.append("\033[37m" + line + "\033[0m")
-		else:
-			printlines.append(line)
-
-			# If it's not a comment and we haven't inserted yet
-			if not inserted:
-				# Set both the comment and the linking line
-				line_comment = "# Howdy IR face recognition\n"
-				line_link	= "auth	sufficient			pam_python.so /lib/security/howdy/pam.py\n\n"
-
-				# Add them to the output without any markup
-				outlines.append(line_comment)
-				outlines.append(line_link)
-
-				# Make the print orange to make it clear what's being added
-				printlines.append("\033[33m" + line_comment + "\033[0m")
-				printlines.append("\033[33m" + line_link + "\033[0m")
-
-				# Mark as inserted
-				inserted = True
-
-		# Go to the next line
-		line = fp.readline()
-
-# Print a file Header
-print("\033[33m" + ">>> START OF /etc/pam.d/common-auth" + "\033[0m")
-
-# Loop though all printing lines and use the enters from the file
-for line in printlines:
-	print(line, end="")
-
-# Print a footer
-print("\033[33m" + ">>> END OF /etc/pam.d/common-auth" + "\033[0m" + "\n")
-
-# Do not prompt for a yes if we're in no promt mode
-if "HOWDY_NO_PROMPT" not in os.environ:
-	# Ask the user if this change is okay
-	print("Lines will be insterted in /etc/pam.d/common-auth as shown above")
-	ans = input("Apply this change? [y/N]: ")
-
-	# Abort the whole thing if it's not
-	if ans.lower().strip() != "y" or ans.lower().strip() == "yes":
-		print("Interpreting as a \"NO\", aborting")
-		sys.exit(1)
-
-print("Adding lines to PAM\n")
-
-# Write to PAM
-common_auth = open("/etc/pam.d/common-auth", "w")
-common_auth.write("".join(outlines))
-common_auth.close()
 
 # Sign off
 print("Installation complete.")

debian/preinst

@@ -9,6 +9,7 @@ def col(id):
 	if id == 3: return "\033[31m"
 	return "\033[0m"
 
+
 import subprocess
 import time
 import sys
@@ -24,7 +25,7 @@ if "upgrade" in sys.argv:
 
 		# Let the user know so he knows where to look on a failed install
 		print("Backup of Howdy config file created in /tmp/howdy_config_backup_v" + sys.argv[2] + ".ini")
-	except e:
+	except subprocess.CalledProcessError:
 		print("Could not make an backup of old Howdy config file")
 
 	# Don't continue setup when we're just upgrading

debian/prerm

@@ -2,17 +2,11 @@
 # Executed on deinstallation
 # Completely remove howdy from the system
 
-def col(id):
-	"""Add color escape sequences"""
-	if id == 1: return "\033[32m"
-	if id == 2: return "\033[33m"
-	if id == 3: return "\033[31m"
-	return "\033[0m"
-
 # Import required modules
 import subprocess
 import sys
 import os
+from shutil import rmtree
 
 # Only run when we actually want to remove
 if "remove" not in sys.argv and "purge" not in sys.argv:
@@ -24,24 +18,28 @@ if not os.path.exists("/lib/security/howdy/cli"):
 
 # Remove files and symlinks
 try:
-	subprocess.call(["rm /usr/local/bin/howdy"], shell=True)
+	os.unlink('/usr/local/bin/howdy')
-except e:
+except Exception:
 	print("Can't remove executable")
 try:
-	subprocess.call(["rm /usr/share/bash-completion/completions/howdy"], shell=True)
+	os.unlink('/usr/share/bash-completion/completions/howdy')
-except e:
+except Exception:
 	print("Can't remove autocompletion script")
+
+# Refresh and remove howdy from pam-config
 try:
-	subprocess.call(["rm -rf /lib/security/howdy"], shell=True)
+	subprocess.call(["pam-auth-update --package"], shell=True)
-except e:
+	subprocess.call(["rm /usr/share/pam-configs/howdy"], shell=True)
+	subprocess.call(["pam-auth-update --package"], shell=True)
+except Exception:
+	print("Can't remove pam module")
+
+# Remove full installation folder, just to be sure
+try:
+	rmtree('/lib/security/howdy')
+except Exception:
 	# This error is normal
 	pass
 
-# Remove face_recognition and dlib
+# Remove dlib
-subprocess.call(["pip3 uninstall face_recognition face_recognition_models dlib -y --no-cache-dir"], shell=True)
+subprocess.call(['pip3', 'uninstall', 'dlib', '-y', '--no-cache-dir'])
-
-# Print a tearbending message
-print(col(2) + """
-There are still lines in /etc/pam.d/common-auth that can't be removed automatically
-Run "nano /etc/pam.d/common-auth" to remove them by hand\
-""" + col(0))

debian/source/options

@@ -1,6 +1,7 @@
 tar-ignore = ".git"
-tar-ignore = "models"
 tar-ignore = ".gitignore"
+tar-ignore = ".github"
+tar-ignore = "models"
+tar-ignore = "tests"
 tar-ignore = "README.md"
-tar-ignore = "LICENSE"
 tar-ignore = ".travis.yml"

src/cli.py

@@ -4,7 +4,6 @@
 # Import required modules
 import sys
 import os
-import subprocess
 import getpass
 import argparse
 import builtins
@@ -12,11 +11,11 @@ import builtins
 # Try to get the original username (not "root") from shell
 try:
 	user = os.getlogin()
-except:
+except Exception:
 	user = os.environ.get("SUDO_USER")
 
 # If that fails, try to get the direct user
-if user == "root" or user == None:
+if user == "root" or user is None:
 	env_user = getpass.getuser().strip()
 
 	# If even that fails, error out
@@ -28,37 +27,37 @@ if user == "root" or user == None:
 
 # Basic command setup
 parser = argparse.ArgumentParser(description="Command line interface for Howdy face authentication.",
-                                 formatter_class=argparse.RawDescriptionHelpFormatter,
+								formatter_class=argparse.RawDescriptionHelpFormatter,
-                                 add_help=False,
+								add_help=False,
-                                 prog="howdy",
+								prog="howdy",
-                                 epilog="For support please visit\nhttps://github.com/boltgolt/howdy")
+								epilog="For support please visit\nhttps://github.com/boltgolt/howdy")
 
 # Add an argument for the command
 parser.add_argument("command",
-                    help="The command option to execute, can be one of the following: add, clear, config, disable, list, remove or test.",
+					help="The command option to execute, can be one of the following: add, clear, config, disable, list, remove or test.",
-                    metavar="command",
+					metavar="command",
-                    choices=["add", "clear", "config", "disable", "list", "remove", "test"])
+					choices=["add", "clear", "config", "disable", "list", "remove", "test"])
 
 # Add an argument for the extra arguments of diable and remove
 parser.add_argument("argument",
-                    help="Either 0 (enable) or 1 (disable) for the disable command, or the model ID for the remove command.",
+					help="Either 0 (enable) or 1 (disable) for the disable command, or the model ID for the remove command.",
-                    nargs="?")
+					nargs="?")
 
 # Add the user flag
 parser.add_argument("-U", "--user",
-                    default=user,
+					default=user,
-                    help="Set the user account to use.")
+					help="Set the user account to use.")
 
 # Add the -y flag
 parser.add_argument("-y",
-                    help="Skip all questions.",
+					help="Skip all questions.",
-                    action="store_true")
+					action="store_true")
 
 # Overwrite the default help message so we can use a uppercase S
 parser.add_argument("-h", "--help",
-                    action="help",
+					action="help",
-                    default=argparse.SUPPRESS,
+					default=argparse.SUPPRESS,
-                    help="Show this help message and exit.")
+					help="Show this help message and exit.")
 
 # If we only have 1 argument we print the help text
 if len(sys.argv) < 2:

src/cli/add.py

@@ -1,33 +1,53 @@
 # Save the face of the user in encoded form
 
 # Import required modules
-import subprocess
 import time
 import os
 import sys
 import json
-import cv2
 import configparser
 import builtins
+import cv2
+import numpy as np
 
-# Try to import face_recognition and give a nice error if we can't
+# Try to import dlib and give a nice error if we can't
 # Add should be the first point where import issues show up
 try:
-	import face_recognition
+	import dlib
 except ImportError as err:
 	print(err)
 
-	print("\nCan't import the face_recognition module, check the output of")
+	print("\nCan't import the dlib module, check the output of")
-	print("pip3 show face_recognition")
+	print("pip3 show dlib")
 	sys.exit(1)
 
-# Get the absolute path to the current file
+# Get the absolute path to the current directory
-path = os.path.dirname(os.path.abspath(__file__))
+path = os.path.abspath(__file__ + "/..")
+
+# Test if at lest 1 of the data files is there and abort if it's not
+if not os.path.isfile(path + "/../dlib-data/shape_predictor_5_face_landmarks.dat"):
+	print("Data files have not been downloaded, please run the following commands:")
+	print("\n\tcd " + os.path.realpath(path + "/../dlib-data"))
+	print("\tsudo ./install.sh\n")
+	sys.exit(1)
 
 # Read config from disk
 config = configparser.ConfigParser()
 config.read(path + "/../config.ini")
 
+if not os.path.exists(config.get("video", "device_path")):
+	print("Camera path is not configured correctly, please edit the 'device_path' config value.")
+	sys.exit(1)
+
+use_cnn = config.getboolean("core", "use_cnn", fallback=False)
+if use_cnn:
+	face_detector = dlib.cnn_face_detection_model_v1(path + "/../dlib-data/mmod_human_face_detector.dat")
+else:
+	face_detector = dlib.get_frontal_face_detector()
+
+pose_predictor = dlib.shape_predictor(path + "/../dlib-data/shape_predictor_5_face_landmarks.dat")
+face_encoder = dlib.face_recognition_model_v1(path + "/../dlib-data/dlib_face_recognition_resnet_model_v1.dat")
+
 user = builtins.howdy_user
 # The permanent file to store the encoded model in
 enc_file = path + "/../models/" + user + ".dat"
@@ -47,8 +67,8 @@ except FileNotFoundError:
 
 # Print a warning if too many encodings are being added
 if len(encodings) > 3:
-	print("WARNING: Every additional model slows down the face recognition engine")
+	print("NOTICE: Each additional model slows down the face recognition engine slightly")
-	print("Press ctrl+C to cancel\n")
+	print("Press Ctrl+C to cancel\n")
 
 print("Adding face model for the user " + user)
 
@@ -56,15 +76,15 @@ print("Adding face model for the user " + user)
 label = "Initial model"
 
 # If models already exist, set that default label
-if len(encodings) > 0:
+if encodings:
 	label = "Model #" + str(len(encodings) + 1)
 
 # Keep de default name if we can't ask questions
 if builtins.howdy_args.y:
-	print("Using default label \"" + label + "\" because of -y flag")
+	print('Using default label "%s" because of -y flag' % (label, ))
 else:
 	# Ask the user for a custom label
-	label_in = input("Enter a label for this new model [" + label + "]: ")
+	label_in = input("Enter a label for this new model [" + label + "] (max 24 characters): ")
 
 	# Set the custom label (if any) and limit it to 24 characters
 	if label_in != "":
@@ -78,22 +98,35 @@ insert_model = {
 	"data": []
 }
 
-# Open the camera
+# Check if the user explicitly set ffmpeg as recorder
-video_capture = cv2.VideoCapture(config.get("video", "device_path"))
+if config.get("video", "recording_plugin") == "ffmpeg":
+	# Set the capture source for ffmpeg
+	from recorders.ffmpeg_reader import ffmpeg_reader
+	video_capture = ffmpeg_reader(config.get("video", "device_path"), config.get("video", "device_format"))
+elif config.get("video", "recording_plugin") == "pyv4l2":
+	# Set the capture source for pyv4l2
+	from recorders.pyv4l2_reader import pyv4l2_reader
+	video_capture = pyv4l2_reader(config.get("video", "device_path"), config.get("video", "device_format"))
+else:
+	# Start video capture on the IR camera through OpenCV
+	video_capture = cv2.VideoCapture(config.get("video", "device_path"))
 
 # Force MJPEG decoding if true
-if config.get("video", "force_mjpeg") == "true":
+if config.getboolean("video", "force_mjpeg", fallback=False):
+	# Set a magic number, will enable MJPEG but is badly documentated
 	video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237)
 
 # Set the frame width and height if requested
-if int(config.get("video", "frame_width")) != -1:
+fw = config.getint("video", "frame_width", fallback=-1)
-	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, int(config.get("video", "frame_width")))
+fh = config.getint("video", "frame_height", fallback=-1)
+if fw != -1:
+	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw)
 
-if int(config.get("video", "frame_height")) != -1:
+if fh != -1:
-	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, int(config.get("video", "frame_height")))
+	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh)
 
 # Request a frame to wake the camera up
-video_capture.read()
+video_capture.grab()
 
 print("\nPlease look straight into the camera")
 
@@ -104,40 +137,53 @@ time.sleep(2)
 enc = []
 # Count the amount or read frames
 frames = 0
+dark_threshold = config.getfloat("video", "dark_threshold")
 
 # Loop through frames till we hit a timeout
 while frames < 60:
-	frames += 1
-
 	# Grab a single frame of video
 	# Don't remove ret, it doesn't work without it
 	ret, frame = video_capture.read()
+	gsframe = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
 
-	# Get the encodings in the frame
+	# Create a histogram of the image with 8 values
-	enc = face_recognition.face_encodings(frame)
+	hist = cv2.calcHist([gsframe], [0], None, [8], [0, 256])
+	# All values combined for percentage calculation
+	hist_total = np.sum(hist)
+
+	# If the image is fully black or the frame exceeds threshold,
+	# skip to the next frame
+	if hist_total == 0 or (hist[0] / hist_total * 100 > dark_threshold):
+		continue
+
+	frames += 1
+
+	# Get all faces from that frame as encodings
+	face_locations = face_detector(gsframe, 1)
 
 	# If we've found at least one, we can continue
-	if len(enc) > 0:
+	if face_locations:
 		break
 
-# If 0 faces are detected we can't continue
+video_capture.release()
-if len(enc) == 0:
+
+# If more than 1 faces are detected we can't know wich one belongs to the user
+if len(face_locations) > 1:
+	print("Multiple faces detected, aborting")
+	sys.exit(1)
+elif not face_locations:
 	print("No face detected, aborting")
 	sys.exit(1)
 
-# If more than 1 faces are detected we can't know wich one belongs to the user
+face_location = face_locations[0]
-if len(enc) > 1:
+if use_cnn:
-	print("Multiple faces detected, aborting")
+	face_location = face_location.rect
-	sys.exit(1)
 
-# Totally clean array that can be exported as JSON
+# Get the encodings in the frame
-clean_enc = []
+face_landmark = pose_predictor(frame, face_location)
+face_encoding = np.array(face_encoder.compute_face_descriptor(frame, face_landmark, 1))
 
-# Copy the values into a clean array so we can export it as JSON later on
+insert_model["data"].append(face_encoding.tolist())
-for point in enc[0]:
-	clean_enc.append(point)
-
-insert_model["data"].append(clean_enc)
 
 # Insert full object into the list
 encodings.append(insert_model)

src/cli/config.py

@@ -17,4 +17,4 @@ elif "EDITOR" in os.environ:
 	editor = os.environ["EDITOR"]
 
 # Open the editor as a subprocess and fork it
-subprocess.call([editor, os.path.dirname(os.path.realpath(__file__))  + "/../config.ini"])
+subprocess.call([editor, os.path.dirname(os.path.realpath(__file__)) + "/../config.ini"])

src/cli/disable.py

@@ -16,7 +16,7 @@ config = configparser.ConfigParser()
 config.read(config_path)
 
 # Check if enough arguments have been passed
-if builtins.howdy_args.argument == None:
+if builtins.howdy_args.argument is None:
 	print("Please add a 0 (enable) or a 1 (disable) as an argument")
 	sys.exit(1)
 

src/cli/list.py

@@ -8,7 +8,7 @@ import time
 import builtins
 
 # Get the absolute path and the username
-path = os.path.dirname(os.path.realpath(__file__))  + "/.."
+path = os.path.dirname(os.path.realpath(__file__)) + "/.."
 user = builtins.howdy_user
 
 # Check if the models file has been created yet

src/cli/remove.py

@@ -7,11 +7,11 @@ import json
 import builtins
 
 # Get the absolute path and the username
-path = os.path.dirname(os.path.realpath(__file__))  + "/.."
+path = os.path.dirname(os.path.realpath(__file__)) + "/.."
 user = builtins.howdy_user
 
 # Check if enough arguments have been passed
-if builtins.howdy_args.argument == None:
+if builtins.howdy_args.argument is None:
 	print("Please add the ID of the model you want to remove as an argument")
 	print("You can find the IDs by running:")
 	print("\n\thowdy list\n")

src/cli/test.py

@@ -1,14 +1,12 @@
 # Show a windows with the video stream and testing information
 
 # Import required modules
-import face_recognition
-import cv2
 import configparser
 import os
 import sys
-import json
-import numpy
 import time
+import cv2
+import dlib
 
 # Get the absolute path to the current file
 path = os.path.dirname(os.path.abspath(__file__))
@@ -17,19 +15,27 @@ path = os.path.dirname(os.path.abspath(__file__))
 config = configparser.ConfigParser()
 config.read(path + "/../config.ini")
 
+if config.get("video", "recording_plugin") != "opencv":
+	print("Howdy has been configured to use a recorder which doesn't support the test command yet")
+	print("Aborting")
+	sys.exit(12)
+
 # Start capturing from the configured webcam
 video_capture = cv2.VideoCapture(config.get("video", "device_path"))
 
 # Force MJPEG decoding if true
-if config.get("video", "force_mjpeg") == "true":
+if config.getboolean("video", "force_mjpeg", fallback=False):
+	# Set a magic number, will enable MJPEG but is badly documented
 	video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237)
 
 # Set the frame width and height if requested
-if int(config.get("video", "frame_width")) != -1:
+fw = config.getint("video", "frame_width", fallback=-1)
-	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, int(config.get("video", "frame_width")))
+fh = config.getint("video", "frame_height", fallback=-1)
+if fw != -1:
+	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw)
 
-if int(config.get("video", "frame_height")) != -1:
+if fh != -1:
-	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, int(config.get("video", "frame_height")))
+	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh)
 
 # Let the user know what's up
 print("""
@@ -39,6 +45,7 @@ Press ctrl+C in this terminal to quit
 Click on the image to enable or disable slow mode
 """)
 
+
 def mouse(event, x, y, flags, param):
 	"""Handle mouse events"""
 	global slow_mode
@@ -47,6 +54,21 @@ def mouse(event, x, y, flags, param):
 	if event == cv2.EVENT_LBUTTONDOWN:
 		slow_mode = not slow_mode
 
+
+def print_text(line_number, text):
+	"""Print the status text by line number"""
+	cv2.putText(overlay, text, (10, height - 10 - (10 * line_number)), cv2.FONT_HERSHEY_SIMPLEX, .3, (0, 255, 0), 0, cv2.LINE_AA)
+
+use_cnn = config.getboolean('core', 'use_cnn', fallback=False)
+if use_cnn:
+	face_detector = dlib.cnn_face_detection_model_v1(
+		path + '/../dlib-data/mmod_human_face_detector.dat'
+	)
+else:
+	face_detector = dlib.get_frontal_face_detector()
+
+clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
+
 # Open the window and attach a a mouse listener
 cv2.namedWindow("Howdy Test")
 cv2.setMouseCallback("Howdy Test", mouse)
@@ -61,26 +83,31 @@ sec_frames = 0
 fps = 0
 # The current second we're counting
 sec = int(time.time())
+# recognition time
+rec_tm = 0
 
 # Wrap everything in an keyboard interupt handler
 try:
 	while True:
-		# Inclement the frames
+		frame_tm = time.time()
+
+		# Increment the frames
 		total_frames += 1
 		sec_frames += 1
 
 		# Id we've entered a new second
-		if sec != int(time.time()):
+		if sec != int(frame_tm):
 			# Set the last seconds FPS
 			fps = sec_frames
 
 			# Set the new second and reset the counter
-			sec = int(time.time())
+			sec = int(frame_tm)
 			sec_frames = 0
 
-
 		# Grab a single frame of video
-		ret, frame = (video_capture.read())
+		ret, frame = video_capture.read()
+		frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+		frame = clahe.apply(frame)
 		# Make a frame to put overlays in
 		overlay = frame.copy()
 
@@ -94,7 +121,7 @@ try:
 		# Fill with the overal containing percentage
 		hist_perc = []
 
-		# Loop though all values to calculate a pensentage and add it to the overlay
+		# Loop though all values to calculate a percentage and add it to the overlay
 		for index, value in enumerate(hist):
 			value_perc = float(value[0]) / hist_total * 100
 			hist_perc.append(value_perc)
@@ -109,14 +136,11 @@ try:
 		# Draw a stripe indicating the dark threshold
 		cv2.rectangle(overlay, (8, 35), (20, 36), (255, 0, 0), thickness=cv2.FILLED)
 
-		def print_text(line_number, text):
-			"""Print the status text by line number"""
-			cv2.putText(overlay, text, (10, height - 10 - (10 * line_number)), cv2.FONT_HERSHEY_SIMPLEX, .3, (0, 255, 0), 0, cv2.LINE_AA)
-
 		# Print the statis in the bottom left
-		print_text(0, "RESOLUTION: " + str(height) + "x" + str(width))
+		print_text(0, "RESOLUTION: %dx%d" % (height, width))
-		print_text(1, "FPS: " + str(fps))
+		print_text(1, "FPS: %d" % (fps, ))
-		print_text(2, "FRAMES: " + str(total_frames))
+		print_text(2, "FRAMES: %d" % (total_frames, ))
+		print_text(3, "RECOGNITION: %dms" % (round(rec_tm * 1000), ))
 
 		# Show that slow mode is on, if it's on
 		if slow_mode:
@@ -130,17 +154,22 @@ try:
 			# SHow that this is an active frame
 			cv2.putText(overlay, "SCAN FRAME", (width - 68, 16), cv2.FONT_HERSHEY_SIMPLEX, .3, (0, 255, 0), 0, cv2.LINE_AA)
 
+			rec_tm = time.time()
 			# Get the locations of all faces and their locations
-			face_locations = face_recognition.face_locations(frame)
+			face_locations = face_detector(frame, 1) # upsample 1 time
+			rec_tm = time.time() - rec_tm
 
 			# Loop though all faces and paint a circle around them
 			for loc in face_locations:
+				if use_cnn:
+					loc = loc.rect
+
 				# Get the center X and Y from the rectangular points
-				x = int((loc[1] - loc[3]) / 2) + loc[3]
+				x = int((loc.right() - loc.left()) / 2) + loc.left()
-				y = int((loc[2] - loc[0]) / 2) + loc[0]
+				y = int((loc.bottom() - loc.top()) / 2) + loc.top()
 
 				# Get the raduis from the with of the square
-				r = (loc[1] - loc[3]) / 2
+				r = (loc.right() - loc.left()) / 2
 				# Add 20% padding
 				r = int(r + (r * 0.2))
 
@@ -158,9 +187,11 @@ try:
 		if cv2.waitKey(1) != -1:
 			raise KeyboardInterrupt()
 
+		frame_time = time.time() - frame_tm
+
 		# Delay the frame if slowmode is on
 		if slow_mode:
-			time.sleep(.55)
+			time.sleep(.5 - frame_time)
 
 # On ctrl+C
 except KeyboardInterrupt:

src/compare.py

@@ -5,33 +5,62 @@
 import time
 
 # Start timing
-timings = [time.time()]
+timings = {
+	"st": time.time()
+}
 
 # Import required modules
-import cv2
 import sys
 import os
 import json
-import math
 import configparser
+import cv2
+import dlib
+import numpy as np
+import _thread as thread
+
+
+def init_detector(lock):
+	"""Start face detector, encoder and predictor in a new thread"""
+	global face_detector, pose_predictor, face_encoder
+
+	# Test if at lest 1 of the data files is there and abort if it's not
+	if not os.path.isfile(PATH + "/dlib-data/shape_predictor_5_face_landmarks.dat"):
+		print("Data files have not been downloaded, please run the following commands:")
+		print("\n\tcd " + PATH + "/dlib-data")
+		print("\tsudo ./install.sh\n")
+		lock.release()
+		sys.exit(1)
+
+	# Use the CNN detector if enabled
+	if use_cnn:
+		face_detector = dlib.cnn_face_detection_model_v1(PATH + "/dlib-data/mmod_human_face_detector.dat")
+	else:
+		face_detector = dlib.get_frontal_face_detector()
+
+	# Start the others regardless
+	pose_predictor = dlib.shape_predictor(PATH + "/dlib-data/shape_predictor_5_face_landmarks.dat")
+	face_encoder = dlib.face_recognition_model_v1(PATH + "/dlib-data/dlib_face_recognition_resnet_model_v1.dat")
+
+	# Note the time it took to initialize detectors
+	timings["ll"] = time.time() - timings["ll"]
+	lock.release()
 
-# Read config from disk
-config = configparser.ConfigParser()
-config.read(os.path.dirname(os.path.abspath(__file__)) + "/config.ini")
 
 def stop(status):
 	"""Stop the execution and close video stream"""
 	video_capture.release()
 	sys.exit(status)
 
-# Make sure we were given an username to tast against
-try:
-	if not isinstance(sys.argv[1], str):
-		sys.exit(1)
-except IndexError:
-	sys.exit(1)
 
-# The username of the authenticating user
+# Make sure we were given an username to tast against
+if len(sys.argv) < 2:
+	sys.exit(12)
+
+# Get the absolute path to the current directory
+PATH = os.path.abspath(__file__ + "/..")
+
+# The username of the user being authenticated
 user = sys.argv[1]
 # The model file contents
 models = []
@@ -39,10 +68,19 @@ models = []
 encodings = []
 # Amount of ingnored dark frames
 dark_tries = 0
+# Total amount of frames captured
+frames = 0
+# face recognition/detection instances
+face_detector = None
+pose_predictor = None
+face_encoder = None
 
 # Try to load the face model from the models folder
 try:
-	models = json.load(open(os.path.dirname(os.path.abspath(__file__)) + "/models/" + user + ".dat"))
+	models = json.load(open(PATH + "/models/" + user + ".dat"))
+
+	for model in models:
+		encodings += model["data"]
 except FileNotFoundError:
 	sys.exit(10)
 
@@ -50,127 +88,176 @@ except FileNotFoundError:
 if len(models) < 1:
 	sys.exit(10)
 
-# Put all models together into 1 array
+# Read config from disk
-for model in models:
+config = configparser.ConfigParser()
-	encodings += model["data"]
+config.read(PATH + "/config.ini")
 
-# Add the time needed to start the script
+# Get all config values needed
-timings.append(time.time())
+use_cnn = config.getboolean("core", "use_cnn", fallback=False)
+timeout = config.getint("video", "timout", fallback=5)
+dark_threshold = config.getfloat("video", "dark_threshold", fallback=50.0)
+video_certainty = config.getfloat("video", "certainty", fallback=3.5) / 10
+end_report = config.getboolean("debug", "end_report", fallback=False)
+
+# Save the time needed to start the script
+timings["in"] = time.time() - timings["st"]
+
+# Import face recognition, takes some time
+timings["ll"] = time.time()
+
+# Start threading and wait for init to finish
+lock = thread.allocate_lock()
+lock.acquire()
+thread.start_new_thread(init_detector, (lock, ))
 
 # Start video capture on the IR camera
-video_capture = cv2.VideoCapture(config.get("video", "device_path"))
+timings["ic"] = time.time()
+
+# Check if the user explicitly set ffmpeg as recorder
+if config.get("video", "recording_plugin") == "ffmpeg":
+	# Set the capture source for ffmpeg
+	from recorders.ffmpeg_reader import ffmpeg_reader
+	video_capture = ffmpeg_reader(config.get("video", "device_path"), config.get("video", "device_format"))
+elif config.get("video", "recording_plugin") == "pyv4l2":
+	# Set the capture source for pyv4l2
+	from recorders.pyv4l2_reader import pyv4l2_reader
+	video_capture = pyv4l2_reader(config.get("video", "device_path"), config.get("video", "device_format"))
+else:
+	# Start video capture on the IR camera through OpenCV
+	video_capture = cv2.VideoCapture(config.get("video", "device_path"))
 
 # Force MJPEG decoding if true
-if config.get("video", "force_mjpeg") == "true":
+if config.getboolean("video", "force_mjpeg", fallback=False):
-	# Set a magic number, will enable MJPEG but is badly documentated
+	# Set a magic number, will enable MJPEG but is badly documented
+	# 1196444237 is "GPJM" in ASCII
 	video_capture.set(cv2.CAP_PROP_FOURCC, 1196444237)
 
 # Set the frame width and height if requested
-if int(config.get("video", "frame_width")) != -1:
+fw = config.getint("video", "frame_width", fallback=-1)
-	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, int(config.get("video", "frame_width")))
+fh = config.getint("video", "frame_height", fallback=-1)
-
+if fw != -1:
-if int(config.get("video", "frame_height")) != -1:
+	video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, fw)
-	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, int(config.get("video", "frame_height")))
+if fh != -1:
+	video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, fh)
 
 # Capture a single frame so the camera becomes active
 # This will let the camera adjust its light levels while we're importing for faster scanning
-video_capture.read()
+video_capture.grab()
 
 # Note the time it took to open the camera
-timings.append(time.time())
+timings["ic"] = time.time() - timings["ic"]
 
-# Import face recognition, takes some time
+# wait for thread to finish
-import face_recognition
+lock.acquire()
-timings.append(time.time())
+lock.release()
+del lock
 
 # Fetch the max frame height
-max_height = int(config.get("video", "max_height"))
+max_height = config.getfloat("video", "max_height", fallback=0.0)
+# Get the height of the image
+height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 1
+
+# Calculate the amount the image has to shrink
+scaling_factor = (max_height / height) or 1
+
+# Fetch config settings out of the loop
+timeout = config.getint("video", "timeout")
+dark_threshold = config.getfloat("video", "dark_threshold")
+end_report = config.getboolean("debug", "end_report")
 
 # Start the read loop
 frames = 0
+timings["fr"] = time.time()
+
 while True:
 	# Increment the frame count every loop
 	frames += 1
 
 	# Stop if we've exceded the time limit
-	if time.time() - timings[3] > int(config.get("video", "timout")):
+	if time.time() - timings["fr"] > timeout:
 		stop(11)
 
 	# Grab a single frame of video
-	# Don't remove ret, it doesn't work without it
 	ret, frame = video_capture.read()
 
+	try:
+		# Convert from color to grayscale
+		# First processing of frame, so frame errors show up here
+		gsframe = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+	except cv2.error:
+		print("\nUnknown camera, please check your 'device_path' config value.\n")
+		raise
+
 	# Create a histogram of the image with 8 values
-	hist = cv2.calcHist([frame], [0], None, [8], [0, 256])
+	hist = cv2.calcHist([gsframe], [0], None, [8], [0, 256])
 	# All values combined for percentage calculation
-	hist_total = int(sum(hist)[0])
+	hist_total = np.sum(hist)
 
-	# If the image is fully black, skip to the next frame
+	# If the image is fully black or the frame exceeds threshold,
-	if hist_total == 0:
+	# skip to the next frame
+	if hist_total == 0 or (hist[0] / hist_total * 100 > dark_threshold):
 		dark_tries += 1
 		continue
 
-	# Scrip the frame if it exceeds the threshold
-	if float(hist[0]) / hist_total * 100 > float(config.get("video", "dark_threshold")):
-		dark_tries += 1
-		continue
-
-	# Get the height and with of the image
-	height, width = frame.shape[:2]
-
 	# If the hight is too high
-	if max_height < height:
+	if scaling_factor != 1:
-		# Calculate the amount the image has to shrink
-		scaling_factor = max_height / float(height)
 		# Apply that factor to the frame
 		frame = cv2.resize(frame, None, fx=scaling_factor, fy=scaling_factor, interpolation=cv2.INTER_AREA)
-
+		gsframe = cv2.resize(gsframe, None, fx=scaling_factor, fy=scaling_factor, interpolation=cv2.INTER_AREA)
-	# Save the new size for diagnostics
-	scale_height, scale_width = frame.shape[:2]
 
 	# Get all faces from that frame as encodings
-	face_encodings = face_recognition.face_encodings(frame)
+	# Upsamples 1 time
+	face_locations = face_detector(gsframe, 1)
 
 	# Loop through each face
-	for face_encoding in face_encodings:
+	for fl in face_locations:
+		if use_cnn:
+			fl = fl.rect
+
+		# Fetch the faces in the image
+		face_landmark = pose_predictor(frame, fl)
+		face_encoding = np.array(face_encoder.compute_face_descriptor(frame, face_landmark, 1))
+
 		# Match this found face against a known face
-		matches = face_recognition.face_distance(encodings, face_encoding)
+		matches = np.linalg.norm(encodings - face_encoding, axis=1)
 
-		# Check if any match is certain enough to be the user we're looking for
+		# Get best match
-		match_index = 0
+		match_index = np.argmin(matches)
-		for match in matches:
+		match = matches[match_index]
-			match_index += 1
 
-			# Try to find a match that's confident enough
+		# Check if a match that's confident enough
-			if match * 10 < float(config.get("video", "certainty")) and match > 0:
+		if 0 < match < video_certainty:
-				timings.append(time.time())
+			timings["tt"] = time.time() - timings["st"]
+			timings["fr"] = time.time() - timings["fr"]
 
-				# If set to true in the config, print debug text
+			# If set to true in the config, print debug text
-				if config.get("debug", "end_report") == "true":
+			if end_report:
-					def print_timing(label, offset):
+				def print_timing(label, k):
-						"""Helper function to print a timing from the list"""
+					"""Helper function to print a timing from the list"""
-						print("  " + label + ": " + str(round((timings[1 + offset] - timings[offset]) * 1000)) + "ms")
+					print("  %s: %dms" % (label, round(timings[k] * 1000)))
 
-					print("Time spend")
+				# Print a nice timing report
-					print_timing("Starting up", 0)
+				print("Time spent")
-					print_timing("Opening the camera", 1)
+				print_timing("Starting up", "in")
-					print_timing("Importing face_recognition", 2)
+				print("  Open cam + load libs: %dms" % (round(max(timings["ll"], timings["ic"]) * 1000, )))
-					print_timing("Searching for known face", 3)
+				print_timing("  Opening the camera", "ic")
+				print_timing("  Importing recognition libs", "ll")
+				print_timing("Searching for known face", "fr")
+				print_timing("Total time", "tt")
 
-					print("\nResolution")
+				print("\nResolution")
-					print("  Native: " + str(height) + "x" + str(width))
+				width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 1
-					print("  Used: " + str(scale_height) + "x" + str(scale_width))
+				print("  Native: %dx%d" % (height, width))
+				# Save the new size for diagnostics
+				scale_height, scale_width = frame.shape[:2]
+				print("  Used: %dx%d" % (scale_height, scale_width))
 
-					# Show the total number of frames and calculate the FPS by deviding it by the total scan time
+				# Show the total number of frames and calculate the FPS by deviding it by the total scan time
-					print("\nFrames searched: " + str(frames) + " (" + str(round(float(frames) / (timings[4] - timings[3]), 2)) + " fps)")
+				print("\nFrames searched: %d (%.2f fps)" % (frames, frames / timings["fr"]))
-					print("Dark frames ignored: " + str(dark_tries))
+				print("Dark frames ignored: %d " % (dark_tries, ))
-					print("Certainty of winning frame: " + str(round(match * 10, 3)))
+				print("Certainty of winning frame: %.3f" % (match * 10, ))
 
-					# Catch older 3-encoding models
+				print("Winning model: %d (\"%s\")" % (match_index, models[match_index]["label"]))
-					if not match_index in models:
-						match_index = 0
 
-					print("Winning model: " + str(match_index) + " (\"" + models[match_index]["label"] + "\")")
+			# End peacefully
-
+			stop(0)
-				# End peacefully
-				stop(0)

src/config.ini

@@ -2,6 +2,9 @@
 # Press CTRL + X to save in the nano editor
 
 [core]
+# Print that face detection is being attempted
+detection_notice = false
+
 # Do not print anything when a face verification succeeds
 no_confirmation = false
 
@@ -14,33 +17,34 @@ ignore_ssh = true
 
 # Auto dismiss lock screen on confirmation
 # Will run loginctl unlock-sessions after every auth
-# Expirimental, can behave incorrectly on some systems
+# Experimental, can behave incorrectly on some systems
 dismiss_lockscreen = false
 
 # Disable howdy in the PAM
 # The howdy command will still function
 disabled = false
 
+# Use CNN instead of HOG
+# CNN model is much more accurate than the HOG based model, but takes much more
+# computational power to run, and is meant to be executed on a GPU to attain reasonable speed.
+use_cnn = false
+
 [video]
 # The certainty of the detected face belonging to the user of the account
 # On a scale from 1 to 10, values above 5 are not recommended
 certainty = 3.5
 
 # The number of seconds to search before timing out
-timout = 4
+timeout = 4
 
 # The path of the device to capture frames from
-# Should be set automatically by the installer
+# Should be set automatically by an installer if your distro has one
 device_path = none
 
 # Scale down the video feed to this maximum height
 # Speeds up face recognition but can make it less precise
 max_height = 320
 
-# Force the use of Motion JPEG when decoding frames, fixes issues with
-# YUYV raw frame deconding
-force_mjpeg = false
-
 # Set the camera input profile to this width and height
 # The largest profile will be used if set to -1
 # Automatically ignored if not a valid profile
@@ -53,6 +57,19 @@ frame_height = -1
 # The lower this setting is, the more dark frames are ignored
 dark_threshold = 50
 
+# The recorder to use. Can be either opencv (default), ffmpeg or pyv4l2.
+# Switching from the default opencv to ffmpeg can help with grayscale issues.
+recording_plugin = opencv
+
+# Video format used by ffmpeg. Options include vfwcap or v4l2.
+# FFMPEG only.
+device_format = v4l2
+
+# Force the use of Motion JPEG when decoding frames, fixes issues with YUYV
+# raw frame decoding.
+# OPENCV only.
+force_mjpeg = false
+
 [debug]
 # Show a short but detailed diagnostic report in console
 end_report = false

src/dlib-data/.gitignore

@@ -0,0 +1,2 @@
+*.dat
+*.dat.bz2

src/dlib-data/Readme.md

@@ -0,0 +1,7 @@
+Download and unpack `dlib` data files from https://github.com/davisking/dlib-models repository:
+```shell
+wget https://github.com/davisking/dlib-models/raw/master/dlib_face_recognition_resnet_model_v1.dat.bz2
+wget https://github.com/davisking/dlib-models/raw/master/mmod_human_face_detector.dat.bz2
+wget https://github.com/davisking/dlib-models/raw/master/shape_predictor_5_face_landmarks.dat.bz2
+bunzip *bz2
+```

src/dlib-data/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+echo "Downloading 3 required data files..."
+
+# Check if wget is installed
+if hash wget;then
+	# Check if wget supports the option to only show the progress bar
+	wget --help | grep -q "\--show-progress" && \
+		_PROGRESS_OPT="-q --show-progress" || _PROGRESS_OPT=""
+
+	# Download the archives
+	wget $_PROGRESS_OPT --tries 5 https://github.com/davisking/dlib-models/raw/master/dlib_face_recognition_resnet_model_v1.dat.bz2
+	wget $_PROGRESS_OPT --tries 5 https://github.com/davisking/dlib-models/raw/master/mmod_human_face_detector.dat.bz2
+	wget $_PROGRESS_OPT --tries 5 https://github.com/davisking/dlib-models/raw/master/shape_predictor_5_face_landmarks.dat.bz2
+
+# Otherwise fall back on curl
+else
+	curl --location --retry 5 --output dlib_face_recognition_resnet_model_v1.dat.bz2 https://github.com/davisking/dlib-models/raw/master/dlib_face_recognition_resnet_model_v1.dat.bz2
+	curl --location --retry 5 --output mmod_human_face_detector.dat.bz2 https://github.com/davisking/dlib-models/raw/master/mmod_human_face_detector.dat.bz2
+	curl --location --retry 5 --output shape_predictor_5_face_landmarks.dat.bz2 https://github.com/davisking/dlib-models/raw/master/shape_predictor_5_face_landmarks.dat.bz2
+fi
+
+# Uncompress the data files and delete the original archive
+echo "Unpacking..."
+bzip2 -d *.bz2

src/pam-config/howdy

@@ -0,0 +1,6 @@
+Name: Howdy
+Default: yes
+Priority: 512
+Auth-Type: Primary
+Auth:
+	[success=end default=ignore]        pam_python.so /lib/security/howdy/pam.py

src/pam.py

@@ -12,38 +12,46 @@ import ConfigParser
 config = ConfigParser.ConfigParser()
 config.read(os.path.dirname(os.path.abspath(__file__)) + "/config.ini")
 
+
 def doAuth(pamh):
 	"""Starts authentication in a seperate process"""
 
 	# Abort is Howdy is disabled
-	if config.get("core", "disabled") == "true":
+	if config.getboolean("core", "disabled"):
 		sys.exit(0)
 
 	# Abort if we're in a remote SSH env
-	if config.get("core", "ignore_ssh") == "true":
+	if config.getboolean("core", "ignore_ssh"):
 		if "SSH_CONNECTION" in os.environ or "SSH_CLIENT" in os.environ or "SSHD_OPTS" in os.environ:
 			sys.exit(0)
 
+	# Alert the user that we are doing face detection
+	if config.get("core", "detection_notice") == "true":
+		pamh.conversation(pamh.Message(pamh.PAM_TEXT_INFO, "Attempting face detection"))
+
 	# Run compare as python3 subprocess to circumvent python version and import issues
 	status = subprocess.call(["/usr/bin/python3", os.path.dirname(os.path.abspath(__file__)) + "/compare.py", pamh.get_user()])
 
 	# Status 10 means we couldn't find any face models
 	if status == 10:
-		if config.get("core", "suppress_unknown") != "true":
+		if not config.getboolean("core", "suppress_unknown"):
 			pamh.conversation(pamh.Message(pamh.PAM_ERROR_MSG, "No face model known"))
 		return pamh.PAM_USER_UNKNOWN
 	# Status 11 means we exceded the maximum retry count
-	if status == 11:
+	elif status == 11:
 		pamh.conversation(pamh.Message(pamh.PAM_ERROR_MSG, "Face detection timeout reached"))
 		return pamh.PAM_AUTH_ERR
+	# Status 12 means we aborted
+	elif status == 12:
+		return pamh.PAM_AUTH_ERR
 	# Status 0 is a successful exit
-	if status == 0:
+	elif status == 0:
 		# Show the success message if it isn't suppressed
-		if config.get("core", "no_confirmation") != "true":
+		if not config.getboolean("core", "no_confirmation"):
 			pamh.conversation(pamh.Message(pamh.PAM_TEXT_INFO, "Identified face as " + pamh.get_user()))
 
 		# Try to dismiss the lock screen if enabled
-		if config.get("core", "dismiss_lockscreen") == "true":
+		if config.get("core", "dismiss_lockscreen"):
 			# Run it as root with a timeout of 1s, and never ask for a password through the UI
 			subprocess.Popen(["sudo", "timeout", "1", "loginctl", "unlock-sessions", "--no-ask-password"])
 
@@ -53,18 +61,22 @@ def doAuth(pamh):
 	pamh.conversation(pamh.Message(pamh.PAM_ERROR_MSG, "Unknown error: " + str(status)))
 	return pamh.PAM_SYSTEM_ERR
 
+
 def pam_sm_authenticate(pamh, flags, args):
 	"""Called by PAM when the user wants to authenticate, in sudo for example"""
 	return doAuth(pamh)
 
+
 def pam_sm_open_session(pamh, flags, args):
 	"""Called when starting a session, such as su"""
 	return doAuth(pamh)
 
+
 def pam_sm_close_session(pamh, flags, argv):
 	"""We don't need to clean anyting up at the end of a session, so returns true"""
 	return pamh.PAM_SUCCESS
 
+
 def pam_sm_setcred(pamh, flags, argv):
 	"""We don't need set any credentials, so returns true"""
 	return pamh.PAM_SUCCESS

src/recorders/ffmpeg_reader.py

@@ -0,0 +1,132 @@
+# Class that simulates the functionality of opencv so howdy can use ffmpeg seamlessly
+
+# Import required modules
+import numpy
+import sys
+import re
+from subprocess import Popen, PIPE
+from cv2 import CAP_PROP_FRAME_WIDTH
+from cv2 import CAP_PROP_FRAME_HEIGHT
+
+try:
+	import ffmpeg
+except ImportError:
+	print("Missing ffmpeg module, please run:")
+	print(" pip3 install ffmpeg-python\n")
+	sys.exit(12)
+
+
+class ffmpeg_reader:
+	""" This class was created to look as similar to the openCV features used in Howdy as possible for overall code cleanliness. """
+
+	def __init__(self, device_path, device_format, numframes=10):
+		self.device_path = device_path
+		self.device_format = device_format
+		self.numframes = numframes
+		self.video = ()
+		self.num_frames_read = 0
+		self.height = 0
+		self.width = 0
+		self.init_camera = True
+
+	def set(self, prop, setting):
+		""" Setter method for height and width """
+		if prop == CAP_PROP_FRAME_WIDTH:
+			self.width = setting
+		elif prop == CAP_PROP_FRAME_HEIGHT:
+			self.height = setting
+
+	def get(self, prop):
+		""" Getter method for height and width """
+		if prop == CAP_PROP_FRAME_WIDTH:
+			return self.width
+		elif prop == CAP_PROP_FRAME_HEIGHT:
+			return self.height
+
+	def probe(self):
+		""" Probe the video device to get height and width info """
+
+		# Running this command on ffmpeg unfortunately returns with an exit code of 1, which is silly.
+		# Returns an error code of 1 and this text:  "/dev/video2: Immediate exit requested"
+		args = ["ffmpeg", "-f", self.device_format, "-list_formats", "all", "-i", self.device_path]
+		process = Popen(args, stdout=PIPE, stderr=PIPE)
+		out, err = process.communicate()
+		return_code = process.poll()
+
+		# Worst case scenario, err will equal en empty byte string, b'', so probe will get set to [] here.
+		regex = re.compile(r"\s\d{3,4}x\d{3,4}")
+		probe = regex.findall(str(err.decode("utf-8")))
+
+		if not return_code == 1 or len(probe) < 1:
+			# Could not determine the resolution from ffmpeg call. Reverting to ffmpeg.probe()
+			probe = ffmpeg.probe(self.device_path)
+			height = probe["streams"][0]["height"]
+			width = probe["streams"][0]["width"]
+		else:
+			(height, width) = [x.strip() for x in probe[0].split("x")]
+
+		# Set height and width from probe if they haven't been set already
+		if height.isdigit() and self.get(CAP_PROP_FRAME_HEIGHT) == 0:
+			self.set(CAP_PROP_FRAME_HEIGHT, int(height))
+		if width.isdigit() and self.get(CAP_PROP_FRAME_WIDTH) == 0:
+			self.set(CAP_PROP_FRAME_WIDTH, int(width))
+
+	def record(self, numframes):
+		""" Record a video, saving it to self.video array for processing later """
+
+		# Eensure we have set our width and height before we record, otherwise our numpy call will fail
+		if self.get(CAP_PROP_FRAME_WIDTH) == 0 or self.get(CAP_PROP_FRAME_HEIGHT) == 0:
+			self.probe()
+
+		# Ensure num_frames_read is reset to 0
+		self.num_frames_read = 0
+
+		# Record a predetermined amount of frames from the camera
+		stream, ret = (
+			ffmpeg
+			.input(self.device_path, format=self.device_format)
+			.output("pipe:", format="rawvideo", pix_fmt="rgb24", vframes=numframes)
+			.run(capture_stdout=True, quiet=True)
+		)
+		self.video = (
+			numpy
+			.frombuffer(stream, numpy.uint8)
+			.reshape([-1, self.width, self.height, 3])
+		)
+
+	def read(self):
+		""" Read a sigle frame from the self.video array. Will record a video if array is empty. """
+
+		# First time we are called, we want to initialize the camera by probing it, to ensure we have height/width
+		# and then take numframes of video to fill the buffer for faster recognition.
+		if self.init_camera:
+			self.init_camera = False
+			self.video = ()
+			self.record(self.numframes)
+			return 0, self.video
+
+		# If we are called and self.video is empty, we should record self.numframes to fill the video buffer
+		if self.video == ():
+			self.record(self.numframes)
+
+		# If we've read max frames, but still are being requested to read more, we simply record another batch.
+		# Note, the video array is 0 based, so if numframes is 10, we must subtract 1 or run into an array index
+		# error.
+		if self.num_frames_read >= (self.numframes - 1):
+			self.record(self.numframes)
+
+		# Add one to num_frames_read. If we were at 0, that's fine as frame 0 is almost 100% going to be black
+		# as the IR lights aren't fully active yet anyways. Saves us one iteration in the while loop ni add/compare.py.
+		self.num_frames_read += 1
+
+		# Return a single frame of video
+		return 0, self.video[self.num_frames_read]
+
+	def release(self):
+		""" Empty our array. If we had a hold on the camera, we would give it back here. """
+		self.video = ()
+		self.num_frames_read = 0
+
+	def grab(self):
+		""" Redirect grab() to read() for compatibility """
+		self.read()

src/recorders/pyv4l2_reader.py

@@ -0,0 +1,102 @@
+# Class that simulates the functionality of opencv so howdy can use v4l2 devices seamlessly
+
+# Import required modules. lib4l-dev package is also required.
+from recorders import v4l2
+import fcntl
+import numpy
+import sys
+from cv2 import cvtColor, COLOR_GRAY2BGR, CAP_PROP_FRAME_WIDTH, CAP_PROP_FRAME_HEIGHT
+
+try:
+	from v4l2.frame import Frame
+except ImportError:
+	print("Missing pyv4l2 module, please run:")
+	print(" pip3 install pyv4l2\n")
+	sys.exit(13)
+
+
+class pyv4l2_reader:
+	""" This class was created to look as similar to the openCV features used in Howdy as possible for overall code cleanliness. """
+
+	# Init
+	def __init__(self, device_name, device_format):
+		self.device_name = device_name
+		self.device_format = device_format
+		self.height = 0
+		self.width = 0
+		self.probe()
+		self.frame = ""
+
+	def set(self, prop, setting):
+		""" Setter method for height and width """
+		if prop == CAP_PROP_FRAME_WIDTH:
+			self.width = setting
+		elif prop == CAP_PROP_FRAME_HEIGHT:
+			self.height = setting
+
+	def get(self, prop):
+		""" Getter method for height and width """
+		if prop == CAP_PROP_FRAME_WIDTH:
+			return self.width
+		elif prop == CAP_PROP_FRAME_HEIGHT:
+			return self.height
+
+	def probe(self):
+		""" Probe the video device to get height and width info """
+
+		vd = open(self.device_name, 'r')
+		fmt = v4l2.v4l2_format()
+		fmt.type = v4l2.V4L2_BUF_TYPE_VIDEO_CAPTURE
+		ret = fcntl.ioctl(vd, v4l2.VIDIOC_G_FMT, fmt)
+		vd.close()
+		if ret == 0:
+			height = fmt.fmt.pix.height
+			width = fmt.fmt.pix.width
+		else:
+			# Could not determine the resolution from ioctl call. Reverting to slower ffmpeg.probe() method
+			import ffmpeg
+			probe = ffmpeg.probe(self.device_name)
+			height = int(probe['streams'][0]['height'])
+			width = int(probe['streams'][0]['width'])
+
+		if self.get(CAP_PROP_FRAME_HEIGHT) == 0:
+			self.set(CAP_PROP_FRAME_HEIGHT, int(height))
+
+		if self.get(CAP_PROP_FRAME_WIDTH) == 0:
+			self.set(CAP_PROP_FRAME_WIDTH, int(width))
+
+	def record(self):
+		""" Start recording """
+		self.frame = Frame(self.device_name)
+
+	def grab(self):
+		""" Read a sigle frame from the IR camera. """
+		self.read()
+
+	def read(self):
+		""" Read a sigle frame from the IR camera. """
+
+		if not self.frame:
+			self.record()
+
+		# Grab a raw frame from the camera
+		frame_data = self.frame.get_frame()
+
+		# Convert the raw frame_date to a numpy array
+		img = (numpy.frombuffer(frame_data, numpy.uint8))
+
+		# Convert the numpy array to a proper grayscale image array
+		img_bgr = cvtColor(img, COLOR_GRAY2BGR)
+
+		# Convert the grayscale image array into a proper RGB style numpy array
+		img2 = (numpy.frombuffer(img_bgr, numpy.uint8).reshape([352, 352, 3]))
+
+		# Return a single frame of video
+		return 0, img2
+
+	def release(self):
+		""" Empty our array. If we had a hold on the camera, we would give it back here. """
+		self.video = ()
+		self.num_frames_read = 0
+		if self.frame:
+			self.frame.close()

tests/compare.sh

@@ -0,0 +1,29 @@
+# TEST MODEL-FRAME COMPARE FUNCTIONS
+set -o xtrace
+set -e
+
+# Make sure howdy is clean before starting
+sudo howdy clear -y || true
+
+# Learn match 1
+sudo sed -i "s,device_path.*,device_path = $PWD\/tests\/video\/match1.m4v,g" /lib/security/howdy/config.ini
+sudo howdy add -y
+
+# Text compare matching with same camera input
+sudo python3 /lib/security/howdy/compare.py $USER
+
+# Change to match 2 and compare against the modal of match 1, which should fail
+sudo sed -i "s,device_path.*,device_path = $PWD\/tests\/video\/match2.m4v,g" /lib/security/howdy/config.ini
+! sudo python3 /lib/security/howdy/compare.py $USER
+
+# Add match 2 as a model to compare both 1 and 2 at the same time
+sudo howdy add -y
+sudo python3 /lib/security/howdy/compare.py $USER
+
+# Compare against a camera with no visible face
+sudo sed -i "s,device_path.*,device_path = $PWD\/tests\/video\/noMatch.m4v,g" /lib/security/howdy/config.ini
+! sudo python3 /lib/security/howdy/compare.py $USER
+
+# Clean up
+sudo howdy clear -y
+sudo sed -i "s,device_path.*,device_path = none,g" /lib/security/howdy/config.ini

tests/importing.sh

@@ -0,0 +1,9 @@
+# TEST INSTALLATION OF DEPENDENCIES
+set -o xtrace
+set -e
+
+# Confirm the cv2 module has been installed correctly
+sudo /usr/bin/env python3 -c "import cv2; print(cv2.__version__);"
+
+# Confirm the dlib module has been installed correctly
+sudo /usr/bin/env python3 -c "import dlib; print(dlib.__version__);"

tests/pam.sh

@@ -0,0 +1,32 @@
+# TEST THE PAM INTEGRATION
+set -o xtrace
+set -e
+
+# Make sure howdy is clean before starting
+sudo howdy clear -y || true
+
+# Change active camera to match video 1
+sudo sed -i "s,device_path.*,device_path = $PWD/tests\/video\/match1.m4v,g" /lib/security/howdy/config.ini
+
+# Let howdy add the match face
+sudo howdy add -y
+
+# Test the PAM auth
+timeout 10 pamtester login $USER authenticate
+
+# Clear the face models and change the camera to video 2
+sudo howdy clear -y
+sudo sed -i "s,device_path.*,device_path = $PWD\/tests\/video\/match2.m4v,g" /lib/security/howdy/config.ini
+
+# Let howdy add the match face
+sudo howdy add -y
+
+# Try to open a elevated session through PAM
+timeout 10 pamtester login $USER open_session
+
+# Verify we can close sessions, even though howdy does not use this PAM function
+timeout 10 pamtester login $USER close_session
+
+# Clean up
+sudo howdy clear -y
+sudo sed -i "s,device_path.*,device_path = none,g" /lib/security/howdy/config.ini

tests/passthrough.sh

@@ -0,0 +1,7 @@
+# TEST USER SUDO PASSTHOUGH (NON-ROOT)
+set -o xtrace
+set -e
+
+# Check if the username passthough works correctly with sudo
+howdy | ack-grep --passthru --color "current active user: travis"
+sudo howdy | ack-grep --passthru --color "current active user: travis"