update to PIT script and README

improvements to PIRdbWriteGate.py
gatecounter db initialized by python script
updated .env.template
python scripts now run by gatecounter service
gatecounter service build from Dockerfile

.env.template

@@ -25,3 +25,5 @@ GRAFANA_DB_NAME=
 #must match value in grafana.ini
 GRAFANA_DB_ROOT_PW=
 
+GATECOUNTER_SCRIPT=
+EMAIL_ADDRESS=

Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3
+WORKDIR /usr/src/app
+RUN pip install --no-cache-dir sqlalchemy RPi.GPIO mysqlclient Adafruit-MCP3008 Adafruit-GPIO
+ENTRYPOINT [ "python3" ]

README.md

@@ -16,7 +16,7 @@ Please also, in a developer's text editor e.g. NotePad++ or Microsoft Visual Stu
 * Register for [DuckDNS](https://www.duckdns.org/) and have your subdomain name and token ready
 * Make sure ports 80 and 443 are accessible on your host machine and your machine has a connection to the Internet
 * Copy .env.template to .env with `cp .env.template .env` *DO NOT COMMIT AND PUSH .env TO A PUBLIC GIT REPOSITORY UNLESS YOU WANT TO GET HACKED!!!*
-* Edit the files .env, .configs/traefik.toml, and .configs/grafana.ini, updating configuration values with your desired configuration
+* Edit the files .env and .configs/grafana.ini, updating configuration values with your desired configuration
 * Run `docker-compose config` from this directory to doublecheck that docker-compose.yaml file contains no syntax errors and that all your options from .env were correctly filled in
 
 ## Creating Your Stack
@@ -32,6 +32,7 @@ To Stop or (re)Start a container in your stack without removing it, run `docker-
 Run the following commands to update the images your containers use and recreate/restart the containers using them
 
 ```bash
+docker-compose build
 docker-compose pull
 docker-compose up -d
 docker image prune -f

configs/traefik.toml

@@ -1,90 +0,0 @@
-debug = false
-
-logLevel = "ERROR"
-defaultEntryPoints = ["https","http"]
-
-[entryPoints]
-  [entryPoints.http]
-  address = ":80"
-    [entryPoints.http.redirect]
-    entryPoint = "https"
-  [entryPoints.https]
-  address = ":443"
-  [entryPoints.https.tls]
-#  [entryPoints.traefik]
-#  address = ":8080"
-
-[retry]
-
-# Traefik logs
-# Enabled by default and log to stdout
-#
-# Optional
-#
-# [traefikLog]
-
-# Sets the filepath for the traefik log. If not specified, stdout will be used.
-# Intermediate directories are created if necessary.
-#
-# Optional
-# Default: os.Stdout
-#
-# filePath = "log/traefik.log"
-
-# Format is either "json" or "common".
-#
-# Optional
-# Default: "common"
-#
-# format = "common"
-
-# Enable access logs
-# By default it will write to stdout and produce logs in the textual
-# Common Log Format (CLF), extended with additional fields.
-#
-# Optional
-#
-# [accessLog]
-
-# Sets the file path for the access log. If not specified, stdout will be used.
-# Intermediate directories are created if necessary.
-#
-# Optional
-# Default: os.Stdout
-#
-# filePath = "/path/to/log/log.txt"
-
-# Format is either "json" or "common".
-#
-# Optional
-# Default: "common"
-#
-# format = "common"
-
-################################################################
-# Web configuration backend
-################################################################
-
-# Enable web configuration backend
-# https://docs.traefik.io/configuration/api/
-#[api]
-#entryPoint = "traefik"
-#dashboard = true
-
-[file]
-directory = "/etc/traefik/rules"
-watch = true
-
-[docker]
-endpoint = "unix:///var/run/docker.sock" 
-domain = "yoursubdomain.duckdns.org"
-watch = true
-exposedbydefault = false
-
-[acme]
-email = "you@youremail.com"
-storage = "/etc/traefik/acme/acme.json"
-entryPoint = "https"
-OnHostRule = true
-[acme.dnsChallenge]
-provider = "duckdns"

docker-compose.yaml

@@ -1,6 +1,32 @@
 version: '3.7' #specifies the version of the compose-file specification to use.  Refer to the compose-file reference for more info https://docs.docker.com/compose/compose-file/
 #this section specifies the various services that comprise the project
 services:
+  gatecounter:
+    build:
+      context: .  #build a custom python container to run the gatecounter script
+    container_name: gatecounter
+    privileged: true  #enable access to GPIO from Docker Container
+    volumes:
+      #make the scripts acessible to the container, read-only
+      - ${PWD}/gatecounter-scripts:/usr/src/app:ro
+    labels:
+      - traefik.enable=false
+    networks:
+      - gatecounter
+    restart: unless-stopped
+    depends_on: 
+      - gatecounter-db
+    command:
+      - "${GATECOUNTER_SCRIPT:?The name of a gatecounter script in the gatecounter-scripts directory is required.  Please edit .env and add a value for GATECOUNTER_SCRIPT}"
+      - "-H"
+      - "gatecounter-db"
+      - "-d"
+      - "${MYSQL_DB_NAME}"
+      - "-u"
+      - "${MYSQL_USER}"
+      - "-p"
+      - "${MYSQL_USER_PW}"
+
   #this service will be the mysql database that detections will be logged to
   gatecounter-db: #how this service will be referenced in this file
     image: yobasystems/alpine-mariadb:armhf
@@ -13,8 +39,7 @@ services:
       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
       TZ: ${TZ:-America/New_York}
     volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/config #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
+      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
     restart: unless-stopped #keep this service running unless told explicitly to stop
     networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
       - gatecounter
@@ -25,41 +50,13 @@ services:
       interval: 30s
       timeout: 10s
       retries: 5
-    ports:
-      - "3306:3306" #connects port 3306 of the host (left) to 3306 of this container (right) making it accessible to things outside of our docker virtual network
     expose:
       - "3306"
 
-  gatecounter-db-init: #how this service will be referenced in this file
-    image: yobasystems/alpine-mariadb:armhf
-    environment: #set environment variables for this service.  These will initialize a database
-    #these environment variables will specify how the gate counter script will connect to the db to record data
-      MYSQL_DATABASE: ${MYSQL_DB_NAME:-gatecounter}
-      MYSQL_USER: ${MYSQL_USER:-gatecounter}
-      MYSQL_PASSWORD: ${MYSQL_USER_PW:?a non-admin database password is requred.  Please edit .env with this value}
-      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
-      TZ: ${TZ:-America/New_York}
-    volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
-    networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
-      - gatecounter
-    labels: #can be used to communicate info about this service to other services
-      - traefik.enable=false #tells traefik reverse proxy to ignore this container, do not proxy requests to it
-      - com.docker.compose.oneoff=true
-    entrypoint:
-      - /bin/sh
-      - -c
-      - "\"mysql -h $$HOSTNAME -u root -D ${MYSQL_DB_NAME} -p${MYSQL_ROOT_PW} < /docker-entrypoint-initdb.d/db-init.sql\""
-    depends_on:
-      - gatecounter-db
-    expose:
-      - "3306"
-
-
   grafana:
-    image: grafana/grafana-arm32v7-linux
+    image: grafana/grafana:latest
     container_name: grafana #redundant, would have defaulted to the service name anyway
+    restart: unless-stopped
     volumes:
       - ./configs/grafana.ini:/etc/grafana/grafana.ini #maps grafana.ini in this directory into the container
       - grafana_data:/var/lib/grafana
@@ -68,15 +65,19 @@ services:
       - grafana_plugins:/var/lib/grafana/plugins
       - grafana_provisioning:/etc/grafana/provisioning
     labels:
-      - traefik.enable=true #enable forwarding of http requests to this container
-      - traefik.frontend.rule=Host:${GRAFANA_DOMAIN_NAME} #when a request is received for this domain...
-      - traefik.backend=grafana #forward the request to this container...
-      - traefik.port=3000 #on this port...
-      - traefik.protocol=http #forwarding the request in plain http on the internal virtual network
+      - traefik.enable=true #enable forwarding of requests to this container
+      - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME}`) #when a request is received for this domain, forward the request to this container...
+      - traefik.http.routers.grafana-http.entrypoints=http
+      - traefik.http.routers.grafana-http.middlewares=https-only #redirect all http requests to https
+      - traefik.http.routers.grafana-https.entrypoints=https
+      - traefik.http.routers.grafana-https.tls=true
+      - traefik.http.routers.grafana-https.tls.certResolver=gatecounter
+      - traefik.http.services.grafana.loadbalancer.server.port=3000 #on this port...
     expose:
       - "3000" #makes this port accessible to other containers on the same network, but not availble directly on the host system
     depends_on: #specifies which containers must be up and running before this one can be started
       - reverse-proxy
+      - grafana-db
       - gatecounter-db
     environment:
       GF_SERVER_ROOT_URL: https://${GRAFANA_DOMAIN_NAME}
@@ -122,17 +123,29 @@ services:
       - default #put this service on the built-in docker bridge network
 
   reverse-proxy:
-    image: traefik:v1.7
-    container_name: traefik-gc #referenced in ./configs/traefik.toml by this name in [api] section
+    image: traefik:latest
+    container_name: traefik
+    command:
+      - "--api=false"
+      - "--entryPoints.http.address=:80"
+      - "--entryPoints.https.address=:443"
+      - "--providers.docker=true"
+      - "--accesslog=true"
+      - "--log=true"
+      - "--log.level=INFO"
+      - "--certificatesResolvers.gatecounter.acme.email=${EMAIL_ADDRESS:?An email address to use to obtain a SSL Cert is required.   Please edit .env with this value}"
+      - "--certificatesResolvers.gatecounter.acme.storage=/etc/traefik/acme/acme.json"
+      - "--certificatesResolvers.gatecounter.acme.dnsChallenge=true"
+      - "--certificatesResolvers.gatecounter.acme.dnsChallenge.provider=duckdns"
+    labels:
+      - "traefik.http.middlewares.https-only.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.https-only.redirectscheme.permanent=true"      
     restart: unless-stopped #Docker will automatically restart this container unless you intentionally stopped it
     ports:
       - 80:80
       - 443:443
-#      - 8080:8080 #admin web UI port
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock  #allows traefik to monitor for changes and to read labels
-      - ./configs/traefik.toml:/etc/traefik/traefik.toml #traefik config file
-      - ./rules:/etc/traefik/rules
       - traefik-cert-gc:/etc/traefik/acme/ #volume for storing LetsEncrypt cets
     #The following section allows you to deifne services which must be started before this service can start
     depends_on:

gatecounter-scripts/PIRdbWriteGate.py

@@ -1,89 +1,123 @@
-# Written By Johnathan Cintron and Devlyn Courtier for the HCCC Library
-
-#!/usr/bin/python
-
+#!/usr/bin/python3
+import collections
+import logging
+import subprocess
 import sys
-import MySQLdb
-from time import sleep
+
+from argparse import ArgumentParser
+from concurrent.futures import ThreadPoolExecutor, CancelledError, wait
 from datetime import datetime
+from queue import SimpleQueue
+
 import RPi.GPIO as GPIO
-from getpass import getpass
-from multiprocessing import Queue
-from concurrent.futures import ProcessPoolExecutor, CancelledError, wait
+
+from sqlalchemy import Column, DateTime, Integer, Table, create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+log = logging.getLogger("sqlalchemy")
+info = logging.StreamHandler(sys.stdout)
+info.addFilter(lambda x: x.levelno <= logging.WARNING)
+errors = logging.StreamHandler(sys.stderr)
+errors.addFilter(lambda x: x.levelno >= logging.ERROR)
+log.setLevel(logging.DEBUG)
+log.addHandler(info)
+log.addHandler(errors)
+
+
+Base = declarative_base()
+
+class PIR_Detection(Base):
+    __tablename__ = "PIRSTATS"
+
+    timestamp = Column('timestamp', DateTime, nullable=False, primary_key=True)
+    count = Column('count', Integer, nullable=False)
+
+
+Detection=collections.namedtuple("Detection", ['timestamp','count'])
 
 class PIRgate:
-	def __init__(self, hostname, username, password, database):
-		# Set RPi GPIO Mode
-		GPIO.setmode(GPIO.BCM)
+    def __init__(self, hostname, username, password, database):
+        # Set RPi GPIO Mode
+        GPIO.setmode(GPIO.BCM)
 
-		# Setup GPIO in and out pins
-		self.PIR_PIN = 7
-		GPIO.setup(self.PIR_PIN, GPIO.IN)
-		# End GPIO setup
-		self.counts=Queue()
-		self._pool=ProcessPoolExecutor()
+        # Setup GPIO in and out pins
+        self.PIR_PIN = 7
+        GPIO.setup(self.PIR_PIN, GPIO.IN)
+        # End GPIO setup
+        self._pool=ThreadPoolExecutor()
+        self._detection_queue=SimpleQueue()
+        if not hostname:
+            stdout,stderr = subprocess.Popen(['docker',
+                                              'inspect',
+                                              '-f',
+                                              "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
+                                              'gatecounter-db'],
+                                              stdout=subprocess.PIPE,
+                                              stderr=subprocess.STDOUT).communicate()
+            host = stdout.decode().strip()
+        else:
+            host = hostname
+        self.db_engine = create_engine(f"mysql://{username}:{password}@{host}/{database}")
+        Base.metadata.create_all(bind=self.db_engine)
+        self._session_factory = sessionmaker(bind=self.db_engine)
+        self.Session = scoped_session(self._session_factory)
 
-	def start(self):
-		try:
-			self.event_listener = self._pool.submit(self.listen_for_events)
-			self.db_writer = self._pool.submit(self.write_to_db, hostname, username, password, database)
-		except KeyboardInterrupt:
-			print("\nCtrl-C pressed cleaning up GPIO")
-			self.event_listener.cancel()
-			self.db_writer.cancel()
-			GPIO.cleanup()
-		finally:
-			wait([self.event_listener,self.db_writer])
-			GPIO.cleanup()
+    def start(self):
+        GPIO.add_event_detect(self.PIR_PIN, GPIO.RISING, callback=lambda c: self._detection_queue.put(Detection(datetime.now(),1)))
+        with self._pool:
+            try:
+                self.db_writer = self._pool.submit(self.write_to_db)
+                wait([self.db_writer])
+            except KeyboardInterrupt:
+                print("\nCtrl-C pressed cleaning up GPIO")
+                raise
+            finally:
+                GPIO.cleanup()
 
-	def listen_for_events(self):
-		count = 0
-		while True:
-			try:
-				if GPIO.input(self.PIR_PIN):
-					count += 1
-				curr_date = datetime.now()
-				if (curr_date.minute % 10 == 0) and (curr_date.second == 0):
-					self.counts.put_nowait((curr_date,count))
-					count = 0
-			except (KeyboardInterrupt,CancelledError):
-				break
-
-
-	def write_to_db(self, hostname, username, password, database):
-		while True:
-			try:
-				time, count = self.counts.get()
-				with MySQLdb.connect(hostname,username,password,database) as db:
-					try:
-						db.cursor().execute("INSERT INTO PIRSTATS (datetime, gatecount) VALUES ('%s', '%d')" % (time.isoformat(' '), count))
-					except (KeyboardInterrupt,CancelledError):
-						db.rollback()
-						break
-					except:
-						db.rollback()
-						self.counts.put(time,count) #put the data back in queue to try writing it again
-					else:
-						db.commit()
-					finally:
-						db.close()
-			except (KeyboardInterrupt,CancelledError):
-				break
+    def write_to_db(self):
+        while True:
+            try:
+                detection = self._detection_queue.get()
+                session = self.Session()
+                session.add(PIR_Detection(timestamp=detection.timestamp, count=detection.count))
+            except KeyboardInterrupt:
+                session.rollback()
+                raise
+            except:
+                session.rollback()
+                self._detection_queue.put(detection) #return detection to queue to try again
+            else:
+                session.commit()
+            finally:
+                session.remove()
 
 if __name__ == "__main__":
-	while True:
-		try:
-			hostname = input("DB Hostname: ")
-			database = input("Database: ")
-			username = input("Username: ")
-			password = getpass()
-			#just check the credentials by connecting to the db and closing
-			MySQLdb.connect(hostname, username, password, database).close()
-		except KeyboardInterrupt:
-			sys.exit(1)
-		except:
-			print("\nProblem connecting to the database.  Check your credentials and try again \n")
-			continue
-		else:
-			break
-	PIRgate(hostname, username, password, database).start()
+    parser = ArgumentParser(description="Begin PIR detections")
+    parser.add_argument("-H", "--hostname",default="")
+    parser.add_argument("-d", "--database",default="gatecounter")
+    parser.add_argument("-u", "--username", required=True)
+    parser.add_argument("-p", "--password", required=True)
+    if sys.argv[0].startswith("python"):
+        args = parser.parse_args(sys.argv[2:])
+    else:
+        args = parser.parse_args(sys.argv[1:])
+    if not args.hostname:
+        stdout,stderr = subprocess.Popen(['docker',
+                                          'inspect',
+                                          '-f',
+                                          "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
+                                          'gatecounter-db'],
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.STDOUT).communicate()
+        host = stdout.decode().strip()
+    else:
+        host = args.hostname
+    try:
+        PIRgate(args.hostname, args.username, args.password, args.database).start()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except:
+        print("\nProblem connecting to the database.  Check your credentials and try again \n")
+        sys.exit(2)
+

gatecounter-test.toml

@@ -1,10 +0,0 @@
-[backends]
-[backends.gatecounter]
-    [backends.gatecounter.servers.gatecounter-server]
-    url = "http://grafana:3000"
-[frontends]
-[frontends.gatecounter]
-entryPoints = ["http"]
-backend = "gatecounter"
-    [frontends.gatecounter.routes.test]
-    rule = "HostRegexp:grafana.{hostname:[a-z]+}{suffix:(\.local|\.home)}"

sql/db-init.sql

@@ -1,17 +0,0 @@
-create table if not exists PIRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists ULTRASTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists LDRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);