PIRgate now logs immediately for better visualization in Grafana

Gatecounter service added
Gatecounter db now initialized by the gatecounter itself courtesy of SQLAlchemy

Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3
+WORKDIR /usr/src/app
+RUN pip install --no-cache-dir sqlalchemy RPi.GPIO mysqlclient Adafruit-MCP3008 Adafruit-GPIO
+ENTRYPOINT [ "python3" ]

docker-compose.yaml

@@ -1,6 +1,32 @@
 version: '3.7' #specifies the version of the compose-file specification to use.  Refer to the compose-file reference for more info https://docs.docker.com/compose/compose-file/
 #this section specifies the various services that comprise the project
 services:
+  gatecounter:
+    build:
+      context: .  #build a custom python container to run the gatecounter script
+    container_name: gatecounter
+    privileged: true  #enable access to GPIO from Docker Container
+    volumes:
+      #make the scripts acessible to the container, read-only
+      - ${PWD}/gatecounter-scripts:/usr/src/app:ro
+    labels:
+      - traefik.enable=false
+    networks:
+      - gatecounter
+    restart: unless-stopped
+    depends_on: 
+      - gatecounter-db
+    command:
+      - "${GATECOUNTER_SCRIPT:?The name of a gatecounter script in the gatecounter-scripts directory is required.  Please edit .env and add a value for GATECOUNTER_SCRIPT}"
+      - "-H"
+      - "gatecounter-db"
+      - "-d"
+      - "${MYSQL_DB_NAME}"
+      - "-u"
+      - "${MYSQL_USER}"
+      - "-p"
+      - "${MYSQL_USER_PW}"
+
   #this service will be the mysql database that detections will be logged to
   gatecounter-db: #how this service will be referenced in this file
     image: yobasystems/alpine-mariadb:armhf
@@ -13,8 +39,7 @@ services:
       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
       TZ: ${TZ:-America/New_York}
     volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/config #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
+      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
     restart: unless-stopped #keep this service running unless told explicitly to stop
     networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
       - gatecounter
@@ -28,34 +53,10 @@ services:
     expose:
       - "3306"
 
-  gatecounter-db-init: #how this service will be referenced in this file
-    image: yobasystems/alpine-mariadb:armhf
-    environment: #set environment variables for this service.  These will initialize a database
-    #these environment variables will specify how the gate counter script will connect to the db to record data
-      MYSQL_DATABASE: ${MYSQL_DB_NAME:-gatecounter}
-      MYSQL_USER: ${MYSQL_USER:-gatecounter}
-      MYSQL_PASSWORD: ${MYSQL_USER_PW:?a non-admin database password is requred.  Please edit .env with this value}
-      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
-      TZ: ${TZ:-America/New_York}
-    volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
-    networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
-      - gatecounter
-    labels: #can be used to communicate info about this service to other services
-      - traefik.enable=false #tells traefik reverse proxy to ignore this container, do not proxy requests to it
-      - com.docker.compose.oneoff=true
-    entrypoint:
-      - /bin/sh
-      - -c
-      - "\"mysql -h gatecounter-db -u root -D ${MYSQL_DB_NAME} -p${MYSQL_ROOT_PW} < /docker-entrypoint-initdb.d/db-init.sql\""
-    depends_on:
-      - gatecounter-db
-
-
   grafana:
-    image: grafana/grafana-arm32v7-linux
+    image: grafana/grafana:6.4.3
     container_name: grafana #redundant, would have defaulted to the service name anyway
+    restart: unless-stopped
     volumes:
       - ./configs/grafana.ini:/etc/grafana/grafana.ini #maps grafana.ini in this directory into the container
       - grafana_data:/var/lib/grafana
@@ -65,7 +66,7 @@ services:
       - grafana_provisioning:/etc/grafana/provisioning
     labels:
       - traefik.enable=true #enable forwarding of requests to this container
-      - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME`) #when a request is received for this domain, forward the request to this container...
+      - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME}`) #when a request is received for this domain, forward the request to this container...
       - traefik.http.routers.grafana-http.entrypoints=http
       - traefik.http.routers.grafana-http.middlewares=https-only #redirect all http requests to https
       - traefik.http.routers.grafana-https.entrypoints=https
@@ -129,11 +130,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock  #allows traefik to monitor for changes and to read labels
       - ./certs/:/certs/:ro
       - ./configs/traefik:/etc/traefik/custom:ro
-    #The following section allows you to deifne services which must be started before this service can start
-    depends_on:
-      - dynamic-dns
-    environment:
-      DUCKDNS_TOKEN: ${DUCKDNS_TOKEN:?Please provide a duckdns token for your domain.  Please edit .env with this value} #allows traefik to obtain ssl certs for your domain(s) automatically enabling you to use https for security
     networks:
       - gatecounter
 
@@ -143,7 +139,6 @@ services:
 volumes:
   gatecounter-db:
   grafana-db:
-  traefik-cert-gc:
   grafana_data:
   grafana_home:
   grafana_logs:

gatecounter-scripts/PIRdbWriteGate.py

@@ -1,89 +1,123 @@
-# Written By Johnathan Cintron and Devlyn Courtier for the HCCC Library
+#!/usr/bin/python3
-
+import collections
-#!/usr/bin/python
+import logging
-
+import subprocess
 import sys
-import MySQLdb
+
-from time import sleep
+from argparse import ArgumentParser
+from concurrent.futures import ThreadPoolExecutor, CancelledError, wait
 from datetime import datetime
+from queue import Queue
+
 import RPi.GPIO as GPIO
-from getpass import getpass
+
-from multiprocessing import Queue
+from sqlalchemy import Column, DateTime, Integer, Table, create_engine
-from concurrent.futures import ProcessPoolExecutor, CancelledError, wait
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+log = logging.getLogger("sqlalchemy")
+info = logging.StreamHandler(sys.stdout)
+info.addFilter(lambda x: x.levelno <= logging.WARNING)
+errors = logging.StreamHandler(sys.stderr)
+errors.addFilter(lambda x: x.levelno >= logging.ERROR)
+log.setLevel(logging.DEBUG)
+log.addHandler(info)
+log.addHandler(errors)
+
+
+Base = declarative_base()
+
+class PIR_Detection(Base):
+    __tablename__ = "PIRSTATS"
+
+    time = Column('datetime', DateTime, nullable=False, primary_key=True)
+    count = Column('count', Integer, nullable=False)
+
+
+Detection=collections.namedtuple("Detection", ['time','count'])
 
 class PIRgate:
-	def __init__(self, hostname, username, password, database):
+    def __init__(self, hostname, username, password, database):
-		# Set RPi GPIO Mode
+        # Set RPi GPIO Mode
-		GPIO.setmode(GPIO.BCM)
+        GPIO.setmode(GPIO.BCM)
 
-		# Setup GPIO in and out pins
+        # Setup GPIO in and out pins
-		self.PIR_PIN = 7
+        self.PIR_PIN = 7
-		GPIO.setup(self.PIR_PIN, GPIO.IN)
+        GPIO.setup(self.PIR_PIN, GPIO.IN)
-		# End GPIO setup
+        # End GPIO setup
-		self.counts=Queue()
+        self._pool=ThreadPoolExecutor()
-		self._pool=ProcessPoolExecutor()
+        self._detection_queue=Queue()
+        if not hostname:
+            stdout,stderr = subprocess.Popen(['docker',
+                                              'inspect',
+                                              '-f',
+                                              "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
+                                              'gatecounter-db'],
+                                              stdout=subprocess.PIPE,
+                                              stderr=subprocess.STDOUT).communicate()
+            host = stdout.decode().strip()
+        else:
+            host = hostname
+        self.db_engine = create_engine(f"mysql://{username}:{password}@{host}/{database}")
+        Base.metadata.create_all(bind=self.db_engine)
+        self._session_factory = sessionmaker(bind=self.db_engine)
+        self.Session = scoped_session(self._session_factory)
 
-	def start(self):
+    def start(self):
-		try:
+        GPIO.add_event_detect(self.PIR_PIN, GPIO.RISING, callback=lambda c: self._detection_queue.put(Detection(datetime.now(),1)))
-			self.event_listener = self._pool.submit(self.listen_for_events)
+        with self._pool:
-			self.db_writer = self._pool.submit(self.write_to_db, hostname, username, password, database)
+            try:
-		except KeyboardInterrupt:
+                self.db_writer = self._pool.submit(self.write_to_db)
-			print("\nCtrl-C pressed cleaning up GPIO")
+                wait([self.db_writer])
-			self.event_listener.cancel()
+            except KeyboardInterrupt:
-			self.db_writer.cancel()
+                print("\nCtrl-C pressed cleaning up GPIO")
-			GPIO.cleanup()
+                raise
-		finally:
+            finally:
-			wait([self.event_listener,self.db_writer])
+                GPIO.cleanup()
-			GPIO.cleanup()
 
-	def listen_for_events(self):
+    def write_to_db(self):
-		count = 0
+        while True:
-		while True:
+            try:
-			try:
+                detection = self._detection_queue.get()
-				if GPIO.input(self.PIR_PIN):
+                session = self.Session()
-					count += 1
+                session.add(PIR_Detection(time=detection.datetime, count=detection.count))
-				curr_date = datetime.now()
+            except KeyboardInterrupt:
-				if (curr_date.minute % 10 == 0) and (curr_date.second == 0):
+                session.rollback()
-					self.counts.put_nowait((curr_date,count))
+                raise
-					count = 0
+            except:
-			except (KeyboardInterrupt,CancelledError):
+                session.rollback()
-				break
+                self._detection_queue.put(detection) #return detection to queue to try again
-
+            else:
-
+                session.commit()
-	def write_to_db(self, hostname, username, password, database):
+            finally:
-		while True:
+                session.remove()
-			try:
-				time, count = self.counts.get()
-				with MySQLdb.connect(hostname,username,password,database) as db:
-					try:
-						db.cursor().execute("INSERT INTO PIRSTATS (datetime, gatecount) VALUES ('%s', '%d')" % (time.isoformat(' '), count))
-					except (KeyboardInterrupt,CancelledError):
-						db.rollback()
-						break
-					except:
-						db.rollback()
-						self.counts.put(time,count) #put the data back in queue to try writing it again
-					else:
-						db.commit()
-					finally:
-						db.close()
-			except (KeyboardInterrupt,CancelledError):
-				break
 
 if __name__ == "__main__":
-	while True:
+    parser = ArgumentParser(description="Begin PIR detections")
-		try:
+    parser.add_argument("-H", "--hostname",default="")
-			hostname = input("DB Hostname: ")
+    parser.add_argument("-d", "--database",default="gatecounter")
-			database = input("Database: ")
+    parser.add_argument("-u", "--username", required=True)
-			username = input("Username: ")
+    parser.add_argument("-p", "--password", required=True)
-			password = getpass()
+    if sys.argv[0].startswith("python"):
-			#just check the credentials by connecting to the db and closing
+        args = parser.parse_args(sys.argv[2:])
-			MySQLdb.connect(hostname, username, password, database).close()
+    else:
-		except KeyboardInterrupt:
+        args = parser.parse_args(sys.argv[1:])
-			sys.exit(1)
+    if not args.hostname:
-		except:
+        stdout,stderr = subprocess.Popen(['docker',
-			print("\nProblem connecting to the database.  Check your credentials and try again \n")
+                                          'inspect',
-			continue
+                                          '-f',
-		else:
+                                          "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
-			break
+                                          'gatecounter-db'],
-	PIRgate(hostname, username, password, database).start()
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.STDOUT).communicate()
+        host = stdout.decode().strip()
+    else:
+        host = args.hostname
+    try:
+        PIRgate(args.hostname, args.username, args.password, args.database).start()
+    except KeyboardInterrupt:
+        sys.exit(1)
+    except:
+        print("\nProblem connecting to the database.  Check your credentials and try again \n")
+        sys.exit(2)
+

sql/db-init.sql

@@ -1,17 +0,0 @@
-create table if not exists PIRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists ULTRASTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists LDRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);