added script to create certs

Gatecounter service added
Gatecounter db now initialized by the gatecounter itself courtesy of SQLAlchemy

.gitignore

@@ -100,3 +100,4 @@ ENV/
 # mypy
 .mypy_cache/
 .vscode/settings.json
+certs/*

Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3
+WORKDIR /usr/src/app
+RUN pip install --no-cache-dir sqlalchemy RPi.GPIO mysqlclient Adafruit-MCP3008 Adafruit-GPIO
+ENTRYPOINT [ "python3" ]

configs/traefik.toml

@@ -1,90 +0,0 @@
-debug = false
-
-logLevel = "ERROR"
-defaultEntryPoints = ["https","http"]
-
-[entryPoints]
-  [entryPoints.http]
-  address = ":80"
-    [entryPoints.http.redirect]
-    entryPoint = "https"
-  [entryPoints.https]
-  address = ":443"
-  [entryPoints.https.tls]
-#  [entryPoints.traefik]
-#  address = ":8080"
-
-[retry]
-
-# Traefik logs
-# Enabled by default and log to stdout
-#
-# Optional
-#
-# [traefikLog]
-
-# Sets the filepath for the traefik log. If not specified, stdout will be used.
-# Intermediate directories are created if necessary.
-#
-# Optional
-# Default: os.Stdout
-#
-# filePath = "log/traefik.log"
-
-# Format is either "json" or "common".
-#
-# Optional
-# Default: "common"
-#
-# format = "common"
-
-# Enable access logs
-# By default it will write to stdout and produce logs in the textual
-# Common Log Format (CLF), extended with additional fields.
-#
-# Optional
-#
-# [accessLog]
-
-# Sets the file path for the access log. If not specified, stdout will be used.
-# Intermediate directories are created if necessary.
-#
-# Optional
-# Default: os.Stdout
-#
-# filePath = "/path/to/log/log.txt"
-
-# Format is either "json" or "common".
-#
-# Optional
-# Default: "common"
-#
-# format = "common"
-
-################################################################
-# Web configuration backend
-################################################################
-
-# Enable web configuration backend
-# https://docs.traefik.io/configuration/api/
-#[api]
-#entryPoint = "traefik"
-#dashboard = true
-
-[file]
-directory = "/etc/traefik/rules"
-watch = true
-
-[docker]
-endpoint = "unix:///var/run/docker.sock" 
-domain = "yoursubdomain.duckdns.org"
-watch = true
-exposedbydefault = false
-
-[acme]
-email = "you@youremail.com"
-storage = "/etc/traefik/acme/acme.json"
-entryPoint = "https"
-OnHostRule = true
-[acme.dnsChallenge]
-provider = "duckdns"

configs/traefik/traefik.toml

@@ -0,0 +1,7 @@
+tls:
+  stores:
+    default:
+      defaultCertificate:
+        certFile: /certs/gatecounter.crt
+        keyFile: /certs/gatecounter.key
+

docker-compose.yaml

@@ -1,6 +1,32 @@
 version: '3.7' #specifies the version of the compose-file specification to use.  Refer to the compose-file reference for more info https://docs.docker.com/compose/compose-file/
 #this section specifies the various services that comprise the project
 services:
+  gatecounter:
+    build:
+      context: .  #build a custom python container to run the gatecounter script
+    container_name: gatecounter
+    privileged: true  #enable access to GPIO from Docker Container
+    volumes:
+      #make the scripts acessible to the container, read-only
+      - ${PWD}/gatecounter-scripts:/usr/src/app:ro
+    labels:
+      - traefik.enable=false
+    networks:
+      - gatecounter
+    restart: unless-stopped
+    depends_on: 
+      - gatecounter-db
+    command:
+      - "${GATECOUNTER_SCRIPT:?The name of a gatecounter script in the gatecounter-scripts directory is required.  Please edit .env and add a value for GATECOUNTER_SCRIPT}"
+      - "-H"
+      - "gatecounter-db"
+      - "-d"
+      - "${MYSQL_DB_NAME}"
+      - "-u"
+      - "${MYSQL_USER}"
+      - "-p"
+      - "${MYSQL_USER_PW}"
+
   #this service will be the mysql database that detections will be logged to
   gatecounter-db: #how this service will be referenced in this file
     image: yobasystems/alpine-mariadb:armhf
@@ -13,8 +39,7 @@ services:
       MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
       TZ: ${TZ:-America/New_York}
     volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/config #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
+      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
     restart: unless-stopped #keep this service running unless told explicitly to stop
     networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
       - gatecounter
@@ -25,41 +50,13 @@ services:
       interval: 30s
       timeout: 10s
       retries: 5
-    ports:
-      - "3306:3306" #connects port 3306 of the host (left) to 3306 of this container (right) making it accessible to things outside of our docker virtual network
     expose:
       - "3306"
 
-  gatecounter-db-init: #how this service will be referenced in this file
-    image: yobasystems/alpine-mariadb:armhf
-    environment: #set environment variables for this service.  These will initialize a database
-    #these environment variables will specify how the gate counter script will connect to the db to record data
-      MYSQL_DATABASE: ${MYSQL_DB_NAME:-gatecounter}
-      MYSQL_USER: ${MYSQL_USER:-gatecounter}
-      MYSQL_PASSWORD: ${MYSQL_USER_PW:?a non-admin database password is requred.  Please edit .env with this value}
-      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred.  Please edit .env with this value}
-      TZ: ${TZ:-America/New_York}
-    volumes: #specify where data to be peristed will be stored on host and where it resides within the service
-      - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service
-      - ./sql:/docker-entrypoint-initdb.d
-    networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address
-      - gatecounter
-    labels: #can be used to communicate info about this service to other services
-      - traefik.enable=false #tells traefik reverse proxy to ignore this container, do not proxy requests to it
-      - com.docker.compose.oneoff=true
-    entrypoint:
-      - /bin/sh
-      - -c
-      - "\"mysql -h $$HOSTNAME -u root -D ${MYSQL_DB_NAME} -p${MYSQL_ROOT_PW} < /docker-entrypoint-initdb.d/db-init.sql\""
-    depends_on:
-      - gatecounter-db
-    expose:
-      - "3306"
-
-
   grafana:
-    image: grafana/grafana-arm32v7-linux
+    image: grafana/grafana:6.4.3
     container_name: grafana #redundant, would have defaulted to the service name anyway
+    restart: unless-stopped
     volumes:
       - ./configs/grafana.ini:/etc/grafana/grafana.ini #maps grafana.ini in this directory into the container
       - grafana_data:/var/lib/grafana
@@ -68,11 +65,13 @@ services:
       - grafana_plugins:/var/lib/grafana/plugins
       - grafana_provisioning:/etc/grafana/provisioning
     labels:
-      - traefik.enable=true #enable forwarding of http requests to this container
-      - traefik.frontend.rule=Host:${GRAFANA_DOMAIN_NAME} #when a request is received for this domain...
-      - traefik.backend=grafana #forward the request to this container...
-      - traefik.port=3000 #on this port...
-      - traefik.protocol=http #forwarding the request in plain http on the internal virtual network
+      - traefik.enable=true #enable forwarding of requests to this container
+      - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME}`) #when a request is received for this domain, forward the request to this container...
+      - traefik.http.routers.grafana-http.entrypoints=http
+      - traefik.http.routers.grafana-http.middlewares=https-only #redirect all http requests to https
+      - traefik.http.routers.grafana-https.entrypoints=https
+      - traefik.http.routers.grafana-https.tls=true
+      - traefik.http.services.grafana.loadbalancer.server.port=3000 #on this port...
     expose:
       - "3000" #makes this port accessible to other containers on the same network, but not availble directly on the host system
     depends_on: #specifies which containers must be up and running before this one can be started
@@ -107,38 +106,30 @@ services:
     expose:
       - "3306" #this database only needs to be accessible internally to grafana
 
-  #this container makes sure a domain you register for free on https://duckdns.org always points to where this is running
-  dynamic-dns:
-    image: lsioarmhf/duckdns
-    container_name: duckdns
-    environment:
-      SUBDOMAINS: ${DUCKDNS_SUBDOMAIN:?Please provide a duckdns subdomain for your project.  Please edit .env with this value} #replace with the domain you registered.
-      TOKEN: ${DUCKDNS_TOKEN:?Please provide a duckdns token for your domain.  Please edit .env with this value} #the token duckDNS provides you for domain updates
-      TZ: ${TZ:-America/New_York}
-    labels:
-      - traefik.enable=false #tells traefik reverse proxy to ignore this container, do not proxy requests to it
-    restart: always #if this container stops for any reason, docker will restart it automatically
-    networks:
-      - default #put this service on the built-in docker bridge network
-
   reverse-proxy:
-    image: traefik:v1.7
-    container_name: traefik-gc #referenced in ./configs/traefik.toml by this name in [api] section
+    image: traefik:latest
+    container_name: traefik
+    command:
+      - "--api=false"
+      - "--entryPoints.http.address=:80"
+      - "--entryPoints.https.address=:443"
+      - "--providers.docker=true"
+      - "--accesslog=true"
+      - "--log=true"
+      - "--log.level=INFO"
+      - "--providers.file.directory=/etc/traefik/custom/"
+    labels:
+      - "com.ouroboros.enable=true" #enables watchtower for auto updates
+      - "traefik.http.middlewares.https-only.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.https-only.redirectscheme.permanent=true"      
     restart: unless-stopped #Docker will automatically restart this container unless you intentionally stopped it
     ports:
       - 80:80
       - 443:443
-#      - 8080:8080 #admin web UI port
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock  #allows traefik to monitor for changes and to read labels
-      - ./configs/traefik.toml:/etc/traefik/traefik.toml #traefik config file
-      - ./rules:/etc/traefik/rules
-      - traefik-cert-gc:/etc/traefik/acme/ #volume for storing LetsEncrypt cets
-    #The following section allows you to deifne services which must be started before this service can start
-    depends_on:
-      - dynamic-dns
-    environment:
-      DUCKDNS_TOKEN: ${DUCKDNS_TOKEN:?Please provide a duckdns token for your domain.  Please edit .env with this value} #allows traefik to obtain ssl certs for your domain(s) automatically enabling you to use https for security
+      - ./certs/:/certs/:ro
+      - ./configs/traefik:/etc/traefik/custom:ro
     networks:
       - gatecounter
 
@@ -148,7 +139,6 @@ services:
 volumes:
   gatecounter-db:
   grafana-db:
-  traefik-cert-gc:
   grafana_data:
   grafana_home:
   grafana_logs:

gatecounter-scripts/PIRdbWriteGate.py

@@ -1,15 +1,40 @@
-# Written By Johnathan Cintron and Devlyn Courtier for the HCCC Library
-
-#!/usr/bin/python
-
+#!/usr/bin/python3
+import collections
+import logging
+import subprocess
 import sys
-import MySQLdb
-from time import sleep
+
+from argparse import ArgumentParser
+from concurrent.futures import ThreadPoolExecutor, CancelledError, wait
 from datetime import datetime
+from queue import Queue
+
 import RPi.GPIO as GPIO
-from getpass import getpass
-from multiprocessing import Queue
-from concurrent.futures import ProcessPoolExecutor, CancelledError, wait
+
+from sqlalchemy import Column, DateTime, Integer, Table, create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+log = logging.getLogger("sqlalchemy")
+info = logging.StreamHandler(sys.stdout)
+info.addFilter(lambda x: x.levelno <= logging.WARNING)
+errors = logging.StreamHandler(sys.stderr)
+errors.addFilter(lambda x: x.levelno >= logging.ERROR)
+log.setLevel(logging.DEBUG)
+log.addHandler(info)
+log.addHandler(errors)
+
+
+Base = declarative_base()
+
+class PIR_Detection(Base):
+    __tablename__ = "PIRSTATS"
+
+    time = Column('datetime', DateTime, nullable=False, primary_key=True)
+    count = Column('count', Integer, nullable=False)
+
+
+Detection=collections.namedtuple("Detection", ['time','count'])
 
 class PIRgate:
     def __init__(self, hostname, username, password, database):
@@ -20,70 +45,79 @@ class PIRgate:
         self.PIR_PIN = 7
         GPIO.setup(self.PIR_PIN, GPIO.IN)
         # End GPIO setup
-		self.counts=Queue()
-		self._pool=ProcessPoolExecutor()
+        self._pool=ThreadPoolExecutor()
+        self._detection_queue=Queue()
+        if not hostname:
+            stdout,stderr = subprocess.Popen(['docker',
+                                              'inspect',
+                                              '-f',
+                                              "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
+                                              'gatecounter-db'],
+                                              stdout=subprocess.PIPE,
+                                              stderr=subprocess.STDOUT).communicate()
+            host = stdout.decode().strip()
+        else:
+            host = hostname
+        self.db_engine = create_engine(f"mysql://{username}:{password}@{host}/{database}")
+        Base.metadata.create_all(bind=self.db_engine)
+        self._session_factory = sessionmaker(bind=self.db_engine)
+        self.Session = scoped_session(self._session_factory)
 
     def start(self):
+        GPIO.add_event_detect(self.PIR_PIN, GPIO.RISING, callback=lambda c: self._detection_queue.put(Detection(datetime.now(),1)))
+        with self._pool:
             try:
-			self.event_listener = self._pool.submit(self.listen_for_events)
-			self.db_writer = self._pool.submit(self.write_to_db, hostname, username, password, database)
+                self.db_writer = self._pool.submit(self.write_to_db)
+                wait([self.db_writer])
             except KeyboardInterrupt:
                 print("\nCtrl-C pressed cleaning up GPIO")
-			self.event_listener.cancel()
-			self.db_writer.cancel()
-			GPIO.cleanup()
+                raise
             finally:
-			wait([self.event_listener,self.db_writer])
                 GPIO.cleanup()
 
-	def listen_for_events(self):
-		count = 0
+    def write_to_db(self):
         while True:
             try:
-				if GPIO.input(self.PIR_PIN):
-					count += 1
-				curr_date = datetime.now()
-				if (curr_date.minute % 10 == 0) and (curr_date.second == 0):
-					self.counts.put_nowait((curr_date,count))
-					count = 0
-			except (KeyboardInterrupt,CancelledError):
-				break
-
-
-	def write_to_db(self, hostname, username, password, database):
-		while True:
-			try:
-				time, count = self.counts.get()
-				with MySQLdb.connect(hostname,username,password,database) as db:
-					try:
-						db.cursor().execute("INSERT INTO PIRSTATS (datetime, gatecount) VALUES ('%s', '%d')" % (time.isoformat(' '), count))
-					except (KeyboardInterrupt,CancelledError):
-						db.rollback()
-						break
+                detection = self._detection_queue.get()
+                session = self.Session()
+                session.add(PIR_Detection(time=detection.datetime, count=detection.count))
+            except KeyboardInterrupt:
+                session.rollback()
+                raise
             except:
-						db.rollback()
-						self.counts.put(time,count) #put the data back in queue to try writing it again
+                session.rollback()
+                self._detection_queue.put(detection) #return detection to queue to try again
             else:
-						db.commit()
+                session.commit()
             finally:
-						db.close()
-			except (KeyboardInterrupt,CancelledError):
-				break
+                session.remove()
 
 if __name__ == "__main__":
-	while True:
+    parser = ArgumentParser(description="Begin PIR detections")
+    parser.add_argument("-H", "--hostname",default="")
+    parser.add_argument("-d", "--database",default="gatecounter")
+    parser.add_argument("-u", "--username", required=True)
+    parser.add_argument("-p", "--password", required=True)
+    if sys.argv[0].startswith("python"):
+        args = parser.parse_args(sys.argv[2:])
+    else:
+        args = parser.parse_args(sys.argv[1:])
+    if not args.hostname:
+        stdout,stderr = subprocess.Popen(['docker',
+                                          'inspect',
+                                          '-f',
+                                          "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
+                                          'gatecounter-db'],
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.STDOUT).communicate()
+        host = stdout.decode().strip()
+    else:
+        host = args.hostname
     try:
-			hostname = input("DB Hostname: ")
-			database = input("Database: ")
-			username = input("Username: ")
-			password = getpass()
-			#just check the credentials by connecting to the db and closing
-			MySQLdb.connect(hostname, username, password, database).close()
+        PIRgate(args.hostname, args.username, args.password, args.database).start()
     except KeyboardInterrupt:
         sys.exit(1)
     except:
         print("\nProblem connecting to the database.  Check your credentials and try again \n")
-			continue
-		else:
-			break
-	PIRgate(hostname, username, password, database).start()
+        sys.exit(2)
+

gatecounter-test.toml

@@ -1,10 +0,0 @@
-[backends]
-[backends.gatecounter]
-    [backends.gatecounter.servers.gatecounter-server]
-    url = "http://grafana:3000"
-[frontends]
-[frontends.gatecounter]
-entryPoints = ["http"]
-backend = "gatecounter"
-    [frontends.gatecounter.routes.test]
-    rule = "HostRegexp:grafana.{hostname:[a-z]+}{suffix:(\.local|\.home)}"

generate_certs.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+openssl req -newkey rsa:4096 \
+            -x509 \
+            -sha256 \
+            -days 36500 \
+            -nodes \
+            -out ./certs/gatecounter.crt \
+            -keyout ./certs/gatecounter.key

sql/db-init.sql

@@ -1,17 +0,0 @@
-create table if not exists PIRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists ULTRASTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);
-
-create table if not exists LDRSTATS (
-datetime DATETIME not NULL,
-gatecount INT,
-PRIMARY KEY (datetime)
-);