From fa48b3357e126722cd0d7d47933ab866ae709913 Mon Sep 17 00:00:00 2001 From: Brennen Raimer <> Date: Sun, 3 Nov 2019 13:49:46 -0500 Subject: [PATCH] PIRgate now logs immediately for better visualization in Grafana Gatecounter service added Gatecounter db now initialized by the gatecounter itself courtesy of SQLAlchemy --- Dockerfile | 4 + docker-compose.yaml | 65 ++++----- gatecounter-scripts/PIRdbWriteGate.py | 192 +++++++++++++++----------- sql/db-init.sql | 17 --- 4 files changed, 147 insertions(+), 131 deletions(-) create mode 100644 Dockerfile delete mode 100644 sql/db-init.sql diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c4047c8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3 +WORKDIR /usr/src/app +RUN pip install --no-cache-dir sqlalchemy RPi.GPIO mysqlclient Adafruit-MCP3008 Adafruit-GPIO +ENTRYPOINT [ "python3" ] diff --git a/docker-compose.yaml b/docker-compose.yaml index 90694c4..2c71b54 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,32 @@ version: '3.7' #specifies the version of the compose-file specification to use. Refer to the compose-file reference for more info https://docs.docker.com/compose/compose-file/ #this section specifies the various services that comprise the project services: + gatecounter: + build: + context: . #build a custom python container to run the gatecounter script + container_name: gatecounter + privileged: true #enable access to GPIO from Docker Container + volumes: + #make the scripts acessible to the container, read-only + - ${PWD}/gatecounter-scripts:/usr/src/app:ro + labels: + - traefik.enable=false + networks: + - gatecounter + restart: unless-stopped + depends_on: + - gatecounter-db + command: + - "${GATECOUNTER_SCRIPT:?The name of a gatecounter script in the gatecounter-scripts directory is required. Please edit .env and add a value for GATECOUNTER_SCRIPT}" + - "-H" + - "gatecounter-db" + - "-d" + - "${MYSQL_DB_NAME}" + - "-u" + - "${MYSQL_USER}" + - "-p" + - "${MYSQL_USER_PW}" + #this service will be the mysql database that detections will be logged to gatecounter-db: #how this service will be referenced in this file image: yobasystems/alpine-mariadb:armhf @@ -13,8 +39,7 @@ services: MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred. Please edit .env with this value} TZ: ${TZ:-America/New_York} volumes: #specify where data to be peristed will be stored on host and where it resides within the service - - gatecounter-db:/config #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service - - ./sql:/docker-entrypoint-initdb.d + - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service restart: unless-stopped #keep this service running unless told explicitly to stop networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address - gatecounter @@ -28,34 +53,10 @@ services: expose: - "3306" - gatecounter-db-init: #how this service will be referenced in this file - image: yobasystems/alpine-mariadb:armhf - environment: #set environment variables for this service. These will initialize a database - #these environment variables will specify how the gate counter script will connect to the db to record data - MYSQL_DATABASE: ${MYSQL_DB_NAME:-gatecounter} - MYSQL_USER: ${MYSQL_USER:-gatecounter} - MYSQL_PASSWORD: ${MYSQL_USER_PW:?a non-admin database password is requred. Please edit .env with this value} - MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PW:?an admin database password is requred. Please edit .env with this value} - TZ: ${TZ:-America/New_York} - volumes: #specify where data to be peristed will be stored on host and where it resides within the service - - gatecounter-db:/var/lib/mysql #left of the : is the name of a docker volume to store data in, right of it is where it is located in the service - - ./sql:/docker-entrypoint-initdb.d - networks: #virtual network for services to connect to each other through. necessary to resolve their container_name to their virtual ip address - - gatecounter - labels: #can be used to communicate info about this service to other services - - traefik.enable=false #tells traefik reverse proxy to ignore this container, do not proxy requests to it - - com.docker.compose.oneoff=true - entrypoint: - - /bin/sh - - -c - - "\"mysql -h gatecounter-db -u root -D ${MYSQL_DB_NAME} -p${MYSQL_ROOT_PW} < /docker-entrypoint-initdb.d/db-init.sql\"" - depends_on: - - gatecounter-db - - grafana: - image: grafana/grafana-arm32v7-linux + image: grafana/grafana:6.4.3 container_name: grafana #redundant, would have defaulted to the service name anyway + restart: unless-stopped volumes: - ./configs/grafana.ini:/etc/grafana/grafana.ini #maps grafana.ini in this directory into the container - grafana_data:/var/lib/grafana @@ -65,7 +66,7 @@ services: - grafana_provisioning:/etc/grafana/provisioning labels: - traefik.enable=true #enable forwarding of requests to this container - - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME`) #when a request is received for this domain, forward the request to this container... + - traefik.http.routers.grafana-http.rule=Host(`${GRAFANA_DOMAIN_NAME}`) #when a request is received for this domain, forward the request to this container... - traefik.http.routers.grafana-http.entrypoints=http - traefik.http.routers.grafana-http.middlewares=https-only #redirect all http requests to https - traefik.http.routers.grafana-https.entrypoints=https @@ -129,11 +130,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock #allows traefik to monitor for changes and to read labels - ./certs/:/certs/:ro - ./configs/traefik:/etc/traefik/custom:ro - #The following section allows you to deifne services which must be started before this service can start - depends_on: - - dynamic-dns - environment: - DUCKDNS_TOKEN: ${DUCKDNS_TOKEN:?Please provide a duckdns token for your domain. Please edit .env with this value} #allows traefik to obtain ssl certs for your domain(s) automatically enabling you to use https for security networks: - gatecounter @@ -143,7 +139,6 @@ services: volumes: gatecounter-db: grafana-db: - traefik-cert-gc: grafana_data: grafana_home: grafana_logs: diff --git a/gatecounter-scripts/PIRdbWriteGate.py b/gatecounter-scripts/PIRdbWriteGate.py index 9780f94..c23085c 100755 --- a/gatecounter-scripts/PIRdbWriteGate.py +++ b/gatecounter-scripts/PIRdbWriteGate.py @@ -1,89 +1,123 @@ -# Written By Johnathan Cintron and Devlyn Courtier for the HCCC Library - -#!/usr/bin/python - +#!/usr/bin/python3 +import collections +import logging +import subprocess import sys -import MySQLdb -from time import sleep + +from argparse import ArgumentParser +from concurrent.futures import ThreadPoolExecutor, CancelledError, wait from datetime import datetime +from queue import Queue + import RPi.GPIO as GPIO -from getpass import getpass -from multiprocessing import Queue -from concurrent.futures import ProcessPoolExecutor, CancelledError, wait + +from sqlalchemy import Column, DateTime, Integer, Table, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import scoped_session, sessionmaker + +log = logging.getLogger("sqlalchemy") +info = logging.StreamHandler(sys.stdout) +info.addFilter(lambda x: x.levelno <= logging.WARNING) +errors = logging.StreamHandler(sys.stderr) +errors.addFilter(lambda x: x.levelno >= logging.ERROR) +log.setLevel(logging.DEBUG) +log.addHandler(info) +log.addHandler(errors) + + +Base = declarative_base() + +class PIR_Detection(Base): + __tablename__ = "PIRSTATS" + + time = Column('datetime', DateTime, nullable=False, primary_key=True) + count = Column('count', Integer, nullable=False) + + +Detection=collections.namedtuple("Detection", ['time','count']) class PIRgate: - def __init__(self, hostname, username, password, database): - # Set RPi GPIO Mode - GPIO.setmode(GPIO.BCM) + def __init__(self, hostname, username, password, database): + # Set RPi GPIO Mode + GPIO.setmode(GPIO.BCM) - # Setup GPIO in and out pins - self.PIR_PIN = 7 - GPIO.setup(self.PIR_PIN, GPIO.IN) - # End GPIO setup - self.counts=Queue() - self._pool=ProcessPoolExecutor() + # Setup GPIO in and out pins + self.PIR_PIN = 7 + GPIO.setup(self.PIR_PIN, GPIO.IN) + # End GPIO setup + self._pool=ThreadPoolExecutor() + self._detection_queue=Queue() + if not hostname: + stdout,stderr = subprocess.Popen(['docker', + 'inspect', + '-f', + "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", + 'gatecounter-db'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).communicate() + host = stdout.decode().strip() + else: + host = hostname + self.db_engine = create_engine(f"mysql://{username}:{password}@{host}/{database}") + Base.metadata.create_all(bind=self.db_engine) + self._session_factory = sessionmaker(bind=self.db_engine) + self.Session = scoped_session(self._session_factory) - def start(self): - try: - self.event_listener = self._pool.submit(self.listen_for_events) - self.db_writer = self._pool.submit(self.write_to_db, hostname, username, password, database) - except KeyboardInterrupt: - print("\nCtrl-C pressed cleaning up GPIO") - self.event_listener.cancel() - self.db_writer.cancel() - GPIO.cleanup() - finally: - wait([self.event_listener,self.db_writer]) - GPIO.cleanup() + def start(self): + GPIO.add_event_detect(self.PIR_PIN, GPIO.RISING, callback=lambda c: self._detection_queue.put(Detection(datetime.now(),1))) + with self._pool: + try: + self.db_writer = self._pool.submit(self.write_to_db) + wait([self.db_writer]) + except KeyboardInterrupt: + print("\nCtrl-C pressed cleaning up GPIO") + raise + finally: + GPIO.cleanup() - def listen_for_events(self): - count = 0 - while True: - try: - if GPIO.input(self.PIR_PIN): - count += 1 - curr_date = datetime.now() - if (curr_date.minute % 10 == 0) and (curr_date.second == 0): - self.counts.put_nowait((curr_date,count)) - count = 0 - except (KeyboardInterrupt,CancelledError): - break - - - def write_to_db(self, hostname, username, password, database): - while True: - try: - time, count = self.counts.get() - with MySQLdb.connect(hostname,username,password,database) as db: - try: - db.cursor().execute("INSERT INTO PIRSTATS (datetime, gatecount) VALUES ('%s', '%d')" % (time.isoformat(' '), count)) - except (KeyboardInterrupt,CancelledError): - db.rollback() - break - except: - db.rollback() - self.counts.put(time,count) #put the data back in queue to try writing it again - else: - db.commit() - finally: - db.close() - except (KeyboardInterrupt,CancelledError): - break + def write_to_db(self): + while True: + try: + detection = self._detection_queue.get() + session = self.Session() + session.add(PIR_Detection(time=detection.datetime, count=detection.count)) + except KeyboardInterrupt: + session.rollback() + raise + except: + session.rollback() + self._detection_queue.put(detection) #return detection to queue to try again + else: + session.commit() + finally: + session.remove() if __name__ == "__main__": - while True: - try: - hostname = input("DB Hostname: ") - database = input("Database: ") - username = input("Username: ") - password = getpass() - #just check the credentials by connecting to the db and closing - MySQLdb.connect(hostname, username, password, database).close() - except KeyboardInterrupt: - sys.exit(1) - except: - print("\nProblem connecting to the database. Check your credentials and try again \n") - continue - else: - break - PIRgate(hostname, username, password, database).start() \ No newline at end of file + parser = ArgumentParser(description="Begin PIR detections") + parser.add_argument("-H", "--hostname",default="") + parser.add_argument("-d", "--database",default="gatecounter") + parser.add_argument("-u", "--username", required=True) + parser.add_argument("-p", "--password", required=True) + if sys.argv[0].startswith("python"): + args = parser.parse_args(sys.argv[2:]) + else: + args = parser.parse_args(sys.argv[1:]) + if not args.hostname: + stdout,stderr = subprocess.Popen(['docker', + 'inspect', + '-f', + "'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'", + 'gatecounter-db'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).communicate() + host = stdout.decode().strip() + else: + host = args.hostname + try: + PIRgate(args.hostname, args.username, args.password, args.database).start() + except KeyboardInterrupt: + sys.exit(1) + except: + print("\nProblem connecting to the database. Check your credentials and try again \n") + sys.exit(2) + diff --git a/sql/db-init.sql b/sql/db-init.sql deleted file mode 100644 index e748c72..0000000 --- a/sql/db-init.sql +++ /dev/null @@ -1,17 +0,0 @@ -create table if not exists PIRSTATS ( -datetime DATETIME not NULL, -gatecount INT, -PRIMARY KEY (datetime) -); - -create table if not exists ULTRASTATS ( -datetime DATETIME not NULL, -gatecount INT, -PRIMARY KEY (datetime) -); - -create table if not exists LDRSTATS ( -datetime DATETIME not NULL, -gatecount INT, -PRIMARY KEY (datetime) -); \ No newline at end of file