Initial commit - not the cleanest of projects, but it works

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.pyc
+*.swp
+settings.ini
+venv/
+.vscode

certreport.py

@@ -0,0 +1,52 @@
+from classes.config import MonsterConfig
+
+from datetime import datetime, timedelta
+
+import argparse
+import csv
+
+
+config = MonsterConfig("settings.ini")
+
+certs = []
+
+before = datetime.utcnow().replace(hour=0,minute=0,second=0,microsecond=0)
+after = before - timedelta(days=1)
+
+for vessel in config.vessels:
+    users = dict()
+
+    ocourses = vessel.getCourses()
+    courses = dict()
+
+    for ocourse in ocourses:
+        courses[ocourse["id"]] = ocourse
+
+    for ocert in vessel.getCerts(after=after.timestamp(), before=before.timestamp()):
+        if ocert["cert"]:
+            cert = dict()
+
+            user_id = ocert["userid"]
+            if not (user := users.get(user_id)):
+                user = vessel.getUsers(id=user_id)[user_id]
+                users[user_id] = user
+
+            cert["user_name"] = f'{user["firstname"]} {user["lastname"]}'
+            cert["user_email"] = user["email"]
+            cert["user_pin"] = user["custom_fields"].get("pin")
+            cert["course_id"] = ocert["cert"]["course"]
+            cert["course_shortname"] = courses[ocert["cert"]["course"]]["shortname"]
+            cert["course_fullname"] = courses[ocert["cert"]["course"]]["fullname"]
+            cert["code"] = ocert["code"]
+            cert["time_created"] = datetime.utcfromtimestamp(ocert["timecreated"]).strftime('%Y-%m-%d %H:%M:%S')
+            cert["vessel"] = vessel.name
+            certs.append(cert)
+
+certs = sorted(certs, key=lambda d: d["time_created"])
+
+keys = ["user_name", "user_email", "user_pin", "course_id", "course_shortname", "course_fullname", "code", "time_created", "vessel"]
+
+with open('test.csv', 'a') as output_file:
+    dict_writer = csv.DictWriter(output_file, restval="", fieldnames=keys, delimiter=';')
+    dict_writer.writeheader()
+    dict_writer.writerows(certs)

classes/config.py

@@ -0,0 +1,38 @@
+import configparser
+
+from pathlib import Path
+from typing import Union
+
+from classes.vessel import Vessel
+
+
+class MonsterConfig:
+    def readFile(self, path: Union[str, Path]) -> None:
+        """Read .ini file into MonsterConfig object
+
+        Args:
+            path (str, pathlib.Path): Location of the .ini file to read
+              (absolute or relative to the working directory)
+
+        Raises:
+            ValueError: Raised if the passed file is not a ContentMonster .ini
+            IOError: Raised if the file cannot be read from the provided path
+        """
+        parser = configparser.ConfigParser()
+        parser.read(str(path))
+
+        if not "MONSTER" in parser.sections():
+            raise ValueError("Config file does not contain a MONSTER section!")
+
+        for section in parser.sections():
+            # Read Vessels from the config file
+            if section.startswith("Vessel"):
+                self.vessels.append(Vessel.fromConfig(parser[section]))
+
+    def __init__(self, path: Union[str, Path]) -> None:
+        """Initialize a new (empty) MonsterConfig object
+        """
+        self.vessels = []
+
+        if path:
+            self.readFile(path)

classes/connection.py

@@ -0,0 +1,50 @@
+import paramiko as pikuniku  # :P
+
+from paramiko.client import SSHClient, WarningPolicy
+from sshtunnel import SSHTunnelForwarder
+
+from typing import Union, Optional
+from contextlib import closing
+
+import socket
+
+
+class Connection:
+    """Class representing an SSH/SFTP connection to a Vessel
+    """
+
+    def __init__(self, vessel):
+        """Initialize a new Connection to a Vessel
+
+        Args:
+            vessel (classes.vessel.Vessel): Vessel object to open connection to
+        """
+        self._vessel = vessel
+        self._client = SSHClient()
+        self._client.load_system_host_keys()
+        self._client.set_missing_host_key_policy(WarningPolicy)
+        self._client.connect(vessel.host, 22, vessel.ssh_username,
+                             vessel.ssh_password, timeout=vessel.ssh_timeout,
+                             passphrase=vessel.ssh_passphrase)
+        self._transport = self._client.get_transport()
+        self._transport.set_keepalive(10)
+        self._sftp = self._client.open_sftp()
+        self._process = None
+
+    def forward_tcp(self, remote=3306):
+        self._process = SSHTunnelForwarder(
+            (self._vessel.host, 22),
+            ssh_username=self._vessel.ssh_username,
+            ssh_private_key_password=self._vessel.ssh_passphrase,
+            remote_bind_address=("127.0.0.1", remote))
+
+        self._process.start()
+        return self._process.local_bind_port
+
+    def __del__(self):
+        """Close SSH connection when ending Connection
+        """
+        self._client.close()
+
+        if self._process:
+            self._process.close()

classes/database.py

@@ -0,0 +1,68 @@
+import MySQLdb
+import MySQLdb.cursors
+
+from typing import Union, Optional
+
+from classes.connection import Connection
+
+
+class Database:
+    """Class wrapping MySQL database connection 
+    """
+
+    def __init__(self, vessel):
+        """Initialize a new Database object
+        """
+        self.vessel = vessel
+        self._con = None
+        self._ssh = None
+
+        self._connect()
+
+    def _execute(self, query: str, parameters: Optional[tuple] = None, ctype: Optional[MySQLdb.cursors.BaseCursor] = None) -> None:
+        """Execute a query on the database
+
+        Args:
+            query (str): SQL query to execute
+            parameters (tuple, optional): Parameters to use to replace
+              placeholders in the query, if any. Defaults to None.
+        """
+        cur = self.getCursor(ctype)
+        cur.execute(query, parameters)
+        self.commit()  # Instantly commit after every (potential) write action
+        return cur.fetchall()
+
+    def _connect(self):
+        if self.vessel.ssh:
+           self._ssh = Connection(self.vessel)
+           port = self._ssh.forward_tcp(3306)
+           host = "127.0.0.1"
+        else:
+           port = 3306
+           host = self.vessel.host
+
+        self._con = MySQLdb.connect(host=host, user=self.vessel.username,
+                passwd=self.vessel.password, db=self.vessel.database, port=port)
+
+    def commit(self) -> None:
+        """Commit the current database transaction
+
+        N.B.: Commit instantly after every write action to make the database
+        "thread-safe". Connections will time out if the database is locked for
+        more than five seconds.
+        """
+        self._con.commit()
+
+    def getCursor(self, ctype: Optional[MySQLdb.cursors.BaseCursor] = None) -> MySQLdb.cursors.BaseCursor:
+        """Return a cursor to operate on the MySQL database
+
+        Returns:
+            MySQLdb.Cursor: Cursor object to execute queries on
+        """
+        return self._con.cursor(ctype)
+
+    def __del__(self):
+        """Close database connection on removal of the Database object
+        """
+        self._con.close()
+

classes/logger.py

@@ -0,0 +1,21 @@
+import logging
+import threading
+
+from datetime import datetime
+
+
+class Logger:
+    @staticmethod
+    def _format(message: str, severity: str) -> str:
+        thread = threading.current_thread().name
+        datestr = str(datetime.now())
+        return f"{datestr} - {thread} - {severity} - {message}"
+
+    def debug(self, message: str) -> None:
+        print(self.__class__()._format(message, "DEBUG"))
+
+    def info(self, message: str) -> None:
+        print(self.__class__()._format(message, "INFO"))
+
+    def error(self, message: str) -> None:
+        print(self.__class__()._format(message, "ERROR"))

classes/vessel.py

@@ -0,0 +1,180 @@
+from classes.database import Database
+
+from configparser import SectionProxy
+from typing import Optional, Union
+from datetime import datetime
+
+from MySQLdb.cursors import DictCursor
+
+from const import *
+
+
+class Vessel:
+    """Class describing a Vessel
+    """
+    @classmethod
+    def fromConfig(cls, config: SectionProxy):
+        """Create Vessel object from a Vessel section in the Config file
+
+        Args:
+            config (configparser.SectionProxy): Vessel section defining a 
+              Vessel
+
+        Raises:
+            ValueError: Raised if section does not contain Address parameter
+
+        Returns:
+            classes.vessel.Vessel: Vessel object for the vessel specified in
+              the config section
+        """
+
+        host = None
+        username = None
+        password = None
+        database = None
+        ssh = False
+        ssh_username = None
+        ssh_password = None
+        ssh_timeout = 100
+        ssh_passphrase = None
+
+        if "Username" in config.keys():
+            username = config["Username"]
+
+        if "Password" in config.keys():
+            password = config["Password"]
+
+        if "Database" in config.keys():
+            database = config["Database"]
+
+        if "SSH" in config.keys():
+            if int(config["SSH"]) == 1:
+                ssh = True
+
+        return cls(config.name.split()[1], config["Host"], username, password, database, ssh, ssh_username, ssh_password, ssh_timeout, ssh_passphrase)
+
+    def __init__(self, name: str, host: str, username: Optional[str] = None,
+            password: Optional[str] = None, database: Optional[str] = None,
+            ssh = False, ssh_username = None, ssh_password = None, ssh_timeout = None, ssh_passphrase = None) -> None:
+        """Initialize new Vessel object
+
+        Args:
+            name (str): Name of the Vessel
+        """
+        self.name = name
+        self.host = host
+        self.username = username
+        self.password = password
+        self.database = database
+        self.ssh = ssh
+        self.ssh_username = ssh_username
+        self.ssh_password = ssh_password
+        self.ssh_timeout = ssh_timeout
+        self.ssh_passphrase = ssh_passphrase
+
+        self.db = self.connect()
+
+    def connect(self):
+        return Database(self)
+
+    def getCourses(self) -> list:
+        results = self.db._execute(QUERY_COURSE, ctype=DictCursor)
+        return results
+
+    def getUserInfoFields(self) -> list:
+        results = self.db._execute(QUERY_USER_INFO_FIELD, ctype=DictCursor)
+        return results
+
+    def getUserInfoData(self, field: Optional[int] = None, user: Optional[int] = None) -> list:
+        query = QUERY_USER_INFO_DATA
+        parameters = []
+
+        if field:
+            query += " WHERE fieldid = %s"
+            parameters.append(int(field))
+
+        if user:
+            if query != QUERY_USER_INFO_DATA:
+                query += " AND "
+            else:
+                query += " WHERE "
+
+            query += "userid = %s"
+            parameters.append(int(user))
+
+        results = self.db._execute(query, tuple(parameters), ctype=DictCursor)
+
+        return results
+
+    def getUsers(self, username: Optional[str] = None, id: Optional[int] = None) -> dict:
+        query = QUERY_USER
+        parameters = tuple()
+
+        if username:
+            query += f" WHERE username = %s"
+            parameters = (username,)
+
+        elif id:
+            query += f" WHERE id = %s"
+            parameters = (int(id),)
+
+        results = self.db._execute(query, parameters, ctype=DictCursor)
+
+        users = dict()
+
+        for result in results:
+            user = result
+            result["custom_fields"] = dict()
+            users[user["id"]] = user
+
+        ofields = self.getUserInfoFields()
+
+        for ofield in ofields:
+            odata = self.getUserInfoData(ofield["id"], id)
+
+            for value in odata:
+                users[value["userid"]]["custom_fields"][ofield["shortname"]] = value["data"]
+
+        return users
+
+    def getHTMLCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())):
+        results = self.db._execute(f"{QUERY_HTML_CERT_ISSUES} {QUERY_WHERE_TIMESTAMPS % {'column': 'timecreated', 'after': after, 'before': before}}", ctype=DictCursor)
+        ocerts = self.db._execute(QUERY_HTML_CERT, ctype=DictCursor)
+
+        certs = dict()
+
+        for ocert in ocerts:
+            certs[ocert["id"]] = ocert
+
+        for result in results:
+            try:
+                result["cert"] = certs[result["htmlcertid"]]
+            except KeyError:
+                result["cert"] = None
+
+            result["certtype"] = "htmlcert"
+
+        return results
+
+    def getCustomCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())):
+        results = self.db._execute(f"{QUERY_CUSTOM_CERT_ISSUES} {QUERY_WHERE_TIMESTAMPS % {'column': 'timecreated', 'after': after, 'before': before}}", ctype=DictCursor)
+        ocerts = self.db._execute(QUERY_CUSTOM_CERT, ctype=DictCursor)
+
+        certs = dict()
+
+        for ocert in ocerts:
+            certs[ocert["id"]] = ocert
+
+        for result in results:
+            try:
+                result["cert"] = certs[result["customcertid"]]
+            except KeyError:
+                result["cert"] = None
+
+            result["certtype"] = "customcert"
+
+        return results
+
+    def getCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())):
+        return sorted(self.getHTMLCerts(after, before) + self.getCustomCerts(after, before), key=lambda d: d["timecreated"])
+

const.py

@@ -0,0 +1,13 @@
+QUERY_HTML_CERT_ISSUES = "SELECT * FROM mdl_htmlcert_issues"
+QUERY_HTML_CERT = "SELECT * FROM mdl_htmlcert"
+
+QUERY_CUSTOM_CERT_ISSUES = "SELECT * FROM mdl_customcert_issues"
+QUERY_CUSTOM_CERT = "SELECT * FROM mdl_customcert"
+
+QUERY_USER = "SELECT * FROM mdl_user"
+QUERY_USER_INFO_FIELD = "SELECT * FROM mdl_user_info_field"
+QUERY_USER_INFO_DATA = "SELECT * FROM mdl_user_info_data"
+
+QUERY_COURSE = "SELECT * FROM mdl_course"
+
+QUERY_WHERE_TIMESTAMPS = "WHERE %(column)s >= %(after)i AND %(column)s < %(before)i"

settings.dist.ini

@@ -0,0 +1,16 @@
+[MONSTER]
+
+[Vessel vessel1]
+Host = 10.12.13.14
+Username = monster
+Password = jdsfskafjaf
+Database = moodle
+ssh = 0
+
+[Vessel vessel2]
+Host = 192.168.95192.1
+Username = monster
+Password = jfaskldfjklasfh
+Database = moodle
+ssh = 1
+