From 313c24f72776222a208dbccefe80573a153c3294 Mon Sep 17 00:00:00 2001 From: Klaus-Uwe Mitterer Date: Thu, 25 Nov 2021 13:42:00 +0100 Subject: [PATCH] More documentation strings --- classes/config.py | 35 ++++++++---- classes/database.py | 121 +++++++++++++++++++++++++++++++++++++----- classes/directory.py | 48 ++++++++++++++--- classes/doghandler.py | 56 ++++++++++++++++--- 4 files changed, 224 insertions(+), 36 deletions(-) diff --git a/classes/config.py b/classes/config.py index 13f3b74..890b1de 100644 --- a/classes/config.py +++ b/classes/config.py @@ -1,27 +1,42 @@ import configparser +from pathlib import Path +from typing import Union + from classes.vessel import Vessel from classes.directory import Directory + class MonsterConfig: - @classmethod - def fromFile(cls, path): + def readFile(self, path: Union[str, Path]) -> None: + """Read .ini file into MonsterConfig object + + Args: + path (str, pathlib.Path): Location of the .ini file to read + (absolute or relative to the working directory) + + Raises: + ValueError: Raised if the passed file is not a ContentMonster .ini + IOError: Raised if the file cannot be read from the provided path + """ parser = configparser.ConfigParser() - parser.read(path) + parser.read(str(path)) if not "MONSTER" in parser.sections(): raise ValueError("Config file does not contain a MONSTER section!") - config = cls() - for section in parser.sections(): + # Read Directories from the config file if section.startswith("Directory"): - config.directories.append(Directory.fromConfig(parser[section])) + self.directories.append( + Directory.fromConfig(parser[section])) + + # Read Vessels from the config file elif section.startswith("Vessel"): - config.vessels.append(Vessel.fromConfig(parser[section])) + self.vessels.append(Vessel.fromConfig(parser[section])) - return config - - def __init__(self): + def __init__(self) -> None: + """Initialize a new (empty) MonsterConfig object + """ self.directories = [] self.vessels = [] diff --git a/classes/database.py b/classes/database.py index f3423eb..b442706 100644 --- a/classes/database.py +++ b/classes/database.py @@ -2,26 +2,57 @@ import sqlite3 import pathlib import uuid +from typing import Union, Optional + class Database: - def __init__(self, filename=None): + """Class wrapping sqlite3 database connection + """ + + def __init__(self, filename: Optional[Union[str, pathlib.Path]] = None): + """Initialize a new Database object + + Args: + filename (str, pathlib.Path, optional): Filename of the sqlite3 + database to use. If None, use "database.sqlite3" in project base + directory. Defaults to None. + """ filename = filename or pathlib.Path( __file__).parent.parent.absolute() / "database.sqlite3" self._con = sqlite3.connect(filename) self.migrate() - def _execute(self, query, parameters=None): + def _execute(self, query: str, parameters: Optional[tuple] = None) -> None: + """Execute a query on the database + + Args: + query (str): SQL query to execute + parameters (tuple, optional): Parameters to use to replace + placeholders in the query, if any. Defaults to None. + """ cur = self.getCursor() cur.execute(query, parameters) - self.commit() + self.commit() # Instantly commit after every write action - def commit(self): - return self._con.commit() + def commit(self) -> None: + """Commit the current database transaction + """ + self._con.commit() - def getCursor(self): + def getCursor(self) -> sqlite3.Cursor: + """Return a cursor to operate on the sqlite3 database + + Returns: + sqlite3.Cursor: Cursor object to execute queries on + """ return self._con.cursor() - def getVersion(self): + def getVersion(self) -> int: + """Return the current version of the ContentMonster database + + Returns: + int: Version of the last applied database migration + """ cur = self.getCursor() try: cur.execute( @@ -31,7 +62,15 @@ class Database: except (sqlite3.OperationalError, AssertionError): return 0 - def getFileUUID(self, fileobj): + def getFileUUID(self, fileobj) -> str: + """Retrieve unique identifier for File object + + Args: + fileobj (classes.file.File): File object to retrieve UUID for + + Returns: + str: UUID for passed File object + """ hash = fileobj.getHash() cur = self.getCursor() @@ -39,42 +78,96 @@ class Database: (fileobj.directory.name, fileobj.name)) fileuuid = None + + # If file with same name and directory exists for result in cur.fetchall(): + + # If it has the same hash, it is the same file -> return its UUID if result[1] == hash: fileuuid = result[0] + + # If not, it is a file that can no longer exist -> delete it else: self.removeFileByUUID(result[0]) + # Return found UUID or generate a new one return fileuuid or self.addFile(fileobj, hash) - def addFile(self, fileobj, hash=None): + def addFile(self, fileobj, hash: Optional[str] = None) -> str: + """Adds a new File object to the database + + Args: + fileobj (classes.file.File): File object to add to database + hash (str, optional): Checksum of the file, if already known. + Defaults to None. + + Returns: + str: UUID of the new File record + """ hash = hash or fileobj.getHash() fileuuid = str(uuid.uuid4()) self._execute("INSERT INTO contentmonster_file(uuid, directory, name, checksum) VALUES (?, ?, ?, ?)", (fileuuid, fileobj.directory.name, fileobj.name, hash)) return fileuuid - def getFileByUUID(self, fileuuid): + def getFileByUUID(self, fileuuid: str) -> Optional[tuple[str, str, str]]: + """Get additional information on a File by its UUID + + Args: + fileuuid (str): The UUID of the File to retrieve from the database + + Returns: + tuple: A tuple consisting of (directory, name, checksum), where + "directory" is the name of the Directory object the File is + located in, "name" is the filename (basename) of the File and + checksum is the SHA256 hash of the file at the time of insertion + into the database. None is returned if no such record is found. + """ cur = self.getCursor() cur.execute( "SELECT directory, name, checksum FROM contentmonster_file WHERE uuid = ?", (fileuuid,)) if (result := cur.fetchone()): return result - def removeFileByUUID(self, fileuuid): + def removeFileByUUID(self, fileuuid: str) -> None: + """Remove a File from the database based on UUID + + Args: + fileuuid (str): The UUID of the File to remove from the database + """ self._execute( "DELETE FROM contentmonster_file WHERE uuid = ?", (fileuuid,)) def logCompletion(self, file, vessel): + """Log the completion of a File upload + + Args: + file (classes.file.File): The File object that has been uploaded + vessel (classes.vessel.Vessel): The Vessel the File has been + uploaded to + """ self._execute( "INSERT INTO contentmonster_file_log(file, vessel) VALUES(?, ?)", (file.uuid, vessel.name)) - def getCompletionForVessel(self, vessel): + def getCompletionForVessel(self, vessel) -> list[Optional[str]]: + """Get completed uploads for a vessel + + Args: + vessel (classes.vessel.Vessel): The Vessel object to retrieve + uploaded files for + + Returns: + list: List of UUIDs of Files that have been successfully uploaded + """ cur = self.getCursor() cur.execute( "SELECT file FROM contentmonster_file_log WHERE vessel = ?", (vessel.name,)) + + return [f[0] for f in cur.fetchall()] - def migrate(self): + def migrate(self) -> None: + """Apply database migrations + """ cur = self.getCursor() if self.getVersion() == 0: @@ -93,4 +186,6 @@ class Database: self.commit() def __del__(self): + """Close database connection on removal of the Database object + """ self._con.close() diff --git a/classes/directory.py b/classes/directory.py index b0fe374..9633f8b 100644 --- a/classes/directory.py +++ b/classes/directory.py @@ -3,22 +3,58 @@ from classes.file import File import os import pathlib +from configparser import SectionProxy +from typing import Union + + class Directory: + """Class representing a Directory on the local filesystem + """ @classmethod - def fromConfig(cls, config): + def fromConfig(cls, config: SectionProxy) -> Directory: # pylint: disable=undefined-variable + """Create Directory object from a Directory section in the Config file + + Args: + config (configparser.SectionProxy): Configuration section defining + a Directory + + Raises: + ValueError: Raised if section does not contain Location parameter + + Returns: + classes.directory.Directory: Directory object for the location + specified in the config section + """ if "Location" in config.keys(): return cls(config.name.split()[1], config["Location"]) else: - raise ValueError("Definition for Directory " + config.name.split()[1] + " does not contain Location!") + raise ValueError("Definition for Directory " + + config.name.split()[1] + " does not contain Location!") - def __init__(self, name, location): + def __init__(self, name: str, location: Union[str, pathlib.Path]): + """Initialize a new Directory object + + Args: + name (str): Name of the Directory object + location (str, pathlib.Path): Filesystem location of the Directory + + Raises: + ValueError: Raised if passed location does not exist or is not a + directory + """ self.name = name if os.path.isdir(location): self.location = pathlib.Path(location) else: - raise ValueError(f"Location {location} for Directory {name} does not exist or is not a directory.") + raise ValueError( + f"Location {location} for Directory {name} does not exist or is not a directory.") - def getFiles(self): + def getFiles(self) -> list[File]: + """Get all Files in Directory + + Returns: + list: List of names (str) of files within the Directory + """ files = [f for f in os.listdir(self.location) if os.path.isfile] - return [File(f, self) for f in files] \ No newline at end of file + return [File(f, self) for f in files] diff --git a/classes/doghandler.py b/classes/doghandler.py index c71159e..a585c18 100644 --- a/classes/doghandler.py +++ b/classes/doghandler.py @@ -1,28 +1,70 @@ -from watchdog.events import FileSystemEventHandler +from watchdog.events import (FileSystemEventHandler, FileSystemEvent, + FileCreatedEvent, FileDeletedEvent, + FileModifiedEvent, FileMovedEvent) + +from multiprocessing import Queue import os.path class DogHandler(FileSystemEventHandler): - def __init__(self, directory, queue, *args, **kwargs): + """Class implementing a watchdog event handler + """ + + def __init__(self, directory, queue: Queue, *args, **kwargs) -> None: + """Initialize a new DogHandler object + + Args: + directory (classes.directory.Directory): Directory to watch + queue (multiprocessing.Queue): Queue to put detected events on + """ print("Initialized") super().__init__(*args, **kwargs) self._directory = directory self._queue = queue - def dispatch(self, event): + def dispatch(self, event: FileSystemEvent): + """Dispatch events to the appropriate event handlers + + Args: + event (watchdog.events.FileSystemEvent): Event to handle + """ if not event.is_directory: super().dispatch(event) - def on_created(self, event): + def on_created(self, event: FileCreatedEvent): + """Put file creation events on the queue + + Args: + event (watchdog.events.FileCreatedEvent): Event describing the + created file + """ self._queue.put((self._directory, os.path.basename(event.src_path))) - def on_modified(self, event): + def on_modified(self, event: FileModifiedEvent): + """Put file modification events on the queue + + Args: + event (watchdog.events.FileModifiedEvent): Event describing the + modified file + """ self._queue.put((self._directory, os.path.basename(event.src_path))) - def on_moved(self, event): + def on_moved(self, event: FileMovedEvent): + """Put file move events on the queue + + Args: + event (watchdog.events.FileMovedEvent): Event describing the moved + file (source and destination) + """ self._queue.put((self._directory, os.path.basename(event.src_path))) self._queue.put((self._directory, os.path.basename(event.dest_path))) - def on_deleted(self, event): + def on_deleted(self, event: FileDeletedEvent): + """Put file deletion events on the queue + + Args: + event (watchdog.events.FileDeletedEvent): Event describing the + deleted file + """ self._queue.put((self._directory, os.path.basename(event.src_path)))