More documentation strings
This commit is contained in:
parent
48bc92653d
commit
313c24f727
4 changed files with 224 additions and 36 deletions
|
@ -1,27 +1,42 @@
|
||||||
import configparser
|
import configparser
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
from classes.vessel import Vessel
|
from classes.vessel import Vessel
|
||||||
from classes.directory import Directory
|
from classes.directory import Directory
|
||||||
|
|
||||||
|
|
||||||
class MonsterConfig:
|
class MonsterConfig:
|
||||||
@classmethod
|
def readFile(self, path: Union[str, Path]) -> None:
|
||||||
def fromFile(cls, path):
|
"""Read .ini file into MonsterConfig object
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (str, pathlib.Path): Location of the .ini file to read
|
||||||
|
(absolute or relative to the working directory)
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: Raised if the passed file is not a ContentMonster .ini
|
||||||
|
IOError: Raised if the file cannot be read from the provided path
|
||||||
|
"""
|
||||||
parser = configparser.ConfigParser()
|
parser = configparser.ConfigParser()
|
||||||
parser.read(path)
|
parser.read(str(path))
|
||||||
|
|
||||||
if not "MONSTER" in parser.sections():
|
if not "MONSTER" in parser.sections():
|
||||||
raise ValueError("Config file does not contain a MONSTER section!")
|
raise ValueError("Config file does not contain a MONSTER section!")
|
||||||
|
|
||||||
config = cls()
|
|
||||||
|
|
||||||
for section in parser.sections():
|
for section in parser.sections():
|
||||||
|
# Read Directories from the config file
|
||||||
if section.startswith("Directory"):
|
if section.startswith("Directory"):
|
||||||
config.directories.append(Directory.fromConfig(parser[section]))
|
self.directories.append(
|
||||||
|
Directory.fromConfig(parser[section]))
|
||||||
|
|
||||||
|
# Read Vessels from the config file
|
||||||
elif section.startswith("Vessel"):
|
elif section.startswith("Vessel"):
|
||||||
config.vessels.append(Vessel.fromConfig(parser[section]))
|
self.vessels.append(Vessel.fromConfig(parser[section]))
|
||||||
|
|
||||||
return config
|
def __init__(self) -> None:
|
||||||
|
"""Initialize a new (empty) MonsterConfig object
|
||||||
def __init__(self):
|
"""
|
||||||
self.directories = []
|
self.directories = []
|
||||||
self.vessels = []
|
self.vessels = []
|
||||||
|
|
|
@ -2,26 +2,57 @@ import sqlite3
|
||||||
import pathlib
|
import pathlib
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
from typing import Union, Optional
|
||||||
|
|
||||||
|
|
||||||
class Database:
|
class Database:
|
||||||
def __init__(self, filename=None):
|
"""Class wrapping sqlite3 database connection
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, filename: Optional[Union[str, pathlib.Path]] = None):
|
||||||
|
"""Initialize a new Database object
|
||||||
|
|
||||||
|
Args:
|
||||||
|
filename (str, pathlib.Path, optional): Filename of the sqlite3
|
||||||
|
database to use. If None, use "database.sqlite3" in project base
|
||||||
|
directory. Defaults to None.
|
||||||
|
"""
|
||||||
filename = filename or pathlib.Path(
|
filename = filename or pathlib.Path(
|
||||||
__file__).parent.parent.absolute() / "database.sqlite3"
|
__file__).parent.parent.absolute() / "database.sqlite3"
|
||||||
self._con = sqlite3.connect(filename)
|
self._con = sqlite3.connect(filename)
|
||||||
self.migrate()
|
self.migrate()
|
||||||
|
|
||||||
def _execute(self, query, parameters=None):
|
def _execute(self, query: str, parameters: Optional[tuple] = None) -> None:
|
||||||
|
"""Execute a query on the database
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query (str): SQL query to execute
|
||||||
|
parameters (tuple, optional): Parameters to use to replace
|
||||||
|
placeholders in the query, if any. Defaults to None.
|
||||||
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute(query, parameters)
|
cur.execute(query, parameters)
|
||||||
self.commit()
|
self.commit() # Instantly commit after every write action
|
||||||
|
|
||||||
def commit(self):
|
def commit(self) -> None:
|
||||||
return self._con.commit()
|
"""Commit the current database transaction
|
||||||
|
"""
|
||||||
|
self._con.commit()
|
||||||
|
|
||||||
def getCursor(self):
|
def getCursor(self) -> sqlite3.Cursor:
|
||||||
|
"""Return a cursor to operate on the sqlite3 database
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
sqlite3.Cursor: Cursor object to execute queries on
|
||||||
|
"""
|
||||||
return self._con.cursor()
|
return self._con.cursor()
|
||||||
|
|
||||||
def getVersion(self):
|
def getVersion(self) -> int:
|
||||||
|
"""Return the current version of the ContentMonster database
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: Version of the last applied database migration
|
||||||
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
try:
|
try:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
|
@ -31,7 +62,15 @@ class Database:
|
||||||
except (sqlite3.OperationalError, AssertionError):
|
except (sqlite3.OperationalError, AssertionError):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def getFileUUID(self, fileobj):
|
def getFileUUID(self, fileobj) -> str:
|
||||||
|
"""Retrieve unique identifier for File object
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fileobj (classes.file.File): File object to retrieve UUID for
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: UUID for passed File object
|
||||||
|
"""
|
||||||
hash = fileobj.getHash()
|
hash = fileobj.getHash()
|
||||||
|
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
|
@ -39,42 +78,96 @@ class Database:
|
||||||
(fileobj.directory.name, fileobj.name))
|
(fileobj.directory.name, fileobj.name))
|
||||||
|
|
||||||
fileuuid = None
|
fileuuid = None
|
||||||
|
|
||||||
|
# If file with same name and directory exists
|
||||||
for result in cur.fetchall():
|
for result in cur.fetchall():
|
||||||
|
|
||||||
|
# If it has the same hash, it is the same file -> return its UUID
|
||||||
if result[1] == hash:
|
if result[1] == hash:
|
||||||
fileuuid = result[0]
|
fileuuid = result[0]
|
||||||
|
|
||||||
|
# If not, it is a file that can no longer exist -> delete it
|
||||||
else:
|
else:
|
||||||
self.removeFileByUUID(result[0])
|
self.removeFileByUUID(result[0])
|
||||||
|
|
||||||
|
# Return found UUID or generate a new one
|
||||||
return fileuuid or self.addFile(fileobj, hash)
|
return fileuuid or self.addFile(fileobj, hash)
|
||||||
|
|
||||||
def addFile(self, fileobj, hash=None):
|
def addFile(self, fileobj, hash: Optional[str] = None) -> str:
|
||||||
|
"""Adds a new File object to the database
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fileobj (classes.file.File): File object to add to database
|
||||||
|
hash (str, optional): Checksum of the file, if already known.
|
||||||
|
Defaults to None.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: UUID of the new File record
|
||||||
|
"""
|
||||||
hash = hash or fileobj.getHash()
|
hash = hash or fileobj.getHash()
|
||||||
fileuuid = str(uuid.uuid4())
|
fileuuid = str(uuid.uuid4())
|
||||||
self._execute("INSERT INTO contentmonster_file(uuid, directory, name, checksum) VALUES (?, ?, ?, ?)",
|
self._execute("INSERT INTO contentmonster_file(uuid, directory, name, checksum) VALUES (?, ?, ?, ?)",
|
||||||
(fileuuid, fileobj.directory.name, fileobj.name, hash))
|
(fileuuid, fileobj.directory.name, fileobj.name, hash))
|
||||||
return fileuuid
|
return fileuuid
|
||||||
|
|
||||||
def getFileByUUID(self, fileuuid):
|
def getFileByUUID(self, fileuuid: str) -> Optional[tuple[str, str, str]]:
|
||||||
|
"""Get additional information on a File by its UUID
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fileuuid (str): The UUID of the File to retrieve from the database
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple: A tuple consisting of (directory, name, checksum), where
|
||||||
|
"directory" is the name of the Directory object the File is
|
||||||
|
located in, "name" is the filename (basename) of the File and
|
||||||
|
checksum is the SHA256 hash of the file at the time of insertion
|
||||||
|
into the database. None is returned if no such record is found.
|
||||||
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT directory, name, checksum FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
"SELECT directory, name, checksum FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
||||||
if (result := cur.fetchone()):
|
if (result := cur.fetchone()):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def removeFileByUUID(self, fileuuid):
|
def removeFileByUUID(self, fileuuid: str) -> None:
|
||||||
|
"""Remove a File from the database based on UUID
|
||||||
|
|
||||||
|
Args:
|
||||||
|
fileuuid (str): The UUID of the File to remove from the database
|
||||||
|
"""
|
||||||
self._execute(
|
self._execute(
|
||||||
"DELETE FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
"DELETE FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
||||||
|
|
||||||
def logCompletion(self, file, vessel):
|
def logCompletion(self, file, vessel):
|
||||||
|
"""Log the completion of a File upload
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file (classes.file.File): The File object that has been uploaded
|
||||||
|
vessel (classes.vessel.Vessel): The Vessel the File has been
|
||||||
|
uploaded to
|
||||||
|
"""
|
||||||
self._execute(
|
self._execute(
|
||||||
"INSERT INTO contentmonster_file_log(file, vessel) VALUES(?, ?)", (file.uuid, vessel.name))
|
"INSERT INTO contentmonster_file_log(file, vessel) VALUES(?, ?)", (file.uuid, vessel.name))
|
||||||
|
|
||||||
def getCompletionForVessel(self, vessel):
|
def getCompletionForVessel(self, vessel) -> list[Optional[str]]:
|
||||||
|
"""Get completed uploads for a vessel
|
||||||
|
|
||||||
|
Args:
|
||||||
|
vessel (classes.vessel.Vessel): The Vessel object to retrieve
|
||||||
|
uploaded files for
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: List of UUIDs of Files that have been successfully uploaded
|
||||||
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT file FROM contentmonster_file_log WHERE vessel = ?", (vessel.name,))
|
"SELECT file FROM contentmonster_file_log WHERE vessel = ?", (vessel.name,))
|
||||||
|
|
||||||
def migrate(self):
|
return [f[0] for f in cur.fetchall()]
|
||||||
|
|
||||||
|
def migrate(self) -> None:
|
||||||
|
"""Apply database migrations
|
||||||
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
|
|
||||||
if self.getVersion() == 0:
|
if self.getVersion() == 0:
|
||||||
|
@ -93,4 +186,6 @@ class Database:
|
||||||
self.commit()
|
self.commit()
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
|
"""Close database connection on removal of the Database object
|
||||||
|
"""
|
||||||
self._con.close()
|
self._con.close()
|
||||||
|
|
|
@ -3,22 +3,58 @@ from classes.file import File
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
|
||||||
|
from configparser import SectionProxy
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
|
||||||
class Directory:
|
class Directory:
|
||||||
|
"""Class representing a Directory on the local filesystem
|
||||||
|
"""
|
||||||
@classmethod
|
@classmethod
|
||||||
def fromConfig(cls, config):
|
def fromConfig(cls, config: SectionProxy) -> Directory: # pylint: disable=undefined-variable
|
||||||
|
"""Create Directory object from a Directory section in the Config file
|
||||||
|
|
||||||
|
Args:
|
||||||
|
config (configparser.SectionProxy): Configuration section defining
|
||||||
|
a Directory
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: Raised if section does not contain Location parameter
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
classes.directory.Directory: Directory object for the location
|
||||||
|
specified in the config section
|
||||||
|
"""
|
||||||
if "Location" in config.keys():
|
if "Location" in config.keys():
|
||||||
return cls(config.name.split()[1], config["Location"])
|
return cls(config.name.split()[1], config["Location"])
|
||||||
else:
|
else:
|
||||||
raise ValueError("Definition for Directory " + config.name.split()[1] + " does not contain Location!")
|
raise ValueError("Definition for Directory " +
|
||||||
|
config.name.split()[1] + " does not contain Location!")
|
||||||
|
|
||||||
def __init__(self, name, location):
|
def __init__(self, name: str, location: Union[str, pathlib.Path]):
|
||||||
|
"""Initialize a new Directory object
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name (str): Name of the Directory object
|
||||||
|
location (str, pathlib.Path): Filesystem location of the Directory
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: Raised if passed location does not exist or is not a
|
||||||
|
directory
|
||||||
|
"""
|
||||||
self.name = name
|
self.name = name
|
||||||
|
|
||||||
if os.path.isdir(location):
|
if os.path.isdir(location):
|
||||||
self.location = pathlib.Path(location)
|
self.location = pathlib.Path(location)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Location {location} for Directory {name} does not exist or is not a directory.")
|
raise ValueError(
|
||||||
|
f"Location {location} for Directory {name} does not exist or is not a directory.")
|
||||||
|
|
||||||
def getFiles(self):
|
def getFiles(self) -> list[File]:
|
||||||
|
"""Get all Files in Directory
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: List of names (str) of files within the Directory
|
||||||
|
"""
|
||||||
files = [f for f in os.listdir(self.location) if os.path.isfile]
|
files = [f for f in os.listdir(self.location) if os.path.isfile]
|
||||||
return [File(f, self) for f in files]
|
return [File(f, self) for f in files]
|
|
@ -1,28 +1,70 @@
|
||||||
from watchdog.events import FileSystemEventHandler
|
from watchdog.events import (FileSystemEventHandler, FileSystemEvent,
|
||||||
|
FileCreatedEvent, FileDeletedEvent,
|
||||||
|
FileModifiedEvent, FileMovedEvent)
|
||||||
|
|
||||||
|
from multiprocessing import Queue
|
||||||
|
|
||||||
import os.path
|
import os.path
|
||||||
|
|
||||||
|
|
||||||
class DogHandler(FileSystemEventHandler):
|
class DogHandler(FileSystemEventHandler):
|
||||||
def __init__(self, directory, queue, *args, **kwargs):
|
"""Class implementing a watchdog event handler
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, directory, queue: Queue, *args, **kwargs) -> None:
|
||||||
|
"""Initialize a new DogHandler object
|
||||||
|
|
||||||
|
Args:
|
||||||
|
directory (classes.directory.Directory): Directory to watch
|
||||||
|
queue (multiprocessing.Queue): Queue to put detected events on
|
||||||
|
"""
|
||||||
print("Initialized")
|
print("Initialized")
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self._directory = directory
|
self._directory = directory
|
||||||
self._queue = queue
|
self._queue = queue
|
||||||
|
|
||||||
def dispatch(self, event):
|
def dispatch(self, event: FileSystemEvent):
|
||||||
|
"""Dispatch events to the appropriate event handlers
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event (watchdog.events.FileSystemEvent): Event to handle
|
||||||
|
"""
|
||||||
if not event.is_directory:
|
if not event.is_directory:
|
||||||
super().dispatch(event)
|
super().dispatch(event)
|
||||||
|
|
||||||
def on_created(self, event):
|
def on_created(self, event: FileCreatedEvent):
|
||||||
|
"""Put file creation events on the queue
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event (watchdog.events.FileCreatedEvent): Event describing the
|
||||||
|
created file
|
||||||
|
"""
|
||||||
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
||||||
|
|
||||||
def on_modified(self, event):
|
def on_modified(self, event: FileModifiedEvent):
|
||||||
|
"""Put file modification events on the queue
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event (watchdog.events.FileModifiedEvent): Event describing the
|
||||||
|
modified file
|
||||||
|
"""
|
||||||
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
||||||
|
|
||||||
def on_moved(self, event):
|
def on_moved(self, event: FileMovedEvent):
|
||||||
|
"""Put file move events on the queue
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event (watchdog.events.FileMovedEvent): Event describing the moved
|
||||||
|
file (source and destination)
|
||||||
|
"""
|
||||||
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
||||||
self._queue.put((self._directory, os.path.basename(event.dest_path)))
|
self._queue.put((self._directory, os.path.basename(event.dest_path)))
|
||||||
|
|
||||||
def on_deleted(self, event):
|
def on_deleted(self, event: FileDeletedEvent):
|
||||||
|
"""Put file deletion events on the queue
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event (watchdog.events.FileDeletedEvent): Event describing the
|
||||||
|
deleted file
|
||||||
|
"""
|
||||||
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
||||||
|
|
Loading…
Reference in a new issue