Current status, fixes #1
This commit is contained in:
parent
d5adbbc1fa
commit
90b53f8ee7
4 changed files with 133 additions and 79 deletions
|
@ -1,11 +1,14 @@
|
||||||
|
import re
|
||||||
|
|
||||||
class Directory:
|
class Directory:
|
||||||
def __init__(self, name, source, destination, sourcebackup=None, destinationbackup=None):
|
def __init__(self, name, source, destination, sourcebackup=None, destinationbackup=None, regex=None):
|
||||||
self.name = name
|
self.name = name
|
||||||
self.source = source
|
self.source = source
|
||||||
self.destination = destination
|
self.destination = destination
|
||||||
self.sourcebackup = sourcebackup
|
self.sourcebackup = sourcebackup
|
||||||
self.destinationbackup = destinationbackup
|
self.destinationbackup = destinationbackup
|
||||||
|
self.regex = re.compile(regex)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_config(cls, section):
|
def from_config(cls, section):
|
||||||
return cls(section.name.split()[1], section["SourceDirectory"], section["DestinationDirectory"], section.get("SourceBackup"), section.get("DestinationBackup"))
|
return cls(section.name.split()[1], section["SourceDirectory"], section["DestinationDirectory"], section.get("SourceBackup"), section.get("DestinationBackup"), section.get("Regex"))
|
||||||
|
|
106
communicator.py
Normal file
106
communicator.py
Normal file
|
@ -0,0 +1,106 @@
|
||||||
|
from classes.config import Config
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
|
from gnupg import GPG
|
||||||
|
|
||||||
|
def log_string(input_string, log_level="INFO"):
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
print(f"[{now.strftime('%Y-%m-%d %H:%M:%S.%f')}][{log_level}] {input_string}")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
config = Config.from_file(Path(__file__).parent / "settings.ini")
|
||||||
|
|
||||||
|
for directory in config.directories:
|
||||||
|
log_string(f"Start processing {directory.name}")
|
||||||
|
source = Path(directory.source)
|
||||||
|
|
||||||
|
for newfile in source.iterdir():
|
||||||
|
if newfile.is_file() and (time.time() - newfile.stat().st_mtime > 20):
|
||||||
|
log_string(f"Found file {newfile.name}")
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
raw = newfile.read_text(encoding="utf-8").encode("utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raw = newfile.read_text(encoding="Windows-1252").encode("utf-8")
|
||||||
|
|
||||||
|
if config.server.theirkey and config.server.ourkey:
|
||||||
|
log_string(f"Encrypting for {config.server.theirkey}...")
|
||||||
|
encrypted = GPG().encrypt(raw, config.server.theirkey, sign=config.server.ourkey, always_trust=True).data
|
||||||
|
else:
|
||||||
|
log_string("Not encrypting. Do not use in production.", "WARNING")
|
||||||
|
encrypted = raw.encode()
|
||||||
|
|
||||||
|
upfl = BytesIO(encrypted)
|
||||||
|
uppath = Path(config.server.inpath) / f"{newfile.name}.pgp"
|
||||||
|
|
||||||
|
with config.server.get_sftp_client() as sftp:
|
||||||
|
log_string(f"Uploading to {uppath}...")
|
||||||
|
sftp.putfo(upfl, str(uppath))
|
||||||
|
|
||||||
|
if directory.sourcebackup:
|
||||||
|
log_string(f"Backing up...")
|
||||||
|
newfile.rename(Path(directory.sourcebackup) / newfile.name)
|
||||||
|
else:
|
||||||
|
log_string(f"Deleting input file...")
|
||||||
|
newfile.unlink()
|
||||||
|
log_string("Completed processing {newfile.name}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log_string(f"Something went wrong uploading file {newfile.name} from {directory.name}: {e}", "ERROR")
|
||||||
|
|
||||||
|
try:
|
||||||
|
log_string("Start downloading files from server")
|
||||||
|
with config.server.get_sftp_client() as sftp:
|
||||||
|
for response in sftp.listdir(config.server.outpath):
|
||||||
|
rpath = str(Path(config.server.outpath) / response)
|
||||||
|
if (time.time() - sftp.stat(rpath).st_mtime < 20):
|
||||||
|
continue
|
||||||
|
|
||||||
|
log_string(f"Found file {response}, downloading...")
|
||||||
|
|
||||||
|
encrypted = sftp.open(rpath).read()
|
||||||
|
|
||||||
|
if config.server.ourkey:
|
||||||
|
log_string("Decrypting...")
|
||||||
|
decrypted = GPG().decrypt(encrypted).data.decode()
|
||||||
|
else:
|
||||||
|
log_string("Not decrypting. Do not use in production.", "WARN")
|
||||||
|
decrypted = str(encrypted)
|
||||||
|
|
||||||
|
founddir = None
|
||||||
|
|
||||||
|
for directory in config.directories:
|
||||||
|
if directory.regex.search(response).group():
|
||||||
|
log_string(f"File seems to belong to {directory.name}")
|
||||||
|
founddir = directory
|
||||||
|
break
|
||||||
|
|
||||||
|
if not founddir:
|
||||||
|
founddir = directory
|
||||||
|
|
||||||
|
outpath = Path(directory.destination) / response
|
||||||
|
|
||||||
|
if outpath.suffix == ".pgp":
|
||||||
|
outpath = outpath.with_suffix("")
|
||||||
|
|
||||||
|
assert not outpath.exists()
|
||||||
|
|
||||||
|
outpath.write_text(decrypted)
|
||||||
|
|
||||||
|
if founddir.destinationbackup:
|
||||||
|
log_string("Backing up...")
|
||||||
|
(Path(founddir.destinationbackup) / outpath.name).write_text(decrypted)
|
||||||
|
|
||||||
|
log_string(f"Deleting file {response} from server...")
|
||||||
|
sftp.remove(rpath)
|
||||||
|
|
||||||
|
log_string(f"Done processing {response}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log_string(f"Something went wrong downloading files from the server: {e}", "ERROR")
|
||||||
|
|
77
daemon.py
77
daemon.py
|
@ -1,77 +0,0 @@
|
||||||
from classes.config import Config
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
from io import BytesIO
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
from gnupg import GPG
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
config = Config.from_file(Path(__file__).parent / "settings.ini")
|
|
||||||
|
|
||||||
for directory in config.directories:
|
|
||||||
source = Path(directory.source)
|
|
||||||
|
|
||||||
for newfile in source.iterdir():
|
|
||||||
if newfile.is_file() and (time.time() - newfile.stat().st_mtime > 60):
|
|
||||||
try:
|
|
||||||
raw = newfile.read_text()
|
|
||||||
encrypted: str = GPG().encrypt(raw, config.server.theirkey, sign=config.server.ourkey, always_trust=True)
|
|
||||||
upfl = BytesIO(encrypted.data)
|
|
||||||
uppath = Path(config.server.inpath) / f"{directory.name}_{newfile.name}.pgp"
|
|
||||||
|
|
||||||
with config.server.get_sftp_client() as sftp:
|
|
||||||
sftp.putfo(upfl, str(uppath))
|
|
||||||
|
|
||||||
if directory.sourcebackup:
|
|
||||||
newfile.rename(Path(directory.sourcebackup) / newfile.name)
|
|
||||||
else:
|
|
||||||
newfile.unlink()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Something went wrong uploading file {newfile.name} from {directory.name}: {e}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
with config.server.get_sftp_client() as sftp:
|
|
||||||
for response in sftp.listdir(config.server.outpath):
|
|
||||||
rpath = str(Path(config.server.outpath) / response)
|
|
||||||
if (time.time() - sftp.stat(rpath).st_mtime < 60):
|
|
||||||
continue
|
|
||||||
|
|
||||||
encrypted: bytes = sftp.open(rpath).read()
|
|
||||||
decrypted = GPG().decrypt(encrypted).data.decode()
|
|
||||||
|
|
||||||
dirname = response.split("_")[0]
|
|
||||||
|
|
||||||
founddir = None
|
|
||||||
|
|
||||||
for directory in config.directories:
|
|
||||||
if directory.name == dirname:
|
|
||||||
founddir = directory
|
|
||||||
break
|
|
||||||
|
|
||||||
if not founddir:
|
|
||||||
founddir = directory
|
|
||||||
|
|
||||||
if founddir.name == dirname:
|
|
||||||
outfile = response.split("_")[1:]
|
|
||||||
else:
|
|
||||||
outfile = response
|
|
||||||
|
|
||||||
outpath = Path(directory.destination) / outfile
|
|
||||||
|
|
||||||
if outpath.suffix == ".pgp":
|
|
||||||
outpath = outpath.with_suffix("")
|
|
||||||
|
|
||||||
assert not outpath.exists()
|
|
||||||
|
|
||||||
outpath.write_text(decrypted)
|
|
||||||
|
|
||||||
if founddir.destinationbackup:
|
|
||||||
(Path(founddir.destinationbackup) / outpath.name).write_text(decrypted)
|
|
||||||
|
|
||||||
sftp.remove(rpath)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Something went wrong downloading files from the server: {e}")
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
[Directory Payment]
|
||||||
|
SourceDirectory = /srv/monsterwell/payment/
|
||||||
|
DestinationDirectory = /srv/monsterwell/payment-echo/
|
||||||
|
SourceBackup = /var/backups/monsterwell/payment/
|
||||||
|
DestinationBackup = /var/backups/monsterwell/payment-echo/
|
||||||
|
Regex = ^(?:(?!registration).)*
|
||||||
|
|
||||||
|
[Directory Registration]
|
||||||
|
SourceDirectory = /srv/monsterwell/registration/
|
||||||
|
DestinationDirectory = /srv/monsterwell/registration-echo/
|
||||||
|
SourceBackup = /var/backups/monsterwell/registration/
|
||||||
|
DestinationBackup = /var/backups/monsterwell/registration-echo/
|
||||||
|
Regex = ^registration
|
||||||
|
|
||||||
|
[Server]
|
||||||
|
OurKey = our@gpg.key
|
||||||
|
TheirKey = their@gpg.key
|
||||||
|
Host = their.host.com
|
||||||
|
Username = username
|
||||||
|
Password = värisikret
|
||||||
|
InPath = /in/
|
||||||
|
OutPath = /out/
|
Loading…
Reference in a new issue