From 27fe413d11b36048d11b76032fd0cd7f6eda3946 Mon Sep 17 00:00:00 2001 From: Klaus-Uwe Mitterer Date: Sun, 24 May 2020 17:44:27 +0200 Subject: [PATCH] Some refactoring to get cron running Moved dbsettings documentation to Gitlab wiki --- celerybeat.pid | 1 + core/apps.py | 1 - core/classes/cron.py | 38 ++++++++++++ core/cron.py | 22 +++++++ core/helpers/auth.py | 7 ++- core/helpers/cron.py | 13 ++++ core/helpers/mail.py | 5 +- core/management/commands/setupcron.py | 9 +++ core/models/cron.py | 6 ++ core/modules/cron.py | 18 ++++++ core/tasks/__init__.py | 3 +- core/tasks/cron.py | 28 +++++++++ dbsettings_keys.md | 87 --------------------------- expephalon/celery.py | 9 --- requirements.txt | 1 + 15 files changed, 147 insertions(+), 101 deletions(-) create mode 100644 celerybeat.pid create mode 100644 core/classes/cron.py create mode 100644 core/cron.py create mode 100644 core/helpers/cron.py create mode 100644 core/management/commands/setupcron.py create mode 100644 core/models/cron.py create mode 100644 core/modules/cron.py create mode 100644 core/tasks/cron.py delete mode 100644 dbsettings_keys.md diff --git a/celerybeat.pid b/celerybeat.pid new file mode 100644 index 0000000..fdc3ad0 --- /dev/null +++ b/celerybeat.pid @@ -0,0 +1 @@ +7348 diff --git a/core/apps.py b/core/apps.py index 26f78a8..48f4f7b 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,5 +1,4 @@ from django.apps import AppConfig - class CoreConfig(AppConfig): name = 'core' diff --git a/core/classes/cron.py b/core/classes/cron.py new file mode 100644 index 0000000..986ddd1 --- /dev/null +++ b/core/classes/cron.py @@ -0,0 +1,38 @@ +from django.utils import timezone + +from core.models.cron import CronLog + +from dbsettings.functions import getValue + +from parse_crontab import CronTab + +class Cronjob: + def __init__(self, name, crondef, lock=getValue("core.cron.lock", 300)): + self.name = name + self.crondef = crondef + self.lock = lock + + @property + def is_running(self): + now = timezone.now() + maxage = now - timezone.timedelta(seconds=self.lock) + CronLog.objects.filter(task=self.name, locked=True, execution__lt=maxage).update(locked=False) + return True if CronLog.objects.filter(task=self.name, locked=True) else False + + @property + def next_run(self): + runs = CronLog.objects.filter(task=self.name) + if not runs: + CronLog.objects.create(task=self.name, locked=False) + return self.next_run + + lastrun = runs.latest("execution").execution + return lastrun + timezone.timedelta(seconds=CronTab(self.crondef).next(lastrun)) + + @property + def is_due(self): + return self.next_run <= timezone.now() + + def run(self): + from core.tasks.cron import run_cron + run_cron.delay(self.name) \ No newline at end of file diff --git a/core/cron.py b/core/cron.py new file mode 100644 index 0000000..a438b42 --- /dev/null +++ b/core/cron.py @@ -0,0 +1,22 @@ +from core.classes.cron import Cronjob +from core.helpers.auth import clear_login_log + +CRONDEFINITIONS = [] +CRONFUNCTIONS = {} + +### Demonstration Cronjob + +def debug_job(): + return "Test" + +debug_cron = Cronjob("core.debug_job", "* * * * *") + +# CRONFUNCTIONS["core.debug_job"] = debug_job +# CRONDEFINITIONS.append(debug_cron) + +### Remove old entries from the login log + +loginlog_cron = Cronjob("core.clear_login_log", "* * * * *") + +CRONFUNCTIONS["core.clear_login_log"] = clear_login_log +CRONDEFINITIONS.append(loginlog_cron) \ No newline at end of file diff --git a/core/helpers/auth.py b/core/helpers/auth.py index 4b2fbb6..3d3064c 100644 --- a/core/helpers/auth.py +++ b/core/helpers/auth.py @@ -5,6 +5,7 @@ from core.helpers.request import get_client_ip from django.urls import reverse from django.contrib import messages +from django.utils import timezone from dbsettings.functions import getValue @@ -19,4 +20,8 @@ def login_fail(request, user=None, message=None): messages.error(request, message) def login_success(request, user): - LoginLog.objects.create(user=user, ip=get_client_ip(request), success=True) \ No newline at end of file + LoginLog.objects.create(user=user, ip=get_client_ip(request), success=True) + +def clear_login_log(maxage=int(getValue("core.auth.ratelimit.period", 600))): + timestamp = timezone.now() - timezone.timedelta(seconds=maxage) + LoginLog.objects.filter(timestamp__lt=timestamp).delete() \ No newline at end of file diff --git a/core/helpers/cron.py b/core/helpers/cron.py new file mode 100644 index 0000000..eb34b84 --- /dev/null +++ b/core/helpers/cron.py @@ -0,0 +1,13 @@ +from django_celery_beat.models import PeriodicTask, IntervalSchedule + +def setup_cron(): + schedule, created = IntervalSchedule.objects.get_or_create( + every=1, + period=IntervalSchedule.MINUTES, + ) + + PeriodicTask.objects.get_or_create( + interval=schedule, + name='Expephacron', + task='cron', + ) \ No newline at end of file diff --git a/core/helpers/mail.py b/core/helpers/mail.py index fc6f98e..7f0d978 100644 --- a/core/helpers/mail.py +++ b/core/helpers/mail.py @@ -1,7 +1,5 @@ from django.conf import settings -from core.modules.mail import providers, templates -from core.tasks.mail import send_mail as send_mail_task from core.exceptions.mail import NoSuchTemplate from dbsettings.functions import getValue @@ -9,12 +7,14 @@ from dbsettings.functions import getValue import os.path def get_provider_by_name(name, fallback=True): + from core.modules.mail import providers return providers.get(name, None) or providers["smtp"] def get_default_provider(fallback=True): return get_provider_by_name(getValue("core.email.provider", "smtp"), fallback) def send_mail(provider=get_default_provider(), **kwargs): + from core.tasks.mail import send_mail as send_mail_task provider = get_provider_by_name(provider) if type(provider) == str else provider return send_mail_task.delay(provider, **kwargs) @@ -22,6 +22,7 @@ def simple_send_mail(subject, content, recipients, cc=[], bcc=[], headers={}): return send_mail(subject=subject, content=content, recipients=recipients, cc=cc, bcc=bcc, headers=headers) def get_template(template_name, format="txt", **kwargs): + from core.modules.mail import templates try: template = templates[template_name][format] except KeyError: diff --git a/core/management/commands/setupcron.py b/core/management/commands/setupcron.py new file mode 100644 index 0000000..97e08ea --- /dev/null +++ b/core/management/commands/setupcron.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand, CommandError + +from core.helpers.cron import setup_cron + +class Command(BaseCommand): + help = 'Enables the cron system' + + def handle(self, *args, **options): + setup_cron() \ No newline at end of file diff --git a/core/models/cron.py b/core/models/cron.py new file mode 100644 index 0000000..63d1a13 --- /dev/null +++ b/core/models/cron.py @@ -0,0 +1,6 @@ +from django.db.models import Model, CharField, DateTimeField, BooleanField + +class CronLog(Model): + task = CharField(max_length=255) + execution = DateTimeField(auto_now_add=True) + locked = BooleanField(default=True) diff --git a/core/modules/cron.py b/core/modules/cron.py new file mode 100644 index 0000000..d79e27e --- /dev/null +++ b/core/modules/cron.py @@ -0,0 +1,18 @@ +import importlib + +from django.conf import settings + +cronfunctions = {} + +crondefinitions = [] + +for module in ["core"] + settings.EXPEPHALON_MODULES: + try: + moc = importlib.import_module(f"{module}.cron") + for name, fun in moc.CRONFUNCTIONS.items(): + if name in cronfunctions.keys(): + raise ValueError(f"Error in {module}: Cron function with name {name} already registered!") + cronfunctions[name] = fun + crondefinitions += moc.CRONDEFINITIONS + except (AttributeError, ModuleNotFoundError): + continue \ No newline at end of file diff --git a/core/tasks/__init__.py b/core/tasks/__init__.py index 33d4183..6d18f37 100644 --- a/core/tasks/__init__.py +++ b/core/tasks/__init__.py @@ -1 +1,2 @@ -from core.tasks.mail import * \ No newline at end of file +from core.tasks.mail import send_mail +from core.tasks.cron import process_crons, run_cron \ No newline at end of file diff --git a/core/tasks/cron.py b/core/tasks/cron.py new file mode 100644 index 0000000..6ff7c8f --- /dev/null +++ b/core/tasks/cron.py @@ -0,0 +1,28 @@ +from celery import shared_task, task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + +@task(name="cron") +def process_crons(): + from core.modules.cron import crondefinitions + + for definition in crondefinitions: + print(definition.next_run) + if definition.is_due and not definition.is_running: + definition.run() + +@shared_task +def run_cron(name, *args, **kwargs): + from core.models.cron import CronLog + from core.modules.cron import cronfunctions + + log = CronLog.objects.create(task=name) + try: + output = cronfunctions[name]() + if output: + logger.debug(f"[{name}] {output}") + except Exception as e: + logger.error(f"[{name}] {str(e)}") + log.locked = False + log.save() \ No newline at end of file diff --git a/dbsettings_keys.md b/dbsettings_keys.md deleted file mode 100644 index 0d7b37b..0000000 --- a/dbsettings_keys.md +++ /dev/null @@ -1,87 +0,0 @@ -This document includes all database config keys currently used by Expephalon itself. - -Third-party modules must not use keys in the core.\*, custom.\*, dbsettings.\* or expephalon.\* namespaces, as well as in the namespaces of official modules (chat.\*, demomodule.\*, kumisms.\*, playsms.\*, smsotp.\*, totp.\*), and should use namespaces which are specific enough to prevent collisions with other third-party modules or future official modules. - -The custom.\* is reserved for settings created manually by administrators for use in custom scripts or other reasons. - -[[_TOC_]] - -## Base configuration - -### core.base_url - -**Description**: URL under which this Expephalon installation is available, including protocol, generally without trailing slash, used for generation of absolute URLs - -**Default value:** http://localhost:8000 - -### core.title - -**Description**: Title of the Expephalon installation, used in page titles and anywhere else the name of the site is referenced - -**Default value:** Expephalon - -## Authentication system - -### core.auth.otp.max_age - -**Description:** Maximum time from starting to finishing a one-time-password flow (in seconds) - -**Default value:** 300 - -### core.auth.pwreset.max_age - -**Description:** Maximum time between creation and usage of a password reset token (in seconds) - -**Default value:** 86400 - -### core.auth.ratelimit.attempts - -**Description:** Maximum number of invalid login attempts in last core.auth.ratelimit.period seconds from individual IP before blocking - -**Default value:** 5 - -### core.auth.ratelimit.block - -**Description:** Period for which to block further login attempts from individual IP after c.a.r.attempts failures in c.a.r.period seconds (in seconds) - -**Default value:** 3600 - -### core.auth.ratelimit.period - -**Description:** Period in which to check for previous failed login attempts for ratelimiting (in seconds) - -**Default value:** 600 - -## Mail - -### core.mail.sender - -**Description:** Email address to be used as sender of outgoing mail - -**Default value:** "Expephalon" \ - -### core.smtp.host - -**Description:** Hostname of the SMTP server to be used for outgoing mail - -**Default value:** localhost - -### core.smtp.username - -**Description:** Username to authenticate to the SMTP server with - -**Default value:** (None) - -### core.smtp.password - -**Description:** Password to authenticate to the SMTP server with - -**Default value:** (None) - -## SMS - -### core.sms.default - -**Description:** Name of the default SMS provider to be used by Expephalon - must be the unique return value of the provider's get_name property - -**Default value:** (None, effectively disabling SMS - doesn't make sense without an SMS provider module installed) diff --git a/expephalon/celery.py b/expephalon/celery.py index 2ae3c51..1b9b445 100644 --- a/expephalon/celery.py +++ b/expephalon/celery.py @@ -2,20 +2,11 @@ import os from celery import Celery -# set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'expephalon.settings') app = Celery('expephalon') -# Using a string here means the worker doesn't have to serialize -# the configuration object to child processes. -# - namespace='CELERY' means all celery-related configuration keys -# should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -# Load task modules from all registered Django app configs. app.autodiscover_tasks() -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 85795ee..acbcb4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ django-celery-beat python-memcached django-countries pyuca +git+https://kumig.it/kumisystems/parse_crontab.git