feat: Implement background scheduler initialization to prevent race conditions in multi-worker deployments

This commit is contained in:
2026-04-18 12:39:07 +02:00
parent 5d9069e690
commit ac3e48da3d
10 changed files with 97 additions and 8 deletions
+3 -1
View File
@@ -2,4 +2,6 @@ dist
logs logs
certs certs
build build
.venv .venv
__pycache__
.pyc
View File
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+52 -7
View File
@@ -1234,17 +1234,62 @@ def update_appointment_statuses():
except Exception as e: except Exception as e:
app.logger.error(f"Automatic appointment status update failed: {e}") app.logger.error(f"Automatic appointment status update failed: {e}")
# Schedule jobs # Schedule jobs - only start scheduler if this is the main process or a single-worker deployment
# This prevents race conditions in multi-worker Gunicorn environments
scheduler = BackgroundScheduler() scheduler = BackgroundScheduler()
if cfg.SCHEDULER_ENABLED: _scheduler_initialized = False
scheduler.add_job(func=create_daily_backup, trigger="interval", hours=cfg.BACKUP_INTERVAL_HOURS)
scheduler.add_job(func=update_appointment_statuses, trigger="interval", minutes=cfg.SCHEDULER_INTERVAL_MIN) def _initialize_scheduler():
scheduler.add_job(func=create_return_reminders, trigger="interval", minutes=cfg.SCHEDULER_INTERVAL_MIN) """Initialize the background scheduler in a safe way for multi-worker deployments."""
scheduler.start() global _scheduler_initialized
if _scheduler_initialized or not cfg.SCHEDULER_ENABLED:
return
try:
# For multi-worker Gunicorn, check if we're in a reasonable scenario
# Using a lock file to ensure only one instance starts the scheduler
scheduler_lock_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.scheduler_lock')
try:
# Try to create the lock file - only succeeds if it doesn't exist
lock_fd = os.open(scheduler_lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
os.close(lock_fd)
should_start = True
except FileExistsError:
should_start = False
app.logger.warning("Scheduler lock exists - another process is already running the scheduler")
if should_start:
scheduler.add_job(func=create_daily_backup, trigger="interval", hours=cfg.BACKUP_INTERVAL_HOURS)
scheduler.add_job(func=update_appointment_statuses, trigger="interval", minutes=cfg.SCHEDULER_INTERVAL_MIN)
scheduler.add_job(func=create_return_reminders, trigger="interval", minutes=cfg.SCHEDULER_INTERVAL_MIN)
scheduler.start()
_scheduler_initialized = True
app.logger.info(f"Scheduler started successfully (interval={cfg.SCHEDULER_INTERVAL_MIN} min)")
else:
app.logger.info("Scheduler skipped - another worker instance is running it")
except Exception as e:
app.logger.error(f"Failed to initialize scheduler: {e}")
_scheduler_initialized = False
# Initialize scheduler on app startup
_initialize_scheduler()
# Register shutdown handler to stop scheduler when app is terminated # Register shutdown handler to stop scheduler when app is terminated
import atexit import atexit
atexit.register(lambda: scheduler.shutdown() if cfg.SCHEDULER_ENABLED else None) def _shutdown_scheduler():
if cfg.SCHEDULER_ENABLED and _scheduler_initialized:
try:
scheduler.shutdown()
lock_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.scheduler_lock')
try:
os.remove(lock_path)
except Exception:
pass
except Exception as e:
app.logger.error(f"Error during scheduler shutdown: {e}")
atexit.register(_shutdown_scheduler)
def allowed_file(filename, file_content=None, max_size_mb=cfg.MAX_UPLOAD_MB): def allowed_file(filename, file_content=None, max_size_mb=cfg.MAX_UPLOAD_MB):
""" """
+42
View File
@@ -0,0 +1,42 @@
"""
Gunicorn configuration for Inventarsystem.
This configuration ensures that:
1. The BackgroundScheduler runs reliably in only one worker process
2. Appointment status updates and reminders work correctly
3. Multi-worker deployments don't cause race conditions
"""
import os
import sys
from pathlib import Path
# Get project root
PROJECT_ROOT = Path(__file__).parent
# Basic configuration
bind = "unix:/tmp/inventarsystem.sock"
workers = 1 # CRITICAL: Only 1 worker to prevent BackgroundScheduler race conditions
worker_class = "sync"
timeout = 60
graceful_timeout = 20
max_requests = 1000
max_requests_jitter = 100
# Logging
accesslog = str(PROJECT_ROOT / "logs" / "access.log")
errorlog = str(PROJECT_ROOT / "logs" / "error.log")
log_level = "info"
capture_output = True
# Worker initialization hook to ensure scheduler starts only once
def on_starting(server):
"""Called just before the master process is initialized."""
print("[GUNICORN] Starting Inventarsystem with scheduler support (1 worker only)")
def when_ready(server):
"""Called just after the server is started."""
print("[GUNICORN] Server is ready. Scheduler should be active in the single worker process.")
# Ensure the logs directory exists
os.makedirs(PROJECT_ROOT / "logs", exist_ok=True)