59 lines
1.8 KiB
Python
59 lines
1.8 KiB
Python
"""Taskiq CV-upload broker — isolated Redis stream so manual uploads
|
|
never sit behind the bulk /email/fetch backlog.
|
|
|
|
Worker: taskiq worker taskiq_management.cv_broker_setup:cv_broker inbox.cv_tasks
|
|
Scheduler: taskiq scheduler taskiq_management.cv_broker_setup:cv_scheduler inbox.cv_tasks
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from dotenv import load_dotenv
|
|
from taskiq import TaskiqScheduler
|
|
from taskiq.middlewares import SmartRetryMiddleware
|
|
from taskiq.schedule_sources import LabelScheduleSource
|
|
from taskiq_redis import (
|
|
ListRedisScheduleSource,
|
|
RedisAsyncResultBackend,
|
|
RedisStreamBroker,
|
|
)
|
|
|
|
from taskiq_management.broker_setup import MAX_RETRIES,RETRY_DELAY
|
|
from taskiq_management.middleware import DeadLetterMiddleware
|
|
|
|
load_dotenv()
|
|
|
|
REDIS_URL=os.getenv("REDIS_URL","redis://localhost:6379/0")
|
|
CV_QUEUE_NAME=os.getenv("TASKIQ_CV_QUEUE_NAME","cv_upload")
|
|
|
|
result_backend=RedisAsyncResultBackend(redis_url=REDIS_URL)
|
|
cv_schedule_source=ListRedisScheduleSource(url=REDIS_URL,prefix="taskiq:schedule:cv")
|
|
|
|
cv_broker=(
|
|
RedisStreamBroker(
|
|
url=REDIS_URL,
|
|
queue_name=CV_QUEUE_NAME,
|
|
consumer_group_name=os.getenv("TASKIQ_CONSUMER_GROUP","taskiq"),
|
|
idle_timeout=int(os.getenv("TASKIQ_IDLE_TIMEOUT_MS","600000")),
|
|
)
|
|
.with_result_backend(result_backend)
|
|
.with_middlewares(
|
|
DeadLetterMiddleware(redis_url=REDIS_URL),
|
|
SmartRetryMiddleware(
|
|
default_retry_count=MAX_RETRIES,
|
|
default_retry_label=True,
|
|
default_delay=RETRY_DELAY,
|
|
use_jitter=True,
|
|
use_delay_exponent=True,
|
|
max_delay_exponent=float(os.getenv("TASKIQ_MAX_DELAY","120")),
|
|
schedule_source=cv_schedule_source,
|
|
),
|
|
)
|
|
)
|
|
|
|
cv_scheduler=TaskiqScheduler(
|
|
broker=cv_broker,
|
|
sources=[cv_schedule_source,LabelScheduleSource(cv_broker)],
|
|
)
|