feat: 添加 AI 索引重建功能,优化相关 API 和工作流,增强内存管理配置
Some checks failed
docker-images / resolve-build-targets (push) Successful in 6s
ui-regression / playwright-regression (push) Successful in 4m43s
docker-images / build-and-push (admin) (push) Successful in 42s
docker-images / submit-indexnow (push) Has been cancelled
docker-images / build-and-push (frontend) (push) Has been cancelled
docker-images / build-and-push (backend) (push) Has started running

This commit is contained in:
2026-04-03 15:48:33 +08:00
parent 1df179c327
commit cf00dc5e8e
15 changed files with 391 additions and 88 deletions

View File

@@ -11,6 +11,7 @@ use crate::{
models::_entities::{notification_deliveries, worker_jobs},
services::subscriptions,
workers::{
ai_reindex::{AiReindexWorker, AiReindexWorkerArgs},
downloader::{DownloadWorker, DownloadWorkerArgs},
notification_delivery::{NotificationDeliveryWorker, NotificationDeliveryWorkerArgs},
},
@@ -27,6 +28,7 @@ pub const JOB_STATUS_CANCELLED: &str = "cancelled";
pub const WORKER_DOWNLOAD_MEDIA: &str = "worker.download_media";
pub const WORKER_NOTIFICATION_DELIVERY: &str = "worker.notification_delivery";
pub const WORKER_AI_REINDEX: &str = "worker.ai_reindex";
pub const TASK_RETRY_DELIVERIES: &str = "task.retry_deliveries";
pub const TASK_SEND_WEEKLY_DIGEST: &str = "task.send_weekly_digest";
pub const TASK_SEND_MONTHLY_DIGEST: &str = "task.send_monthly_digest";
@@ -164,6 +166,7 @@ fn trim_to_option(value: Option<String>) -> Option<String> {
fn queue_name_for(worker_name: &str) -> Option<String> {
match worker_name {
WORKER_AI_REINDEX => Some("ai".to_string()),
WORKER_DOWNLOAD_MEDIA => Some("media".to_string()),
WORKER_NOTIFICATION_DELIVERY => Some("notifications".to_string()),
TASK_RETRY_DELIVERIES => Some("maintenance".to_string()),
@@ -174,6 +177,7 @@ fn queue_name_for(worker_name: &str) -> Option<String> {
fn label_for(worker_name: &str) -> String {
match worker_name {
WORKER_AI_REINDEX => "AI 索引重建".to_string(),
WORKER_DOWNLOAD_MEDIA => "远程媒体下载".to_string(),
WORKER_NOTIFICATION_DELIVERY => "通知投递".to_string(),
TASK_RETRY_DELIVERIES => "重试待投递通知".to_string(),
@@ -185,6 +189,8 @@ fn label_for(worker_name: &str) -> String {
fn description_for(worker_name: &str) -> String {
match worker_name {
WORKER_AI_REINDEX => "按当前站点内容重新生成 AI 检索索引,并分批写入向量数据。"
.to_string(),
WORKER_DOWNLOAD_MEDIA => "抓取远程图片 / PDF 到媒体库,并回写媒体元数据。".to_string(),
WORKER_NOTIFICATION_DELIVERY => "执行订阅通知、测试通知与 digest 投递。".to_string(),
TASK_RETRY_DELIVERIES => "扫描 retry_pending 的通知记录并重新入队。".to_string(),
@@ -196,6 +202,7 @@ fn description_for(worker_name: &str) -> String {
fn tags_for(worker_name: &str) -> Value {
match worker_name {
WORKER_AI_REINDEX => json!(["ai", "reindex"]),
WORKER_DOWNLOAD_MEDIA => json!(["media", "download"]),
WORKER_NOTIFICATION_DELIVERY => json!(["notifications", "delivery"]),
TASK_RETRY_DELIVERIES => json!(["maintenance", "retry"]),
@@ -249,6 +256,7 @@ fn to_job_record(item: worker_jobs::Model) -> WorkerJobRecord {
fn catalog_entries() -> Vec<WorkerCatalogEntry> {
[
(WORKER_AI_REINDEX, JOB_KIND_WORKER, true, true),
(WORKER_DOWNLOAD_MEDIA, JOB_KIND_WORKER, true, true),
(WORKER_NOTIFICATION_DELIVERY, JOB_KIND_WORKER, true, true),
(TASK_RETRY_DELIVERIES, JOB_KIND_TASK, true, true),
@@ -313,6 +321,13 @@ async fn dispatch_download(args_ctx: AppContext, args: DownloadWorkerArgs) {
}
}
async fn dispatch_ai_reindex(args_ctx: AppContext, args: AiReindexWorkerArgs) {
let worker = AiReindexWorker::build(&args_ctx);
if let Err(error) = worker.perform(args).await {
tracing::warn!("ai reindex worker execution failed: {error}");
}
}
async fn dispatch_notification_delivery(
args_ctx: AppContext,
args: NotificationDeliveryWorkerArgs,
@@ -340,6 +355,21 @@ async fn enqueue_download_worker(ctx: &AppContext, args: DownloadWorkerArgs) ->
}
}
async fn enqueue_ai_reindex_worker(ctx: &AppContext, args: AiReindexWorkerArgs) -> Result<()> {
match AiReindexWorker::perform_later(ctx, args.clone()).await {
Ok(_) => Ok(()),
Err(Error::QueueProviderMissing) => {
tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
Ok(())
}
Err(error) => {
tracing::warn!("ai reindex worker queue unavailable, falling back to local task: {error}");
tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
Ok(())
}
}
}
async fn enqueue_notification_worker(
ctx: &AppContext,
args: NotificationDeliveryWorkerArgs,
@@ -717,6 +747,46 @@ pub async fn queue_notification_delivery_job(
get_job_record(ctx, job.id).await
}
pub async fn queue_ai_reindex_job(
ctx: &AppContext,
requested_by: Option<String>,
requested_source: Option<String>,
parent_job_id: Option<i32>,
trigger_mode: Option<String>,
) -> Result<WorkerJobRecord> {
let base_args = AiReindexWorkerArgs { job_id: None };
let payload = serde_json::to_value(&base_args)?;
let job = create_job(
ctx,
CreateWorkerJobInput {
parent_job_id,
job_kind: JOB_KIND_WORKER.to_string(),
worker_name: WORKER_AI_REINDEX.to_string(),
display_name: Some("重建 AI 索引".to_string()),
queue_name: queue_name_for(WORKER_AI_REINDEX),
requested_by,
requested_source,
trigger_mode,
payload: Some(payload),
tags: Some(tags_for(WORKER_AI_REINDEX)),
related_entity_type: Some("ai_index".to_string()),
related_entity_id: Some("site".to_string()),
max_attempts: 1,
},
)
.await?;
enqueue_ai_reindex_worker(
ctx,
AiReindexWorkerArgs {
job_id: Some(job.id),
},
)
.await?;
get_job_record(ctx, job.id).await
}
pub async fn spawn_retry_deliveries_task(
ctx: &AppContext,
limit: Option<u64>,
@@ -810,6 +880,17 @@ pub async fn retry_job(
let payload = item.payload.clone().unwrap_or(Value::Null);
match item.worker_name.as_str() {
WORKER_AI_REINDEX => {
let _ = serde_json::from_value::<AiReindexWorkerArgs>(payload)?;
queue_ai_reindex_job(
ctx,
requested_by,
requested_source,
Some(item.id),
Some("retry".to_string()),
)
.await
}
WORKER_DOWNLOAD_MEDIA => {
let args = serde_json::from_value::<DownloadWorkerArgs>(payload)?;
queue_download_job(