feat: 添加 AI 索引重建功能,优化相关 API 和工作流,增强内存管理配置
Some checks failed
docker-images / resolve-build-targets (push) Successful in 6s
ui-regression / playwright-regression (push) Successful in 4m43s
docker-images / build-and-push (admin) (push) Successful in 42s
docker-images / submit-indexnow (push) Has been cancelled
docker-images / build-and-push (frontend) (push) Has been cancelled
docker-images / build-and-push (backend) (push) Has started running
Some checks failed
docker-images / resolve-build-targets (push) Successful in 6s
ui-regression / playwright-regression (push) Successful in 4m43s
docker-images / build-and-push (admin) (push) Successful in 42s
docker-images / submit-indexnow (push) Has been cancelled
docker-images / build-and-push (frontend) (push) Has been cancelled
docker-images / build-and-push (backend) (push) Has started running
This commit is contained in:
@@ -11,6 +11,7 @@ use crate::{
|
||||
models::_entities::{notification_deliveries, worker_jobs},
|
||||
services::subscriptions,
|
||||
workers::{
|
||||
ai_reindex::{AiReindexWorker, AiReindexWorkerArgs},
|
||||
downloader::{DownloadWorker, DownloadWorkerArgs},
|
||||
notification_delivery::{NotificationDeliveryWorker, NotificationDeliveryWorkerArgs},
|
||||
},
|
||||
@@ -27,6 +28,7 @@ pub const JOB_STATUS_CANCELLED: &str = "cancelled";
|
||||
|
||||
pub const WORKER_DOWNLOAD_MEDIA: &str = "worker.download_media";
|
||||
pub const WORKER_NOTIFICATION_DELIVERY: &str = "worker.notification_delivery";
|
||||
pub const WORKER_AI_REINDEX: &str = "worker.ai_reindex";
|
||||
pub const TASK_RETRY_DELIVERIES: &str = "task.retry_deliveries";
|
||||
pub const TASK_SEND_WEEKLY_DIGEST: &str = "task.send_weekly_digest";
|
||||
pub const TASK_SEND_MONTHLY_DIGEST: &str = "task.send_monthly_digest";
|
||||
@@ -164,6 +166,7 @@ fn trim_to_option(value: Option<String>) -> Option<String> {
|
||||
|
||||
fn queue_name_for(worker_name: &str) -> Option<String> {
|
||||
match worker_name {
|
||||
WORKER_AI_REINDEX => Some("ai".to_string()),
|
||||
WORKER_DOWNLOAD_MEDIA => Some("media".to_string()),
|
||||
WORKER_NOTIFICATION_DELIVERY => Some("notifications".to_string()),
|
||||
TASK_RETRY_DELIVERIES => Some("maintenance".to_string()),
|
||||
@@ -174,6 +177,7 @@ fn queue_name_for(worker_name: &str) -> Option<String> {
|
||||
|
||||
fn label_for(worker_name: &str) -> String {
|
||||
match worker_name {
|
||||
WORKER_AI_REINDEX => "AI 索引重建".to_string(),
|
||||
WORKER_DOWNLOAD_MEDIA => "远程媒体下载".to_string(),
|
||||
WORKER_NOTIFICATION_DELIVERY => "通知投递".to_string(),
|
||||
TASK_RETRY_DELIVERIES => "重试待投递通知".to_string(),
|
||||
@@ -185,6 +189,8 @@ fn label_for(worker_name: &str) -> String {
|
||||
|
||||
fn description_for(worker_name: &str) -> String {
|
||||
match worker_name {
|
||||
WORKER_AI_REINDEX => "按当前站点内容重新生成 AI 检索索引,并分批写入向量数据。"
|
||||
.to_string(),
|
||||
WORKER_DOWNLOAD_MEDIA => "抓取远程图片 / PDF 到媒体库,并回写媒体元数据。".to_string(),
|
||||
WORKER_NOTIFICATION_DELIVERY => "执行订阅通知、测试通知与 digest 投递。".to_string(),
|
||||
TASK_RETRY_DELIVERIES => "扫描 retry_pending 的通知记录并重新入队。".to_string(),
|
||||
@@ -196,6 +202,7 @@ fn description_for(worker_name: &str) -> String {
|
||||
|
||||
fn tags_for(worker_name: &str) -> Value {
|
||||
match worker_name {
|
||||
WORKER_AI_REINDEX => json!(["ai", "reindex"]),
|
||||
WORKER_DOWNLOAD_MEDIA => json!(["media", "download"]),
|
||||
WORKER_NOTIFICATION_DELIVERY => json!(["notifications", "delivery"]),
|
||||
TASK_RETRY_DELIVERIES => json!(["maintenance", "retry"]),
|
||||
@@ -249,6 +256,7 @@ fn to_job_record(item: worker_jobs::Model) -> WorkerJobRecord {
|
||||
|
||||
fn catalog_entries() -> Vec<WorkerCatalogEntry> {
|
||||
[
|
||||
(WORKER_AI_REINDEX, JOB_KIND_WORKER, true, true),
|
||||
(WORKER_DOWNLOAD_MEDIA, JOB_KIND_WORKER, true, true),
|
||||
(WORKER_NOTIFICATION_DELIVERY, JOB_KIND_WORKER, true, true),
|
||||
(TASK_RETRY_DELIVERIES, JOB_KIND_TASK, true, true),
|
||||
@@ -313,6 +321,13 @@ async fn dispatch_download(args_ctx: AppContext, args: DownloadWorkerArgs) {
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch_ai_reindex(args_ctx: AppContext, args: AiReindexWorkerArgs) {
|
||||
let worker = AiReindexWorker::build(&args_ctx);
|
||||
if let Err(error) = worker.perform(args).await {
|
||||
tracing::warn!("ai reindex worker execution failed: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch_notification_delivery(
|
||||
args_ctx: AppContext,
|
||||
args: NotificationDeliveryWorkerArgs,
|
||||
@@ -340,6 +355,21 @@ async fn enqueue_download_worker(ctx: &AppContext, args: DownloadWorkerArgs) ->
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_ai_reindex_worker(ctx: &AppContext, args: AiReindexWorkerArgs) -> Result<()> {
|
||||
match AiReindexWorker::perform_later(ctx, args.clone()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(Error::QueueProviderMissing) => {
|
||||
tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!("ai reindex worker queue unavailable, falling back to local task: {error}");
|
||||
tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue_notification_worker(
|
||||
ctx: &AppContext,
|
||||
args: NotificationDeliveryWorkerArgs,
|
||||
@@ -717,6 +747,46 @@ pub async fn queue_notification_delivery_job(
|
||||
get_job_record(ctx, job.id).await
|
||||
}
|
||||
|
||||
pub async fn queue_ai_reindex_job(
|
||||
ctx: &AppContext,
|
||||
requested_by: Option<String>,
|
||||
requested_source: Option<String>,
|
||||
parent_job_id: Option<i32>,
|
||||
trigger_mode: Option<String>,
|
||||
) -> Result<WorkerJobRecord> {
|
||||
let base_args = AiReindexWorkerArgs { job_id: None };
|
||||
let payload = serde_json::to_value(&base_args)?;
|
||||
|
||||
let job = create_job(
|
||||
ctx,
|
||||
CreateWorkerJobInput {
|
||||
parent_job_id,
|
||||
job_kind: JOB_KIND_WORKER.to_string(),
|
||||
worker_name: WORKER_AI_REINDEX.to_string(),
|
||||
display_name: Some("重建 AI 索引".to_string()),
|
||||
queue_name: queue_name_for(WORKER_AI_REINDEX),
|
||||
requested_by,
|
||||
requested_source,
|
||||
trigger_mode,
|
||||
payload: Some(payload),
|
||||
tags: Some(tags_for(WORKER_AI_REINDEX)),
|
||||
related_entity_type: Some("ai_index".to_string()),
|
||||
related_entity_id: Some("site".to_string()),
|
||||
max_attempts: 1,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
enqueue_ai_reindex_worker(
|
||||
ctx,
|
||||
AiReindexWorkerArgs {
|
||||
job_id: Some(job.id),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
get_job_record(ctx, job.id).await
|
||||
}
|
||||
|
||||
pub async fn spawn_retry_deliveries_task(
|
||||
ctx: &AppContext,
|
||||
limit: Option<u64>,
|
||||
@@ -810,6 +880,17 @@ pub async fn retry_job(
|
||||
let payload = item.payload.clone().unwrap_or(Value::Null);
|
||||
|
||||
match item.worker_name.as_str() {
|
||||
WORKER_AI_REINDEX => {
|
||||
let _ = serde_json::from_value::<AiReindexWorkerArgs>(payload)?;
|
||||
queue_ai_reindex_job(
|
||||
ctx,
|
||||
requested_by,
|
||||
requested_source,
|
||||
Some(item.id),
|
||||
Some("retry".to_string()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
WORKER_DOWNLOAD_MEDIA => {
|
||||
let args = serde_json::from_value::<DownloadWorkerArgs>(payload)?;
|
||||
queue_download_job(
|
||||
|
||||
Reference in New Issue
Block a user