feat: 添加 AI 索引重建功能，优化相关 API 和工作流，增强内存管理配置

backend/src/app.rs

@@ -28,7 +28,10 @@ use crate::{
         ai_chunks, categories, comments, friend_links, posts, reviews, site_settings, tags, users,
     },
     tasks,
-    workers::{downloader::DownloadWorker, notification_delivery::NotificationDeliveryWorker},
+    workers::{
+        ai_reindex::AiReindexWorker, downloader::DownloadWorker,
+        notification_delivery::NotificationDeliveryWorker,
+    },
 };
 
 pub struct App;
@@ -153,6 +156,7 @@ impl Hooks for App {
         Ok(router.layer(cors))
     }
     async fn connect_workers(ctx: &AppContext, queue: &Queue) -> Result<()> {
+        queue.register(AiReindexWorker::build(ctx)).await?;
         queue.register(DownloadWorker::build(ctx)).await?;
         queue
             .register(NotificationDeliveryWorker::build(ctx))

backend/src/controllers/admin_api.rs

@@ -230,8 +230,8 @@ pub struct AdminSiteSettingsResponse {
 
 #[derive(Clone, Debug, Serialize)]
 pub struct AdminAiReindexResponse {
-    pub indexed_chunks: usize,
-    pub last_indexed_at: Option<String>,
+    pub queued: bool,
+    pub job: worker_jobs::WorkerJobRecord,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -1395,15 +1395,30 @@ pub async fn update_site_settings(
 
 #[debug_handler]
 pub async fn reindex_ai(headers: HeaderMap, State(ctx): State<AppContext>) -> Result<Response> {
-    check_auth(&headers)?;
-    let summary = ai::rebuild_index(&ctx).await?;
+    let actor = check_auth(&headers)?;
+    let job = worker_jobs::queue_ai_reindex_job(
+        &ctx,
+        Some(actor.username.clone()),
+        Some(actor.source.clone()),
+        None,
+        Some("manual".to_string()),
+    )
+    .await?;
+
+    admin_audit::log_event(
+        &ctx,
+        Some(&actor),
+        "worker.ai_reindex",
+        "worker_job",
+        Some(job.id.to_string()),
+        Some(job.worker_name.clone()),
+        None,
+    )
+    .await?;
 
     format::json(AdminAiReindexResponse {
-        indexed_chunks: summary.indexed_chunks,
-        last_indexed_at: format_timestamp(
-            summary.last_indexed_at.map(Into::into),
-            "%Y-%m-%d %H:%M:%S UTC",
-        ),
+        queued: true,
+        job,
     })
 }
 

backend/src/controllers/ai.rs

@@ -16,7 +16,7 @@ use std::time::Instant;
 
 use crate::{
     controllers::{admin::check_auth, site_settings},
-    services::{abuse_guard, ai, analytics},
+    services::{abuse_guard, ai, analytics, worker_jobs},
 };
 
 #[derive(Clone, Debug, Deserialize)]
@@ -35,8 +35,8 @@ pub struct AskResponse {
 
 #[derive(Clone, Debug, Serialize)]
 pub struct ReindexResponse {
-    pub indexed_chunks: usize,
-    pub last_indexed_at: Option<String>,
+    pub queued: bool,
+    pub job: worker_jobs::WorkerJobRecord,
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -514,12 +514,19 @@ pub async fn ask_stream(
 
 #[debug_handler]
 pub async fn reindex(headers: HeaderMap, State(ctx): State<AppContext>) -> Result<Response> {
-    check_auth(&headers)?;
-    let summary = ai::rebuild_index(&ctx).await?;
+    let actor = check_auth(&headers)?;
+    let job = worker_jobs::queue_ai_reindex_job(
+        &ctx,
+        Some(actor.username.clone()),
+        Some(actor.source.clone()),
+        None,
+        Some("manual".to_string()),
+    )
+    .await?;
 
     format::json(ReindexResponse {
-        indexed_chunks: summary.indexed_chunks,
-        last_indexed_at: format_timestamp(summary.last_indexed_at),
+        queued: true,
+        job,
     })
 }
 

backend/src/services/ai.rs

@@ -7,7 +7,7 @@ use loco_rs::prelude::*;
 use reqwest::{Client, Url, header::CONTENT_TYPE, multipart};
 use sea_orm::{
     ActiveModelTrait, ConnectionTrait, DbBackend, EntityTrait, FromQueryResult, IntoActiveModel,
-    PaginatorTrait, QueryOrder, Set, Statement,
+    PaginatorTrait, QueryOrder, Set, Statement, TransactionTrait,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::{Value, json};
@@ -36,6 +36,7 @@ const DEFAULT_TOP_K: usize = 4;
 const DEFAULT_CHUNK_SIZE: usize = 1200;
 const DEFAULT_SYSTEM_PROMPT: &str = "你是这个博客的站内 AI 助手。请严格基于提供的博客上下文回答，优先给出准确结论，再补充细节；如果上下文不足，请明确说明。";
 const EMBEDDING_BATCH_SIZE: usize = 32;
+const REINDEX_EMBEDDING_BATCH_SIZE: usize = 4;
 const EMBEDDING_DIMENSION: usize = 384;
 const LOCAL_EMBEDDING_MODEL_LABEL: &str = "fastembed / local all-MiniLM-L6-v2";
 const LOCAL_EMBEDDING_CACHE_DIR: &str = "storage/ai_embedding_models/all-minilm-l6-v2";
@@ -771,6 +772,14 @@ pub fn default_image_model_for_provider(provider: &str) -> &'static str {
 }
 
 async fn embed_texts_locally(inputs: Vec<String>, kind: EmbeddingKind) -> Result<Vec<Vec<f64>>> {
+    embed_texts_locally_with_batch_size(inputs, kind, EMBEDDING_BATCH_SIZE).await
+}
+
+async fn embed_texts_locally_with_batch_size(
+    inputs: Vec<String>,
+    kind: EmbeddingKind,
+    batch_size: usize,
+) -> Result<Vec<Vec<f64>>> {
     tokio::task::spawn_blocking(move || {
         let model = local_embedding_engine()?;
         let prepared = inputs
@@ -783,7 +792,7 @@ async fn embed_texts_locally(inputs: Vec<String>, kind: EmbeddingKind) -> Result
         })?;
 
         let embeddings = guard
-            .embed(prepared, Some(EMBEDDING_BATCH_SIZE))
+            .embed(prepared, Some(batch_size.max(1)))
             .map_err(|error| Error::BadRequest(format!("本地 embedding 生成失败: {error}")))?;
 
         Ok(embeddings
@@ -2555,14 +2564,14 @@ async fn load_runtime_settings(
     })
 }
 
-async fn update_indexed_at(
-    ctx: &AppContext,
+async fn update_indexed_at<C: ConnectionTrait>(
+    db: &C,
     settings: &site_settings::Model,
 ) -> Result<DateTime<Utc>> {
     let now = Utc::now();
     let mut model = settings.clone().into_active_model();
     model.ai_last_indexed_at = Set(Some(now.into()));
-    let _ = model.update(&ctx.db).await?;
+    let _ = model.update(db).await?;
     Ok(now)
 }
 
@@ -2571,14 +2580,8 @@ async fn retrieve_matches(
     settings: &AiRuntimeSettings,
     question: &str,
 ) -> Result<(Vec<ScoredChunk>, usize, Option<DateTime<Utc>>)> {
-    let mut indexed_chunks = ai_chunks::Entity::find().count(&ctx.db).await? as usize;
-    let mut last_indexed_at = settings.raw.ai_last_indexed_at.map(Into::into);
-
-    if indexed_chunks == 0 {
-        let summary = rebuild_index(ctx).await?;
-        indexed_chunks = summary.indexed_chunks;
-        last_indexed_at = summary.last_indexed_at;
-    }
+    let indexed_chunks = ai_chunks::Entity::find().count(&ctx.db).await? as usize;
+    let last_indexed_at = settings.raw.ai_last_indexed_at.map(Into::into);
 
     if indexed_chunks == 0 {
         return Ok((Vec::new(), 0, last_indexed_at));
@@ -2645,61 +2648,62 @@ pub async fn rebuild_index(ctx: &AppContext) -> Result<AiIndexSummary> {
     let posts = content::load_markdown_posts_from_store(ctx).await?;
     let mut chunk_drafts = build_chunks(&posts, settings.chunk_size);
     chunk_drafts.extend(build_profile_chunks(&settings.raw, settings.chunk_size));
-    let embeddings = if chunk_drafts.is_empty() {
-        Vec::new()
-    } else {
-        embed_texts_locally(
-            chunk_drafts
+    let txn = ctx.db.begin().await?;
+
+    txn.execute(Statement::from_string(
+        DbBackend::Postgres,
+        "TRUNCATE TABLE ai_chunks RESTART IDENTITY".to_string(),
+    ))
+    .await?;
+
+    for chunk_batch in chunk_drafts.chunks(REINDEX_EMBEDDING_BATCH_SIZE.max(1)) {
+        let embeddings = embed_texts_locally_with_batch_size(
+            chunk_batch
                 .iter()
                 .map(|chunk| chunk.content.clone())
                 .collect::<Vec<_>>(),
             EmbeddingKind::Passage,
+            REINDEX_EMBEDDING_BATCH_SIZE,
         )
-        .await?
-    };
-
-    ctx.db
-        .execute(Statement::from_string(
-            DbBackend::Postgres,
-            "TRUNCATE TABLE ai_chunks RESTART IDENTITY".to_string(),
-        ))
         .await?;
 
-    for (draft, embedding) in chunk_drafts.iter().zip(embeddings.into_iter()) {
-        let embedding_literal = vector_literal(&embedding)?;
-        let statement = Statement::from_sql_and_values(
-            DbBackend::Postgres,
-            r#"
-            INSERT INTO ai_chunks (
-                source_slug,
-                source_title,
-                source_path,
-                source_type,
-                chunk_index,
-                content,
-                content_preview,
-                embedding,
-                word_count
-            ) VALUES (
-                $1, $2, $3, $4, $5, $6, $7, $8::vector, $9
-            )
-            "#,
-            vec![
-                draft.source_slug.clone().into(),
-                draft.source_title.clone().into(),
-                draft.source_path.clone().into(),
-                draft.source_type.clone().into(),
-                draft.chunk_index.into(),
-                draft.content.clone().into(),
-                draft.content_preview.clone().into(),
-                embedding_literal.into(),
-                draft.word_count.into(),
-            ],
-        );
-        ctx.db.execute(statement).await?;
+        for (draft, embedding) in chunk_batch.iter().zip(embeddings.into_iter()) {
+            let embedding_literal = vector_literal(&embedding)?;
+            let statement = Statement::from_sql_and_values(
+                DbBackend::Postgres,
+                r#"
+                INSERT INTO ai_chunks (
+                    source_slug,
+                    source_title,
+                    source_path,
+                    source_type,
+                    chunk_index,
+                    content,
+                    content_preview,
+                    embedding,
+                    word_count
+                ) VALUES (
+                    $1, $2, $3, $4, $5, $6, $7, $8::vector, $9
+                )
+                "#,
+                vec![
+                    draft.source_slug.clone().into(),
+                    draft.source_title.clone().into(),
+                    draft.source_path.clone().into(),
+                    draft.source_type.clone().into(),
+                    draft.chunk_index.into(),
+                    draft.content.clone().into(),
+                    draft.content_preview.clone().into(),
+                    embedding_literal.into(),
+                    draft.word_count.into(),
+                ],
+            );
+            txn.execute(statement).await?;
+        }
     }
 
-    let last_indexed_at = update_indexed_at(ctx, &settings.raw).await?;
+    let last_indexed_at = update_indexed_at(&txn, &settings.raw).await?;
+    txn.commit().await?;
 
     Ok(AiIndexSummary {
         indexed_chunks: chunk_drafts.len(),

backend/src/services/worker_jobs.rs

@@ -11,6 +11,7 @@ use crate::{
     models::_entities::{notification_deliveries, worker_jobs},
     services::subscriptions,
     workers::{
+        ai_reindex::{AiReindexWorker, AiReindexWorkerArgs},
         downloader::{DownloadWorker, DownloadWorkerArgs},
         notification_delivery::{NotificationDeliveryWorker, NotificationDeliveryWorkerArgs},
     },
@@ -27,6 +28,7 @@ pub const JOB_STATUS_CANCELLED: &str = "cancelled";
 
 pub const WORKER_DOWNLOAD_MEDIA: &str = "worker.download_media";
 pub const WORKER_NOTIFICATION_DELIVERY: &str = "worker.notification_delivery";
+pub const WORKER_AI_REINDEX: &str = "worker.ai_reindex";
 pub const TASK_RETRY_DELIVERIES: &str = "task.retry_deliveries";
 pub const TASK_SEND_WEEKLY_DIGEST: &str = "task.send_weekly_digest";
 pub const TASK_SEND_MONTHLY_DIGEST: &str = "task.send_monthly_digest";
@@ -164,6 +166,7 @@ fn trim_to_option(value: Option<String>) -> Option<String> {
 
 fn queue_name_for(worker_name: &str) -> Option<String> {
     match worker_name {
+        WORKER_AI_REINDEX => Some("ai".to_string()),
         WORKER_DOWNLOAD_MEDIA => Some("media".to_string()),
         WORKER_NOTIFICATION_DELIVERY => Some("notifications".to_string()),
         TASK_RETRY_DELIVERIES => Some("maintenance".to_string()),
@@ -174,6 +177,7 @@ fn queue_name_for(worker_name: &str) -> Option<String> {
 
 fn label_for(worker_name: &str) -> String {
     match worker_name {
+        WORKER_AI_REINDEX => "AI 索引重建".to_string(),
         WORKER_DOWNLOAD_MEDIA => "远程媒体下载".to_string(),
         WORKER_NOTIFICATION_DELIVERY => "通知投递".to_string(),
         TASK_RETRY_DELIVERIES => "重试待投递通知".to_string(),
@@ -185,6 +189,8 @@ fn label_for(worker_name: &str) -> String {
 
 fn description_for(worker_name: &str) -> String {
     match worker_name {
+        WORKER_AI_REINDEX => "按当前站点内容重新生成 AI 检索索引，并分批写入向量数据。"
+            .to_string(),
         WORKER_DOWNLOAD_MEDIA => "抓取远程图片 / PDF 到媒体库，并回写媒体元数据。".to_string(),
         WORKER_NOTIFICATION_DELIVERY => "执行订阅通知、测试通知与 digest 投递。".to_string(),
         TASK_RETRY_DELIVERIES => "扫描 retry_pending 的通知记录并重新入队。".to_string(),
@@ -196,6 +202,7 @@ fn description_for(worker_name: &str) -> String {
 
 fn tags_for(worker_name: &str) -> Value {
     match worker_name {
+        WORKER_AI_REINDEX => json!(["ai", "reindex"]),
         WORKER_DOWNLOAD_MEDIA => json!(["media", "download"]),
         WORKER_NOTIFICATION_DELIVERY => json!(["notifications", "delivery"]),
         TASK_RETRY_DELIVERIES => json!(["maintenance", "retry"]),
@@ -249,6 +256,7 @@ fn to_job_record(item: worker_jobs::Model) -> WorkerJobRecord {
 
 fn catalog_entries() -> Vec<WorkerCatalogEntry> {
     [
+        (WORKER_AI_REINDEX, JOB_KIND_WORKER, true, true),
         (WORKER_DOWNLOAD_MEDIA, JOB_KIND_WORKER, true, true),
         (WORKER_NOTIFICATION_DELIVERY, JOB_KIND_WORKER, true, true),
         (TASK_RETRY_DELIVERIES, JOB_KIND_TASK, true, true),
@@ -313,6 +321,13 @@ async fn dispatch_download(args_ctx: AppContext, args: DownloadWorkerArgs) {
     }
 }
 
+async fn dispatch_ai_reindex(args_ctx: AppContext, args: AiReindexWorkerArgs) {
+    let worker = AiReindexWorker::build(&args_ctx);
+    if let Err(error) = worker.perform(args).await {
+        tracing::warn!("ai reindex worker execution failed: {error}");
+    }
+}
+
 async fn dispatch_notification_delivery(
     args_ctx: AppContext,
     args: NotificationDeliveryWorkerArgs,
@@ -340,6 +355,21 @@ async fn enqueue_download_worker(ctx: &AppContext, args: DownloadWorkerArgs) ->
     }
 }
 
+async fn enqueue_ai_reindex_worker(ctx: &AppContext, args: AiReindexWorkerArgs) -> Result<()> {
+    match AiReindexWorker::perform_later(ctx, args.clone()).await {
+        Ok(_) => Ok(()),
+        Err(Error::QueueProviderMissing) => {
+            tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
+            Ok(())
+        }
+        Err(error) => {
+            tracing::warn!("ai reindex worker queue unavailable, falling back to local task: {error}");
+            tokio::spawn(dispatch_ai_reindex(ctx.clone(), args));
+            Ok(())
+        }
+    }
+}
+
 async fn enqueue_notification_worker(
     ctx: &AppContext,
     args: NotificationDeliveryWorkerArgs,
@@ -717,6 +747,46 @@ pub async fn queue_notification_delivery_job(
     get_job_record(ctx, job.id).await
 }
 
+pub async fn queue_ai_reindex_job(
+    ctx: &AppContext,
+    requested_by: Option<String>,
+    requested_source: Option<String>,
+    parent_job_id: Option<i32>,
+    trigger_mode: Option<String>,
+) -> Result<WorkerJobRecord> {
+    let base_args = AiReindexWorkerArgs { job_id: None };
+    let payload = serde_json::to_value(&base_args)?;
+
+    let job = create_job(
+        ctx,
+        CreateWorkerJobInput {
+            parent_job_id,
+            job_kind: JOB_KIND_WORKER.to_string(),
+            worker_name: WORKER_AI_REINDEX.to_string(),
+            display_name: Some("重建 AI 索引".to_string()),
+            queue_name: queue_name_for(WORKER_AI_REINDEX),
+            requested_by,
+            requested_source,
+            trigger_mode,
+            payload: Some(payload),
+            tags: Some(tags_for(WORKER_AI_REINDEX)),
+            related_entity_type: Some("ai_index".to_string()),
+            related_entity_id: Some("site".to_string()),
+            max_attempts: 1,
+        },
+    )
+    .await?;
+
+    enqueue_ai_reindex_worker(
+        ctx,
+        AiReindexWorkerArgs {
+            job_id: Some(job.id),
+        },
+    )
+    .await?;
+    get_job_record(ctx, job.id).await
+}
+
 pub async fn spawn_retry_deliveries_task(
     ctx: &AppContext,
     limit: Option<u64>,
@@ -810,6 +880,17 @@ pub async fn retry_job(
     let payload = item.payload.clone().unwrap_or(Value::Null);
 
     match item.worker_name.as_str() {
+        WORKER_AI_REINDEX => {
+            let _ = serde_json::from_value::<AiReindexWorkerArgs>(payload)?;
+            queue_ai_reindex_job(
+                ctx,
+                requested_by,
+                requested_source,
+                Some(item.id),
+                Some("retry".to_string()),
+            )
+            .await
+        }
         WORKER_DOWNLOAD_MEDIA => {
             let args = serde_json::from_value::<DownloadWorkerArgs>(payload)?;
             queue_download_job(

backend/src/workers/ai_reindex.rs

@@ -0,0 +1,55 @@
+use loco_rs::prelude::*;
+use serde::{Deserialize, Serialize};
+
+use crate::services::{ai, worker_jobs};
+
+pub struct AiReindexWorker {
+    pub ctx: AppContext,
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+pub struct AiReindexWorkerArgs {
+    #[serde(default)]
+    pub job_id: Option<i32>,
+}
+
+#[async_trait]
+impl BackgroundWorker<AiReindexWorkerArgs> for AiReindexWorker {
+    fn build(ctx: &AppContext) -> Self {
+        Self { ctx: ctx.clone() }
+    }
+
+    fn tags() -> Vec<String> {
+        vec!["ai".to_string(), "reindex".to_string()]
+    }
+
+    async fn perform(&self, args: AiReindexWorkerArgs) -> Result<()> {
+        if let Some(job_id) = args.job_id {
+            if !worker_jobs::begin_job_execution(&self.ctx, job_id).await? {
+                return Ok(());
+            }
+
+            match ai::rebuild_index(&self.ctx).await {
+                Ok(summary) => {
+                    worker_jobs::mark_job_succeeded(
+                        &self.ctx,
+                        job_id,
+                        Some(serde_json::json!({
+                            "indexed_chunks": summary.indexed_chunks,
+                            "last_indexed_at": summary.last_indexed_at.map(|value| value.to_rfc3339()),
+                        })),
+                    )
+                    .await?;
+                    Ok(())
+                }
+                Err(error) => {
+                    worker_jobs::mark_job_failed(&self.ctx, job_id, error.to_string()).await?;
+                    Err(error)
+                }
+            }
+        } else {
+            ai::rebuild_index(&self.ctx).await?;
+            Ok(())
+        }
+    }
+}

backend/src/workers/mod.rs

@@ -1,2 +1,3 @@
+pub mod ai_reindex;
 pub mod downloader;
 pub mod notification_delivery;