From 8c38f1c5a49da38c5ca3a87ce3b033a0b45c2a86 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 30 Oct 2026 14:10:03 -0700 Subject: [PATCH 1/4] fix(code-index): isolate parsing in a terminable workspace worker --- knip.json | 2 +- src/esbuild.mjs | 13 +- src/i18n/locales/ca/embeddings.json | 1 + src/i18n/locales/de/embeddings.json | 1 + src/i18n/locales/en/embeddings.json | 1 + src/i18n/locales/es/embeddings.json | 1 + src/i18n/locales/fr/embeddings.json | 1 + src/i18n/locales/hi/embeddings.json | 1 + src/i18n/locales/id/embeddings.json | 1 + src/i18n/locales/it/embeddings.json | 1 + src/i18n/locales/ja/embeddings.json | 1 + src/i18n/locales/ko/embeddings.json | 1 + src/i18n/locales/nl/embeddings.json | 1 + src/i18n/locales/pl/embeddings.json | 1 + src/i18n/locales/pt-BR/embeddings.json | 1 + src/i18n/locales/ru/embeddings.json | 1 + src/i18n/locales/tr/embeddings.json | 1 + src/i18n/locales/vi/embeddings.json | 1 + src/i18n/locales/zh-CN/embeddings.json | 1 + src/i18n/locales/zh-TW/embeddings.json | 1 + .../__tests__/index-write-gate.spec.ts | 48 +++++ .../code-index/__tests__/manager.spec.ts | 71 +++++++ .../code-index/__tests__/orchestrator.spec.ts | 75 +++++++ .../__tests__/parse-worker-client.spec.ts | 153 ++++++++++++++ src/services/code-index/constants/index.ts | 2 + .../embedders/__tests__/openai.spec.ts | 57 +++-- src/services/code-index/embedders/openai.ts | 19 +- src/services/code-index/index-write-gate.ts | 51 +++++ .../code-index/interfaces/embedder.ts | 2 +- .../code-index/interfaces/file-processor.ts | 5 +- src/services/code-index/manager.ts | 62 +++++- src/services/code-index/orchestrator.ts | 101 ++++++--- .../code-index/parse-worker-client.ts | 165 +++++++++++++++ src/services/code-index/parse-worker.ts | 22 ++ src/services/code-index/parser-errors.ts | 22 ++ .../processors/__tests__/file-watcher.spec.ts | 100 +++++++++ .../processors/__tests__/parser.spec.ts | 36 ++++ .../processors/__tests__/scanner.spec.ts | 135 +++++++++++- .../code-index/processors/file-watcher.ts | 170 +++++++++------ src/services/code-index/processors/parser.ts | 194 ++++++++++-------- src/services/code-index/processors/scanner.ts | 110 ++++++---- src/services/code-index/service-factory.ts | 23 ++- .../__tests__/qdrant-client.spec.ts | 7 + .../code-index/vector-store/qdrant-client.ts | 1 + src/services/tree-sitter/languageParser.ts | 12 +- 45 files changed, 1411 insertions(+), 265 deletions(-) create mode 100644 src/services/code-index/__tests__/index-write-gate.spec.ts create mode 100644 src/services/code-index/__tests__/parse-worker-client.spec.ts create mode 100644 src/services/code-index/index-write-gate.ts create mode 100644 src/services/code-index/parse-worker-client.ts create mode 100644 src/services/code-index/parse-worker.ts create mode 100644 src/services/code-index/parser-errors.ts diff --git a/knip.json b/knip.json index 4c63f171cc4..d71404708a0 100644 --- a/knip.json +++ b/knip.json @@ -12,7 +12,7 @@ ], "workspaces": { "src": { - "entry": ["extension.ts"], + "entry": ["extension.ts", "services/code-index/parse-worker.ts"], "project": ["**/*.ts"] }, "webview-ui": { diff --git a/src/esbuild.mjs b/src/esbuild.mjs index 6089ac3306d..03af536c1d1 100644 --- a/src/esbuild.mjs +++ b/src/esbuild.mjs @@ -133,18 +133,23 @@ async function main() { outdir: "dist/workers", } - const [extensionCtx, workerCtx] = await Promise.all([ + const [extensionCtx, workerCtx, parserCtx] = await Promise.all([ esbuild.context(extensionConfig), esbuild.context(workerConfig), + esbuild.context({ + ...buildOptions, + entryPoints: ["services/code-index/parse-worker.ts"], + outfile: "dist/parse-worker.js", + }), ]) if (watch) { - await Promise.all([extensionCtx.watch(), workerCtx.watch()]) + await Promise.all([extensionCtx.watch(), workerCtx.watch(), parserCtx.watch()]) copyLocales(srcDir, distDir) setupLocaleWatcher(srcDir, distDir) } else { - await Promise.all([extensionCtx.rebuild(), workerCtx.rebuild()]) - await Promise.all([extensionCtx.dispose(), workerCtx.dispose()]) + await Promise.all([extensionCtx.rebuild(), workerCtx.rebuild(), parserCtx.rebuild()]) + await Promise.all([extensionCtx.dispose(), workerCtx.dispose(), parserCtx.dispose()]) } } diff --git a/src/i18n/locales/ca/embeddings.json b/src/i18n/locales/ca/embeddings.json index 9ceec7d05c3..ed812e9baa7 100644 --- a/src/i18n/locales/ca/embeddings.json +++ b/src/i18n/locales/ca/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "No es poden crear serveis: La indexació de codi no està configurada correctament" }, "orchestrator": { + "parserFailed": "La indexació de codi s’ha aturat en analitzar {{filePath}} ({{category}}). Corregeix el fitxer i inicia la indexació explícitament per tornar-ho a provar, o desactiva-la. No és un error del model o proveïdor.", "indexingFailedNoBlocks": "Indexació fallida: No s'ha indexat cap bloc de codi amb èxit. Això normalment indica un problema de configuració de l'embedder.", "indexingFailedCritical": "Indexació fallida: No s'ha indexat cap bloc de codi amb èxit malgrat trobar fitxers per processar. Això indica una fallida crítica de l'embedder.", "fileWatcherStarted": "Monitor de fitxers iniciat.", diff --git a/src/i18n/locales/de/embeddings.json b/src/i18n/locales/de/embeddings.json index 766d31d5ba0..edf2efd4c0e 100644 --- a/src/i18n/locales/de/embeddings.json +++ b/src/i18n/locales/de/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Kann keine Dienste erstellen: Code-Indizierung ist nicht richtig konfiguriert" }, "orchestrator": { + "parserFailed": "Die Code-Indexierung wurde beim Parsen von {{filePath}} angehalten ({{category}}). Korrigiere die Datei und starte die Indexierung ausdrücklich erneut oder deaktiviere sie. Dies ist kein Modell-/Anbieterfehler.", "indexingFailedNoBlocks": "Indizierung fehlgeschlagen: Keine Code-Blöcke wurden erfolgreich indiziert. Dies deutet normalerweise auf ein Embedder-Konfigurationsproblem hin.", "indexingFailedCritical": "Indizierung fehlgeschlagen: Keine Code-Blöcke wurden erfolgreich indiziert, obwohl zu verarbeitende Dateien gefunden wurden. Dies deutet auf einen kritischen Embedder-Fehler hin.", "fileWatcherStarted": "Datei-Watcher gestartet.", diff --git a/src/i18n/locales/en/embeddings.json b/src/i18n/locales/en/embeddings.json index 7777af9027e..a7ba0ca5a8b 100644 --- a/src/i18n/locales/en/embeddings.json +++ b/src/i18n/locales/en/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Cannot create services: Code indexing is not properly configured" }, "orchestrator": { + "parserFailed": "Code indexing stopped while parsing {{filePath}} ({{category}}). Fix the file and explicitly start indexing to retry, or disable indexing. This is not a model/provider error.", "indexingFailedNoBlocks": "Indexing failed: No code blocks were successfully indexed. This usually indicates an embedder configuration issue.", "indexingFailedCritical": "Indexing failed: No code blocks were successfully indexed despite finding files to process. This indicates a critical embedder failure.", "fileWatcherStarted": "File watcher started.", diff --git a/src/i18n/locales/es/embeddings.json b/src/i18n/locales/es/embeddings.json index 930404de1f5..442a58e39da 100644 --- a/src/i18n/locales/es/embeddings.json +++ b/src/i18n/locales/es/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "No se pueden crear servicios: La indexación de código no está configurada correctamente" }, "orchestrator": { + "parserFailed": "La indexación de código se detuvo al analizar {{filePath}} ({{category}}). Corrige el archivo e inicia la indexación explícitamente para reintentar, o desactívala. No es un error del modelo o proveedor.", "indexingFailedNoBlocks": "Indexación fallida: No se indexaron exitosamente bloques de código. Esto usualmente indica un problema de configuración del incrustador.", "indexingFailedCritical": "Indexación fallida: No se indexaron exitosamente bloques de código a pesar de encontrar archivos para procesar. Esto indica una falla crítica del incrustador.", "fileWatcherStarted": "Monitor de archivos iniciado.", diff --git a/src/i18n/locales/fr/embeddings.json b/src/i18n/locales/fr/embeddings.json index 7de086307ea..81c6ce71d69 100644 --- a/src/i18n/locales/fr/embeddings.json +++ b/src/i18n/locales/fr/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Impossible de créer les services : L'indexation du code n'est pas correctement configurée" }, "orchestrator": { + "parserFailed": "L’indexation du code s’est arrêtée pendant l’analyse de {{filePath}} ({{category}}). Corrige le fichier et relance explicitement l’indexation, ou désactive-la. Ce n’est pas une erreur de modèle ou de fournisseur.", "indexingFailedNoBlocks": "Échec de l'indexation : Aucun bloc de code n'a été indexé avec succès. Cela indique généralement un problème de configuration de l'embedder.", "indexingFailedCritical": "Échec de l'indexation : Aucun bloc de code n'a été indexé avec succès malgré la découverte de fichiers à traiter. Cela indique une défaillance critique de l'embedder.", "fileWatcherStarted": "Surveillant de fichiers démarré.", diff --git a/src/i18n/locales/hi/embeddings.json b/src/i18n/locales/hi/embeddings.json index 9c7f9ca50ae..e0c4498a45a 100644 --- a/src/i18n/locales/hi/embeddings.json +++ b/src/i18n/locales/hi/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "सेवाएं नहीं बना सकते: कोड इंडेक्सिंग ठीक से कॉन्फ़िगर नहीं है" }, "orchestrator": { + "parserFailed": "{{filePath}} को पार्स करते समय कोड इंडेक्सिंग रुक गई ({{category}})। फ़ाइल ठीक करके दोबारा कोशिश करने के लिए इंडेक्सिंग खुद शुरू करो, या इसे बंद करो। यह मॉडल/प्रदाता की त्रुटि नहीं है.", "indexingFailedNoBlocks": "इंडेक्सिंग असफल: कोई भी कोड ब्लॉक सफलतापूर्वक इंडेक्स नहीं हुआ। यह आमतौर पर एम्बेडर कॉन्फ़िगरेशन समस्या को दर्शाता है।", "indexingFailedCritical": "इंडेक्सिंग असफल: प्रोसेस करने के लिए फाइलें मिलने के बावजूद कोई भी कोड ब्लॉक सफलतापूर्वक इंडेक्स नहीं हुआ। यह एक गंभीर एम्बेडर विफलता को दर्शाता है।", "fileWatcherStarted": "फाइल वॉचर शुरू हुआ।", diff --git a/src/i18n/locales/id/embeddings.json b/src/i18n/locales/id/embeddings.json index 955a039effe..449ab9388da 100644 --- a/src/i18n/locales/id/embeddings.json +++ b/src/i18n/locales/id/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Tidak dapat membuat layanan: Pengindeksan kode tidak dikonfigurasi dengan benar" }, "orchestrator": { + "parserFailed": "Pengindeksan kode berhenti saat mengurai {{filePath}} ({{category}}). Perbaiki file lalu mulai pengindeksan secara manual untuk mencoba lagi, atau nonaktifkan pengindeksan. Ini bukan kesalahan model/penyedia.", "indexingFailedNoBlocks": "Pengindeksan gagal: Tidak ada blok kode yang berhasil diindeks. Ini biasanya menunjukkan masalah konfigurasi embedder.", "indexingFailedCritical": "Pengindeksan gagal: Tidak ada blok kode yang berhasil diindeks meskipun menemukan file untuk diproses. Ini menunjukkan kegagalan kritis embedder.", "fileWatcherStarted": "Pemantau file dimulai.", diff --git a/src/i18n/locales/it/embeddings.json b/src/i18n/locales/it/embeddings.json index b7314c244dc..f0d16ff7c5e 100644 --- a/src/i18n/locales/it/embeddings.json +++ b/src/i18n/locales/it/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Impossibile creare i servizi: L'indicizzazione del codice non è configurata correttamente" }, "orchestrator": { + "parserFailed": "L’indicizzazione del codice si è interrotta durante l’analisi di {{filePath}} ({{category}}). Correggi il file e avvia esplicitamente l’indicizzazione per riprovare, oppure disattivala. Non è un errore del modello o del provider.", "indexingFailedNoBlocks": "Indicizzazione fallita: Nessun blocco di codice è stato indicizzato con successo. Questo di solito indica un problema di configurazione dell'embedder.", "indexingFailedCritical": "Indicizzazione fallita: Nessun blocco di codice è stato indicizzato con successo nonostante siano stati trovati file da elaborare. Questo indica un errore critico dell'embedder.", "fileWatcherStarted": "Monitoraggio file avviato.", diff --git a/src/i18n/locales/ja/embeddings.json b/src/i18n/locales/ja/embeddings.json index ce7150cf1ca..bdcac2a63d5 100644 --- a/src/i18n/locales/ja/embeddings.json +++ b/src/i18n/locales/ja/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "サービスを作成できません: コードインデックスが正しく設定されていません" }, "orchestrator": { + "parserFailed": "{{filePath}} の解析中にコードのインデックス作成が停止しました（{{category}}）。ファイルを修正してインデックス作成を手動で再開するか、無効にしてください。モデルやプロバイダーのエラーではありません。", "indexingFailedNoBlocks": "インデックス作成に失敗しました：コードブロックが正常にインデックス化されませんでした。これは通常、エンベッダーの設定問題を示しています。", "indexingFailedCritical": "インデックス作成に失敗しました：処理するファイルが見つかったにもかかわらず、コードブロックが正常にインデックス化されませんでした。これは重大なエンベッダーの障害を示しています。", "fileWatcherStarted": "ファイルウォッチャーが開始されました。", diff --git a/src/i18n/locales/ko/embeddings.json b/src/i18n/locales/ko/embeddings.json index 436fa985c02..2b737c2c974 100644 --- a/src/i18n/locales/ko/embeddings.json +++ b/src/i18n/locales/ko/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "서비스를 생성할 수 없습니다: 코드 인덱싱이 올바르게 구성되지 않았습니다" }, "orchestrator": { + "parserFailed": "{{filePath}} 구문 분석 중 코드 인덱싱이 중지되었습니다({{category}}). 파일을 수정하고 인덱싱을 직접 시작해 다시 시도하거나 인덱싱을 비활성화하세요. 모델/공급자 오류가 아닙니다.", "indexingFailedNoBlocks": "인덱싱 실패: 코드 블록이 성공적으로 인덱싱되지 않았습니다. 이는 일반적으로 임베더 구성 문제를 나타냅니다.", "indexingFailedCritical": "인덱싱 실패: 처리할 파일을 찾았음에도 불구하고 코드 블록이 성공적으로 인덱싱되지 않았습니다. 이는 중요한 임베더 오류를 나타냅니다.", "fileWatcherStarted": "파일 감시자가 시작되었습니다.", diff --git a/src/i18n/locales/nl/embeddings.json b/src/i18n/locales/nl/embeddings.json index 01e68683d3a..5f04da3f3ed 100644 --- a/src/i18n/locales/nl/embeddings.json +++ b/src/i18n/locales/nl/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Kan geen services maken: Code-indexering is niet correct geconfigureerd" }, "orchestrator": { + "parserFailed": "Code-indexering is gestopt tijdens het parseren van {{filePath}} ({{category}}). Herstel het bestand en start de indexering zelf opnieuw, of schakel indexering uit. Dit is geen model-/providerfout.", "indexingFailedNoBlocks": "Indexering mislukt: Geen codeblokken werden succesvol geïndexeerd. Dit duidt meestal op een embedder configuratieprobleem.", "indexingFailedCritical": "Indexering mislukt: Geen codeblokken werden succesvol geïndexeerd ondanks het vinden van bestanden om te verwerken. Dit duidt op een kritieke embedder fout.", "fileWatcherStarted": "Bestandsmonitor gestart.", diff --git a/src/i18n/locales/pl/embeddings.json b/src/i18n/locales/pl/embeddings.json index 0ef846b2cc9..68b368d17c4 100644 --- a/src/i18n/locales/pl/embeddings.json +++ b/src/i18n/locales/pl/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Nie można utworzyć usług: Indeksowanie kodu nie jest poprawnie skonfigurowane" }, "orchestrator": { + "parserFailed": "Indeksowanie kodu zatrzymało się podczas parsowania {{filePath}} ({{category}}). Popraw plik i ręcznie uruchom indeksowanie ponownie albo je wyłącz. To nie jest błąd modelu ani dostawcy.", "indexingFailedNoBlocks": "Indeksowanie nie powiodło się: Żadne bloki kodu nie zostały pomyślnie zaindeksowane. To zwykle wskazuje na problem z konfiguracją embeddera.", "indexingFailedCritical": "Indeksowanie nie powiodło się: Żadne bloki kodu nie zostały pomyślnie zaindeksowane pomimo znalezienia plików do przetworzenia. To wskazuje na krytyczny błąd embeddera.", "fileWatcherStarted": "Monitor plików uruchomiony.", diff --git a/src/i18n/locales/pt-BR/embeddings.json b/src/i18n/locales/pt-BR/embeddings.json index 9cdf775e76e..fe3deed35bc 100644 --- a/src/i18n/locales/pt-BR/embeddings.json +++ b/src/i18n/locales/pt-BR/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Não é possível criar serviços: A indexação de código não está configurada corretamente" }, "orchestrator": { + "parserFailed": "A indexação de código parou ao analisar {{filePath}} ({{category}}). Corrige o arquivo e inicia a indexação explicitamente para tentar de novo, ou desativa a indexação. Não é um erro do modelo/provedor.", "indexingFailedNoBlocks": "Indexação falhou: Nenhum bloco de código foi indexado com sucesso. Isso geralmente indica um problema de configuração do embedder.", "indexingFailedCritical": "Indexação falhou: Nenhum bloco de código foi indexado com sucesso apesar de encontrar arquivos para processar. Isso indica uma falha crítica do embedder.", "fileWatcherStarted": "Monitor de arquivos iniciado.", diff --git a/src/i18n/locales/ru/embeddings.json b/src/i18n/locales/ru/embeddings.json index 873b1c06308..86eed7d6c58 100644 --- a/src/i18n/locales/ru/embeddings.json +++ b/src/i18n/locales/ru/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Невозможно создать сервисы: Индексация кода не настроена должным образом" }, "orchestrator": { + "parserFailed": "Индексация кода остановлена при разборе {{filePath}} ({{category}}). Исправь файл и явно запусти индексацию снова или отключи её. Это не ошибка модели или провайдера.", "indexingFailedNoBlocks": "Индексация не удалась: Ни один блок кода не был успешно проиндексирован. Это обычно указывает на проблему конфигурации эмбеддера.", "indexingFailedCritical": "Индексация не удалась: Ни один блок кода не был успешно проиндексирован, несмотря на обнаружение файлов для обработки. Это указывает на критическую ошибку эмбеддера.", "fileWatcherStarted": "Наблюдатель файлов запущен.", diff --git a/src/i18n/locales/tr/embeddings.json b/src/i18n/locales/tr/embeddings.json index 30b703a93f1..4cac4a791b8 100644 --- a/src/i18n/locales/tr/embeddings.json +++ b/src/i18n/locales/tr/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Hizmetler oluşturulamıyor: Kod indeksleme düzgün yapılandırılmamış" }, "orchestrator": { + "parserFailed": "{{filePath}} ayrıştırılırken kod dizinleme durdu ({{category}}). Dosyayı düzeltip yeniden denemek için dizinlemeyi elle başlat veya dizinlemeyi devre dışı bırak. Bu bir model/sağlayıcı hatası değil.", "indexingFailedNoBlocks": "İndeksleme başarısız: Hiçbir kod bloğu başarıyla indekslenemedi. Bu genellikle bir embedder yapılandırma sorunu olduğunu gösterir.", "indexingFailedCritical": "İndeksleme başarısız: İşlenecek dosyalar bulunmasına rağmen hiçbir kod bloğu başarıyla indekslenemedi. Bu kritik bir embedder hatası olduğunu gösterir.", "fileWatcherStarted": "Dosya izleyici başlatıldı.", diff --git a/src/i18n/locales/vi/embeddings.json b/src/i18n/locales/vi/embeddings.json index c92ebba2765..3bd9d584d20 100644 --- a/src/i18n/locales/vi/embeddings.json +++ b/src/i18n/locales/vi/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "Không thể tạo dịch vụ: Lập chỉ mục mã không được cấu hình đúng cách" }, "orchestrator": { + "parserFailed": "Lập chỉ mục mã đã dừng khi phân tích {{filePath}} ({{category}}). Sửa tệp rồi tự khởi động lại việc lập chỉ mục để thử lại, hoặc tắt lập chỉ mục. Đây không phải lỗi mô hình/nhà cung cấp.", "indexingFailedNoBlocks": "Lập chỉ mục thất bại: Không có khối mã nào được lập chỉ mục thành công. Điều này thường cho thấy vấn đề cấu hình embedder.", "indexingFailedCritical": "Lập chỉ mục thất bại: Không có khối mã nào được lập chỉ mục thành công mặc dù đã tìm thấy tệp để xử lý. Điều này cho thấy lỗi nghiêm trọng của embedder.", "fileWatcherStarted": "Trình theo dõi tệp đã khởi động.", diff --git a/src/i18n/locales/zh-CN/embeddings.json b/src/i18n/locales/zh-CN/embeddings.json index b4f4eaad1d1..00ea2b6854c 100644 --- a/src/i18n/locales/zh-CN/embeddings.json +++ b/src/i18n/locales/zh-CN/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "无法创建服务：代码索引未正确配置" }, "orchestrator": { + "parserFailed": "解析 {{filePath}} 时，代码索引已停止（{{category}}）。请修复文件并手动启动索引以重试，或禁用索引。这不是模型或提供商错误。", "indexingFailedNoBlocks": "索引失败：没有代码块被成功索引。这通常表示 Embedder 配置问题。", "indexingFailedCritical": "索引失败：尽管找到了要处理的文件，但没有代码块被成功索引。这表示 Embedder 出现严重故障。", "fileWatcherStarted": "文件监控已启动。", diff --git a/src/i18n/locales/zh-TW/embeddings.json b/src/i18n/locales/zh-TW/embeddings.json index 26845ed9488..cd0f5e19b2d 100644 --- a/src/i18n/locales/zh-TW/embeddings.json +++ b/src/i18n/locales/zh-TW/embeddings.json @@ -63,6 +63,7 @@ "codeIndexingNotConfigured": "無法建立服務：程式碼索引未正確設定" }, "orchestrator": { + "parserFailed": "剖析 {{filePath}} 時，程式碼索引已停止（{{category}}）。請修正檔案並手動啟動索引以重試，或停用索引。這不是模型或供應商錯誤。", "indexingFailedNoBlocks": "索引失敗：沒有程式碼區塊被成功索引。這通常表示 Embedder 設定問題。", "indexingFailedCritical": "索引失敗：儘管找到了要處理的檔案，但沒有程式碼區塊被成功索引。這表示 Embedder 出現嚴重故障。", "fileWatcherStarted": "檔案監控已啟動。", diff --git a/src/services/code-index/__tests__/index-write-gate.spec.ts b/src/services/code-index/__tests__/index-write-gate.spec.ts new file mode 100644 index 00000000000..2e771bf87ce --- /dev/null +++ b/src/services/code-index/__tests__/index-write-gate.spec.ts @@ -0,0 +1,48 @@ +import { IndexWriteGate } from "../index-write-gate" + +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} + +describe("IndexWriteGate", () => { + it("refuses late embedding writes and all subsequent mutations after abort", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + attempt.abort() + const write = vi.fn() + await expect(gate.run(attempt.signal, write, ["file.ts"])).rejects.toMatchObject({ name: "AbortError" }) + expect(write).not.toHaveBeenCalled() + }) + + it.each(["resolve", "reject"] as const)( + "orders a retry behind an issued old write that will %s", + async (outcome) => { + const gate = new IndexWriteGate() + const oldAttempt = new AbortController() + const nextAttempt = new AbortController() + const held = deferred() + const oldWrite = vi.fn(() => held.promise) + const pending = gate.run(oldAttempt.signal, oldWrite, ["file.ts"]).catch((error) => error) + await vi.waitFor(() => expect(oldWrite).toHaveBeenCalled()) + oldAttempt.abort() + const newWrite = vi.fn() + const retry = gate.run(nextAttempt.signal, newWrite) + await Promise.resolve() + expect(newWrite).not.toHaveBeenCalled() + if (outcome === "resolve") held.resolve() + else held.reject(new Error("request timeout")) + await pending + await retry + expect(newWrite).toHaveBeenCalledOnce() + expect(gate.needsReconciliation("file.ts")).toBe(outcome === "reject") + gate.reconciled("file.ts") + expect(gate.needsReconciliation("file.ts")).toBe(false) + }, + ) +}) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 0a145c4a096..b7cda907dfa 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -153,6 +153,77 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { CodeIndexManager.disposeAll() }) + describe("parser failure lifecycle", () => { + it("blocks automatic initialize and settings recreation while latched", async () => { + const internal = manager as any + internal.parserFailed = true + internal.parserFailureMessage = "Parsing bad.ts failed; retry or disable indexing" + internal._configManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), + isFeatureEnabled: true, + isFeatureConfigured: true, + } + const recreate = vi.spyOn(internal, "_recreateServices") + await manager.initialize({} as any) + await manager.handleSettingsChange() + expect(recreate).not.toHaveBeenCalled() + expect(internal._stateManager.setSystemState).toHaveBeenCalledWith("Error", internal.parserFailureMessage) + }) + + it.each(["fatal", "cancel", "disable"])( + "replaces a stopped service set on explicit restart: %s", + async (reason) => { + const internal = manager as any + internal._configManager = { isFeatureEnabled: true, isFeatureConfigured: true } + internal._cacheManager = {} + internal._searchService = {} + internal._stateManager.getCurrentStatus.mockReturnValue({ systemStatus: "Standby" }) + const old = { startIndexing: vi.fn(), stopIndexing: vi.fn() } + internal._orchestrator = old + if (reason === "fatal") { + internal.parserFailed = true + internal.servicesStopped = true + } else if (reason === "cancel") manager.stopIndexing() + else { + await manager.setWorkspaceEnabled(false) + await manager.setWorkspaceEnabled(true) + } + const replacement = { startIndexing: vi.fn() } + vi.spyOn(internal, "_recreateServices").mockImplementation(async () => { + internal._orchestrator = replacement + internal.servicesStopped = false + }) + await manager.startIndexing() + expect(internal._recreateServices).toHaveBeenCalledOnce() + expect(replacement.startIndexing).toHaveBeenCalledOnce() + expect(old.startIndexing).not.toHaveBeenCalled() + expect(internal.parserFailed).toBe(false) + internal._orchestrator = undefined + }, + ) + // Real post-failure state is Error; the test above forces Standby and so skips recoverFromError. + it("starts the replacement orchestrator on retry while state is still Error", async () => { + const internal = manager as any + internal._configManager = { isFeatureEnabled: true, isFeatureConfigured: true } + internal._cacheManager = {} + internal._searchService = {} + internal._stateManager.getCurrentStatus.mockReturnValue({ systemStatus: "Error" }) + internal._orchestrator = { startIndexing: vi.fn(), stopIndexing: vi.fn() } + internal.parserFailed = true + internal.servicesStopped = true + const replacement = { startIndexing: vi.fn(), stopIndexing: vi.fn() } + vi.spyOn(internal, "_recreateServices").mockImplementation(async () => { + internal._orchestrator = replacement + internal.servicesStopped = false + }) + await manager.startIndexing() + expect(replacement.stopIndexing).not.toHaveBeenCalled() + expect(replacement.startIndexing).toHaveBeenCalledOnce() + expect(internal._orchestrator).toBe(replacement) + internal._orchestrator = undefined + }) + }) + describe("handleSettingsChange", () => { it("should not throw when called on uninitialized manager (regression test)", async () => { // This is the core regression test: handleSettingsChange() should not throw diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index 320d13e06fd..ab0dfbdcfa1 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, vi } from "vitest" import { CodeIndexOrchestrator } from "../orchestrator" +import { ParserFatalError } from "../parser-errors" // Mock vscode workspace so startIndexing passes workspace check vi.mock("vscode", () => { @@ -94,6 +95,80 @@ describe("CodeIndexOrchestrator - error path cleanup gating", () => { } }) + it.each([false, true])( + "fail-stops a fatal parser during initial/incremental scan (%s), preserves data and drops late callbacks", + async (existing) => { + vectorStore.initialize.mockResolvedValue(false) + vectorStore.hasIndexedData.mockResolvedValue(existing) + let lateProgress!: (count: number) => void + scanner.scanDirectory.mockImplementation( + async (_path: string, _error: unknown, progress: (count: number) => void) => { + lateProgress = progress + throw new ParserFatalError("parser-abort", "bad.ts") + }, + ) + const parser = { stop: vi.fn() } + const failed = vi.fn() + const stopped = vi.fn() + const orchestrator = new CodeIndexOrchestrator( + configManager, + stateManager, + workspacePath, + cacheManager, + vectorStore, + scanner, + fileWatcher, + parser, + failed, + stopped, + ) + await orchestrator.startIndexing() + expect(stateManager.state).toBe("Error") + expect(failed).toHaveBeenCalledOnce() + expect(parser.stop).toHaveBeenCalledOnce() + expect(fileWatcher.dispose).toHaveBeenCalledOnce() + expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() + expect(vectorStore.clearCollection).not.toHaveBeenCalled() + lateProgress(100) + expect(stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() + await orchestrator.startIndexing() + expect(scanner.scanDirectory).toHaveBeenCalledOnce() + }, + ) + + it("fail-stops watcher parsing once and ignores captured late progress", async () => { + vectorStore.initialize.mockResolvedValue(false) + vectorStore.hasIndexedData.mockResolvedValue(true) + scanner.scanDirectory.mockResolvedValue({ stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 }) + let fatal!: (error: ParserFatalError) => void + fileWatcher.onParserFatal = vi.fn((listener) => { + fatal = listener + return { dispose: vi.fn() } + }) + const failed = vi.fn() + const parser = { stop: vi.fn() } + const orchestrator = new CodeIndexOrchestrator( + configManager, + stateManager, + workspacePath, + cacheManager, + vectorStore, + scanner, + fileWatcher, + parser, + failed, + ) + await orchestrator.startIndexing() + const progress = fileWatcher.onBatchProgressUpdate.mock.calls[0][0] + fatal(new ParserFatalError("parse-timeout", "hang.ts")) + fatal(new ParserFatalError("parser-abort", "other.ts")) + progress({ processedInBatch: 1, totalInBatch: 1 }) + expect(failed).toHaveBeenCalledOnce() + expect(stateManager.state).toBe("Error") + expect(stateManager.reportFileQueueProgress).not.toHaveBeenCalled() + expect(parser.stop).toHaveBeenCalledOnce() + }) + it("should not call clearCollection() or clear cache when initialize() fails (indexing not started)", async () => { // Arrange: fail at initialize() vectorStore.initialize.mockRejectedValue(new Error("Qdrant unreachable")) diff --git a/src/services/code-index/__tests__/parse-worker-client.spec.ts b/src/services/code-index/__tests__/parse-worker-client.spec.ts new file mode 100644 index 00000000000..2292a19773c --- /dev/null +++ b/src/services/code-index/__tests__/parse-worker-client.spec.ts @@ -0,0 +1,153 @@ +import { EventEmitter } from "node:events" +import { Worker } from "node:worker_threads" +import { ParseWorkerClient, type ParseRequest } from "../parse-worker-client" + +class MockWorker extends EventEmitter { + requests: ParseRequest[] = [] + terminate = vi.fn().mockResolvedValue(0) + postMessage(request: ParseRequest): void { + this.requests.push(request) + } +} + +const options = { + content: "function example() { return 'a sufficiently long function body for indexing'; }", + fileHash: "hash", +} + +async function drain(): Promise { + for (let turn = 0; turn < 10; turn++) await Promise.resolve() +} + +describe("ParseWorkerClient", () => { + afterEach(() => vi.useRealTimers()) + + it("interrupts an actual busy worker while the main event loop keeps running", async () => { + const worker = new Worker( + "require('node:worker_threads').parentPort.on('message', () => { while (true) {} })", + { eval: true }, + ) + const client = new ParseWorkerClient(() => worker, 200) + const exited = new Promise((resolve) => worker.once("exit", () => resolve())) + const started = performance.now() + const pending = client.parseFile("busy.ts", options).catch((error) => error) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(performance.now() - started).toBeLessThan(500) + expect(await pending).toMatchObject({ category: "parse-timeout" }) + await exited + expect(performance.now() - started).toBeLessThan(2000) + }) + + it("dispatches FIFO and starts each deadline at dispatch, not enqueue", async () => { + vi.useFakeTimers() + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker, 100) + const first = client.parseFile("first.ts", options) + const second = client.parseFile("second.ts", options) + await drain() + expect(worker.requests.map((request) => request.filePath)).toEqual(["first.ts"]) + await vi.advanceTimersByTimeAsync(90) + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + await vi.advanceTimersByTimeAsync(90) + expect(worker.terminate).not.toHaveBeenCalled() + worker.emit("message", { id: worker.requests[1].id, blocks: [] }) + await second + client.stop() + }) + + it.each(["Aborted()", "memory access out of bounds", "error", "exit", "hang"])( + "contains %s and terminally rejects queued work", + async (failure) => { + vi.useFakeTimers() + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker, 100) + const active = client.parseFile("bad.ts", options).catch((error) => error) + const queued = client.parseFile("queued.ts", options).catch((error) => error) + await drain() + if (failure === "hang") await vi.advanceTimersByTimeAsync(100) + else if (failure === "error") worker.emit("error", new Error("crash")) + else if (failure === "exit") worker.emit("exit", 1) + else worker.emit("message", { id: worker.requests[0].id, error: { message: failure, fatal: false } }) + expect(await active).toMatchObject({ + name: "ParserFatalError", + filePath: "bad.ts", + category: failure === "hang" ? "parse-timeout" : "parser-abort", + }) + expect(await queued).toMatchObject({ name: "AbortError" }) + await expect(client.parseFile("later.md", options)).rejects.toMatchObject({ name: "AbortError" }) + expect(worker.requests).toHaveLength(1) + expect(worker.terminate).toHaveBeenCalledTimes(1) + }, + ) + + it("removes a cancelled queue entry without stopping another caller", async () => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const controller = new AbortController() + const first = client.parseFile("first.ts", options) + const cancelled = client + .parseFile("cancelled.ts", { ...options, signal: controller.signal }) + .catch((error) => error) + const last = client.parseFile("last.ts", options) + await drain() + controller.abort() + expect(await cancelled).toMatchObject({ name: "AbortError" }) + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + expect(worker.requests[1].filePath).toBe("last.ts") + worker.emit("message", { id: worker.requests[1].id, blocks: [] }) + await last + client.stop() + }) + + it.each(["stop", "abort"])( + "%s synchronously settles active and queued requests without waiting for termination", + async (action) => { + const worker = new MockWorker() + worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + const controller = new AbortController() + const active = client + .parseFile("active.ts", { ...options, signal: controller.signal }) + .catch((error) => error) + const queued = client.parseFile("queued.ts", options).catch((error) => error) + await drain() + if (action === "stop") client.stop() + else controller.abort() + expect(await active).toMatchObject({ name: "AbortError" }) + expect(await queued).toMatchObject({ name: "AbortError" }) + client.stop() + expect(worker.terminate).toHaveBeenCalledTimes(1) + }, + ) + + // Spec FR6 / design Decision 3: a worker crash is a parser failure, even when no request is in flight. + it.each(["error", "exit"])("reports an idle worker %s as a parser failure on the next parse", async (failure) => { + const worker = new MockWorker() + const client = new ParseWorkerClient(() => worker) + const first = client.parseFile("first.ts", options) + await drain() + worker.emit("message", { id: worker.requests[0].id, blocks: [] }) + await first + if (failure === "error") worker.emit("error", new Error("crash")) + else worker.emit("exit", 1) + await expect(client.parseFile("next.ts", options)).rejects.toMatchObject({ name: "ParserFatalError" }) + }) + + it("keeps independent workspaces isolated and a replacement usable", async () => { + const failedWorker = new MockWorker() + const otherWorker = new MockWorker() + const failed = new ParseWorkerClient(() => failedWorker) + const other = new ParseWorkerClient(() => otherWorker) + const failure = failed.parseFile("bad.ts", options).catch((error) => error) + const success = other.parseFile("good.ts", options) + await drain() + failedWorker.emit("exit", 1) + await failure + otherWorker.emit("message", { id: otherWorker.requests[0].id, blocks: [] }) + await expect(success).resolves.toEqual([]) + expect(otherWorker.terminate).not.toHaveBeenCalled() + other.stop() + }) +}) diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 6f0e0fe7e62..e1f5609189d 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -1,6 +1,8 @@ import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types" /**Parser */ +export const PARSE_TIMEOUT_MS = 10_000 +export const INDEX_STOP_LIMIT_MS = 2_000 export const MAX_BLOCK_CHARS = 1000 export const MIN_BLOCK_CHARS = 50 export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split diff --git a/src/services/code-index/embedders/__tests__/openai.spec.ts b/src/services/code-index/embedders/__tests__/openai.spec.ts index 22b8d098903..49b3a5e59d2 100644 --- a/src/services/code-index/embedders/__tests__/openai.spec.ts +++ b/src/services/code-index/embedders/__tests__/openai.spec.ts @@ -82,6 +82,19 @@ describe("OpenAiEmbedder", () => { describe("createEmbeddings", () => { const testModelId = "text-embedding-3-small" + it("passes cancellation to the SDK and does not retry an aborted request", async () => { + const controller = new AbortController() + mockEmbeddingsCreate.mockImplementation(async (_request, options) => { + expect((options as { signal: AbortSignal }).signal).toBe(controller.signal) + controller.abort() + throw new DOMException("Cancelled", "AbortError") + }) + await expect(embedder.createEmbeddings(["text"], undefined, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }) + expect(mockEmbeddingsCreate).toHaveBeenCalledOnce() + }) + it("should create embeddings for a single text", async () => { const testTexts = ["Hello world"] const mockResponse = { @@ -92,10 +105,13 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: testModelId, + }, + undefined, + ) expect(result).toEqual({ embeddings: [[0.1, 0.2, 0.3]], usage: { promptTokens: 10, totalTokens: 15 }, @@ -112,10 +128,13 @@ describe("OpenAiEmbedder", () => { const result = await embedder.createEmbeddings(testTexts) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: testModelId, + }, + undefined, + ) expect(result).toEqual({ embeddings: [ [0.1, 0.2, 0.3], @@ -136,10 +155,13 @@ describe("OpenAiEmbedder", () => { await embedder.createEmbeddings(testTexts, customModel) - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: testTexts, - model: customModel, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: testTexts, + model: customModel, + }, + undefined, + ) }) it("should handle missing usage data gracefully", async () => { @@ -195,10 +217,13 @@ describe("OpenAiEmbedder", () => { expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`exceeds maximum token limit`)) // Verify only normal texts were processed - expect(mockEmbeddingsCreate).toHaveBeenCalledWith({ - input: [normalText, "another normal"], - model: testModelId, - }) + expect(mockEmbeddingsCreate).toHaveBeenCalledWith( + { + input: [normalText, "another normal"], + model: testModelId, + }, + undefined, + ) expect(result.embeddings).toHaveLength(2) }) diff --git a/src/services/code-index/embedders/openai.ts b/src/services/code-index/embedders/openai.ts index 0a2a43a86df..3bf6587ffc2 100644 --- a/src/services/code-index/embedders/openai.ts +++ b/src/services/code-index/embedders/openai.ts @@ -45,7 +45,8 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -113,7 +114,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -132,13 +133,18 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { private async _embedBatchWithRetries( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { try { - const response = await this.embeddingsClient.embeddings.create({ - input: batchTexts, - model: model, - }) + signal?.throwIfAborted() + const response = await this.embeddingsClient.embeddings.create( + { + input: batchTexts, + model: model, + }, + signal ? { signal } : undefined, + ) return { embeddings: response.data.map((item) => item.embedding), @@ -148,6 +154,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { }, } } catch (error: any) { + signal?.throwIfAborted() const hasMoreAttempts = attempts < MAX_RETRIES - 1 // Check if it's a rate limit error diff --git a/src/services/code-index/index-write-gate.ts b/src/services/code-index/index-write-gate.ts new file mode 100644 index 00000000000..6b674479510 --- /dev/null +++ b/src/services/code-index/index-write-gate.ts @@ -0,0 +1,51 @@ +import { throwIfAborted } from "./parser-errors" + +export class IndexWriteGate { + private readonly issued = new Map>>() + private readonly uncertain = new Set() + + async ready(signal: AbortSignal): Promise { + throwIfAborted(signal) + const earlier = [...this.issued].filter(([owner]) => owner !== signal).flatMap(([, writes]) => [...writes]) + await Promise.allSettled(earlier) + throwIfAborted(signal) + } + + needsReconciliation(filePath: string): boolean { + return this.uncertain.has(filePath) + } + + reconciled(filePath: string): void { + this.uncertain.delete(filePath) + } + + async run( + signal: AbortSignal, + write: () => Result | Promise, + files: string[] = [], + ): Promise { + await this.ready(signal) + throwIfAborted(signal) + let writes = this.issued.get(signal) + if (!writes) { + writes = new Set() + this.issued.set(signal, writes) + } + const pending = Promise.resolve().then(() => { + throwIfAborted(signal) + return write() + }) + writes.add(pending) + try { + return await pending + } catch (error) { + if (!(error instanceof Error && error.name === "AbortError")) { + for (const filePath of files) this.uncertain.add(filePath) + } + throw error + } finally { + writes.delete(pending) + if (writes.size === 0) this.issued.delete(signal) + } + } +} diff --git a/src/services/code-index/interfaces/embedder.ts b/src/services/code-index/interfaces/embedder.ts index aa8515d6986..695b648e3a2 100644 --- a/src/services/code-index/interfaces/embedder.ts +++ b/src/services/code-index/interfaces/embedder.ts @@ -9,7 +9,7 @@ export interface IEmbedder { * @param model Optional model ID to use for embeddings * @returns Promise resolving to an EmbeddingResponse */ - createEmbeddings(texts: string[], model?: string): Promise + createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise /** * Validates the embedder configuration by testing connectivity and credentials. diff --git a/src/services/code-index/interfaces/file-processor.ts b/src/services/code-index/interfaces/file-processor.ts index 8ecdc518c8d..e309eaf69b1 100644 --- a/src/services/code-index/interfaces/file-processor.ts +++ b/src/services/code-index/interfaces/file-processor.ts @@ -1,5 +1,6 @@ import * as vscode from "vscode" import { PointStruct } from "./vector-store" +import type { ParserFatalError } from "../parser-errors" /** * Interface for code file parser @@ -18,6 +19,7 @@ export interface ICodeParser { maxBlockLines?: number content?: string fileHash?: string + signal?: AbortSignal }, ): Promise } @@ -54,7 +56,8 @@ export interface IFileWatcher extends vscode.Disposable { /** * Initializes the file watcher */ - initialize(): Promise + initialize(signal?: AbortSignal): Promise + readonly onParserFatal?: vscode.Event /** * Event emitted when a batch of files begins processing. diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index ede0b0e18fa..dc7c702316c 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -13,6 +13,7 @@ import fs from "fs/promises" import ignore from "ignore" import path from "path" import { t } from "../../i18n" +import { IndexWriteGate } from "./index-write-gate" export class CodeIndexManager { // --- Singleton Implementation --- @@ -28,6 +29,11 @@ export class CodeIndexManager { // Flag to prevent race conditions during error recovery private _isRecoveringFromError = false + private parserFailed = false + private parserFailureMessage = "" + private servicesStopped = false + private disabledSinceFailure = false + private readonly writeGate = new IndexWriteGate() public static getInstance(context: vscode.ExtensionContext, workspacePath?: string): CodeIndexManager | undefined { // Resolve the workspace folder to get both fsPath and the real URI @@ -108,6 +114,13 @@ export class CodeIndexManager { public async setWorkspaceEnabled(enabled: boolean): Promise { await this.context.workspaceState.update(this._workspaceEnabledKey(), enabled) + if (!enabled) { + this.disabledSinceFailure = true + this.stopIndexing() + } else if (this.disabledSinceFailure) { + this.parserFailed = false + this.disabledSinceFailure = false + } } public get autoEnableDefault(): boolean { @@ -168,9 +181,8 @@ export class CodeIndexManager { // 2. Check if feature is enabled if (!this.isFeatureEnabled) { - if (this._orchestrator) { - this._orchestrator.stopWatcher() - } + this.disabledSinceFailure = true + this.stopIndexing() return { requiresRestart } } @@ -187,6 +199,15 @@ export class CodeIndexManager { return { requiresRestart } } + if (this.disabledSinceFailure) { + this.parserFailed = false + this.disabledSinceFailure = false + } + if (this.parserFailed) { + this._stateManager.setSystemState("Error", this.parserFailureMessage) + return { requiresRestart } + } + // 5. CacheManager Initialization if (!this._cacheManager) { this._cacheManager = new CacheManager(this.context, this.workspacePath) @@ -194,7 +215,7 @@ export class CodeIndexManager { } // 6. Determine if Core Services Need Recreation - const needsServiceRecreation = !this._serviceFactory || requiresRestart + const needsServiceRecreation = !this._serviceFactory || requiresRestart || this.servicesStopped if (needsServiceRecreation) { await this._recreateServices() @@ -224,9 +245,10 @@ export class CodeIndexManager { return } - // Check if we're in error state and recover if needed - const currentStatus = this.getCurrentStatus() - if (currentStatus.systemStatus === "Error") { + if (this.parserFailed || this.servicesStopped) { + this.parserFailed = false + await this._recreateServices() + } else if (this.getCurrentStatus().systemStatus === "Error") { await this.recoverFromError() // After recovery, we need to reinitialize since recoverFromError clears all services @@ -244,6 +266,7 @@ export class CodeIndexManager { public stopIndexing(): void { if (this._orchestrator) { this._orchestrator.stopIndexing() + this.servicesStopped = true } } @@ -256,6 +279,7 @@ export class CodeIndexManager { } if (this._orchestrator) { this._orchestrator.stopWatcher() + this.servicesStopped = true } } @@ -281,6 +305,7 @@ export class CodeIndexManager { this._isRecoveringFromError = true try { + this._orchestrator?.stopIndexing() // Clear error state this._stateManager.setSystemState("Standby", "") } catch (error) { @@ -317,7 +342,6 @@ export class CodeIndexManager { } this.assertInitialized() await this._orchestrator!.clearIndexData() - await this._cacheManager!.clearCacheFile() } // --- Private Helpers --- @@ -358,6 +382,7 @@ export class CodeIndexManager { this._configManager!, this.workspacePath, this._cacheManager!, + this.writeGate, ) const ignoreInstance = ignore() @@ -384,7 +409,7 @@ export class CodeIndexManager { await rooIgnoreController.initialize() // (Re)Create shared service instances - const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices( + const { embedder, vectorStore, scanner, fileWatcher, parser } = this._serviceFactory.createServices( this.context, this._cacheManager!, ignoreInstance, @@ -408,7 +433,17 @@ export class CodeIndexManager { vectorStore, scanner, fileWatcher, + parser, + (_error, message) => { + this.parserFailed = true + this.parserFailureMessage = message + }, + () => { + this.servicesStopped = true + }, + this.writeGate, ) + this.servicesStopped = false // (Re)Initialize search service this._searchService = new CodeIndexSearchService( @@ -437,12 +472,18 @@ export class CodeIndexManager { // If feature is disabled, stop the service (including any active scan) if (!isFeatureEnabled) { + this.disabledSinceFailure = true this.stopIndexing() this._stateManager.setSystemState("Standby", "Code indexing is disabled") return } - if (requiresRestart && isFeatureEnabled && isFeatureConfigured) { + if (this.disabledSinceFailure) { + this.parserFailed = false + this.disabledSinceFailure = false + } + if (this.parserFailed) return + if ((requiresRestart || this.servicesStopped) && isFeatureEnabled && isFeatureConfigured) { try { // Ensure cacheManager is initialized before recreating services if (!this._cacheManager) { @@ -452,6 +493,7 @@ export class CodeIndexManager { // Recreate services with new configuration await this._recreateServices() + if (this.isWorkspaceEnabled) void this._orchestrator?.startIndexing() } catch (error) { // Error state already set in _recreateServices console.error("Failed to recreate services:", error) diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index a9ca2f11c0d..6d9fbdbe1d8 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -6,6 +6,8 @@ import { IFileWatcher, IVectorStore, BatchProcessingSummary } from "./interfaces import { DirectoryScanner } from "./processors" import { CacheManager } from "./cache-manager" import { t } from "../../i18n" +import { ParserFatalError, throwIfAborted } from "./parser-errors" +import { IndexWriteGate } from "./index-write-gate" /** * Manages the code indexing workflow, coordinating between different services and managers. @@ -15,6 +17,9 @@ export class CodeIndexOrchestrator { private _isProcessing: boolean = false private _abortController: AbortController | null = null + public stopped = false + private parserFailure?: ParserFatalError + constructor( private readonly configManager: CodeIndexConfigManager, private readonly stateManager: CodeIndexStateManager, @@ -23,8 +28,25 @@ export class CodeIndexOrchestrator { private readonly vectorStore: IVectorStore, private readonly scanner: DirectoryScanner, private readonly fileWatcher: IFileWatcher, + private readonly parser?: { stop(): void }, + private readonly onFatal?: (error: ParserFatalError, message: string) => void, + private readonly onStopped?: () => void, + private readonly writeGate = new IndexWriteGate(), ) {} + private failStop(error: ParserFatalError): void { + if (this.parserFailure || this._abortController?.signal.aborted) return + this.parserFailure = error + this._abortController?.abort() + this.stopWatcher() + const message = t("embeddings:orchestrator.parserFailed", { + filePath: error.filePath, + category: error.category, + }) + this.stateManager.setSystemState("Error", message) + this.onFatal?.(error, message) + } + /** * Starts the file watcher if not already running. */ @@ -36,11 +58,14 @@ export class CodeIndexOrchestrator { this.stateManager.setSystemState("Indexing", "Initializing file watcher...") try { - await this.fileWatcher.initialize() + const signal = this._abortController!.signal + await this.fileWatcher.initialize(signal) + throwIfAborted(signal) this._fileWatcherSubscriptions = [ this.fileWatcher.onDidStartBatchProcessing((filePaths: string[]) => {}), this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { + if (signal.aborted) return if (totalInBatch > 0 && this.stateManager.state !== "Indexing") { this.stateManager.setSystemState("Indexing", "Processing file changes...") } @@ -63,6 +88,7 @@ export class CodeIndexOrchestrator { } }), this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { + if (signal.aborted) return if (summary.batchError) { console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) } else { @@ -75,6 +101,9 @@ export class CodeIndexOrchestrator { } }), ] + if (this.fileWatcher.onParserFatal) { + this._fileWatcherSubscriptions.push(this.fileWatcher.onParserFatal((error) => this.failStop(error))) + } } catch (error) { console.error("[CodeIndexOrchestrator] Failed to start file watcher:", error) throw error @@ -89,6 +118,7 @@ export class CodeIndexOrchestrator { * Initiates the indexing process (initial scan and starts watcher). */ public async startIndexing(): Promise { + if (this.stopped) return // Check if workspace is available first if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) { this.stateManager.setSystemState("Error", t("embeddings:orchestrator.indexingRequiresWorkspace")) @@ -124,19 +154,22 @@ export class CodeIndexOrchestrator { let indexingStarted = false try { - const collectionCreated = await this.vectorStore.initialize() + const collectionCreated = await this.writeGate.run(signal, () => this.vectorStore.initialize()) + throwIfAborted(signal) // Successfully connected to Qdrant indexingStarted = true if (collectionCreated) { - await this.cacheManager.clearCacheFile() + await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) } // Check if the collection already has indexed data // If it does, we can skip the full scan and just start the watcher const hasExistingData = await this.vectorStore.hasIndexedData() + throwIfAborted(signal) + if (hasExistingData && !collectionCreated) { // Collection exists with data - run incremental scan to catch any new/changed files // This handles files added while workspace was closed or Qdrant was inactive @@ -146,18 +179,20 @@ export class CodeIndexOrchestrator { this.stateManager.setSystemState("Indexing", "Checking for new or modified files...") // Mark as incomplete at the start of incremental scan - await this.vectorStore.markIndexingIncomplete() + await this.writeGate.run(signal, () => this.vectorStore.markIndexingIncomplete()) let cumulativeBlocksIndexed = 0 let cumulativeBlocksFoundSoFar = 0 let batchErrors: Error[] = [] const handleFileParsed = (fileBlockCount: number) => { + if (signal.aborted) return cumulativeBlocksFoundSoFar += fileBlockCount this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } const handleBlocksIndexed = (indexedCount: number) => { + if (signal.aborted) return cumulativeBlocksIndexed += indexedCount this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } @@ -177,12 +212,7 @@ export class CodeIndexOrchestrator { signal, ) - if (signal.aborted) { - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) - return - } + if (signal.aborted) return if (!result) { throw new Error("Incremental scan failed, is scanner initialized?") @@ -200,7 +230,8 @@ export class CodeIndexOrchestrator { await this._startWatcher() // Mark indexing as complete after successful incremental scan - await this.vectorStore.markIndexingComplete() + await this.writeGate.run(signal, () => this.vectorStore.markIndexingComplete()) + throwIfAborted(signal) this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) } else { @@ -208,18 +239,20 @@ export class CodeIndexOrchestrator { this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") // Mark as incomplete at the start of full scan - await this.vectorStore.markIndexingIncomplete() + await this.writeGate.run(signal, () => this.vectorStore.markIndexingIncomplete()) let cumulativeBlocksIndexed = 0 let cumulativeBlocksFoundSoFar = 0 let batchErrors: Error[] = [] const handleFileParsed = (fileBlockCount: number) => { + if (signal.aborted) return cumulativeBlocksFoundSoFar += fileBlockCount this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } const handleBlocksIndexed = (indexedCount: number) => { + if (signal.aborted) return cumulativeBlocksIndexed += indexedCount this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) } @@ -238,12 +271,7 @@ export class CodeIndexOrchestrator { signal, ) - if (signal.aborted) { - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) - return - } + if (signal.aborted) return if (!result) { throw new Error("Scan failed, is scanner initialized?") @@ -289,24 +317,28 @@ export class CodeIndexOrchestrator { await this._startWatcher() // Mark indexing as complete after successful full scan - await this.vectorStore.markIndexingComplete() + await this.writeGate.run(signal, () => this.vectorStore.markIndexingComplete()) + throwIfAborted(signal) this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) } } catch (error: any) { + if (error instanceof ParserFatalError) { + this.failStop(error) + return + } + if (this.parserFailure) return // Handle abort gracefully — not an error, just a user-initiated stop if (error?.name === "AbortError" || signal.aborted) { console.log("[CodeIndexOrchestrator] Indexing aborted by user.") - await this.cacheManager.flush() - this.stopWatcher() - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) + if (!this.stopped) this.stopIndexing() return } console.error("[CodeIndexOrchestrator] Error during indexing:", error) if (indexingStarted) { try { - await this.vectorStore.clearCollection() + await this.writeGate.run(signal, () => this.vectorStore.clearCollection()) } catch (cleanupError) { console.error("[CodeIndexOrchestrator] Failed to clean up after error:", cleanupError) } @@ -316,7 +348,7 @@ export class CodeIndexOrchestrator { // If we never connected to Qdrant, preserve cache for incremental scan when it comes back if (indexingStarted) { // Indexing started but failed mid-way - clear cache to avoid cache-Qdrant mismatch - await this.cacheManager.clearCacheFile() + await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) console.log( "[CodeIndexOrchestrator] Indexing failed after starting. Clearing cache to avoid inconsistency.", ) @@ -327,6 +359,7 @@ export class CodeIndexOrchestrator { ) } + if (signal.aborted) return this.stateManager.setSystemState( "Error", t("embeddings:orchestrator.failedDuringInitialScan", { @@ -336,7 +369,6 @@ export class CodeIndexOrchestrator { this.stopWatcher() } finally { this._isProcessing = false - this._abortController = null } } @@ -344,18 +376,20 @@ export class CodeIndexOrchestrator { * Stops any in-progress indexing by aborting the scan and stopping the file watcher. */ public stopIndexing(): void { - if (this._abortController) { - this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStoppedPartial")) - this._abortController.abort() - this._abortController = null - } + this._abortController?.abort() this.stopWatcher() + if (!this.parserFailure) + this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) } /** * Stops the file watcher and cleans up resources. */ public stopWatcher(): void { + this._abortController?.abort() + this.parser?.stop() + this.stopped = true + this.onStopped?.() this.fileWatcher.dispose() this._fileWatcherSubscriptions.forEach((sub) => sub.dispose()) this._fileWatcherSubscriptions = [] @@ -374,11 +408,12 @@ export class CodeIndexOrchestrator { this._isProcessing = true try { - await this.stopWatcher() + this.stopIndexing() + const cleanupSignal = new AbortController().signal try { if (this.configManager.isFeatureConfigured) { - await this.vectorStore.deleteCollection() + await this.writeGate.run(cleanupSignal, () => this.vectorStore.deleteCollection()) } else { console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") } @@ -387,7 +422,7 @@ export class CodeIndexOrchestrator { this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) } - await this.cacheManager.clearCacheFile() + await this.writeGate.run(cleanupSignal, () => this.cacheManager.clearCacheFile()) if (this.stateManager.state !== "Error") { this.stateManager.setSystemState("Standby", "Index data cleared successfully.") diff --git a/src/services/code-index/parse-worker-client.ts b/src/services/code-index/parse-worker-client.ts new file mode 100644 index 00000000000..9cd6ee54f52 --- /dev/null +++ b/src/services/code-index/parse-worker-client.ts @@ -0,0 +1,165 @@ +import { Worker } from "node:worker_threads" +import * as path from "node:path" +import { CodeParser } from "./processors/parser" +import type { CodeBlock, ICodeParser } from "./interfaces" +import { abortError, ParserFatalError, isParserAbort } from "./parser-errors" +import { PARSE_TIMEOUT_MS } from "./constants" + +export interface ParseRequest { + id: number + filePath: string + content: string + fileHash: string +} + +export interface ParseReply { + id: number + blocks?: CodeBlock[] + fallback?: boolean + error?: { message: string; fatal: boolean } +} + +export interface ParseWorker { + postMessage(request: ParseRequest): void + on(event: "message", listener: (reply: ParseReply) => void): unknown + on(event: "error", listener: (error: Error) => void): unknown + on(event: "exit", listener: (code: number) => void): unknown + terminate(): Promise +} + +interface PendingParse { + request: ParseRequest + resolve: (blocks: CodeBlock[]) => void + reject: (error: Error) => void + signal?: AbortSignal + cancel: () => void +} + +export class ParseWorkerClient implements ICodeParser { + private worker?: ParseWorker + private queue: PendingParse[] = [] + private active?: PendingParse + private timer?: NodeJS.Timeout + private nextId = 0 + private stopped = false + private readonly parser: CodeParser + private idleFailure?: Pick + + constructor( + private readonly workerFactory: () => ParseWorker = () => new Worker(path.join(__dirname, "parse-worker.js")), + private readonly timeoutMs = PARSE_TIMEOUT_MS, + ) { + this.parser = new CodeParser((filePath, content, fileHash, signal) => + this.enqueue(filePath, content, fileHash, signal), + ) + } + + parseFile( + filePath: string, + options?: { content?: string; fileHash?: string; signal?: AbortSignal }, + ): Promise { + const stopError = this.getStopError(filePath, options?.signal) + if (stopError) return Promise.reject(stopError) + return this.parser.parseFile(filePath, options) + } + + private getStopError(filePath: string, signal?: AbortSignal): Error | undefined { + if (signal?.aborted) return abortError() + if (this.idleFailure) { + const { category, message } = this.idleFailure + this.idleFailure = undefined + return new ParserFatalError(category, filePath, message) + } + if (this.stopped) return abortError() + return undefined + } + + private enqueue(filePath: string, content: string, fileHash: string, signal?: AbortSignal): Promise { + const stopError = this.getStopError(filePath, signal) + if (stopError) return Promise.reject(stopError) + return new Promise((resolve, reject) => { + const pending: PendingParse = { + request: { id: ++this.nextId, filePath, content, fileHash }, + resolve, + reject, + signal, + cancel: () => { + if (this.active === pending) this.stop() + else { + this.queue = this.queue.filter((entry) => entry !== pending) + this.settle(pending, abortError()) + } + }, + } + signal?.addEventListener("abort", pending.cancel, { once: true }) + this.queue.push(pending) + this.dispatch() + }) + } + + private settle(pending: PendingParse, error?: Error, blocks: CodeBlock[] = []): void { + pending.signal?.removeEventListener("abort", pending.cancel) + if (error) pending.reject(error) + else pending.resolve(blocks) + } + + private dispatch(): void { + if (this.stopped || this.active || this.queue.length === 0) return + this.active = this.queue.shift()! + try { + if (!this.worker) { + this.worker = this.workerFactory() + this.worker.on("message", (reply) => { + if (this.stopped || reply.id !== this.active?.request.id) return + if (reply.error?.fatal || (reply.error && isParserAbort(reply.error.message))) { + this.fail("parser-abort", reply.error?.message) + return + } + clearTimeout(this.timer) + const completed = this.active + this.active = undefined + const blocks = !reply.fallback + ? reply.blocks + : this.parser.fallbackContent( + completed.request.filePath, + completed.request.content, + completed.request.fileHash, + ) + this.settle(completed, reply.error ? new Error(reply.error.message) : undefined, blocks) + this.dispatch() + }) + this.worker.on("error", (error) => this.fail("parser-abort", error.message)) + this.worker.on("exit", (code) => this.fail("parser-abort", `Parse worker exited (${code})`)) + } + this.timer = setTimeout(() => this.fail("parse-timeout"), this.timeoutMs) + this.worker.postMessage(this.active.request) + } catch (error) { + this.fail("parser-abort", String(error)) + } + } + + private fail(category: ParserFatalError["category"], message?: string): void { + if (this.stopped) return + const failed = this.active + this.active = undefined + if (failed) this.settle(failed, new ParserFatalError(category, failed.request.filePath, message)) + else this.idleFailure = { category, message: message ?? category } + this.stop() + } + + stop(): void { + if (this.stopped) return + this.stopped = true + clearTimeout(this.timer) + if (this.active) this.settle(this.active, abortError()) + this.active = undefined + for (const pending of this.queue) this.settle(pending, abortError()) + this.queue = [] + void this.worker?.terminate().catch(() => undefined) + this.worker = undefined + } + + dispose(): void { + this.stop() + } +} diff --git a/src/services/code-index/parse-worker.ts b/src/services/code-index/parse-worker.ts new file mode 100644 index 00000000000..79eba5b7bf0 --- /dev/null +++ b/src/services/code-index/parse-worker.ts @@ -0,0 +1,22 @@ +import { parentPort } from "node:worker_threads" +import { CodeParser } from "./processors/parser" +import { isParserAbort } from "./parser-errors" +import type { ParseRequest, ParseReply } from "./parse-worker-client" + +const parser = new CodeParser(undefined, __dirname) +parentPort!.on("message", async (request: ParseRequest) => { + let reply: ParseReply + try { + const blocks = await parser.parseFile(request.filePath, { + content: request.content, + fileHash: request.fileHash, + }) + reply = { id: request.id, blocks, fallback: parser.fallbackNeeded } + } catch (error) { + reply = { + id: request.id, + error: { message: error instanceof Error ? error.message : String(error), fatal: isParserAbort(error) }, + } + } + parentPort!.postMessage(reply) +}) diff --git a/src/services/code-index/parser-errors.ts b/src/services/code-index/parser-errors.ts new file mode 100644 index 00000000000..74d454242ed --- /dev/null +++ b/src/services/code-index/parser-errors.ts @@ -0,0 +1,22 @@ +export class ParserFatalError extends Error { + constructor( + public readonly category: "parser-abort" | "parse-timeout", + public readonly filePath: string, + message?: string, + ) { + super(message ?? category) + this.name = "ParserFatalError" + } +} + +export function abortError(): DOMException { + return new DOMException("Indexing aborted", "AbortError") +} + +export function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw abortError() +} + +export function isParserAbort(error: unknown): boolean { + return /Aborted\(|memory access out of bounds/i.test(error instanceof Error ? error.message : String(error)) +} diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 06bfca53d67..621333bccc6 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -3,6 +3,7 @@ import * as vscode from "vscode" import { FileWatcher } from "../file-watcher" +import { ParserFatalError } from "../../parser-errors" // Mock dependencies vi.mock("../../cache-manager") @@ -122,6 +123,105 @@ describe("FileWatcher", () => { mockEmbedder, mockVectorStore, mockIgnoreInstance, + undefined, + undefined, + { parseFile: vi.fn().mockResolvedValue([]) }, + ) + }) + + describe("attempt ownership", () => { + function createWatcher(parser: any, signal?: AbortSignal) { + const watcher = new FileWatcher( + "/mock/workspace", + mockContext, + mockCacheManager, + mockEmbedder, + mockVectorStore, + mockIgnoreInstance, + { validateAccess: () => true } as any, + 1, + parser, + ) + return { watcher, initialize: () => watcher.initialize(signal) } + } + + it("reports a fatal parser once, disposes events, and starts no more queued files", async () => { + const fatal = new ParserFatalError("parser-abort", "/mock/workspace/bad.ts") + const parser = { parseFile: vi.fn().mockRejectedValue(fatal) } + const { watcher, initialize } = createWatcher(parser) + await initialize() + const report = vi.spyOn((watcher as any).parserFatal, "fire") + await expect(watcher.processFile("/mock/workspace/bad.ts")).rejects.toBe(fatal) + expect(report).toHaveBeenCalledOnce() + expect(mockWatcher.dispose).toHaveBeenCalledOnce() + await mockOnDidChange({ fsPath: "/mock/workspace/next.ts" }) + await (watcher as any).triggerBatchProcessing() + expect(parser.parseFile).toHaveBeenCalledOnce() + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + }) + + it("drops embeddings released after the attempt stops", async () => { + const parser = { + parseFile: vi.fn().mockResolvedValue([ + { + file_path: "/mock/workspace/file.ts", + content: "function test() {}", + start_line: 1, + end_line: 1, + }, + ]), + } + const controller = new AbortController() + const { watcher, initialize } = createWatcher(parser, controller.signal) + await initialize() + let release!: (value: any) => void + mockEmbedder.createEmbeddings.mockImplementation( + () => + new Promise((resolve) => { + release = resolve + }), + ) + const pending = (watcher as any) + .processBatch(new Map([["file", { uri: { fsPath: "/mock/workspace/file.ts" }, type: "change" }]])) + .catch((error: Error) => error) + await vi.waitFor(() => expect(mockEmbedder.createEmbeddings).toHaveBeenCalledOnce()) + controller.abort() + watcher.dispose() + release({ embeddings: [[1, 2, 3]] }) + await pending + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + }) + + it.each(["fatal", "cancel", "disable"])( + "a replacement watcher processes scan-following changes after %s", + async () => { + const parser = { + parseFile: vi.fn().mockResolvedValue([ + { + file_path: "/mock/workspace/file.ts", + content: "function test() {}", + start_line: 1, + end_line: 1, + }, + ]), + } + const old = createWatcher(parser) + await old.initialize() + old.watcher.dispose() + const replacement = createWatcher(parser) + await replacement.initialize() + await mockOnDidChange({ fsPath: "/mock/workspace/file.ts" }) + await (replacement.watcher as any).triggerBatchProcessing() + expect(mockVectorStore.upsertPoints).toHaveBeenCalledOnce() + expect(mockCacheManager.updateHash).toHaveBeenCalledOnce() + expect((replacement.watcher as any)._onBatchProgressUpdate.fire).toHaveBeenCalled() + expect(mockVectorStore.deletePointsByFilePath.mock.invocationCallOrder[0]).toBeLessThan( + mockVectorStore.upsertPoints.mock.invocationCallOrder[0], + ) + replacement.watcher.dispose() + }, ) }) diff --git a/src/services/code-index/processors/__tests__/parser.spec.ts b/src/services/code-index/processors/__tests__/parser.spec.ts index 15b75e33819..0eface745bc 100644 --- a/src/services/code-index/processors/__tests__/parser.spec.ts +++ b/src/services/code-index/processors/__tests__/parser.spec.ts @@ -30,6 +30,7 @@ const mockLanguageParser = { js: { parser: { parse: vi.fn((content: string) => ({ + delete: vi.fn(), rootNode: { text: content, startPosition: { row: 0 }, @@ -56,6 +57,41 @@ describe("CodeParser", () => { vi.mocked(readFile).mockResolvedValue("// default test content") }) + it("releases a tree even when query extraction throws", async () => { + const release = vi.fn() + const failure = new Error("memory access out of bounds") + ;(loadRequiredLanguageParsers as any).mockResolvedValue({ + js: { + parser: { parse: () => ({ rootNode: {}, delete: release }) }, + query: { + captures: () => { + throw failure + }, + }, + }, + }) + await expect(parser.parseFile("test.js", { content: "function example() { return 123 }" })).rejects.toBe( + failure, + ) + expect(release).toHaveBeenCalledOnce() + }) + + it("keeps fallback chunking scaling proportional to input size", () => { + const source = "const example = value => value.toUpperCase();\n".repeat(750) + const measure = (content: string) => { + const samples: number[] = [] + for (let sample = 0; sample < 5; sample++) { + const started = performance.now() + for (let repeat = 0; repeat < 40; repeat++) parser.fallbackContent("test.txt", content, "hash") + samples.push(performance.now() - started) + } + return Math.min(...samples) + } + for (let warmup = 0; warmup < 20; warmup++) parser.fallbackContent("test.txt", source, "hash") + const baseline = measure(source) + expect(measure(source.repeat(4)) / baseline).toBeLessThan(9) + }) + describe("parseFile", () => { it("should return empty array for unsupported extensions", async () => { const result = await parser.parseFile("test.unsupported") diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index cce67a4c4b0..8ed63c43602 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -2,6 +2,8 @@ import { DirectoryScanner } from "../scanner" import { stat } from "fs/promises" +import { ParserFatalError } from "../../parser-errors" +import { IndexWriteGate } from "../../index-write-gate" vi.mock("fs/promises", () => ({ default: { @@ -386,6 +388,128 @@ describe("DirectoryScanner", () => { expect(points[2].payload.segmentHash).toBe("unique-segment-hash-3") }) + it.each(["success", "abort", "failure"])( + "keeps a multi-batch file incomplete until every batch succeeds: %s", + async (outcome) => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file.js"], false]) + const blocks = Array.from({ length: 3 }, (_, blockNumber) => ({ + file_path: "test/file.js", + content: `function block${blockNumber}() { return 1 }`, + start_line: blockNumber + 1, + end_line: blockNumber + 1, + identifier: null, + type: "function", + fileHash: "hash", + segmentHash: `segment-${blockNumber}`, + })) + mockCodeParser.parseFile.mockResolvedValue(blocks) + let release!: () => void + const held = new Promise((resolve) => { + release = resolve + }) + mockVectorStore.upsertPoints.mockImplementation(async (points: any[]) => { + if (points[0].payload.segmentHash === "segment-0") { + await held + if (outcome === "failure") throw new Error("store failure") + } + }) + const controller = new AbortController() + const scanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + 1, + ) + const pending = scanner + .scanDirectory("/test", vi.fn(), undefined, undefined, controller.signal) + .catch((error) => error) + await vi.waitFor(() => expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(3)) + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(mockVectorStore.deletePointsByFilePath.mock.invocationCallOrder[0]).toBeLessThan( + mockVectorStore.upsertPoints.mock.invocationCallOrder[0], + ) + if (outcome === "abort") controller.abort() + release() + await pending + if (outcome === "success") expect(mockCacheManager.updateHash).toHaveBeenCalledOnce() + else expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + }, + ) + + it("discards embedding results released after abort without issuing an upsert or hash", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file.js"], false]) + mockCodeParser.parseFile.mockResolvedValue([ + { + file_path: "test/file.js", + content: "function test() { return 1 }", + start_line: 1, + end_line: 1, + identifier: null, + type: "function", + fileHash: "hash", + segmentHash: "segment", + }, + ]) + let release!: (value: any) => void + mockEmbedder.createEmbeddings.mockImplementation( + () => + new Promise((resolve) => { + release = resolve + }), + ) + const controller = new AbortController() + const pending = scanner + .scanDirectory("/test", undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await vi.waitFor(() => expect(mockEmbedder.createEmbeddings).toHaveBeenCalledOnce()) + controller.abort() + release({ embeddings: [[1, 2, 3]] }) + await pending + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + }) + + it("reconciles an uncertain unchanged file before replacement upserts", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file.js"], false]) + const gate = new IndexWriteGate() + await gate + .run(new AbortController().signal, () => Promise.reject(new Error("timeout")), ["test/file.js"]) + .catch(() => undefined) + const { createHash } = await import("crypto") + mockCacheManager.getHash.mockReturnValue(createHash("sha256").update("test content").digest("hex")) + const scanner = new DirectoryScanner( + mockEmbedder, + mockVectorStore, + mockCodeParser, + mockCacheManager, + mockIgnoreInstance, + 1, + gate, + ) + await scanner.scanDirectory("/test") + expect(mockCodeParser.parseFile).toHaveBeenCalledOnce() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("test/file.js") + expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("test/file.js") + expect(gate.needsReconciliation("test/file.js")).toBe(false) + }) + + it("propagates fatal parser errors without treating them as per-file errors", async () => { + const { listFiles } = await import("../../../glob/list-files") + vi.mocked(listFiles).mockResolvedValue([["test/file.js"], false]) + const fatal = new ParserFatalError("parser-abort", "test/file.js") + mockCodeParser.parseFile.mockRejectedValue(fatal) + const onError = vi.fn() + await expect(scanner.scanDirectory("/test", onError)).rejects.toBe(fatal) + expect(onError).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + }) + it("should stop processing files when signal is aborted", async () => { const { listFiles } = await import("../../../glob/list-files") vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js", "test/file3.js"], false]) @@ -394,11 +518,10 @@ describe("DirectoryScanner", () => { const controller = new AbortController() controller.abort() - const result = await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) - - // No files should have been processed since signal was already aborted + await expect( + scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }) expect(mockCodeParser.parseFile).not.toHaveBeenCalled() - expect(result.stats.processed).toBe(0) }) it("should stop processing batches when signal is aborted mid-scan", async () => { @@ -443,7 +566,9 @@ describe("DirectoryScanner", () => { const controller = new AbortController() controller.abort() - await scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal) + await expect( + scanner.scanDirectory("/test", undefined, undefined, undefined, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }) // Deleted file cleanup should not have run expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index cede6bcd200..af61b752638 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -19,7 +19,10 @@ import { PointStruct, BatchProcessingSummary, } from "../interfaces" -import { codeParser } from "./parser" +import { ParseWorkerClient } from "../parse-worker-client" +import type { ICodeParser } from "../interfaces" +import { IndexWriteGate } from "../index-write-gate" +import { ParserFatalError, throwIfAborted } from "../parser-errors" import { CacheManager } from "../cache-manager" import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" @@ -39,6 +42,13 @@ export class FileWatcher implements IFileWatcher { private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10 private readonly batchSegmentThreshold: number + private readonly localAbortController = new AbortController() + private signal = this.localAbortController.signal + private processingBatch = false + private disposed = false + private fatalReported = false + private readonly parserFatal = new vscode.EventEmitter() + public readonly onParserFatal = this.parserFatal.event private readonly _onDidStartBatchProcessing = new vscode.EventEmitter() private readonly _onBatchProgressUpdate = new vscode.EventEmitter<{ processedInBatch: number @@ -79,6 +89,8 @@ export class FileWatcher implements IFileWatcher { ignoreInstance?: Ignore, ignoreController?: RooIgnoreController, batchSegmentThreshold?: number, + private readonly parser: ICodeParser = new ParseWorkerClient(), + private readonly writeGate = new IndexWriteGate(), ) { this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) if (ignoreInstance) { @@ -103,7 +115,10 @@ export class FileWatcher implements IFileWatcher { /** * Initializes the file watcher */ - async initialize(): Promise { + async initialize(signal?: AbortSignal): Promise { + if (this.disposed) throw new DOMException("Indexing aborted", "AbortError") + this.signal = signal ?? this.signal + throwIfAborted(this.signal) // Create file watcher const filePattern = new vscode.RelativePattern( this.workspacePath, @@ -121,6 +136,10 @@ export class FileWatcher implements IFileWatcher { * Disposes the file watcher */ dispose(): void { + if (this.disposed) return + this.disposed = true + this.localAbortController.abort() + this.parserFatal.dispose() this.fileWatcher?.dispose() if (this.batchProcessDebounceTimer) { clearTimeout(this.batchProcessDebounceTimer) @@ -136,6 +155,7 @@ export class FileWatcher implements IFileWatcher { * @param uri URI of the created file */ private async handleFileCreated(uri: vscode.Uri): Promise { + if (this.disposed || this.signal.aborted) return this.accumulatedEvents.set(uri.fsPath, { uri, type: "create" }) this.scheduleBatchProcessing() } @@ -145,6 +165,7 @@ export class FileWatcher implements IFileWatcher { * @param uri URI of the changed file */ private async handleFileChanged(uri: vscode.Uri): Promise { + if (this.disposed || this.signal.aborted) return this.accumulatedEvents.set(uri.fsPath, { uri, type: "change" }) this.scheduleBatchProcessing() } @@ -154,6 +175,7 @@ export class FileWatcher implements IFileWatcher { * @param uri URI of the deleted file */ private async handleFileDeleted(uri: vscode.Uri): Promise { + if (this.disposed || this.signal.aborted) return this.accumulatedEvents.set(uri.fsPath, { uri, type: "delete" }) this.scheduleBatchProcessing() } @@ -162,6 +184,7 @@ export class FileWatcher implements IFileWatcher { * Schedules batch processing with debounce */ private scheduleBatchProcessing(): void { + if (this.disposed || this.signal.aborted) return if (this.batchProcessDebounceTimer) { clearTimeout(this.batchProcessDebounceTimer) } @@ -172,7 +195,7 @@ export class FileWatcher implements IFileWatcher { * Triggers processing of accumulated events */ private async triggerBatchProcessing(): Promise { - if (this.accumulatedEvents.size === 0) { + if (this.disposed || this.signal.aborted || this.processingBatch || this.accumulatedEvents.size === 0) { return } @@ -182,30 +205,18 @@ export class FileWatcher implements IFileWatcher { const filePathsInBatch = Array.from(eventsToProcess.keys()) this._onDidStartBatchProcessing.fire(filePathsInBatch) - await this.processBatch(eventsToProcess) - } - - /** - * Processes a batch of accumulated events - * @param eventsToProcess Map of events to process - */ - private async _handleBatchDeletions( - batchResults: FileProcessingResult[], - processedCountInBatch: number, - totalFilesInBatch: number, - pathsToExplicitlyDelete: string[], - filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }>, - ): Promise<{ overallBatchError?: Error; clearedPaths: Set; processedCount: number }> { - let overallBatchError: Error | undefined - const allPathsToClearFromDB = new Set(pathsToExplicitlyDelete) - - for (const fileDetail of filesToUpsertDetails) { - if (fileDetail.originalType === "change") { - allPathsToClearFromDB.add(fileDetail.path) + this.processingBatch = true + try { + await this.processBatch(eventsToProcess) + } catch (error) { + if (error instanceof ParserFatalError) this.reportFatal(error) + else if (!this.signal.aborted && !this.disposed) { + this._onDidFinishBatchProcessing.fire({ processedFiles: [], batchError: error as Error }) } + } finally { + this.processingBatch = false + if (this.accumulatedEvents.size > 0) this.scheduleBatchProcessing() } - - return { overallBatchError, clearedPaths: allPathsToClearFromDB, processedCount: processedCountInBatch } } private async _processFilesAndPrepareUpserts( @@ -236,6 +247,7 @@ export class FileWatcher implements IFileWatcher { const result = await this.processFile(fileDetail.path) return { path: fileDetail.path, result: result, error: undefined } } catch (e) { + if (e instanceof ParserFatalError || (e instanceof Error && e.name === "AbortError")) throw e const error = e as Error console.error(`[FileWatcher] Unhandled exception processing file ${fileDetail.path}:`, e) return { path: fileDetail.path, result: undefined, error: error } @@ -243,6 +255,17 @@ export class FileWatcher implements IFileWatcher { }) const settledChunkResults = await Promise.allSettled(chunkProcessingPromises) + const fatal = settledChunkResults.find( + (result) => result.status === "rejected" && result.reason instanceof ParserFatalError, + ) + if (fatal?.status === "rejected") throw fatal.reason + throwIfAborted(this.signal) + if (this.disposed) + return { + pointsForBatchUpsert: [], + successfullyProcessedForUpsert: [], + processedCount: processedCountInBatch, + } for (const settledResult of settledChunkResults) { let resultPath: string | undefined @@ -293,11 +316,12 @@ export class FileWatcher implements IFileWatcher { if (!pathsToExplicitlyDelete.includes(resultPath || "")) { processedCountInBatch++ } - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: resultPath, - }) + if (processedCountInBatch < totalFilesInBatch) + this._onBatchProgressUpdate.fire({ + processedInBatch: processedCountInBatch, + totalInBatch: totalFilesInBatch, + currentFile: resultPath, + }) } } @@ -308,22 +332,15 @@ export class FileWatcher implements IFileWatcher { } } - private async _executeBatchUpsertOperations( - pointsForBatchUpsert: PointStruct[], - successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }>, - batchResults: FileProcessingResult[], - overallBatchError?: Error, - ): Promise { - return overallBatchError - } - private async processBatch( eventsToProcess: Map, ): Promise { + throwIfAborted(this.signal) + await this.writeGate.ready(this.signal) const batchResults: FileProcessingResult[] = [] let processedCountInBatch = 0 const totalFilesInBatch = eventsToProcess.size - let overallBatchError: Error | undefined + const overallBatchError: Error | undefined = undefined // Initial progress update this._onBatchProgressUpdate.fire({ @@ -348,17 +365,6 @@ export class FileWatcher implements IFileWatcher { } } - // Phase 1: Handle deletions - const { overallBatchError: deletionError, processedCount: deletionCount } = await this._handleBatchDeletions( - batchResults, - processedCountInBatch, - totalFilesInBatch, - pathsToExplicitlyDelete, - filesToUpsertDetails, - ) - overallBatchError = deletionError - processedCountInBatch = deletionCount - // Phase 2: Process files and prepare upserts const { pointsForBatchUpsert, @@ -373,14 +379,37 @@ export class FileWatcher implements IFileWatcher { ) processedCountInBatch = upsertCount - // Phase 3: Execute batch upsert - overallBatchError = await this._executeBatchUpsertOperations( - pointsForBatchUpsert, - successfullyProcessedForUpsert, - batchResults, - overallBatchError, - ) - + throwIfAborted(this.signal) + if (this.disposed) return + const replacementPaths = successfullyProcessedForUpsert.map((file) => file.path) + const cleanupPaths = [...new Set([...pathsToExplicitlyDelete, ...replacementPaths])] + for (const filePath of cleanupPaths) { + if (this.vectorStore) { + await this.writeGate.run(this.signal, () => this.vectorStore!.deletePointsByFilePath(filePath), [ + filePath, + ]) + } + await this.writeGate.run(this.signal, () => this.cacheManager.deleteHash(filePath), [filePath]) + } + if (this.vectorStore) { + for ( + let pointOffset = 0; + pointOffset < pointsForBatchUpsert.length; + pointOffset += this.batchSegmentThreshold + ) { + const points = pointsForBatchUpsert.slice(pointOffset, pointOffset + this.batchSegmentThreshold) + await this.writeGate.run(this.signal, () => this.vectorStore!.upsertPoints(points), replacementPaths) + } + } + for (const file of successfullyProcessedForUpsert) { + if (file.newHash) + await this.writeGate.run(this.signal, () => this.cacheManager.updateHash(file.path, file.newHash!), [ + file.path, + ]) + this.writeGate.reconciled(file.path) + batchResults.push({ path: file.path, status: "success" }) + } + throwIfAborted(this.signal) // Finalize this._onDidFinishBatchProcessing.fire({ processedFiles: batchResults, @@ -405,8 +434,17 @@ export class FileWatcher implements IFileWatcher { * @param filePath Path to the file to process * @returns Promise resolving to processing result */ + private reportFatal(error: ParserFatalError): void { + if (this.fatalReported || this.disposed || this.signal.aborted) return + this.fatalReported = true + this.parserFatal.fire(error) + this.dispose() + } + async processFile(filePath: string): Promise { try { + throwIfAborted(this.signal) + await this.writeGate.ready(this.signal) // Get relative path for ignore checks const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) @@ -450,7 +488,7 @@ export class FileWatcher implements IFileWatcher { const newHash = createHash("sha256").update(content).digest("hex") // Check if file has changed - if (this.cacheManager.getHash(filePath) === newHash) { + if (this.cacheManager.getHash(filePath) === newHash && !this.writeGate.needsReconciliation(filePath)) { return { path: filePath, status: "skipped" as const, @@ -459,13 +497,16 @@ export class FileWatcher implements IFileWatcher { } // Parse file - const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash }) + const blocks = await this.parser.parseFile(filePath, { content, fileHash: newHash, signal: this.signal }) + throwIfAborted(this.signal) // Prepare points for batch processing let pointsToUpsert: PointStruct[] = [] if (this.embedder && blocks.length > 0) { const texts = blocks.map((block) => block.content) - const { embeddings } = await this.embedder.createEmbeddings(texts) + const { embeddings } = await this.embedder.createEmbeddings(texts, undefined, this.signal) + + throwIfAborted(this.signal) pointsToUpsert = blocks.map((block, index) => { const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) @@ -492,6 +533,11 @@ export class FileWatcher implements IFileWatcher { pointsToUpsert, } } catch (error) { + if (error instanceof ParserFatalError) { + this.reportFatal(error) + throw error + } + if (error instanceof Error && error.name === "AbortError") throw error return { path: filePath, status: "local_error" as const, diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 2279edd1c67..a1dfd6494a6 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -7,7 +7,7 @@ import { parseMarkdown } from "../../tree-sitter/markdownParser" import { ICodeParser, CodeBlock } from "../interfaces" import { scannerExtensions, shouldUseFallbackChunking } from "../shared/supported-extensions" import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_TOLERANCE_FACTOR } from "../constants" -import { sanitizeErrorMessage } from "../shared/validation-helpers" +import { isParserAbort, throwIfAborted } from "../parser-errors" /** * Implementation of the code parser interface @@ -15,8 +15,16 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers" export class CodeParser implements ICodeParser { private loadedParsers: LanguageParser = {} private pendingLoads: Map> = new Map() - // Markdown files are now supported using the custom markdown parser - // which extracts headers and sections for semantic indexing + public fallbackNeeded = false + constructor( + private readonly parseInWorker?: ( + filePath: string, + content: string, + fileHash: string, + signal?: AbortSignal, + ) => Promise, + private readonly sourceDirectory?: string, + ) {} /** * Parses a code file into code blocks @@ -29,8 +37,11 @@ export class CodeParser implements ICodeParser { options?: { content?: string fileHash?: string + signal?: AbortSignal }, ): Promise { + this.fallbackNeeded = false + throwIfAborted(options?.signal) // Get file extension const ext = path.extname(filePath).toLowerCase() @@ -43,7 +54,7 @@ export class CodeParser implements ICodeParser { let content: string let fileHash: string - if (options?.content) { + if (options?.content !== undefined) { content = options.content fileHash = options.fileHash || this.createFileHash(content) } else { @@ -57,6 +68,16 @@ export class CodeParser implements ICodeParser { } // Parse the file + throwIfAborted(options?.signal) + const extension = path.extname(filePath).toLowerCase() + if ( + this.parseInWorker && + extension !== ".md" && + extension !== ".markdown" && + !shouldUseFallbackChunking(extension) + ) { + return this.parseInWorker(filePath, content, fileHash, options?.signal) + } return this.parseContent(filePath, content, fileHash) } @@ -85,6 +106,10 @@ export class CodeParser implements ICodeParser { * @param fileHash File hash * @returns Array of code blocks */ + public fallbackContent(filePath: string, content: string, fileHash: string): CodeBlock[] { + return this._performFallbackChunking(filePath, content, fileHash, new Set()) + } + private async parseContent(filePath: string, content: string, fileHash: string): Promise { const ext = path.extname(filePath).slice(1).toLowerCase() const seenSegmentHashes = new Set() @@ -106,11 +131,12 @@ export class CodeParser implements ICodeParser { try { await pendingLoad } catch (error) { + if (isParserAbort(error)) throw error console.error(`Error in pending parser load for ${filePath}:`, error) return [] } } else { - const loadPromise = loadRequiredLanguageParsers([filePath]) + const loadPromise = loadRequiredLanguageParsers([filePath], this.sourceDirectory, this.sourceDirectory) this.pendingLoads.set(ext, loadPromise) try { const newParsers = await loadPromise @@ -118,6 +144,7 @@ export class CodeParser implements ICodeParser { this.loadedParsers = { ...this.loadedParsers, ...newParsers } } } catch (error) { + if (isParserAbort(error)) throw error console.error(`Error loading language parser for ${filePath}:`, error) return [] } finally { @@ -134,82 +161,90 @@ export class CodeParser implements ICodeParser { const tree = language.parser.parse(content) - // We don't need to get the query string from languageQueries since it's already loaded - // in the language object - const captures = tree ? language.query.captures(tree.rootNode) : [] + try { + // We don't need to get the query string from languageQueries since it's already loaded + // in the language object + const captures = tree ? language.query.captures(tree.rootNode) : [] - // Check if captures are empty - if (captures.length === 0) { - if (content.length >= MIN_BLOCK_CHARS) { - // Perform fallback chunking if content is large enough - const blocks = this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) - return blocks - } else { - // Return empty if content is too small for fallback - return [] + // Check if captures are empty + if (captures.length === 0) { + if (this.sourceDirectory) { + this.fallbackNeeded = true + return [] + } + if (content.length >= MIN_BLOCK_CHARS) { + // Perform fallback chunking if content is large enough + const blocks = this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes) + return blocks + } else { + // Return empty if content is too small for fallback + return [] + } } - } - const results: CodeBlock[] = [] - - // Process captures if not empty - const queue: Node[] = Array.from(captures).map((capture) => capture.node) - - while (queue.length > 0) { - const currentNode = queue.shift()! - // const lineSpan = currentNode.endPosition.row - currentNode.startPosition.row + 1 // Removed as per lint error - - // Check if the node meets the minimum character requirement - if (currentNode.text.length >= MIN_BLOCK_CHARS) { - // If it also exceeds the maximum character limit, try to break it down - if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) { - if (currentNode.children.filter((child) => child !== null).length > 0) { - // If it has children, process them instead - queue.push(...currentNode.children.filter((child) => child !== null)) + const results: CodeBlock[] = [] + + // Process captures if not empty + const queue: Node[] = Array.from(captures).map((capture) => capture.node) + + for (let queueIndex = 0; queueIndex < queue.length; queueIndex++) { + const currentNode = queue[queueIndex] + // const lineSpan = currentNode.endPosition.row - currentNode.startPosition.row + 1 // Removed as per lint error + + // Check if the node meets the minimum character requirement + if (currentNode.text.length >= MIN_BLOCK_CHARS) { + // If it also exceeds the maximum character limit, try to break it down + if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) { + if (currentNode.children.filter((child) => child !== null).length > 0) { + // If it has children, process them instead + queue.push(...currentNode.children.filter((child) => child !== null)) + } else { + // If it's a leaf node, chunk it + const chunkedBlocks = this._chunkLeafNodeByLines( + currentNode, + filePath, + fileHash, + seenSegmentHashes, + ) + results.push(...chunkedBlocks) + } } else { - // If it's a leaf node, chunk it - const chunkedBlocks = this._chunkLeafNodeByLines( - currentNode, - filePath, - fileHash, - seenSegmentHashes, - ) - results.push(...chunkedBlocks) - } - } else { - // Node meets min chars and is within max chars, create a block - const identifier = - currentNode.childForFieldName("name")?.text || - currentNode.children.find((c) => c?.type === "identifier")?.text || - null - const type = currentNode.type - const start_line = currentNode.startPosition.row + 1 - const end_line = currentNode.endPosition.row + 1 - const content = currentNode.text - const contentPreview = content.slice(0, 100) - const segmentHash = createHash("sha256") - .update(`${filePath}-${start_line}-${end_line}-${content.length}-${contentPreview}`) - .digest("hex") - - if (!seenSegmentHashes.has(segmentHash)) { - seenSegmentHashes.add(segmentHash) - results.push({ - file_path: filePath, - identifier, - type, - start_line, - end_line, - content, - segmentHash, - fileHash, - }) + // Node meets min chars and is within max chars, create a block + const identifier = + currentNode.childForFieldName("name")?.text || + currentNode.children.find((c) => c?.type === "identifier")?.text || + null + const type = currentNode.type + const start_line = currentNode.startPosition.row + 1 + const end_line = currentNode.endPosition.row + 1 + const content = currentNode.text + const contentPreview = content.slice(0, 100) + const segmentHash = createHash("sha256") + .update(`${filePath}-${start_line}-${end_line}-${content.length}-${contentPreview}`) + .digest("hex") + + if (!seenSegmentHashes.has(segmentHash)) { + seenSegmentHashes.add(segmentHash) + results.push({ + file_path: filePath, + identifier, + type, + start_line, + end_line, + content, + segmentHash, + fileHash, + }) + } } } + // Nodes smaller than minBlockChars are ignored } - // Nodes smaller than minBlockChars are ignored - } - return results + return results + } finally { + tree?.delete() + } } /** @@ -228,6 +263,8 @@ export class CodeParser implements ICodeParser { let currentChunkLength = 0 let chunkStartLineIndex = 0 // 0-based index within the `lines` array const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR + const lineOffsets = [0] + for (const line of lines) lineOffsets.push(lineOffsets[lineOffsets.length - 1] + line.length + 1) const finalizeChunk = (endLineIndex: number) => { if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) { @@ -311,10 +348,7 @@ export class CodeParser implements ICodeParser { if (currentChunkLength > 0 && currentChunkLength + lineLength > effectiveMaxChars) { // Re-balancing Logic let splitIndex = i - 1 - let remainderLength = 0 - for (let j = i; j < lines.length; j++) { - remainderLength += lines[j].length + (j < lines.length - 1 ? 1 : 0) - } + const remainderLength = lineOffsets[lines.length] - lineOffsets[i] - 1 if ( currentChunkLength >= MIN_BLOCK_CHARS && @@ -322,10 +356,8 @@ export class CodeParser implements ICodeParser { currentChunkLines.length > 1 ) { for (let k = i - 2; k >= chunkStartLineIndex; k--) { - const potentialChunkLines = lines.slice(chunkStartLineIndex, k + 1) - const potentialChunkLength = potentialChunkLines.join("\n").length + 1 - const potentialNextChunkLines = lines.slice(k + 1) - const potentialNextChunkLength = potentialNextChunkLines.join("\n").length + 1 + const potentialChunkLength = lineOffsets[k + 1] - lineOffsets[chunkStartLineIndex] + const potentialNextChunkLength = lineOffsets[lines.length] - lineOffsets[k + 1] if ( potentialChunkLength >= MIN_BLOCK_CHARS && diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index f25f6c97b37..a809147bb4b 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -28,6 +28,8 @@ import { import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" +import { IndexWriteGate } from "../index-write-gate" +import { ParserFatalError, throwIfAborted } from "../parser-errors" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number @@ -39,6 +41,7 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, + private readonly writeGate = new IndexWriteGate(), ) { // Get the configurable batch size from VSCode settings, fallback to default // If not provided in constructor, try to get from VSCode settings @@ -71,6 +74,11 @@ export class DirectoryScanner implements IDirectoryScanner { onFileParsed?: (fileBlockCount: number) => void, signal?: AbortSignal, ): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> { + signal ??= new AbortController().signal + await this.writeGate.ready(signal) + const fileCompletions = new Map() + const failedFiles = new Set() + let fatalError: ParserFatalError | undefined const directoryPath = directory // Capture workspace context at scan start const scanWorkspace = getWorkspacePathForContext(directoryPath) @@ -127,7 +135,7 @@ export class DirectoryScanner implements IDirectoryScanner { const parsePromises = supportedPaths.map((filePath) => parseLimiter(async () => { // Check abort signal before processing each file - if (signal?.aborted) return + if (signal?.aborted || fatalError) return try { // Check file size @@ -149,14 +157,28 @@ export class DirectoryScanner implements IDirectoryScanner { // Check against cache const cachedFileHash = this.cacheManager.getHash(filePath) const isNewFile = !cachedFileHash - if (cachedFileHash === currentFileHash) { + if (cachedFileHash === currentFileHash && !this.writeGate.needsReconciliation(filePath)) { // File is unchanged skippedCount++ return } // File is new or changed - parse it using the injected parser function - const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash }) + const blocks = await this.codeParser.parseFile(filePath, { + content, + fileHash: currentFileHash, + signal, + }) + throwIfAborted(signal) + if (fatalError) throw fatalError + // Cleanup must precede every replacement batch, not just the batch carrying the hash. + if (this.qdrantClient) { + await this.writeGate.run(signal!, () => this.qdrantClient.deletePointsByFilePath(filePath), [ + filePath, + ]) + } + await this.writeGate.run(signal!, () => this.cacheManager.deleteHash(filePath), [filePath]) + fileCompletions.set(filePath, currentFileHash) const fileBlockCount = blocks.length onFileParsed?.(fileBlockCount) processedCount++ @@ -209,6 +231,8 @@ export class DirectoryScanner implements IDirectoryScanner { scanWorkspace, onError, onBlocksIndexed, + signal, + failedFiles, ), ) activeBatchPromises.add(batchPromise) @@ -241,9 +265,13 @@ export class DirectoryScanner implements IDirectoryScanner { } } else { // Only update hash if not being processed in a batch - await this.cacheManager.updateHash(filePath, currentFileHash) + throwIfAborted(signal) } } catch (error) { + if (error instanceof ParserFatalError) { + fatalError = error + throw error + } // Re-throw AbortError — it's not a file processing error, just a user-initiated stop if (error instanceof DOMException && error.name === "AbortError") { throw error @@ -294,7 +322,16 @@ export class DirectoryScanner implements IDirectoryScanner { // Queue final batch processing const batchPromise = batchLimiter(() => - this.processBatch(batchBlocks, batchTexts, batchFileInfos, scanWorkspace, onError, onBlocksIndexed), + this.processBatch( + batchBlocks, + batchTexts, + batchFileInfos, + scanWorkspace, + onError, + onBlocksIndexed, + signal, + failedFiles, + ), ) activeBatchPromises.add(batchPromise) @@ -311,6 +348,13 @@ export class DirectoryScanner implements IDirectoryScanner { // Wait for all batch processing to complete await Promise.all(activeBatchPromises) + throwIfAborted(signal) + for (const [filePath, fileHash] of fileCompletions) { + if (failedFiles.has(filePath)) continue + await this.writeGate.run(signal, () => this.cacheManager.updateHash(filePath, fileHash), [filePath]) + this.writeGate.reconciled(filePath) + } + // Check abort signal before handling deleted files if (signal?.aborted) { return { @@ -329,8 +373,14 @@ export class DirectoryScanner implements IDirectoryScanner { // File was deleted or is no longer supported/indexed if (this.qdrantClient) { try { - await this.qdrantClient.deletePointsByFilePath(cachedFilePath) - await this.cacheManager.deleteHash(cachedFilePath) + await this.writeGate.run( + signal, + () => this.qdrantClient.deletePointsByFilePath(cachedFilePath), + [cachedFilePath], + ) + await this.writeGate.run(signal, () => this.cacheManager.deleteHash(cachedFilePath), [ + cachedFilePath, + ]) } catch (error: any) { const errorMessage = error instanceof Error ? error.message : String(error) @@ -375,6 +425,8 @@ export class DirectoryScanner implements IDirectoryScanner { scanWorkspace: string, onError?: (error: Error) => void, onBlocksIndexed?: (indexedCount: number) => void, + signal = new AbortController().signal, + failedFiles = new Set(), ): Promise { if (batchBlocks.length === 0) return @@ -385,36 +437,11 @@ export class DirectoryScanner implements IDirectoryScanner { while (attempts < MAX_BATCH_RETRIES && !success) { attempts++ try { - // --- Deletion Step --- - const uniqueFilePaths = [ - ...new Set( - batchFileInfos - .filter((info) => !info.isNew) // Only modified files (not new) - .map((info) => info.filePath), - ), - ] - if (uniqueFilePaths.length > 0) { - try { - await this.qdrantClient.deletePointsByMultipleFilePaths(uniqueFilePaths) - } catch (deleteError: any) { - const errorMessage = deleteError instanceof Error ? deleteError.message : String(deleteError) - - console.error( - `[DirectoryScanner] Failed to delete points for ${uniqueFilePaths.length} files before upsert in workspace ${scanWorkspace}:`, - deleteError, - ) - - // Re-throw with workspace context - throw new Error( - `Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`, - { cause: deleteError }, - ) - } - } - // --- End Deletion Step --- + throwIfAborted(signal) // Create embeddings for batch - const { embeddings } = await this.embedder.createEmbeddings(batchTexts) + const { embeddings } = await this.embedder.createEmbeddings(batchTexts, undefined, signal) + throwIfAborted(signal) // Prepare points for Qdrant const points = batchBlocks.map((block, index) => { @@ -437,15 +464,19 @@ export class DirectoryScanner implements IDirectoryScanner { }) // Upsert points to Qdrant - await this.qdrantClient.upsertPoints(points) + await this.writeGate.run(signal, () => this.qdrantClient.upsertPoints(points), [ + ...new Set(batchBlocks.map((block) => block.file_path)), + ]) + throwIfAborted(signal) onBlocksIndexed?.(batchBlocks.length) // Update hashes for successfully processed files in this batch - for (const fileInfo of batchFileInfos) { - await this.cacheManager.updateHash(fileInfo.filePath, fileInfo.fileHash) - } success = true } catch (error) { + if (signal.aborted || (error instanceof Error && error.name === "AbortError")) { + for (const block of batchBlocks) failedFiles.add(block.file_path) + return + } lastError = error as Error console.error( `[DirectoryScanner] Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}:`, @@ -460,6 +491,7 @@ export class DirectoryScanner implements IDirectoryScanner { } if (!success && lastError) { + for (const block of batchBlocks) failedFiles.add(block.file_path) console.error(`[DirectoryScanner] Failed to process batch after ${MAX_BATCH_RETRIES} attempts`) if (onError) { // Preserve the original error message from embedders which now have detailed i18n messages diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index 4b703a1e865..406f400880e 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -19,7 +19,9 @@ import { VercelAiGatewayEmbedder } from "./embedders/vercel-ai-gateway" import { BedrockEmbedder } from "./embedders/bedrock" import { OpenRouterEmbedder } from "./embedders/openrouter" import { QdrantVectorStore } from "./vector-store/qdrant-client" -import { codeParser, DirectoryScanner, FileWatcher } from "./processors" +import { DirectoryScanner, FileWatcher } from "./processors" +import { ParseWorkerClient } from "./parse-worker-client" +import { IndexWriteGate } from "./index-write-gate" import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" @@ -33,6 +35,7 @@ export class CodeIndexServiceFactory { private readonly configManager: CodeIndexConfigManager, private readonly workspacePath: string, private readonly cacheManager: CacheManager, + private readonly writeGate = new IndexWriteGate(), ) {} /** @@ -183,7 +186,15 @@ export class CodeIndexServiceFactory { // In test environment, vscode.workspace might not be available batchSize = BATCH_SEGMENT_THRESHOLD } - return new DirectoryScanner(embedder, vectorStore, parser, this.cacheManager, ignoreInstance, batchSize) + return new DirectoryScanner( + embedder, + vectorStore, + parser, + this.cacheManager, + ignoreInstance, + batchSize, + this.writeGate, + ) } /** @@ -196,6 +207,7 @@ export class CodeIndexServiceFactory { cacheManager: CacheManager, ignoreInstance: Ignore, rooIgnoreController?: RooIgnoreController, + parser: ICodeParser = new ParseWorkerClient(), ): IFileWatcher { // Get the configurable batch size from VSCode settings let batchSize: number @@ -216,6 +228,8 @@ export class CodeIndexServiceFactory { ignoreInstance, rooIgnoreController, batchSize, + parser, + this.writeGate, ) } @@ -231,7 +245,7 @@ export class CodeIndexServiceFactory { ): { embedder: IEmbedder vectorStore: IVectorStore - parser: ICodeParser + parser: ParseWorkerClient scanner: DirectoryScanner fileWatcher: IFileWatcher } { @@ -241,7 +255,7 @@ export class CodeIndexServiceFactory { const embedder = this.createEmbedder() const vectorStore = this.createVectorStore() - const parser = codeParser + const parser = new ParseWorkerClient() const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance) const fileWatcher = this.createFileWatcher( context, @@ -250,6 +264,7 @@ export class CodeIndexServiceFactory { cacheManager, ignoreInstance, rooIgnoreController, + parser, ) return { diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index ab7b15783e3..44ba0c80868 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -71,6 +71,13 @@ describe("QdrantVectorStore", () => { vectorStore = new QdrantVectorStore(mockWorkspacePath, mockQdrantUrl, mockVectorSize, mockApiKey) }) + it("propagates rejected deletions so the write gate can reconcile their uncertain outcome", async () => { + vi.spyOn(vectorStore, "collectionExists").mockResolvedValue(true) + const failure = new Error("delete request timed out") + mockQdrantClientInstance.delete.mockRejectedValueOnce(failure) + await expect(vectorStore.deletePointsByFilePath("/test/workspace/file.ts")).rejects.toBe(failure) + }) + it("should correctly initialize QdrantClient and collectionName in constructor", () => { expect(QdrantClient).toHaveBeenCalledTimes(1) expect(QdrantClient).toHaveBeenCalledWith({ diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index ba62afc5f81..06a3caae715 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -536,6 +536,7 @@ export class QdrantVectorStore implements IVectorStore { // Include first few file paths for debugging (avoid logging too many) samplePaths: filePaths.slice(0, 3), }) + throw error } } diff --git a/src/services/tree-sitter/languageParser.ts b/src/services/tree-sitter/languageParser.ts index a8ac0a9ead9..22912a09a2e 100644 --- a/src/services/tree-sitter/languageParser.ts +++ b/src/services/tree-sitter/languageParser.ts @@ -75,12 +75,20 @@ Sources: - https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/README.md - https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js */ -export async function loadRequiredLanguageParsers(filesToParse: string[], sourceDirectory?: string) { +export async function loadRequiredLanguageParsers( + filesToParse: string[], + sourceDirectory?: string, + runtimeDirectory?: string, +) { const { Parser, Query } = require("web-tree-sitter") if (!isParserInitialized) { try { - await Parser.init() + await Parser.init( + runtimeDirectory + ? { locateFile: (filename: string) => path.join(runtimeDirectory, filename) } + : undefined, + ) isParserInitialized = true } catch (error) { console.error(`Error initializing parser: ${error instanceof Error ? error.message : error}`) From a21bd5947b30a94d08536b83d587d3ea674ae76b Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 1 Nov 2026 21:03:49 -0800 Subject: [PATCH 2/4] fix(code-index): preserve progress and fence cancelled indexing attempts --- .github/workflows/code-qa.yml | 1 + knip.json | 6 +- ...debase-indexing-packaged-host-test-plan.md | 199 ++ .../codebase-indexing-packaged-host-log.md | 450 +++++ src/esbuild.mjs | 13 + src/i18n/locales/ca/embeddings.json | 1 + src/i18n/locales/de/embeddings.json | 1 + src/i18n/locales/en/embeddings.json | 1 + src/i18n/locales/es/embeddings.json | 1 + src/i18n/locales/fr/embeddings.json | 1 + src/i18n/locales/hi/embeddings.json | 1 + src/i18n/locales/id/embeddings.json | 1 + src/i18n/locales/it/embeddings.json | 1 + src/i18n/locales/ja/embeddings.json | 1 + src/i18n/locales/ko/embeddings.json | 1 + src/i18n/locales/nl/embeddings.json | 1 + src/i18n/locales/pl/embeddings.json | 1 + src/i18n/locales/pt-BR/embeddings.json | 1 + src/i18n/locales/ru/embeddings.json | 1 + src/i18n/locales/tr/embeddings.json | 1 + src/i18n/locales/vi/embeddings.json | 1 + src/i18n/locales/zh-CN/embeddings.json | 1 + src/i18n/locales/zh-TW/embeddings.json | 1 + .../__tests__/cache-manager.spec.ts | 332 ++-- .../__tests__/file-completion.spec.ts | 317 ++++ .../__tests__/generation-allocator.spec.ts | 249 +++ .../__tests__/index-write-gate.spec.ts | 302 ++- .../code-index/__tests__/manager.spec.ts | 120 +- .../code-index/__tests__/orchestrator.spec.ts | 608 +++--- .../parse-worker-client.real.spec.ts | 153 ++ .../__tests__/parse-worker-client.spec.ts | 70 +- .../__tests__/service-factory.spec.ts | 21 + src/services/code-index/cache-manager.ts | 161 +- .../embedders/__tests__/cancellation.spec.ts | 245 +++ src/services/code-index/embedders/bedrock.ts | 20 +- .../code-index/embedders/cancellation.ts | 15 + src/services/code-index/embedders/gemini.ts | 5 +- src/services/code-index/embedders/mistral.ts | 5 +- src/services/code-index/embedders/ollama.ts | 22 +- .../code-index/embedders/openai-compatible.ts | 45 +- src/services/code-index/embedders/openai.ts | 5 +- .../code-index/embedders/openrouter.ts | 23 +- .../code-index/embedders/vercel-ai-gateway.ts | 5 +- src/services/code-index/fault-loop.wasm | Bin 0 -> 39 bytes src/services/code-index/fault-loop.wat | 4 + src/services/code-index/file-completion.ts | 136 ++ .../code-index/generation-allocator.ts | 58 + src/services/code-index/index-write-gate.ts | 165 +- src/services/code-index/interfaces/cache.ts | 11 +- .../code-index/interfaces/file-processor.ts | 3 + .../code-index/interfaces/vector-store.ts | 13 +- src/services/code-index/manager.ts | 72 +- src/services/code-index/orchestrator.ts | 526 ++---- .../code-index/parse-worker-client.ts | 45 +- src/services/code-index/parse-worker.ts | 12 + .../processors/__tests__/file-watcher.spec.ts | 1642 +++++++++++++---- .../__tests__/scanner.recovery.spec.ts | 1061 +++++++++++ .../processors/__tests__/scanner.spec.ts | 68 +- .../code-index/processors/file-watcher.ts | 503 +---- src/services/code-index/processors/scanner.ts | 566 ++---- src/services/code-index/service-factory.ts | 4 + .../vector-store/README.integration.md | 56 + .../__tests__/qdrant-client.integration.ts | 147 ++ .../__tests__/qdrant-client.spec.ts | 167 +- .../code-index/vector-store/qdrant-client.ts | 119 +- .../vector-store/vitest.integration.config.ts | 10 + .../hooks/__tests__/useSelectedModel.spec.ts | 2 +- 67 files changed, 6464 insertions(+), 2336 deletions(-) create mode 100644 operational/test-plans/codebase-indexing-packaged-host-test-plan.md create mode 100644 operational/test-runs/codebase-indexing-packaged-host-log.md create mode 100644 src/services/code-index/__tests__/file-completion.spec.ts create mode 100644 src/services/code-index/__tests__/generation-allocator.spec.ts create mode 100644 src/services/code-index/__tests__/parse-worker-client.real.spec.ts create mode 100644 src/services/code-index/embedders/__tests__/cancellation.spec.ts create mode 100644 src/services/code-index/embedders/cancellation.ts create mode 100644 src/services/code-index/fault-loop.wasm create mode 100644 src/services/code-index/fault-loop.wat create mode 100644 src/services/code-index/file-completion.ts create mode 100644 src/services/code-index/generation-allocator.ts create mode 100644 src/services/code-index/processors/__tests__/scanner.recovery.spec.ts create mode 100644 src/services/code-index/vector-store/README.integration.md create mode 100644 src/services/code-index/vector-store/__tests__/qdrant-client.integration.ts create mode 100644 src/services/code-index/vector-store/vitest.integration.config.ts diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index 1592b15669b..3421363c090 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -45,6 +45,7 @@ jobs: name: platform-unit-test (${{ matrix.name }}) runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: include: - os: ubuntu-latest diff --git a/knip.json b/knip.json index d71404708a0..f1dc13e9116 100644 --- a/knip.json +++ b/knip.json @@ -12,7 +12,11 @@ ], "workspaces": { "src": { - "entry": ["extension.ts", "services/code-index/parse-worker.ts"], + "entry": [ + "extension.ts", + "services/code-index/parse-worker.ts", + "services/code-index/vector-store/vitest.integration.config.ts" + ], "project": ["**/*.ts"] }, "webview-ui": { diff --git a/operational/test-plans/codebase-indexing-packaged-host-test-plan.md b/operational/test-plans/codebase-indexing-packaged-host-test-plan.md new file mode 100644 index 00000000000..1993f5e6d0d --- /dev/null +++ b/operational/test-plans/codebase-indexing-packaged-host-test-plan.md @@ -0,0 +1,199 @@ +# Codebase indexing packaged-host test plan + +## Status and purpose + +**2026-10-31: BLOCKED; runtime cases NOT RUN.** The +[append-only run log](../test-runs/codebase-indexing-packaged-host-log.md) records the actual +Windows live-input failure, successful package construction, provenance caveat, setup evidence, +and remote prerequisites. This plan is a future procedure, not evidence that it ran. +Do not execute acceptance as part of this documentation-only task. + +Decision 18 requires an installed extension package on local Windows and Remote-WSL. +Both must demonstrate successful parsing, bounded parser failure/termination, explicit retry +without reload, and usable editor and unrelated-extension actions during hang and stop. +Automated simulations and extension-host shutdown do not substitute for these observations. + +## Authority read for this plan + +The full main spec/design, parse-worker design, test inventory, and ADRs 0001–0005 were +read before authoring. Architecture decision records (ADRs) preserve changes to the design; +the current design takes precedence over superseded ADR wording. + +- Main spec, especially + functional requirements 1–13 and packaged-host acceptance. +- Main design, Decisions + 5–6, 11a, 13–19 and the assumptions table. Decision 18 owns this plan and log. +- Parse-worker design, + Decisions 10–11: real WASM-loop injection and physical worker exit. +- Test inventory, + particularly T1–T9, T15, T23, and the repository-suite gate. +- ADR-0001: + per-file progress, stopping and partial-stop reporting, provider cancellation. +- ADR-0002: + shared stop promise includes physical worker exit; WASM hang and suite evidence. +- ADR-0003: + open-file records and late-write grace; no cross-attempt wait. +- ADR-0004: + fenced removal and defined grace clock. +- ADR-0005: + completion invalidation must precede removal; supersedes ADR-0004's removal ordering. + +These are local planning references; this operational plan makes no new design decision. + +## Safety and authorization boundaries + +1. Obtain a signed-in, interactive physical or remote console. If input reports a locked, + disconnected, screensaver, or secure desktop, record BLOCKED and stop. Never bypass it. +2. Use only a disposable fixture, task-owned processes, isolated user-data and extension + directories, and a dedicated test index. Do not edit the active workspace, copy its secrets, + change its trust/settings, or reuse its live indexing collection. +3. Keep all fault injection process-local to separate test hosts. Never set persistent + environment variables or inject a fault into the active editor. +4. Do not install a remote server/connector, change services, create a worktree/clone, + commit, push, or edit protected implementation/output/settings paths under the current + constraint. If resumption needs any such action, obtain separate approval first. +5. Close only independently identified task-owned windows/processes. Prefer graceful cleanup; + never broadly terminate all editor processes or use forced termination as worker evidence. +6. Record concise command/output excerpts and artifact paths. Do not paste raw trees, + screenshots, credentials, or full service responses into the log. + +## Preconditions for a later authorized run + +### Package and runtime identity + +- Identify the exact VSIX, SHA256, size, checkout commit, and dirty state. A VSIX is the + installable extension archive. Record packaging and installed-file comparison evidence. +- Verify backend, worker, WASM assets, locales, and webview correspond to the intended + current source. The existing TEMP package has fresh backend bundles but a reused webview + whose source correspondence is unproven. Hash identity alone does not close that gap. +- Do not use the stale bin package reported in the earlier attempt. The successful TEMP + build is preparation evidence, not a fully proven PR-head package. +- Record OS, VS Code version/commit, and actual extension-host Node version per host. + Builder Node and a main-process cache-directory version are not substitutes. +- Verify installation in the isolated profile, then verify Roo activation in the trusted + fixture. A CLI listing alone does not prove activation or worker startup. + +### Live input and data services + +- Trust only the disposable fixture, using an interactive console and explicit operator + authorization. Confirm the Roo view and indexing controls are reachable. +- Select a real approved embedding backend and credentials through the isolated test + configuration. Verify a real embedding response only with authorization; do not use + dummy inference or infer readiness from a model listing. Do not log credentials. +- Confirm the dedicated Qdrant target and collection ownership before any mutation. + Port 6333 health 200 is readiness evidence only, not the real-store T23 result. +- Confirm an unrelated extension is available in the isolated host and name its test action. + Do not assume a bundled editor command is an unrelated-extension action. + +### Remote-WSL + +- Verify an isolated Remote-WSL connector, compatible server, remote extensions location, + trusted disposable Linux fixture, approved embedder reachability, and dedicated store. +- Prove that Roo runs in the remote extension host, not merely a local Windows window + connected to a Linux folder. Record remote Node, package identity, and worker log location. +- The earlier AzureLinux3.0 discovery did not establish these prerequisites. Under the + no-install constraint, stop BLOCKED if they are unavailable; do not silently install them. + +## Enforced limits and evidence + +| Check | Limit / required evidence | +| ------------------------- | ------------------------------------------------------------------------------------ | +| Parse deadline | 10 seconds from dispatch to the worker, not queue entry | +| Cancel/disable stop | 2 seconds from the stop request; record physical exit, not only UI state | +| Timeout/abort termination | Physical worker exit within the 2-second stop bound after termination begins | +| Responsiveness | An editor action and unrelated-extension action complete during hang and during stop | +| Uncertain-write grace | 30 seconds; relevant to recovery/progress checks, not permission to extend stop | + +Capture timestamps for dispatch, fault entry where observable, user stop, timeout/failure, +physical worker exit, UI status, responsiveness actions, and retry. Record elapsed times +and timing resolution. If the short stop phase cannot be observed sufficiently, record the +responsiveness case as NOT RUN rather than claiming success from later interaction. + +Required parse-client exit messages from parse-worker Decision 11: + +```text +parse worker exited after ms +parse worker exit not observed within stop limit +``` + +The first message must be correlated to the tested worker and termination. The second, +or missing physical exit within the stop limit, fails that termination case. Closing the +window or an `Extension host with pid ... exited` line proves neither parse-worker exit +nor bounded cancellation. Keep abort containment distinct from a genuine Tree-sitter abort: +the injection exercises plumbing. The real WASM-loop fault is not a known Tree-sitter hang. + +## Per-host execution matrix — future work only + +Repeat on local Windows and Remote-WSL, with initial-scan and incremental watcher coverage +where applicable. For every row append actual actions, package/host identity, limits, +outputs, outcome, and artifact paths to the run log. + +| Case | Fault environment | Required observations | +| -------------------- | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| Normal parse | Unset | Supported source parses in installed worker; scan completes with current indexed content | +| Fatal abort | `ROO_PARSE_WORKER_FAULT=abort` | Affected file and parser failure reported; scan fails once, no cascade or automatic restart; physical exit recorded | +| Timeout | `ROO_PARSE_WORKER_FAULT=wasm-hang` | Worker enters real WASM loop; 10-second dispatch deadline; scan fails; physical exit within stop bound | +| User cancel | `ROO_PARSE_WORKER_FAULT=wasm-hang` | Cancel before deadline; Stopping then stopped/partial; physical exit within 2 seconds | +| Disable | `ROO_PARSE_WORKER_FAULT=wasm-hang` | Bounded stop; indexing remains disabled and later fixture file events do not restart it | +| Retry without reload | Same fault host, then suitable explicit recovery | New scan and worker start without editor reload; repeated fault remains bounded; no automatic retry loop | +| Hang responsiveness | Real WASM hang active | Named editor and unrelated-extension actions complete while the worker is hung | +| Stop responsiveness | Termination in progress | Both named actions complete during stop, with correlated timing evidence | + +### Session sequence and retry nuance + +1. Launch the normal isolated host with the fault variable unset; prove successful parsing + before interpreting fault results. Use the same verified package on both host types. +2. Launch separate task-owned hosts for abort, timeout, cancel, and disable. Set the fault + variable before the relevant extension host starts. Confirm the remote process receives + the intended variable; do not assume Windows environment propagation into WSL. +3. Use a disposable supported source file to dispatch a real packaged parse. Perform and + timestamp the named editor and unrelated-extension actions during hang and stop. +4. Record failure status and file, bounded physical exit, absence of repeated notification, + and no restart from subsequent fixture events until explicit recovery. +5. Explicitly retry in the same editor session. The worker reads its fault environment at + startup; a replacement worker in that host can inherit the same fault and fail again. + That bounded repeated failure plus a newly started scan/worker demonstrates retry without + reload. Do not claim a successful fault-free retry by silently relaunching the host. +6. Demonstrate successful post-failure recovery without reload only if an approved, + documented test mechanism can remove the fault for the replacement worker. Otherwise + record that narrower recovery result as unverified, while retaining the bounded retry result. +7. Check completed-file progress and unfinished-file retry behavior. Failure must remain + failure; stop must not become successful completion. Idle cancel leaves status/message + unchanged. Keep provider cancellation and retry-integrity evidence separate from worker exit. +8. Record clean task-owned shutdown independently from worker termination. Preserve the log + artifacts and append outcome; do not rewrite earlier BLOCKED entries. + +## Wider acceptance gates + +Decision 18 host evidence is necessary but not sufficient for overall acceptance. +Decision 19 also requires actual green results for all repository suites, including the +indexing regressions and the named gauge test. T15 must use a real WASM-loop worker; +T23 must use a real Qdrant instance, not mocks or a health endpoint. + +The following are design-required future commands, **not commands executed by this task**: + +```text +pnpm test +pnpm --filter @roo-code/cli test:integration +pnpm --filter @roo-code/vscode-e2e test:ci +``` + +Run only in a separately authorized testing session because build/test commands may write +protected outputs or require credentials. Record the actual target and output, both required +CI platform results, and any unrun prerequisite. Measure large-collection generation payload +index creation time as called out by the design assumptions; do not substitute an empty store. + +## Verdict rules + +- **PASS for a case:** all required live observations and correlated physical-exit/timing + evidence exist for that host and exact package. No inferred success from setup or unit tests. +- **FAIL for a case:** an exercised behavior violates a required bound or result, including + missing physical exit within the stop bound. +- **NOT RUN for a case:** execution did not reach that case. State the concrete reason. +- **BLOCKED overall:** any required case/host/suite is unrun, provenance is inadequate, + or a prerequisite prevents meaningful execution. Do not rename a live-input blocker as + package construction failure when packaging succeeded. + +Every entry must separate direct command evidence, artifact reads, explicitly attributed +parent observations, and assumptions. An untested premise cannot approve acceptance. +The current attempt remains BLOCKED until new evidence closes the listed gaps. diff --git a/operational/test-runs/codebase-indexing-packaged-host-log.md b/operational/test-runs/codebase-indexing-packaged-host-log.md new file mode 100644 index 00000000000..c8d1b7db81e --- /dev/null +++ b/operational/test-runs/codebase-indexing-packaged-host-log.md @@ -0,0 +1,450 @@ +# Codebase indexing packaged-host acceptance log + +## 2026-10-31 — BLOCKED: live input unavailable + +**Decision 18 acceptance is BLOCKED, not PASS.** Package construction succeeded and the +isolated Windows installation was verified. The Windows UI attempt stopped before indexing +because injected input could not reach the desktop. Remote-WSL runtime was not attempted. +Every required packaged runtime case remains **NOT RUN** on both hosts. + +Procedure: [packaged-host test plan](../test-plans/codebase-indexing-packaged-host-test-plan.md). +Append later attempts; do not overwrite this outcome. + +### Evidence sources and boundaries + +- **verified: parent UI Operator tool evidence**, supplied in the task handoff. The UI + commands and observations below were performed by the parent, not rerun by this writer. +- **verified: read-only artifact inspection**, performed by this writer through PowerShell + reads and Node hash comparisons. Session outputs retained as tool artifacts + `cmd-1793412866031.txt`, `cmd-1793412873239.txt`, and `cmd-1793412893176.txt`. +- Builder task: `01a18fb3-444a-7515-9fd5-755b1b754d4f`. + Setup task: `01a18fb7-31f9-719a-9828-67f647cecd57`. +- This documentation task ran no acceptance cases, service probes, installation, UI input, + or build. It changed only this report and the linked plan. No commit, push, worktree, + clone, implementation edit, or settings change was performed by this writer. + +Evidence roots (TEMP is the process temporary directory): + +```text +Build: %TEMP%/roo-dirty-vsix-F4aqrC +Host: %TEMP%/roo-packaged-host-6633814049d4482f8b36250b82bfb436 +``` + +The writer read the build retry log, verification summary/log, provenance, build helper, +verification helper, protected snapshots, setup evidence, installed hash records, extension +listing, handoff, and Windows main log. Helpers were inspected, not executed: they write +artifacts and were unsuitable for this read-only verification. + +### Package identity and provenance + +| Field | Recorded value | +| --------------------------------- | ----------------------------------------------------------------------------------- | +| Package | `roo-cline-3.53.6-dirty-8c38f1c5.vsix` under the build root | +| SHA256 | `a06d84f239d4f04802da90a7c02aa3b0751bb07daaeed18f534005ea088a024f` | +| Bytes | 31136789 | +| Recorded checkout head | `8c38f1c5a49da38c5ca3a87ce3b033a0b45c2a86` | +| Recorded branch | `fix/issue-23-isolated-parse-worker` | +| Checkout state | Dirty; not a clean-head reproducible package | +| Builder Node / esbuild | v22.23.1 / 0.25.9 | +| Windows VS Code | 1.144.0, commit `2c33b50b16c472efe58357d1d2a8a5414bc23fbf` | +| Host Node evidence | Main-process cache path contains v24.22.1; extension-host Node version not measured | +| Planned limits, not measured here | Parse 10 s; stop/physical exit 2 s; late-write grace 30 s | + +**verified: PowerShell artifact reads** of `retry-1.log`, `verification.json`, +`verification.log`, and `provenance.json` under the build root: + +```text +PACKAGE_EXIT=0 +POSTPACKAGE_WORKSPACE_UNCHANGED=true +VERIFY_EXIT=0 +"currentBuildInputsVerified": 3366, +"baselineSourceFilesVerified": 3074, +"webviewProvenanceLimit": "Existing build reused; hash and unchanged inputs do not prove original build correspondence." +``` + +The retained build evidence records fresh extension backend, parse-worker, and token-count +bundles built in TEMP using esbuild 0.25.9, without running the source build script. +The staged prepublish script was removed; repository manifest settings were not changed. +The existing webview build was reused. **Correspondence between that webview and current +source is NOT proven. Do not describe the whole VSIX as equal to current source or as a +fully verified PR-head acceptance package.** + +The parent reported that the existing repository bin VSIX was stale and lacked the worker; +that package was not the selected artifact and was not independently inspected by this writer. +The earlier construction difficulty is not the final blocker: the fresh TEMP package succeeded. + +**verified: writer read-only `node -e` SHA256/size comparison** of the selected VSIX: + +```text +VSIX_SHA256=a06d84f239d4f04802da90a7c02aa3b0751bb07daaeed18f534005ea088a024f +VSIX_BYTES=31136789 +``` + +### Isolated installation and service readiness + +**verified: PowerShell reads** of host-root `handoff.json`, +`evidence/setup-evidence.json`, `evidence/installed-hashes.json`, and +`evidence/list-extensions.txt`. The installed hash records were counted, not regenerated: + +```text +rooveterinaryinc.roo-cline@3.53.6 +INSTALLED_FILES=1741 +IDENTICAL_FILES=1740 +"manifestDifference": "Installer added __metadata only; original manifest properties identical" +``` + +The only differing installed hash record is the manifest. The metadata-only explanation +comes from the setup artifact. Installation identity is not evidence that Roo activated +or that the parse worker ran. + +The setup artifact records these earlier probes; this writer did not repeat them: + +- Qdrant health GET on port 6333 returned 200. No collection was created by setup. +- Ollama model discovery on port 11434 failed with WebException. +- Port 4000 health/model GETs returned 200 and advertised `azure/test-embedding`. + Backend authenticity and credential-free real embedding remain unverified. No embedding + request was sent; model advertisement is not embedding success. +- AzureLinux3.0 was running under WSL2. The checked default user server directory was + absent; root-directory inspection was denied. Setup found no verified safe isolated + Remote-WSL launch under the no-install constraint. It did not copy the existing Windows + connector into the isolated extensions directory, install a remote server, or launch remotely. + +### Windows UI attempt — verified parent tool evidence + +The parent used `C:/git/roo-vault/global-skills/automate-windows-ui/ui.cmd`. +Only the following command fragments and results are quoted; they are not writer reruns. + +1. First process launch using the quoted executable failed with WinError 5. +2. A short-path launch exited 9 with bad option `--new-window`, attributed by the parent + to inherited Electron node-mode. The next launch cleared the process environment variables + ELECTRON_RUN_AS_NODE, VSCODE_IPC_HOOK_CLI, VSCODE_PORTABLE, and ROO_PARSE_WORKER_FAULT. +3. The next process launch used Code.exe with `--new-window`, isolated `--user-data-dir` + and `--extensions-dir`, `--skip-welcome`, `--skip-release-notes`, and the disposable fixture. + These were process-local launch changes, not active-profile settings edits. +4. `desktop windows --title fixture` returned count 1, title + `fixture - Visual Studio Code`, PID 31520. Inspection at depth 5 showed only editor chrome; + both elevation indicators were false. Windows Graphics Capture showed the fixture's + notes and TypeScript sample, Restricted Mode, and no visible Roo activity icon. + This did not establish Roo activation or a successful parse. +5. The parent attempted the trust command through live input: + +```text +web keys --window 'fixture - Visual Studio Code' 'ctrl+shift+p, text:Manage Workspace Trust' --confirm +exit 1; ok false +``` + +Exact returned error: + +> nothing was sent: no window owns the foreground, which is what a locked, disconnected, or screensaver desktop looks like. Injected input cannot reach an application in that state: it is delivered to the secure desktop, which this process cannot open. Sign in at the physical or remote console and retry. + +No trust/input mutation occurred. The parent stopped rather than attempting a secure-desktop +bypass. The live-input failure is the immediate Windows blocker; Restricted Mode was not resolved. + +### Cleanup and physical-exit distinction + +**verified: parent tool evidence** — non-forced termination targeted only task-owned PID 31520: + +```text +process kill 31520 +SUCCESS: Sent termination signal to the process with PID 31520. +desktop windows --title fixture +count 0 +``` + +**verified: writer PowerShell `Select-String` read** of +host-root `user-data/logs/20261030T191227/main.log`: + +```text +2026-10-30 19:12:29.388 [info] AgentHostProcessManager: agent host started +2026-10-30 19:13:25.280 [info] Extension host with pid 41216 exited with code: 0, signal: unknown. +``` + +The second line is extension-host shutdown evidence, **NOT parse-worker physical-exit +or 2-second termination evidence**. Neither an agent-host start nor disappearance of the +window proves a parse worker started, hung, or exited. + +### Runtime result matrix + +| Required case | Local Windows | Remote-WSL | +| ----------------------------------------------------------------------- | ------------- | ---------- | +| Successful packaged parsing | NOT RUN | NOT RUN | +| Fatal abort containment and failure reporting | NOT RUN | NOT RUN | +| Real WASM-loop hang, 10-second timeout | NOT RUN | NOT RUN | +| User cancel during hang, stop and physical worker exit within 2 seconds | NOT RUN | NOT RUN | +| Disable during hang and no automatic restart | NOT RUN | NOT RUN | +| Physical worker exit timing for every termination | NOT RUN | NOT RUN | +| Explicit retry without editor reload | NOT RUN | NOT RUN | +| Editor action and unrelated-extension action during hang | NOT RUN | NOT RUN | +| Editor action and unrelated-extension action during stop | NOT RUN | NOT RUN | + +There are no measured parse/stop latencies or worker-exit lines from this attempt. +T15 real-worker tests, T23 real-Qdrant integration, the full repository suites, CLI integration, +editor end-to-end, and the gauge test were not run by this documentation task. No verdict +about their results elsewhere is made. A health check does not satisfy T23. +Large-collection generation-index creation time was not measured here. + +### Read-only protected-file check + +**verified: writer `node -e` fingerprint comparison** against build-root +`protected-verification-after.json`. The command read files and link targets, hashed bytes, +and compared the recorded JSON structure. It wrote nothing and was a separate hash check, +not execution of the builder's verification gate. + +```text +PROTECTED_SCOPE_MATCH=true +PROTECTED_DRIFT_COUNT=0 +``` + +The comparison covered `0`, all root names beginning `.roo`, `myplans`, `myscripts`, and +`src/dist`. Symlink targets were compared without following their contents. Environment +files and the prohibited plans alias were excluded. Thus this check does not certify +unchanged contents behind the vault symlinks. + +A second read-only Node inventory used the original build inventory format from +`before.json` (file hashes/sizes and link targets; excluding dependency/cache directories +and environment files). Actual output: + +```text +BUILD_DIST_CURRENT_SHA256=143fb53bb4efbd2cb6af1b02e30d45bbe05613e3f5236368b4bad31153b3ddf9 +BUILD_DIST_BASELINE_SHA256=143fb53bb4efbd2cb6af1b02e30d45bbe05613e3f5236368b4bad31153b3ddf9 +BUILD_DIST_UNCHANGED=true +``` + +No drift was found within those scopes. The separate protected-snapshot format yielded +`6e78bbee6a1b80753d3a819e3cbb3d3b1f8afab6bddeebf79cf0d16d330ddc94` for `src/dist`; +that is a different serialization, not conflicting file content. + +### Disposition + +**BLOCKED / NOT ACCEPTED.** Resume requires an interactive console, a trusted disposable +fixture, verified current webview packaging, an approved real embedder, and safe isolated +remote-host prerequisites. The linked plan defines separate fault-injection hosts and +required physical-exit evidence. No dummy inference, active-workspace edits, secure-desktop +bypass, or package-failure reclassification is authorized by this report. + +### Final report-only verification + +**verified: writer read-only `node -e` validation after both reports were created.** It +checked report line lengths and relative links, compared the build baseline's recorded +source hashes/link targets, listed additions with `git --no-optional-locks ls-files`, and +rechecked the protected snapshot's recorded entries. Actual output: + +```text +BASELINE_SOURCE_DRIFT=[] +ADDITIONS_SINCE_BUILD_BASELINE=["operational/test-plans/codebase-indexing-packaged-host-test-plan.md","operational/test-runs/codebase-indexing-packaged-host-log.md"] +FINAL_PROTECTED_RECORDED_ENTRY_DRIFT=0 +``` + +Both reports had zero overlong lines and zero broken relative links at that check. +Only the two authorized reports were additions relative to the retained build baseline; +no recorded baseline source or protected-entry drift was observed. Snapshot exclusions and +unfollowed symlink-content limits above still apply. This check is not acceptance execution. + +## 2026-11-01 — Local Windows run: 7 PASS, 1 NOT RUN; overall still BLOCKED + +Run under the user's decision "Approve it, but try the installed-extension check on this +Windows machine first". Scope: local Windows only. Remote-WSL was not attempted. +Tester task, with UI interaction delegated to ui-operator subtasks. All times are UTC. + +**Local Windows: seven cases PASS and stop responsiveness is NOT RUN. No case FAILED.** +**Decision 18 remains BLOCKED / NOT ACCEPTED:** stop responsiveness and every Remote-WSL +case are still unrun. + +### Evidence sources + +- **verified: tester commands**: packaging, install, host launch, extension-host + Chrome DevTools Protocol (CDP) probes, worker console capture, Qdrant reads and cleanup + checks. Artifacts are under `%TEMP%/d18-win/`. Console captures: + `console-normal.log`, `console-abort.log` and `console-hang.log`. +- **attributed: ui-operator observations**: UI clicks, keys and status texts. Subtasks: + `01a198dd`, `01a198e9`, `01a198fe`, `01a19906`, `01a19908` and `01a1990e`. + Each UI timestamp shows when the tool call was dispatched and when it completed. It does + not give the exact moment of the keystroke. Status was sampled about every 2 s. +- **Worker log capture method**: VS Code relays extension-host `console.log` to the + workbench renderer. The host launcher (`host.cjs`) attaches to it on + `--remote-debugging-port` and records each line with a timestamp. A self-test line, + `parse worker CAPTURE_SELFTEST`, was captured at 21:22:16.849Z. That confirmed the path + before the abort run that produced the first worker-exit evidence below. + An earlier attempt to replace `console.log` in the extension host captured nothing, + because the host's own console wrapper stayed in place. + +### Package identity and provenance + +| Field | Value | +| --------------- | ----------------------------------------------------------------------- | +| VSIX | `%TEMP%/d18-win/roo-d18-win.vsix` | +| SHA256 | `D399FE4CEBD47DD9F783EFEC071CE5FE7DD4C5800EFF9156B6BEA4DC37E13491` | +| Bytes | 31027854 (1743 files) | +| Checkout | `fix/issue-23-isolated-parse-worker` @ `8c38f1c5`, dirty: round-2 edits | +| Webview | Rebuilt from the working tree; `index.js` mtime 2026-11-01T20:34:32Z | +| Backend bundles | Rebuilt by `vsce package` prepublish; `parse-worker.js` mtime 20:35:08Z | +| VS Code | 1.144.0, `2c33b50b16c472efe58357d1d2a8a5414bc23fbf` | +| Ext-host Node | v24.22.1, measured inside the extension host through CDP | + +Commands: `pnpm --filter @roo-code/vscode-webview build`, then +`cd src && npx vsce package --no-dependencies --out %TEMP%/d18-win/roo-d18-win.vsix`. +Both builds finished without errors. The package was a dirty working-tree build, not +a reproducible clean-head build. It closes the earlier gap: the webview now comes from +current source. + +The archive was checked for these entries: `extension/dist/extension.js`, +`parse-worker.js`, `fault-loop.wasm`, `tree-sitter.wasm`, `tree-sitter-typescript.wasm` +and `webview-ui/build/assets/index.js`. + +Install target: the isolated `%TEMP%/d18-win/extensions`. +`--list-extensions --show-versions` listed `rooveterinaryinc.roo-cline@3.53.6` and +`dotjoshjohnson.xml@2.5.1`. Activation was proven later by the Roo output channel and the UI. + +### Preconditions + +| Precondition | Outcome | Evidence | +| --------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Interactive desktop, input | MET | `query user`: session 2 Active, over RDP. A Ctrl+Shift+P to the fixture window was VERIFIED | +| Embedding backend | MET | Ollama on 11434 refused the connection. A real POST to `127.0.0.1:4000/v1/embeddings` returned 200 with a 3072-dim vector (`text-embedding-3-large`) | +| Credentials | MET | The local proxy needs no key. The placeholder `local-test` was typed into the isolated profile only. No secret was copied | +| Dedicated Qdrant collection | MET | Before the run there were 4 collections. Task hosts created the fixture collections `ws-aaa501dd5be1802c`, `ws-5e6734420313a64a` and `ws-b57722351f89d767` | +| Unrelated-extension action | MET | XML Tools `dotjoshjohnson.xml@2.5.1`, command "XML Tools: Minify XML" on `data.xml` | + +Setup notes: + +- Roo settings were seeded by `roo-cline.autoImportSettingsPath` in the isolated profile's + `settings.json`, pointing at `%TEMP%/d18-win/roo-import.json`. That file holds only + local URLs and the placeholder key. +- The import skips secret keys. The embedder therefore stayed unconfigured until + `local-test` was typed into the popover, so Start Indexing did nothing in the first + attempts. That is existing import behaviour, not a defect of this PR. +- Workspace trust was disabled only through the isolated profile's settings and + `--disable-workspace-trust`. + +### Hosts + +Each host was a separate task-owned `Code.exe` with its own `--user-data-dir` and fixture. +The shared isolated extensions directory was used throughout. +`ROO_PARSE_WORKER_FAULT` was set only in each launch environment and was read back inside +the extension host: + +| Label | Code.exe PID | Ext-host PID | Fault read back in host | +| ------ | -----------: | -----------: | ----------------------- | +| normal | 18072 | 33872 | `undefined` | +| abort | 43060 | 38580 | `abort` | +| hang | 48796 | 50628 | `wasm-hang` | + +### Per-case results + +Worker evidence lines are quoted from the renderer capture. Each line is printed with the +renderer's colour-format markers, which are omitted here. + +The value in "exited after N ms" is measured inside `ParseWorkerClient` from `stop()`, +which starts termination, to the worker `exit` event. Limits: parse deadline 10 s from +dispatch; stop and physical exit within 2 s. + +**1. Normal parse — PASS.** Host: normal. + +- The fixture had 3 TypeScript files. +- Save started indexing. The status was "Indexing - Initializing services..." at about + 21:02:03Z, "Indexed" at 21:02:15.880Z, and finally "Indexed - File watcher started." +- **verified: Qdrant scroll** of `ws-aaa501dd5be1802c` showed `POINTS=7` and `DIM=3072`. +- Payloads held parsed blocks such as `src\sample1.ts L1-3 function add1(...)` and + `src\sample1.ts L5-10 class Counter1`, and the same for `sample2` and `sample3`. + +**2. Fatal abort — PASS.** Host: abort. + +- Save was dispatched between 21:25:33.761Z and 21:25:37.176Z. +- Physical exit: `21:25:38.939Z parse worker exited after 4 ms`. +- Status, verbatim from the first abort host (PID 46516) at 21:10:19Z: + "Error - Code indexing stopped while parsing ...\fixture-abort\src\sample1.ts + (parser-abort). Fix the file and explicitly start indexing to retry, or disable + indexing. This is not a model/provider error." +- The same Error text was seen on host 43060. +- There was no automatic restart: 62 s on PID 46516, and 20 s + 22 s + 32 s on PID 43060, + sampled about every 2 s. +- No repeated notifications: no failure toasts were observed at all. The failure is shown + as status text. + +**3. Timeout (`wasm-hang`) — PASS.** Host: hang. + +| Run | Worker entered WASM loop | Worker exit line | Loop → exit | +| ------- | ------------------------ | ---------------------------------------------- | ----------- | +| Initial | 21:35:51.493Z | `21:36:01.409Z parse worker exited after 4 ms` | 9.916 s | +| Retry | 21:37:37.602Z | `21:37:47.557Z parse worker exited after 3 ms` | 9.955 s | + +- The worker entered the real WASM loop: `parse worker entering wasm fault loop`. +- Status: "Error - Code indexing stopped while parsing ...\fixture-hang\src\sample2.ts + (parse-timeout). ..." It was first readable at 21:36:15Z. +- Loop entry to exit stayed under 10 s. Termination to exit took 3–4 ms. + +**4. User cancel — PASS**, with one UI limitation. + +- First attempt: the Stop keypress was not delivered. The run timed out instead + (`21:42:18.173Z exited after 3 ms`, 9.946 s after loop entry). That run counts as + timeout evidence only. +- Retry: + - Loop entered at 21:43:12.750Z. + - Stop focused and Enter pressed between 21:43:14.115Z and 21:43:18.891Z. + - Exit: `21:43:18.070Z parse worker exited after 3 ms`. That is 5.320 s after loop + entry, so cancel won before the deadline. + - Final status: "Standby - Indexing stopped. Partial index data preserved." +- Limitation: no "Stopping" frame was captured. The stop took 3 ms; sampling was about 2 s. + +**5. Disable — PASS.** + +- Loop entered at 21:44:29.508Z. +- The checkbox was unchecked between 21:44:30.904Z and 21:44:34.275Z, while the status was + still "Indexing". Save was activated between 21:44:36.317Z and 21:44:37.864Z. +- Exit: `21:44:37.825Z parse worker exited after 3 ms`, 8.317 s after loop entry and + before the deadline. +- Final status: "Standby - Indexing stopped. Partial index data preserved." The checkbox + stayed unchecked. +- No-restart check: + - **verified:** At 21:46:48.130Z the tester appended to `sample1.ts` and created + `sample4.ts`. No worker line followed in the next 25 s. + - **attributed (ui-operator):** a 33 s UI idle showed no "Indexing". + +**6. Retry without reload — PASS.** The bounded retry was proven; fault-free recovery was not. + +- Abort host: an explicit Start Indexing was pressed between 21:27:53.549Z and + 21:27:57.228Z. A new worker started and exited: `21:27:58.115Z exited after 3 ms`. + The second Error named `sample2.ts`. +- Hang host: an explicit Start started a new worker loop at 21:37:37.602Z, which ended in + a bounded exit (see case 3). +- There was no editor reload and no automatic retry loop during the idle checks. +- **Not verified:** a fault-free recovery without reload. No approved mechanism removes the + fault for a replacement worker, which matches plan step 6. + +**7. Hang responsiveness — PASS** (within the stated limits). + +- During the hang window from 21:37:37.602Z to 21:37:47.557Z, Ctrl+P opened `data.xml` + in the editor. +- "XML Tools: Minify XML" was sent between 21:37:40.603Z and 21:37:43.654Z. +- At 21:37:44Z, `data.xml` was confirmed as one line with unsaved changes. +- XML Tools runs in the same extension host as Roo. The edit also proves the editor + buffer was responsive. +- The first hang's `notes.md` open finished between 21:35:48.777Z and 21:35:52.148Z. + That straddles loop entry, so it is not counted. + +**8. Stop responsiveness — NOT RUN.** Each physical stop finished in 3–4 ms, so no editor +or unrelated-extension action could be placed inside a stop phase. As the plan requires, +nothing is claimed from later interaction. + +### Defects + +None found in the tested behaviour. + +Observation, not required by the plan: parser failures appear only as popover status +text, not as a toast. + +### Cleanup and boundaries + +- All 6 task-owned `Code.exe` main PIDs were closed with a non-forced `taskkill /PID`: + 41936, 57632, 18072, 46516, 43060 and 48796. +- **verified:** afterwards, no `Code.exe` process command line contained `d18-win`. +- The user's active editor window was never targeted. +- The task's Qdrant fixture collections were left in place for inspection. The 4 + pre-existing collections were not modified. +- An accidental write: the first precheck command's quoting turned `=>` into a cmd + redirection. That created a repository-root file `ERR` with probe output only and + no secrets. The tester deleted it; `Test-Path` then returned `False`. +- Protected paths: `git status --short` over the protected set showed only the + pre-existing untracked `0`. `git diff --stat HEAD -- .roo myplans myscripts` was empty. + No commit, push, branch or worktree was created. +- `src/dist` and `src/webview-ui/build` were regenerated by the normal build. diff --git a/src/esbuild.mjs b/src/esbuild.mjs index 03af536c1d1..6585e7a7728 100644 --- a/src/esbuild.mjs +++ b/src/esbuild.mjs @@ -140,6 +140,19 @@ async function main() { ...buildOptions, entryPoints: ["services/code-index/parse-worker.ts"], outfile: "dist/parse-worker.js", + plugins: [ + { + name: "copyParserFaultWasm", + setup(build) { + build.onEnd(async () => { + await fs.promises.copyFile( + path.join(srcDir, "services/code-index/fault-loop.wasm"), + path.join(distDir, "fault-loop.wasm"), + ) + }) + }, + }, + ], }), ]) diff --git a/src/i18n/locales/ca/embeddings.json b/src/i18n/locales/ca/embeddings.json index ed812e9baa7..cac831fe6ff 100644 --- a/src/i18n/locales/ca/embeddings.json +++ b/src/i18n/locales/ca/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Ha fallat durant l'escaneig inicial: {{errorMessage}}", "unknownError": "Error desconegut", "indexingRequiresWorkspace": "Indexació requereix una carpeta de workspace oberta", + "indexingStopping": "S'està aturant la indexació...", "indexingStopped": "Indexació aturada per l'usuari.", "indexingStoppedPartial": "Indexació aturada. Dades d'índex parcials conservades." } diff --git a/src/i18n/locales/de/embeddings.json b/src/i18n/locales/de/embeddings.json index edf2efd4c0e..73928caf793 100644 --- a/src/i18n/locales/de/embeddings.json +++ b/src/i18n/locales/de/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Fehler während des ersten Scans: {{errorMessage}}", "unknownError": "Unbekannter Fehler", "indexingRequiresWorkspace": "Indexierung erfordert einen offenen Workspace-Ordner", + "indexingStopping": "Indexierung wird gestoppt...", "indexingStopped": "Indexierung vom Benutzer gestoppt.", "indexingStoppedPartial": "Indexierung gestoppt. Teilweise Indexdaten beibehalten." } diff --git a/src/i18n/locales/en/embeddings.json b/src/i18n/locales/en/embeddings.json index a7ba0ca5a8b..fa05db68702 100644 --- a/src/i18n/locales/en/embeddings.json +++ b/src/i18n/locales/en/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Failed during initial scan: {{errorMessage}}", "unknownError": "Unknown error", "indexingRequiresWorkspace": "Indexing requires an open workspace folder", + "indexingStopping": "Stopping indexing...", "indexingStopped": "Indexing stopped by user.", "indexingStoppedPartial": "Indexing stopped. Partial index data preserved." } diff --git a/src/i18n/locales/es/embeddings.json b/src/i18n/locales/es/embeddings.json index 442a58e39da..9dc36b90e9b 100644 --- a/src/i18n/locales/es/embeddings.json +++ b/src/i18n/locales/es/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Falló durante el escaneo inicial: {{errorMessage}}", "unknownError": "Error desconocido", "indexingRequiresWorkspace": "La indexación requiere una carpeta de workspace abierta", + "indexingStopping": "Deteniendo la indexación...", "indexingStopped": "Indexación detenida por el usuario.", "indexingStoppedPartial": "Indexación detenida. Datos de índice parciales conservados." } diff --git a/src/i18n/locales/fr/embeddings.json b/src/i18n/locales/fr/embeddings.json index 81c6ce71d69..cdedea06ec0 100644 --- a/src/i18n/locales/fr/embeddings.json +++ b/src/i18n/locales/fr/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Échec lors du scan initial : {{errorMessage}}", "unknownError": "Erreur inconnue", "indexingRequiresWorkspace": "L'indexation nécessite l'ouverture d'un dossier workspace", + "indexingStopping": "Arrêt de l'indexation...", "indexingStopped": "Indexation arrêtée par l'utilisateur.", "indexingStoppedPartial": "Indexation arrêtée. Données d'index partielles conservées." } diff --git a/src/i18n/locales/hi/embeddings.json b/src/i18n/locales/hi/embeddings.json index e0c4498a45a..92aa5a0c0eb 100644 --- a/src/i18n/locales/hi/embeddings.json +++ b/src/i18n/locales/hi/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "प्रारंभिक स्कैन के दौरान असफल: {{errorMessage}}", "unknownError": "अज्ञात त्रुटि", "indexingRequiresWorkspace": "इंडेक्सिंग के लिए एक खुला वर्कस्पेस फ़ोल्डर आवश्यक है", + "indexingStopping": "इंडेक्सिंग रोकी जा रही है...", "indexingStopped": "उपयोगकर्ता द्वारा इंडेक्सिंग रोकी गई।", "indexingStoppedPartial": "इंडेक्सिंग रोकी गई। आंशिक इंडेक्स डेटा संरक्षित।" } diff --git a/src/i18n/locales/id/embeddings.json b/src/i18n/locales/id/embeddings.json index 449ab9388da..6470c3eb86a 100644 --- a/src/i18n/locales/id/embeddings.json +++ b/src/i18n/locales/id/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Gagal selama pemindaian awal: {{errorMessage}}", "unknownError": "Kesalahan tidak diketahui", "indexingRequiresWorkspace": "Pengindeksan memerlukan folder workspace yang terbuka", + "indexingStopping": "Menghentikan pengindeksan...", "indexingStopped": "Pengindeksan dihentikan oleh pengguna.", "indexingStoppedPartial": "Pengindeksan dihentikan. Data indeks parsial dipertahankan." } diff --git a/src/i18n/locales/it/embeddings.json b/src/i18n/locales/it/embeddings.json index f0d16ff7c5e..bbcb2673112 100644 --- a/src/i18n/locales/it/embeddings.json +++ b/src/i18n/locales/it/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Fallito durante la scansione iniziale: {{errorMessage}}", "unknownError": "Errore sconosciuto", "indexingRequiresWorkspace": "L'indicizzazione richiede una cartella di workspace aperta", + "indexingStopping": "Interruzione dell'indicizzazione...", "indexingStopped": "Indicizzazione interrotta dall'utente.", "indexingStoppedPartial": "Indicizzazione interrotta. Dati di indice parziali conservati." } diff --git a/src/i18n/locales/ja/embeddings.json b/src/i18n/locales/ja/embeddings.json index bdcac2a63d5..d74a0c1c5f9 100644 --- a/src/i18n/locales/ja/embeddings.json +++ b/src/i18n/locales/ja/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "初期スキャン中に失敗しました：{{errorMessage}}", "unknownError": "不明なエラー", "indexingRequiresWorkspace": "インデックス作成には、開かれたワークスペースフォルダーが必要です", + "indexingStopping": "インデックス作成を停止しています...", "indexingStopped": "ユーザーによりインデックス作成が停止されました。", "indexingStoppedPartial": "インデックス作成が停止されました。部分的なインデックスデータは保持されています。" } diff --git a/src/i18n/locales/ko/embeddings.json b/src/i18n/locales/ko/embeddings.json index 2b737c2c974..d67ddb6538c 100644 --- a/src/i18n/locales/ko/embeddings.json +++ b/src/i18n/locales/ko/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "초기 스캔 중 실패: {{errorMessage}}", "unknownError": "알 수 없는 오류", "indexingRequiresWorkspace": "인덱싱에는 열린 워크스페이스 폴더가 필요합니다", + "indexingStopping": "인덱싱을 중지하는 중...", "indexingStopped": "사용자에 의해 인덱싱이 중지되었습니다.", "indexingStoppedPartial": "인덱싱이 중지되었습니다. 부분 인덱스 데이터가 보존되었습니다." } diff --git a/src/i18n/locales/nl/embeddings.json b/src/i18n/locales/nl/embeddings.json index 5f04da3f3ed..9bb4d84f53c 100644 --- a/src/i18n/locales/nl/embeddings.json +++ b/src/i18n/locales/nl/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Mislukt tijdens initiële scan: {{errorMessage}}", "unknownError": "Onbekende fout", "indexingRequiresWorkspace": "Indexering vereist een geopende workspace map", + "indexingStopping": "Indexering stoppen...", "indexingStopped": "Indexering gestopt door gebruiker.", "indexingStoppedPartial": "Indexering gestopt. Gedeeltelijke indexgegevens bewaard." } diff --git a/src/i18n/locales/pl/embeddings.json b/src/i18n/locales/pl/embeddings.json index 68b368d17c4..8679441be33 100644 --- a/src/i18n/locales/pl/embeddings.json +++ b/src/i18n/locales/pl/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Niepowodzenie podczas początkowego skanowania: {{errorMessage}}", "unknownError": "Nieznany błąd", "indexingRequiresWorkspace": "Indeksowanie wymaga otwartego folderu workspace", + "indexingStopping": "Zatrzymywanie indeksowania...", "indexingStopped": "Indeksowanie zatrzymane przez użytkownika.", "indexingStoppedPartial": "Indeksowanie zatrzymane. Częściowe dane indeksu zachowane." } diff --git a/src/i18n/locales/pt-BR/embeddings.json b/src/i18n/locales/pt-BR/embeddings.json index fe3deed35bc..7793425dc45 100644 --- a/src/i18n/locales/pt-BR/embeddings.json +++ b/src/i18n/locales/pt-BR/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Falhou durante a varredura inicial: {{errorMessage}}", "unknownError": "Erro desconhecido", "indexingRequiresWorkspace": "A indexação requer uma pasta de workspace aberta", + "indexingStopping": "Interrompendo a indexação...", "indexingStopped": "Indexação interrompida pelo usuário.", "indexingStoppedPartial": "Indexação interrompida. Dados de índice parciais preservados." } diff --git a/src/i18n/locales/ru/embeddings.json b/src/i18n/locales/ru/embeddings.json index 86eed7d6c58..8009952d718 100644 --- a/src/i18n/locales/ru/embeddings.json +++ b/src/i18n/locales/ru/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Ошибка во время первоначального сканирования: {{errorMessage}}", "unknownError": "Неизвестная ошибка", "indexingRequiresWorkspace": "Для индексации требуется открытая папка рабочего пространства", + "indexingStopping": "Остановка индексации...", "indexingStopped": "Индексация остановлена пользователем.", "indexingStoppedPartial": "Индексация остановлена. Частичные данные индекса сохранены." } diff --git a/src/i18n/locales/tr/embeddings.json b/src/i18n/locales/tr/embeddings.json index 4cac4a791b8..8c535159e55 100644 --- a/src/i18n/locales/tr/embeddings.json +++ b/src/i18n/locales/tr/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "İlk tarama sırasında başarısız: {{errorMessage}}", "unknownError": "Bilinmeyen hata", "indexingRequiresWorkspace": "İndeksleme açık bir workspace klasörü gerektirir", + "indexingStopping": "İndeksleme durduruluyor...", "indexingStopped": "İndeksleme kullanıcı tarafından durduruldu.", "indexingStoppedPartial": "İndeksleme durduruldu. Kısmi indeks verileri korundu." } diff --git a/src/i18n/locales/vi/embeddings.json b/src/i18n/locales/vi/embeddings.json index 3bd9d584d20..dd7e2496efe 100644 --- a/src/i18n/locales/vi/embeddings.json +++ b/src/i18n/locales/vi/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "Thất bại trong quá trình quét ban đầu: {{errorMessage}}", "unknownError": "Lỗi không xác định", "indexingRequiresWorkspace": "Lập chỉ mục yêu cầu một thư mục workspace đang mở", + "indexingStopping": "Đang dừng lập chỉ mục...", "indexingStopped": "Lập chỉ mục đã bị dừng bởi người dùng.", "indexingStoppedPartial": "Lập chỉ mục đã dừng. Dữ liệu chỉ mục một phần được bảo toàn." } diff --git a/src/i18n/locales/zh-CN/embeddings.json b/src/i18n/locales/zh-CN/embeddings.json index 00ea2b6854c..8a212a066db 100644 --- a/src/i18n/locales/zh-CN/embeddings.json +++ b/src/i18n/locales/zh-CN/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "初始扫描失败：{{errorMessage}}", "unknownError": "未知错误", "indexingRequiresWorkspace": "索引需要打开的工作区文件夹", + "indexingStopping": "正在停止索引...", "indexingStopped": "用户已停止索引。", "indexingStoppedPartial": "索引已停止。部分索引数据已保留。" } diff --git a/src/i18n/locales/zh-TW/embeddings.json b/src/i18n/locales/zh-TW/embeddings.json index cd0f5e19b2d..8f91116943a 100644 --- a/src/i18n/locales/zh-TW/embeddings.json +++ b/src/i18n/locales/zh-TW/embeddings.json @@ -71,6 +71,7 @@ "failedDuringInitialScan": "初始掃描失敗：{{errorMessage}}", "unknownError": "未知錯誤", "indexingRequiresWorkspace": "索引需要開啟的工作區資料夾", + "indexingStopping": "正在停止索引...", "indexingStopped": "使用者已停止索引。", "indexingStoppedPartial": "索引已停止。部分索引資料已保留。" } diff --git a/src/services/code-index/__tests__/cache-manager.spec.ts b/src/services/code-index/__tests__/cache-manager.spec.ts index e61a92f3cc1..690c74d389e 100644 --- a/src/services/code-index/__tests__/cache-manager.spec.ts +++ b/src/services/code-index/__tests__/cache-manager.spec.ts @@ -1,187 +1,213 @@ -import type { Mock } from "vitest" import * as vscode from "vscode" import { createHash } from "crypto" -import debounce from "lodash.debounce" import { CacheManager } from "../cache-manager" - -// Mock safeWriteJson utility -vitest.mock("../../../utils/safeWriteJson", () => ({ - safeWriteJson: vitest.fn().mockResolvedValue(undefined), -})) - -// Import the mocked version import { safeWriteJson } from "../../../utils/safeWriteJson" -// Mock vscode -vitest.mock("vscode", () => ({ - Uri: { - joinPath: vitest.fn(), - }, - workspace: { - fs: { - readFile: vitest.fn(), - writeFile: vitest.fn(), - delete: vitest.fn(), - }, - }, +vi.mock("../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn() })) +vi.mock("vscode", () => ({ + Uri: { joinPath: vi.fn() }, + workspace: { fs: { readFile: vi.fn() } }, })) -// Mock debounce to execute immediately -vitest.mock("lodash.debounce", () => ({ default: vitest.fn((fn) => fn) })) +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} describe("CacheManager", () => { - let mockContext: vscode.ExtensionContext - let mockWorkspacePath: string - let mockCachePath: vscode.Uri - let cacheManager: CacheManager - - beforeEach(() => { - // Reset all mocks - vitest.clearAllMocks() + const cachePath = { fsPath: "/storage/cache.json" } as vscode.Uri + const context = { globalStorageUri: { fsPath: "/storage" } } as vscode.ExtensionContext + let cache: CacheManager + let persisted: unknown - // Mock context - mockWorkspacePath = "/mock/workspace" - mockCachePath = { fsPath: "/mock/storage/cache.json" } as vscode.Uri - mockContext = { - globalStorageUri: { fsPath: "/mock/storage" } as vscode.Uri, - } as vscode.ExtensionContext + function freshCache() { + return new CacheManager(context, "/workspace") + } - // Mock Uri.joinPath - ;(vscode.Uri.joinPath as Mock).mockReturnValue(mockCachePath) - - // Create cache manager instance - cacheManager = new CacheManager(mockContext, mockWorkspacePath) - }) - - describe("constructor", () => { - it("should correctly set up cachePath using Uri.joinPath and crypto.createHash", () => { - const expectedHash = createHash("sha256").update(mockWorkspacePath).digest("hex") - - expect(vscode.Uri.joinPath).toHaveBeenCalledWith( - mockContext.globalStorageUri, - `roo-index-cache-${expectedHash}.json`, - ) + beforeEach(() => { + vi.useFakeTimers() + vi.resetAllMocks() + persisted = undefined + vi.mocked(vscode.Uri.joinPath).mockReturnValue(cachePath) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async () => { + if (persisted === undefined) throw new Error("Missing cache") + return Buffer.from(JSON.stringify(persisted)) }) - - it("should set up debounced save function", () => { - expect(debounce).toHaveBeenCalledWith(expect.any(Function), 1500) + vi.mocked(safeWriteJson).mockImplementation(async (_path, data) => { + persisted = structuredClone(data) }) + cache = freshCache() }) - describe("initialize", () => { - it("should load existing cache file successfully", async () => { - const mockCache = { "file1.ts": "hash1", "file2.ts": "hash2" } - const mockBuffer = Buffer.from(JSON.stringify(mockCache)) - ;(vscode.workspace.fs.readFile as Mock).mockResolvedValue(mockBuffer) - - await cacheManager.initialize() - - expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(mockCachePath) - expect(cacheManager.getAllHashes()).toEqual(mockCache) - }) - - it("should handle missing cache file by creating empty cache", async () => { - ;(vscode.workspace.fs.readFile as Mock).mockRejectedValue(new Error("File not found")) - - await cacheManager.initialize() - - expect(cacheManager.getAllHashes()).toEqual({}) - }) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() }) - describe("hash management", () => { - it("should update hash and trigger save", () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - - expect(cacheManager.getHash(filePath)).toBe(hash) - expect(safeWriteJson).toHaveBeenCalled() - }) - - it("should delete hash and trigger save", () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - cacheManager.deleteHash(filePath) - - expect(cacheManager.getHash(filePath)).toBeUndefined() - expect(safeWriteJson).toHaveBeenCalled() - }) - - it("should return shallow copy of hashes", () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) - const hashes = cacheManager.getAllHashes() - - // Modify the returned object - hashes[filePath] = "modified" - - // Original should remain unchanged - expect(cacheManager.getHash(filePath)).toBe(hash) - }) + it("uses a workspace-specific hashed cache path", () => { + const workspaceHash = createHash("sha256").update("/workspace").digest("hex") + expect(vscode.Uri.joinPath).toHaveBeenCalledWith( + context.globalStorageUri, + `roo-index-cache-${workspaceHash}.json`, + ) }) - describe("saving", () => { - it("should save cache to disk with correct data", async () => { - const filePath = "test.ts" - const hash = "testhash" - - cacheManager.updateHash(filePath, hash) + it("loads legacy hashes without inventing a generation watermark", async () => { + persisted = { "file.ts": "hash" } + await cache.initialize() + expect(cache.loadSucceeded).toBe(true) + expect(cache.getAllHashes()).toEqual(persisted) + expect(cache.generationWatermark).toBeUndefined() + }) - expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, expect.any(Object)) + it("loads open records ahead of stale hashes and records cache-load completion time", async () => { + persisted = { + version: 2, + fileHashes: { "open.ts": "stale", "complete.ts": "hash" }, + openFiles: { "open.ts": true, "deleted.ts": true }, + generationWatermark: 2048, + } + vi.setSystemTime(12345) + await cache.initialize() + expect(cache.loadedAt).toBe(12345) + expect(cache.loadSucceeded).toBe(true) + expect(cache.generationWatermark).toBe(2048) + expect(cache.getHash("open.ts")).toBeUndefined() + expect(cache.getTrackedPaths().sort()).toEqual(["complete.ts", "deleted.ts", "open.ts"]) + const openFiles = cache.getOpenFiles() + delete openFiles["deleted.ts"] + const hashes = cache.getAllHashes() + hashes["complete.ts"] = "changed" + expect(cache.getOpenFiles()).toEqual({ "open.ts": true, "deleted.ts": true }) + expect(cache.getHash("complete.ts")).toBe("hash") + }) - // Verify the saved data - const savedData = (safeWriteJson as Mock).mock.calls[0][1] - expect(savedData).toEqual({ [filePath]: hash }) + it.each([undefined, "invalid JSON", { version: 2, fileHashes: { "file.ts": 4 }, openFiles: {} }])( + "resets unreadable or invalid cache state (%j)", + async (data) => { + cache.completeFile("previous.ts", "old") + persisted = data + if (data === "invalid JSON") { + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValueOnce(Buffer.from(data)) + } + await cache.initialize() + expect(cache.loadSucceeded).toBe(false) + expect(cache.getAllHashes()).toEqual({}) + expect(cache.getOpenFiles()).toEqual({}) + expect(cache.generationWatermark).toBeUndefined() + }, + ) + + it("awaits durable invalidation and reloads an unfinished file without a hash", async () => { + cache.completeFile("file.ts", "hash") + await cache.flush() + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data) => { + await held.promise + persisted = structuredClone(data) }) + const saved = vi.fn() + const invalidation = cache.invalidateFile("file.ts").then(saved) + await vi.advanceTimersByTimeAsync(0) + expect(saved).not.toHaveBeenCalled() + expect(cache.getHash("file.ts")).toBeUndefined() + held.resolve() + await invalidation + const reloaded = freshCache() + await reloaded.initialize() + expect(reloaded.getHash("file.ts")).toBeUndefined() + expect(reloaded.getOpenFiles()).toEqual({ "file.ts": true }) + }) - it("should handle save errors gracefully", async () => { - const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) - ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) - - cacheManager.updateHash("test.ts", "hash") - - // Wait for any pending promises - await new Promise((resolve) => setTimeout(resolve, 0)) - - expect(consoleErrorSpy).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) - - consoleErrorSpy.mockRestore() + it("serializes an already-issued older snapshot before durable invalidation", async () => { + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data) => { + await held.promise + persisted = structuredClone(data) }) + cache.completeFile("file.ts", "old") + await vi.advanceTimersByTimeAsync(1500) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + const invalidation = cache.invalidateFile("file.ts") + await vi.advanceTimersByTimeAsync(0) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + held.resolve() + await invalidation + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) + await vi.advanceTimersByTimeAsync(1500) + expect(safeWriteJson).toHaveBeenCalledTimes(2) }) - describe("clearCacheFile", () => { - it("should clear cache file and reset state", async () => { - cacheManager.updateHash("test.ts", "hash") - - // Reset the mock to ensure safeWriteJson succeeds for clearCacheFile - ;(safeWriteJson as Mock).mockClear() - ;(safeWriteJson as Mock).mockResolvedValue(undefined) - - await cacheManager.clearCacheFile() + it("propagates durable-save failure, retains the open record, and allows a later save", async () => { + cache.completeFile("file.ts", "hash") + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Disk full")) + await expect(cache.invalidateFile("file.ts")).rejects.toThrow("Disk full") + expect(cache.getHash("file.ts")).toBeUndefined() + expect(cache.getOpenFiles()).toEqual({ "file.ts": true }) + await cache.flush() + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: { "file.ts": true } }) + }) - expect(safeWriteJson).toHaveBeenCalledWith(mockCachePath.fsPath, {}) - expect(cacheManager.getAllHashes()).toEqual({}) + it("debounces completion and removes the open record in the same snapshot as its hash", async () => { + await cache.invalidateFile("file.ts") + vi.mocked(safeWriteJson).mockClear() + cache.completeFile("file.ts", "current") + expect(safeWriteJson).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1499) + expect(safeWriteJson).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(safeWriteJson).toHaveBeenCalledWith(cachePath.fsPath, { + version: 2, + fileHashes: { "file.ts": "current" }, + openFiles: {}, + generationWatermark: undefined, }) + }) - it("should handle clear errors gracefully", async () => { - const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) - ;(safeWriteJson as Mock).mockRejectedValue(new Error("Save failed")) + it("tracks completed and open files until removal completes without enumerating paths", async () => { + const enumerate = vi.spyOn(cache, "getTrackedPaths") + cache.completeFile("file.ts", "hash") + expect(cache.isTracked("file.ts")).toBe(true) + expect(cache.isTracked("missing.ts")).toBe(false) + expect(cache.isTracked("toString")).toBe(false) + await cache.invalidateFile("file.ts") + expect(cache.isTracked("file.ts")).toBe(true) + expect(cache.getHash("file.ts")).toBeUndefined() + expect(enumerate).not.toHaveBeenCalled() + expect(cache.getTrackedPaths()).toEqual(["file.ts"]) + cache.completeRemoval("file.ts") + await cache.flush() + expect(cache.isTracked("file.ts")).toBe(false) + expect(cache.getTrackedPaths()).toEqual([]) + expect(persisted).toMatchObject({ fileHashes: {}, openFiles: {} }) + }) - await cacheManager.clearCacheFile() + it("persists the reserved watermark through hash and open-record clears", async () => { + await cache.reserveGenerations(4096) + cache.completeFile("complete.ts", "hash") + await cache.invalidateFile("open.ts") + await cache.clearCacheFile() + expect(cache.generationWatermark).toBe(4096) + expect(persisted).toEqual({ version: 2, fileHashes: {}, openFiles: {}, generationWatermark: 4096 }) + const reloaded = freshCache() + await reloaded.initialize() + expect(reloaded.generationWatermark).toBe(4096) + }) - expect(consoleErrorSpy).toHaveBeenCalledWith( - "Failed to clear cache file:", - expect.any(Error), - mockCachePath, - ) + it("propagates hash-clear errors instead of claiming durable success", async () => { + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) + await expect(cache.clearCacheFile()).rejects.toThrow("Write denied") + }) - consoleErrorSpy.mockRestore() - }) + it("logs debounced-save errors without an unhandled rejection", async () => { + const log = vi.spyOn(console, "error").mockImplementation(() => {}) + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Write denied")) + cache.completeFile("file.ts", "hash") + await vi.advanceTimersByTimeAsync(1500) + expect(log).toHaveBeenCalledWith("Failed to save cache:", expect.any(Error)) }) }) diff --git a/src/services/code-index/__tests__/file-completion.spec.ts b/src/services/code-index/__tests__/file-completion.spec.ts new file mode 100644 index 00000000000..21126986493 --- /dev/null +++ b/src/services/code-index/__tests__/file-completion.spec.ts @@ -0,0 +1,317 @@ +import { v5 as uuidv5 } from "uuid" +import { FileCompletion, FileIndexWriter } from "../file-completion" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" +import { QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" +import type { CodeBlock } from "../interfaces" + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} + +describe("FileCompletion", () => { + it("does not finish when an early batch settles before the final batch is registered", async () => { + const finish = vi.fn().mockResolvedValue(undefined) + const completion = new FileCompletion(finish) + completion.register() + await completion.settled(true) + expect(completion.outstanding).toBe(0) + expect(finish).not.toHaveBeenCalled() + completion.register() + await completion.registered() + expect(finish).not.toHaveBeenCalled() + await completion.settled(true) + expect(finish).toHaveBeenCalledOnce() + }) + + it("finishes an empty file once and rejects registration after the file is closed", async () => { + const finish = vi.fn().mockResolvedValue(undefined) + const completion = new FileCompletion(finish) + await completion.registered() + await completion.registered() + expect(finish).toHaveBeenCalledOnce() + expect(() => completion.register()).toThrow("registration already complete") + }) + + it.each(["before registration", "after registration"])("keeps a failed file incomplete %s", async (timing) => { + const finish = vi.fn() + const completion = new FileCompletion(finish) + completion.register() + completion.register() + if (timing === "after registration") await completion.registered() + await completion.settled(false) + if (timing === "before registration") await completion.registered() + await completion.settled(true) + expect(completion.failed).toBe(true) + expect(completion.outstanding).toBe(0) + expect(finish).not.toHaveBeenCalled() + }) + + it("shares a pending finish and never retries a failed finish", async () => { + const held = deferred() + const finish = vi.fn(() => held.promise) + const completion = new FileCompletion(finish) + const first = completion.registered() + const second = completion.registered() + const firstRejected = expect(first).rejects.toThrow("Verification failed") + const secondRejected = expect(second).rejects.toThrow("Verification failed") + expect(finish).toHaveBeenCalledOnce() + held.reject(new Error("Verification failed")) + await Promise.all([firstRejected, secondRejected]) + expect(completion.failed).toBe(true) + await completion.registered() + expect(finish).toHaveBeenCalledOnce() + }) +}) + +describe("FileIndexWriter", () => { + const filePath = "/workspace/file.ts" + let cache: any + let store: any + let generations: any + let embedder: any + let gate: IndexWriteGate + let attempt: AbortController + let writer: FileIndexWriter + + function blocks(count: number): CodeBlock[] { + return Array.from({ length: count }, (_, offset) => ({ + file_path: filePath, + identifier: null, + type: "function", + start_line: 1, + end_line: 1, + content: ` block ${offset} `, + fileHash: "hash", + segmentHash: `segment-${offset}`, + })) + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + cache = { + invalidateFile: vi.fn().mockResolvedValue(undefined), + completeFile: vi.fn(), + completeRemoval: vi.fn(), + } + store = { + upsertPoints: vi.fn().mockResolvedValue(undefined), + deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), + countPointsForFile: vi.fn().mockResolvedValue({ current: 2, other: 0 }), + } + generations = { allocate: vi.fn().mockResolvedValue(17) } + embedder = { + createEmbeddings: vi.fn(async (texts: string[]) => ({ embeddings: texts.map(() => [0.1, 0.2]) })), + } + gate = new IndexWriteGate() + attempt = new AbortController() + writer = new FileIndexWriter(cache, store, gate, generations, "/workspace") + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it("awaits durable invalidation before any embedding or vector mutation", async () => { + const held = deferred() + cache.invalidateFile.mockReturnValue(held.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + held.resolve() + await pending + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it.each(["replace", "remove"] as const)( + "issues no vector mutation when %s invalidation fails", + async (operation) => { + cache.invalidateFile.mockRejectedValue(new Error("Disk full")) + const pending = + operation === "replace" + ? writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + : writer.remove(filePath, attempt.signal) + await expect(pending).rejects.toThrow("Disk full") + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(false) + }, + ) + + it("upserts every registered batch before fenced cleanup, verification, hash and progress", async () => { + const finalBatch = deferred() + store.upsertPoints.mockResolvedValueOnce(undefined).mockReturnValueOnce(finalBatch.promise) + const onComplete = vi.fn() + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) + await vi.advanceTimersByTimeAsync(0) + expect(store.upsertPoints).toHaveBeenCalledTimes(2) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + finalBatch.resolve() + await pending + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(store.countPointsForFile).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + expect(onComplete).toHaveBeenCalledWith(2) + expect(store.upsertPoints.mock.invocationCallOrder[1]).toBeLessThan( + store.deletePointsByFilePath.mock.invocationCallOrder[0], + ) + expect(store.countPointsForFile.mock.invocationCallOrder[0]).toBeLessThan( + cache.completeFile.mock.invocationCallOrder[0], + ) + expect(embedder.createEmbeddings).toHaveBeenNthCalledWith(1, ["block 0"], undefined, attempt.signal) + const points = store.upsertPoints.mock.calls.flatMap(([batch]: any[]) => batch) + expect(points.map((point: any) => point.id)).toEqual([ + uuidv5("segment-0:17", QDRANT_CODE_BLOCK_NAMESPACE), + uuidv5("segment-1:17", QDRANT_CODE_BLOCK_NAMESPACE), + ]) + expect(points[0].payload).toMatchObject({ filePath: "file.ts", generation: 17, segmentHash: "segment-0" }) + expect(new Set(points.map((point: any) => point.id)).size).toBe(2) + }) + + it.each([ + { current: 1, other: 0 }, + { current: 2, other: 1 }, + ])("does not complete a file with missing or stale points (%j)", async (counts) => { + store.countPointsForFile.mockResolvedValue(counts) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "verification failed", + ) + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it("cleans and verifies a zero-content file without embedding", async () => { + store.countPointsForFile.mockResolvedValue({ current: 0, other: 0 }) + const whitespace = blocks(1) + whitespace[0].content = " \n\t" + await writer.replace(filePath, "hash", whitespace, embedder, attempt.signal) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it("does not issue cleanup or completion after a failed batch", async () => { + store.upsertPoints.mockRejectedValueOnce(new Error("Upsert failed")) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1)).rejects.toThrow( + "Upsert failed", + ) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + }) + + it("rejects incomplete embeddings without issuing an upsert", async () => { + embedder.createEmbeddings.mockResolvedValue({ embeddings: [[0.1]] }) + await expect(writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal)).rejects.toThrow( + "Incomplete embeddings", + ) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it("discards late embedding results after cancellation without issuing subsequent batches", async () => { + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockReturnValueOnce(held.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1) + const rejected = expect(pending).rejects.toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(0) + attempt.abort() + held.resolve({ embeddings: [[0.1]] }) + await rejected + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it.each(["complete", "stale", "cancel"])( + "keeps an uncertain file incomplete through grace (%s)", + async (outcome) => { + gate.recover([filePath]) + const onComplete = vi.fn() + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal, 1, onComplete) + const observed = pending.catch((error) => error) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(store.countPointsForFile).toHaveBeenCalledTimes(1) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(onComplete).not.toHaveBeenCalled() + if (outcome === "cancel") attempt.abort() + if (outcome === "stale") store.countPointsForFile.mockResolvedValue({ current: 2, other: 1 }) + await vi.advanceTimersByTimeAsync(1) + const result = await observed + if (outcome === "complete") { + expect(result).toBeUndefined() + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + expect(onComplete).toHaveBeenCalledOnce() + expect(gate.needsReconciliation(filePath)).toBe(false) + } else { + expect(result).toMatchObject( + outcome === "cancel" + ? { name: "AbortError" } + : { message: expect.stringContaining("verification failed") }, + ) + expect(cache.completeFile).not.toHaveBeenCalled() + expect(onComplete).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + } + }, + ) + + it("extends grace when another uncertainty arrives during final verification", async () => { + gate.recover([filePath]) + const heldCounts = deferred<{ current: number; other: number }>() + store.countPointsForFile.mockResolvedValueOnce({ current: 2, other: 0 }).mockReturnValueOnce(heldCounts.promise) + const pending = writer.replace(filePath, "hash", blocks(2), embedder, attempt.signal) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + gate.recover([filePath]) + heldCounts.resolve({ current: 2, other: 0 }) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(cache.completeFile).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await pending + expect(store.countPointsForFile).toHaveBeenCalledTimes(3) + expect(cache.completeFile).toHaveBeenCalledWith(filePath, "hash") + }) + + it("invalidates a completed file before removal and drops its open record only after success", async () => { + const invalidation = deferred() + const deletion = deferred() + cache.invalidateFile.mockReturnValue(invalidation.promise) + store.deletePointsByFilePath.mockReturnValue(deletion.promise) + const pending = writer.remove(filePath, attempt.signal) + await vi.advanceTimersByTimeAsync(0) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + invalidation.resolve() + await vi.advanceTimersByTimeAsync(0) + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(filePath, 17) + expect(cache.completeRemoval).not.toHaveBeenCalled() + deletion.resolve() + await pending + expect(cache.completeRemoval).toHaveBeenCalledWith(filePath) + }) + + it("retains the open record after an uncertain removal failure", async () => { + store.deletePointsByFilePath.mockRejectedValue(new DOMException("Lost result", "AbortError")) + await expect(writer.remove(filePath, attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + expect(cache.invalidateFile).toHaveBeenCalledWith(filePath) + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(gate.needsReconciliation(filePath)).toBe(true) + }) +}) diff --git a/src/services/code-index/__tests__/generation-allocator.spec.ts b/src/services/code-index/__tests__/generation-allocator.spec.ts new file mode 100644 index 00000000000..e8b31ed5733 --- /dev/null +++ b/src/services/code-index/__tests__/generation-allocator.spec.ts @@ -0,0 +1,249 @@ +import * as vscode from "vscode" +import { CacheManager } from "../cache-manager" +import { GenerationAllocator } from "../generation-allocator" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" +import { safeWriteJson } from "../../../utils/safeWriteJson" + +vi.mock("../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn() })) +vi.mock("vscode", () => ({ + Uri: { joinPath: vi.fn(() => ({ fsPath: "/storage/cache.json" })) }, + workspace: { fs: { readFile: vi.fn() } }, +})) + +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +describe("GenerationAllocator", () => { + let persisted: unknown + let cache: CacheManager + let gate: IndexWriteGate + let attempt: AbortController + let store: any + + async function loadCache(data: unknown) { + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async () => { + if (data === undefined) throw new Error("Missing cache") + return Buffer.from(typeof data === "string" ? data : JSON.stringify(data)) + }) + const loaded = new CacheManager({ globalStorageUri: {} } as vscode.ExtensionContext, "/workspace") + await loaded.initialize() + return loaded + } + + beforeEach(async () => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + vi.mocked(safeWriteJson) + .mockReset() + .mockImplementation(async (_path, data) => { + persisted = structuredClone(data) + }) + persisted = { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: 2048 } + cache = await loadCache(persisted) + gate = new IndexWriteGate() + attempt = new AbortController() + store = { + hasIndexedData: vi.fn().mockResolvedValue(false), + clearCollection: vi.fn().mockResolvedValue(undefined), + } + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it("awaits a durable 1024-generation reservation before initialization completes", async () => { + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data) => { + await held.promise + persisted = structuredClone(data) + }) + const allocator = new GenerationAllocator(cache) + const initialized = vi.fn() + const pending = allocator.initialize(store, attempt.signal, gate).then(initialized) + await vi.advanceTimersByTimeAsync(0) + expect(safeWriteJson).toHaveBeenCalledWith( + "/storage/cache.json", + expect.objectContaining({ generationWatermark: 3072 }), + ) + expect(initialized).not.toHaveBeenCalled() + held.resolve() + await pending + await expect(allocator.allocate()).resolves.toBe(2049) + expect(persisted).toMatchObject({ generationWatermark: 3072 }) + }) + + it("serializes concurrent allocation and reserves the next block only after 1024 uses", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + const generations = await Promise.all(Array.from({ length: 1024 }, () => allocator.allocate())) + expect(generations).toEqual(Array.from({ length: 1024 }, (_, offset) => 2049 + offset)) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data) => { + await held.promise + persisted = structuredClone(data) + }) + const allocated = vi.fn() + const next = allocator.allocate().then(allocated) + await vi.advanceTimersByTimeAsync(0) + expect(allocated).not.toHaveBeenCalled() + expect(safeWriteJson).toHaveBeenLastCalledWith( + "/storage/cache.json", + expect.objectContaining({ generationWatermark: 4096 }), + ) + held.resolve() + await next + expect(allocated).toHaveBeenCalledWith(3073) + }) + + it("resumes above the entire durable block after reload despite a backwards clock and hash clear", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + await expect(allocator.allocate()).resolves.toBe(2049) + await cache.clearCacheFile() + vi.setSystemTime(1) + const reloaded = await loadCache(persisted) + const nextAllocator = new GenerationAllocator(reloaded) + const clock = vi.spyOn(Date, "now").mockImplementation(() => { + throw new Error("Clock must not seed generations") + }) + await nextAllocator.initialize(store, attempt.signal, gate) + await expect(nextAllocator.allocate()).resolves.toBe(3073) + expect(clock).not.toHaveBeenCalled() + }) + + it("propagates reservation failure and retries initialization with a new durable block", async () => { + const allocator = new GenerationAllocator(cache) + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Disk full")) + await expect(allocator.initialize(store, attempt.signal, gate)).rejects.toThrow("Disk full") + expect(persisted).toMatchObject({ generationWatermark: 2048 }) + await allocator.initialize(store, attempt.signal, gate) + await expect(allocator.allocate()).resolves.toBe(2049) + expect(persisted).toMatchObject({ generationWatermark: 4096 }) + }) + + it("does not return an unreserved generation after rollover-save failure", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + await Promise.all(Array.from({ length: 1024 }, () => allocator.allocate())) + vi.mocked(safeWriteJson).mockRejectedValueOnce(new Error("Disk full")) + await expect(allocator.allocate()).rejects.toThrow("Disk full") + expect(persisted).toMatchObject({ generationWatermark: 3072 }) + await expect(allocator.allocate()).resolves.toBe(3073) + expect(persisted).toMatchObject({ generationWatermark: 5120 }) + }) + + it("refuses cancellation during reservation and keeps the allocation queue usable", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + await Promise.all(Array.from({ length: 1024 }, () => allocator.allocate())) + const held = deferred() + vi.mocked(safeWriteJson).mockImplementationOnce(async (_path, data) => { + await held.promise + persisted = structuredClone(data) + }) + const rejected = expect(allocator.allocate(attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(0) + attempt.abort() + held.resolve() + await rejected + await expect(allocator.allocate()).resolves.toBe(3073) + expect(persisted).toMatchObject({ generationWatermark: 5120 }) + }) + + it.each([undefined, "unreadable JSON", { "file.ts": "legacy-hash" }])( + "clears nonempty storage before reserving when ordering is unknown (%j)", + async (data) => { + cache = await loadCache(data) + store.hasIndexedData.mockResolvedValue(true) + const held = deferred() + store.clearCollection.mockReturnValue(held.promise) + const allocator = new GenerationAllocator(cache) + const pending = allocator.initialize(store, attempt.signal, gate) + await vi.advanceTimersByTimeAsync(0) + expect(store.clearCollection).toHaveBeenCalledOnce() + expect(safeWriteJson).not.toHaveBeenCalled() + held.resolve() + await pending + expect(vi.mocked(safeWriteJson).mock.calls.map((call) => call[1])).toEqual([ + { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: undefined }, + { version: 2, fileHashes: {}, openFiles: {}, generationWatermark: 1024 }, + ]) + await expect(allocator.allocate()).resolves.toBe(1) + }, + ) + + it("does not clear an empty collection with unknown ordering", async () => { + cache = await loadCache("unreadable JSON") + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + expect(store.hasIndexedData).toHaveBeenCalledOnce() + expect(store.clearCollection).not.toHaveBeenCalled() + await expect(allocator.allocate()).resolves.toBe(1) + }) + + it("refuses initialization if reset clearing fails and does not reserve generations", async () => { + cache = await loadCache({ "file.ts": "legacy" }) + store.hasIndexedData.mockResolvedValue(true) + store.clearCollection.mockRejectedValue(new Error("Clear failed")) + await expect(new GenerationAllocator(cache).initialize(store, attempt.signal, gate)).rejects.toThrow( + "Clear failed", + ) + expect(safeWriteJson).not.toHaveBeenCalled() + }) + + it("preserves manager-recovered cache-load grace without recovering again at allocator start", async () => { + cache = await loadCache({ + version: 2, + fileHashes: {}, + openFiles: { "file.ts": true }, + generationWatermark: 1024, + }) + gate.recover(Object.keys(cache.getOpenFiles()), cache.loadedAt) + const recover = vi.spyOn(gate, "recover") + await vi.advanceTimersByTimeAsync(10_000) + await new GenerationAllocator(cache).initialize(store, attempt.signal, gate) + expect(recover).not.toHaveBeenCalled() + const done = vi.fn() + const waiting = gate.waitForGrace("file.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 10_001) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await waiting + expect(done).toHaveBeenCalledOnce() + }) + + it("refuses cancelled allocations without consuming a generation", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + attempt.abort() + await expect(allocator.allocate(attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + await expect(allocator.allocate()).resolves.toBe(2049) + }) + + it("initializes once and rejects generation exhaustion without writing an unsafe integer", async () => { + const allocator = new GenerationAllocator(cache) + await allocator.initialize(store, attempt.signal, gate) + await allocator.initialize(store, attempt.signal, gate) + expect(safeWriteJson).toHaveBeenCalledTimes(1) + cache = await loadCache({ + version: 2, + fileHashes: {}, + openFiles: {}, + generationWatermark: Number.MAX_SAFE_INTEGER, + }) + vi.mocked(safeWriteJson).mockClear() + await expect(new GenerationAllocator(cache).initialize(store, attempt.signal, gate)).rejects.toThrow( + "counter exhausted", + ) + expect(safeWriteJson).not.toHaveBeenCalled() + }) +}) diff --git a/src/services/code-index/__tests__/index-write-gate.spec.ts b/src/services/code-index/__tests__/index-write-gate.spec.ts index 2e771bf87ce..350744a55e2 100644 --- a/src/services/code-index/__tests__/index-write-gate.spec.ts +++ b/src/services/code-index/__tests__/index-write-gate.spec.ts @@ -1,4 +1,4 @@ -import { IndexWriteGate } from "../index-write-gate" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../index-write-gate" function deferred() { let resolve!: () => void @@ -11,17 +11,34 @@ function deferred() { } describe("IndexWriteGate", () => { - it("refuses late embedding writes and all subsequent mutations after abort", async () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + }) + + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + it.each(["already aborted", "aborted before issue"])("refuses %s writes without uncertainty", async (timing) => { const gate = new IndexWriteGate() const attempt = new AbortController() - attempt.abort() const write = vi.fn() - await expect(gate.run(attempt.signal, write, ["file.ts"])).rejects.toMatchObject({ name: "AbortError" }) + if (timing === "already aborted") attempt.abort() + const rejected = expect(gate.run(attempt.signal, write, ["file.ts"])).rejects.toMatchObject({ + name: "AbortError", + }) + attempt.abort() + await rejected expect(write).not.toHaveBeenCalled() + expect(gate.uncertainFiles(attempt.signal)).toEqual([]) + expect(gate.needsReconciliation("file.ts")).toBe(false) + expect(gate.hasPending(attempt.signal)).toBe(false) }) it.each(["resolve", "reject"] as const)( - "orders a retry behind an issued old write that will %s", + "never waits across attempts when an old write will %s", async (outcome) => { const gate = new IndexWriteGate() const oldAttempt = new AbortController() @@ -29,20 +46,275 @@ describe("IndexWriteGate", () => { const held = deferred() const oldWrite = vi.fn(() => held.promise) const pending = gate.run(oldAttempt.signal, oldWrite, ["file.ts"]).catch((error) => error) - await vi.waitFor(() => expect(oldWrite).toHaveBeenCalled()) - oldAttempt.abort() - const newWrite = vi.fn() - const retry = gate.run(nextAttempt.signal, newWrite) await Promise.resolve() - expect(newWrite).not.toHaveBeenCalled() + expect(oldWrite).toHaveBeenCalledOnce() + oldAttempt.abort() + const newWrite = vi.fn(() => "current") + await expect(gate.run(nextAttempt.signal, newWrite, ["retry.ts"])).resolves.toBe("current") + expect(newWrite).toHaveBeenCalledOnce() + expect(gate.hasPending(oldAttempt.signal)).toBe(true) + expect(gate.hasPending(nextAttempt.signal)).toBe(false) if (outcome === "resolve") held.resolve() - else held.reject(new Error("request timeout")) + else held.reject(new Error("Request timeout")) await pending - await retry - expect(newWrite).toHaveBeenCalledOnce() + expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(outcome === "reject" ? ["file.ts"] : []) expect(gate.needsReconciliation("file.ts")).toBe(outcome === "reject") - gate.reconciled("file.ts") - expect(gate.needsReconciliation("file.ts")).toBe(false) + expect(gate.needsReconciliation("retry.ts")).toBe(false) + }, + ) + + it("discards a successful write after abort without extending an existing uncertainty deadline", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"]) + const deadline = gate.graceDeadline("file.ts") + await vi.advanceTimersByTimeAsync(1000) + await expect( + gate.run( + attempt.signal, + async () => { + attempt.abort() + return "remote success" + }, + ["file.ts"], + ), + ).rejects.toMatchObject({ name: "AbortError" }) + expect(gate.uncertainFiles(attempt.signal)).toEqual([]) + expect(gate.graceDeadline("file.ts")).toBe(deadline) + }) + + it.each([new Error("Transport failure"), new DOMException("Cancelled remotely", "AbortError")])( + "marks issued rejection uncertain even for $name", + async (failure) => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + await expect( + gate.run(attempt.signal, () => Promise.reject(failure), ["first.ts", "second.ts"]), + ).rejects.toBe(failure) + expect(gate.uncertainFiles(attempt.signal)).toEqual(["first.ts", "second.ts"]) + const done = vi.fn() + const grace = gate.waitForGrace("first.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + expect(done).toHaveBeenCalledOnce() + expect(gate.needsReconciliation("first.ts")).toBe(true) + gate.reconciled("first.ts") + expect(gate.needsReconciliation("first.ts")).toBe(false) }, ) + + it("starts rejection grace when rejection is observed, not when the request was issued", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + const held = deferred() + const pending = gate.run(attempt.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await vi.advanceTimersByTimeAsync(50_000) + held.reject(new Error("Timeout")) + await pending + const done = vi.fn() + const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + }) + + it("starts stop-expiry grace on abandon and tracks only pending writes from that attempt", async () => { + const gate = new IndexWriteGate() + const oldAttempt = new AbortController() + const otherAttempt = new AbortController() + const held = deferred() + const otherHeld = deferred() + const pending = gate.run(oldAttempt.signal, () => held.promise, ["old.ts"]).catch(() => {}) + const otherPending = gate.run(otherAttempt.signal, () => otherHeld.promise, ["other.ts"]) + await vi.advanceTimersByTimeAsync(2000) + oldAttempt.abort() + gate.abandon(oldAttempt.signal) + expect(gate.uncertainFiles(oldAttempt.signal)).toEqual(["old.ts"]) + expect(gate.needsReconciliation("other.ts")).toBe(false) + const done = vi.fn() + const grace = gate.waitForGrace("old.ts", otherAttempt.signal).then(done) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + held.resolve() + otherHeld.resolve() + await Promise.all([pending, otherPending]) + }) + + it("uses cache-load time and extends an active wait without shortening its deadline", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"], Date.now() - 10_000) + const done = vi.fn() + const grace = gate.waitForGrace("file.ts", attempt.signal).then(done) + await vi.advanceTimersByTimeAsync(5000) + gate.recover(["file.ts"]) + gate.recover(["file.ts"], Date.now() - 50_000) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await grace + expect(done).toHaveBeenCalledOnce() + }) + + it("cancels grace promptly and cleans up its timer while retaining uncertainty", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + gate.recover(["file.ts"]) + const rejected = expect(gate.waitForGrace("file.ts", attempt.signal)).rejects.toMatchObject({ + name: "AbortError", + }) + expect(vi.getTimerCount()).toBe(1) + attempt.abort() + await rejected + expect(vi.getTimerCount()).toBe(0) + expect(gate.needsReconciliation("file.ts")).toBe(true) + await expect(gate.waitForGrace("unaffected.ts", attempt.signal)).rejects.toMatchObject({ name: "AbortError" }) + }) + + it("tracks combined watcher cancellation under its orchestrator attempt", async () => { + const gate = new IndexWriteGate() + const owner = new AbortController() + const local = new AbortController() + const combined = AbortSignal.any([owner.signal, local.signal]) + gate.own(combined, owner.signal) + const held = deferred() + const pending = gate.run(combined, () => held.promise, ["watcher.ts"]).catch(() => {}) + await Promise.resolve() + expect(gate.hasPending(owner.signal)).toBe(true) + owner.abort() + gate.abandon(owner.signal) + expect(gate.uncertainFiles(owner.signal)).toEqual(["watcher.ts"]) + held.resolve() + await pending + await gate.settled(owner.signal) + expect(gate.hasPending(owner.signal)).toBe(false) + }) + + it("completion excludes its current owner, live attempts and unrelated aborted files", async () => { + const gate = new IndexWriteGate() + const current = new AbortController() + const local = new AbortController() + const combined = AbortSignal.any([current.signal, local.signal]) + gate.own(combined, current.signal) + const live = new AbortController() + const old = new AbortController() + const held = deferred() + const writes = [ + gate.run(current.signal, () => held.promise, ["file.ts"]), + gate.run(live.signal, () => held.promise, ["file.ts"]), + gate.run(old.signal, () => held.promise, ["unrelated.ts"]).catch(() => {}), + ] + await Promise.resolve() + old.abort() + await expect(gate.waitForPriorWrites("file.ts", combined)).resolves.toBe(false) + expect(gate.graceDeadline("file.ts")).toBe(0) + held.resolve() + await Promise.all(writes) + }) + + it("bounds a prior aborted write without an orchestrator stop and never repeats abandonment", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + const done = vi.fn() + const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) + await vi.advanceTimersByTimeAsync(1999) + expect(done).not.toHaveBeenCalled() + expect(gate.graceDeadline("file.ts")).toBe(0) + await vi.advanceTimersByTimeAsync(1) + await wait + expect(done).toHaveBeenCalledWith(true) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + await vi.advanceTimersByTimeAsync(1000) + gate.abandon(old.signal) + await expect(gate.waitForPriorWrites("file.ts", current.signal)).resolves.toBe(false) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.reject(new Error("Late rejection")) + await pending + expect(gate.graceDeadline("file.ts")).toBe(133_000) + }) + + it("uses the orchestrator expiry without rebuilding it on abort", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + gate.stopAt(old.signal, Date.now() + 2000) + await vi.advanceTimersByTimeAsync(500) + old.abort() + const done = vi.fn() + const wait = gate.waitForPriorWrites("file.ts", current.signal).then(done) + await vi.advanceTimersByTimeAsync(1499) + expect(done).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + await wait + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.resolve() + await pending + }) + + it("uses the abort stop deadline even when completion starts after its expiry", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + await vi.advanceTimersByTimeAsync(5000) + await gate.waitForPriorWrites("file.ts", current.signal) + expect(gate.graceDeadline("file.ts")).toBe(132_000) + held.resolve() + await pending + }) + + it("cancels the prior-outcome wait and clears its timer without creating uncertainty", async () => { + const gate = new IndexWriteGate() + const old = new AbortController() + const current = new AbortController() + const held = deferred() + const pending = gate.run(old.signal, () => held.promise, ["file.ts"]).catch(() => {}) + await Promise.resolve() + old.abort() + const wait = expect(gate.waitForPriorWrites("file.ts", current.signal)).rejects.toMatchObject({ + name: "AbortError", + }) + expect(vi.getTimerCount()).toBe(1) + current.abort() + await wait + expect(vi.getTimerCount()).toBe(0) + expect(gate.graceDeadline("file.ts")).toBe(0) + held.resolve() + await pending + }) + + it("settles only its own attempt and successful writes introduce no uncertainty", async () => { + const gate = new IndexWriteGate() + const attempt = new AbortController() + const other = new AbortController() + const held = deferred() + const pending = gate.run(other.signal, () => held.promise, ["other.ts"]) + await gate.run(attempt.signal, () => 42, ["file.ts"]) + await gate.settled(attempt.signal) + expect(gate.hasPending(other.signal)).toBe(true) + const done = vi.fn() + const settled = gate.settled(other.signal).then(done) + await Promise.resolve() + expect(done).not.toHaveBeenCalled() + held.resolve() + await Promise.all([pending, settled]) + expect(gate.needsReconciliation("file.ts")).toBe(false) + expect(gate.uncertainFiles(other.signal)).toEqual([]) + }) }) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index b7cda907dfa..c02a2ac9d9a 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -1,5 +1,7 @@ import { CodeIndexManager } from "../manager" import { CodeIndexServiceFactory } from "../service-factory" +import { CacheManager } from "../cache-manager" +import { LATE_WRITE_GRACE_MS } from "../index-write-gate" import type { MockedClass } from "vitest" import * as path from "path" @@ -96,6 +98,18 @@ vi.mock("../state-manager", () => ({ })), })) +vi.mock("../generation-allocator", () => ({ GenerationAllocator: vi.fn() })) +vi.mock("../orchestrator", () => ({ + CodeIndexOrchestrator: vi.fn().mockImplementation((...dependencies: any[]) => ({ + startIndexing: vi.fn().mockResolvedValue(undefined), + stopIndexing: vi.fn().mockResolvedValue(undefined), + stopWatcher: vi.fn().mockResolvedValue(undefined), + generationAllocator: dependencies[11], + isCurrentAttempt: dependencies[12], + onFatal: dependencies[8], + onStopped: dependencies[9], + })), +})) vi.mock("../service-factory") const MockedCodeIndexServiceFactory = CodeIndexServiceFactory as MockedClass @@ -153,6 +167,38 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { CodeIndexManager.disposeAll() }) + it.each(["initialize", "settings"])("T12 recovers grace from cache load time through %s", async (entry) => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + const internal = manager as any + internal._configManager = { + loadConfiguration: vi.fn().mockResolvedValue({ requiresRestart: true }), + isFeatureEnabled: true, + isFeatureConfigured: true, + } + const initialize = vi.spyOn(CacheManager.prototype, "initialize").mockImplementation(async function ( + this: CacheManager, + ) { + this.loadedAt = Date.now() + vi.setSystemTime(105_000) + }) + const openFiles = vi.spyOn(CacheManager.prototype, "getOpenFiles").mockReturnValue({ "open.ts": true }) + const recreate = vi.spyOn(internal, "_recreateServices").mockImplementation(async () => { + internal._orchestrator = { startIndexing: vi.fn(), stopWatcher: vi.fn() } + }) + try { + if (entry === "initialize") await manager.initialize({} as any) + else await manager.handleSettingsChange() + expect(internal.writeGate.graceDeadline("open.ts")).toBe(100_000 + LATE_WRITE_GRACE_MS) + expect(recreate).toHaveBeenCalledOnce() + } finally { + initialize.mockRestore() + openFiles.mockRestore() + recreate.mockRestore() + vi.useRealTimers() + } + }) + describe("parser failure lifecycle", () => { it("blocks automatic initialize and settings recreation while latched", async () => { const internal = manager as any @@ -178,7 +224,7 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { internal._cacheManager = {} internal._searchService = {} internal._stateManager.getCurrentStatus.mockReturnValue({ systemStatus: "Standby" }) - const old = { startIndexing: vi.fn(), stopIndexing: vi.fn() } + const old = { startIndexing: vi.fn(), stopIndexing: vi.fn(), stopWatcher: vi.fn(), stopped: true } internal._orchestrator = old if (reason === "fatal") { internal.parserFailed = true @@ -222,6 +268,74 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { expect(internal._orchestrator).toBe(replacement) internal._orchestrator = undefined }) + it.each(["Indexed", "Standby"])("recreates an idle %s service set for an explicit scan", async (state) => { + const internal = manager as any + internal._configManager = { isFeatureEnabled: true, isFeatureConfigured: true } + internal._cacheManager = {} + internal._searchService = {} + internal._orchestrator = { state, stopWatcher: vi.fn() } + const replacement = { startIndexing: vi.fn(), stopWatcher: vi.fn() } + vi.spyOn(internal, "_recreateServices").mockImplementation(async () => { + internal._orchestrator = replacement + }) + await manager.startIndexing() + expect(replacement.startIndexing).toHaveBeenCalledOnce() + internal._orchestrator = undefined + }) + + it("idle cancel does not flag live services for automatic recreation", () => { + const internal = manager as any + internal._orchestrator = { stopped: false, stopIndexing: vi.fn(() => Promise.resolve()) } + manager.stopIndexing() + expect(internal.servicesStopped).toBe(false) + internal._orchestrator = undefined + }) + + it("returns the orchestrator's shared stop promise unchanged", () => { + const shared = Promise.resolve() + const internal = manager as any + internal._orchestrator = { + stopIndexing: vi.fn(() => shared), + stopWatcher: vi.fn(() => shared), + stopped: true, + } + expect(manager.stopIndexing()).toBe(shared) + expect(manager.stopIndexing()).toBe(shared) + expect(manager.stopWatcher()).toBe(shared) + internal._orchestrator = undefined + }) + + it("preserves the shared allocator across recreation and fences an old attempt's callbacks", async () => { + const internal = manager as any + internal._cacheManager = {} + internal._configManager = { isFeatureEnabled: true, isFeatureConfigured: true } + const watcher = { dispose: vi.fn(), initialize: vi.fn() } + MockedCodeIndexServiceFactory.mockImplementation( + () => + ({ + createServices: vi.fn(() => ({ + embedder: {}, + vectorStore: {}, + scanner: {}, + fileWatcher: watcher, + })), + validateEmbedder: vi.fn().mockResolvedValue({ valid: true }), + }) as any, + ) + await internal._recreateServices() + const old = internal._orchestrator + const allocator = internal.generationAllocator + await internal._recreateServices() + expect(internal.generationAllocator).toBe(allocator) + expect(internal._orchestrator.generationAllocator).toBe(allocator) + expect(MockedCodeIndexServiceFactory.mock.calls.at(-1)?.[4]).toBe(allocator) + expect(old.isCurrentAttempt()).toBe(false) + old.onFatal(new Error("old parser"), "old failure") + old.onStopped() + expect(internal.parserFailed).toBe(false) + expect(internal.servicesStopped).toBe(false) + internal._orchestrator = undefined + }) }) describe("handleSettingsChange", () => { @@ -860,8 +974,8 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { await manager.handleSettingsChange() - expect(mockOrchestrator.stopIndexing).toHaveBeenCalled() - expect(mockStateManager.setSystemState).toHaveBeenCalledWith("Standby", "Code indexing is disabled") + expect(mockOrchestrator.stopWatcher).toHaveBeenCalled() + expect(mockStateManager.setSystemState).not.toHaveBeenCalled() }) }) }) diff --git a/src/services/code-index/__tests__/orchestrator.spec.ts b/src/services/code-index/__tests__/orchestrator.spec.ts index ab0dfbdcfa1..d09ec9d3e6d 100644 --- a/src/services/code-index/__tests__/orchestrator.spec.ts +++ b/src/services/code-index/__tests__/orchestrator.spec.ts @@ -1,402 +1,306 @@ -import { describe, it, expect, beforeEach, vi } from "vitest" import { CodeIndexOrchestrator } from "../orchestrator" import { ParserFatalError } from "../parser-errors" +import { INDEX_STOP_LIMIT_MS } from "../constants" + +vi.mock("../generation-allocator", () => ({ GenerationAllocator: vi.fn() })) +vi.mock("vscode", () => ({ workspace: { workspaceFolders: [{ uri: { fsPath: "/workspace" } }] } })) +vi.mock("../../../i18n", () => ({ t: (key: string) => key })) + +function deferred() { + let resolve!: () => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} -// Mock vscode workspace so startIndexing passes workspace check -vi.mock("vscode", () => { - const path = require("path") - const testWorkspacePath = path.join(path.sep, "test", "workspace") - return { - window: { - activeTextEditor: null, - }, - workspace: { - workspaceFolders: [ - { - uri: { fsPath: testWorkspacePath }, - name: "test", - index: 0, - }, - ], - createFileSystemWatcher: vi.fn().mockReturnValue({ - onDidCreate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidChange: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidDelete: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - }), - }, - RelativePattern: vi.fn().mockImplementation((base: string, pattern: string) => ({ base, pattern })), - } -}) - -// Mock i18n translator used in orchestrator messages -vi.mock("../../i18n", () => ({ - t: (key: string, params?: any) => { - if (key === "embeddings:orchestrator.failedDuringInitialScan" && params?.errorMessage) { - return `Failed during initial scan: ${params.errorMessage}` - } - return key - }, -})) - -describe("CodeIndexOrchestrator - error path cleanup gating", () => { - const workspacePath = "/test/workspace" +async function drain() { + for (let iter = 0; iter < 30; iter++) await Promise.resolve() +} - let configManager: any - let stateManager: any - let cacheManager: any - let vectorStore: any +describe("CodeIndexOrchestrator lifecycle", () => { + let state: any + let cache: any + let store: any let scanner: any - let fileWatcher: any + let watcher: any + let worker: any + let gate: any + let allocator: any + let current: boolean + let fatal: ReturnType + let orchestrator: CodeIndexOrchestrator beforeEach(() => { - vi.clearAllMocks() - - configManager = { - isFeatureConfigured: true, - } - - // Minimal state manager that tracks state transitions - let currentState = "Standby" - stateManager = { + vi.useFakeTimers() + let status = "Standby" + state = { get state() { - return currentState + return status }, - setSystemState: vi.fn().mockImplementation((state: string, _msg: string) => { - currentState = state + setSystemState: vi.fn((next: string) => { + status = next }), - reportFileQueueProgress: vi.fn(), reportBlockIndexingProgress: vi.fn(), + reportFileQueueProgress: vi.fn(), } - - cacheManager = { - clearCacheFile: vi.fn().mockResolvedValue(undefined), - flush: vi.fn().mockResolvedValue(undefined), - } - - vectorStore = { - initialize: vi.fn(), - hasIndexedData: vi.fn(), + cache = { getOpenFiles: vi.fn(() => ({})), clearCacheFile: vi.fn(), flush: vi.fn() } + store = { + initialize: vi.fn().mockResolvedValue(false), markIndexingIncomplete: vi.fn(), markIndexingComplete: vi.fn(), - clearCollection: vi.fn().mockResolvedValue(undefined), + clearCollection: vi.fn(), + deleteCollection: vi.fn(), } - scanner = { - scanDirectory: vi.fn(), + hasIncompleteFiles: false, + scanDirectory: vi.fn().mockResolvedValue({ stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 }), } - - fileWatcher = { - initialize: vi.fn().mockResolvedValue(undefined), - onDidStartBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onBatchProgressUpdate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidFinishBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), + watcher = { + isProcessing: false, + initialize: vi.fn(), dispose: vi.fn(), + onDidStartBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + onBatchProgressUpdate: vi.fn(() => ({ dispose: vi.fn() })), + onDidFinishBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + onParserFatal: vi.fn(() => ({ dispose: vi.fn() })), } + worker = { stop: vi.fn(), stopped: Promise.resolve(), logStopTimeout: vi.fn() } + gate = { + run: vi.fn(async (_signal, write) => write()), + settled: vi.fn().mockResolvedValue(undefined), + hasPending: vi.fn(() => false), + abandon: vi.fn(), + stopAt: vi.fn(), + } + allocator = { initialize: vi.fn() } + current = true + fatal = vi.fn() + orchestrator = new CodeIndexOrchestrator( + { isFeatureConfigured: true } as any, + state, + "/workspace", + cache, + store, + scanner, + watcher, + worker, + fatal, + vi.fn(), + gate, + allocator, + () => current, + ) }) - it.each([false, true])( - "fail-stops a fatal parser during initial/incremental scan (%s), preserves data and drops late callbacks", - async (existing) => { - vectorStore.initialize.mockResolvedValue(false) - vectorStore.hasIndexedData.mockResolvedValue(existing) - let lateProgress!: (count: number) => void + afterEach(() => vi.useRealTimers()) + + it("initializes the allocator after the store and before dispatch", async () => { + const order: string[] = [] + store.initialize.mockImplementation(async () => { + order.push("store") + return false + }) + allocator.initialize.mockImplementation(async () => { + order.push("allocator") + }) + scanner.scanDirectory.mockImplementation(async () => { + order.push("scan") + return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } + }) + await orchestrator.startIndexing() + expect(order).toEqual(["store", "allocator", "scan"]) + expect(allocator.initialize).toHaveBeenCalledWith(store, expect.any(AbortSignal), gate) + expect(state.state).toBe("Indexed") + }) + + it("refuses scanning after a durable reservation failure without destroying data", async () => { + allocator.initialize.mockRejectedValue(new Error("Storage reservation failed")) + await orchestrator.startIndexing() + expect(scanner.scanDirectory).not.toHaveBeenCalled() + expect(state.state).toBe("Error") + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + }) + + it.each(["batch error", "incomplete file", "missing blocks"])( + "does not complete a scan with %s", + async (reason) => { scanner.scanDirectory.mockImplementation( - async (_path: string, _error: unknown, progress: (count: number) => void) => { - lateProgress = progress - throw new ParserFatalError("parser-abort", "bad.ts") + async (_directory: string, onError: any, onIndexed: any, onParsed: any) => { + if (reason === "batch error") onError(new Error("save failed")) + if (reason === "missing blocks") { + onParsed(10) + onIndexed(9) + } + return { + stats: { processed: 1, skipped: 0 }, + totalBlockCount: 10, + incompleteFiles: reason === "incomplete file" ? ["bad.ts"] : [], + } }, ) - const parser = { stop: vi.fn() } - const failed = vi.fn() - const stopped = vi.fn() - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - parser, - failed, - stopped, - ) await orchestrator.startIndexing() - expect(stateManager.state).toBe("Error") - expect(failed).toHaveBeenCalledOnce() - expect(parser.stop).toHaveBeenCalledOnce() - expect(fileWatcher.dispose).toHaveBeenCalledOnce() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - lateProgress(100) - expect(stateManager.reportBlockIndexingProgress).not.toHaveBeenCalled() - await orchestrator.startIndexing() - expect(scanner.scanDirectory).toHaveBeenCalledOnce() + expect(store.markIndexingComplete).not.toHaveBeenCalled() + expect(state.state).toBe("Standby") + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) }, ) - it("fail-stops watcher parsing once and ignores captured late progress", async () => { - vectorStore.initialize.mockResolvedValue(false) - vectorStore.hasIndexedData.mockResolvedValue(true) - scanner.scanDirectory.mockResolvedValue({ stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 }) - let fatal!: (error: ParserFatalError) => void - fileWatcher.onParserFatal = vi.fn((listener) => { - fatal = listener - return { dispose: vi.fn() } - }) - const failed = vi.fn() - const parser = { stop: vi.fn() } - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - parser, - failed, - ) + it("queue progress never declares a failed watcher batch Indexed", async () => { await orchestrator.startIndexing() - const progress = fileWatcher.onBatchProgressUpdate.mock.calls[0][0] - fatal(new ParserFatalError("parse-timeout", "hang.ts")) - fatal(new ParserFatalError("parser-abort", "other.ts")) - progress({ processedInBatch: 1, totalInBatch: 1 }) - expect(failed).toHaveBeenCalledOnce() - expect(stateManager.state).toBe("Error") - expect(stateManager.reportFileQueueProgress).not.toHaveBeenCalled() - expect(parser.stop).toHaveBeenCalledOnce() + state.setSystemState.mockClear() + watcher.onDidStartBatchProcessing.mock.calls[0][0](["bad.ts"]) + watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) + expect(state.state).toBe("Indexing") + watcher.onDidFinishBatchProcessing.mock.calls[0][0]({ processedFiles: [{ path: "bad.ts", status: "error" }] }) + expect(state.state).toBe("Standby") + expect(state.setSystemState.mock.calls.some(([next]: string[]) => next === "Indexed")).toBe(false) }) - it("should not call clearCollection() or clear cache when initialize() fails (indexing not started)", async () => { - // Arrange: fail at initialize() - vectorStore.initialize.mockRejectedValue(new Error("Qdrant unreachable")) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - // Act + it("reports partial when cancellation interrupts a watcher before durable invalidation", async () => { await orchestrator.startIndexing() - - // Assert - expect(vectorStore.clearCollection).not.toHaveBeenCalled() - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - - // Error state should be set - expect(stateManager.setSystemState).toHaveBeenCalled() - const lastCall = stateManager.setSystemState.mock.calls[stateManager.setSystemState.mock.calls.length - 1] - expect(lastCall[0]).toBe("Error") - }) - - it("should call clearCollection() and clear cache when an error occurs after initialize() succeeds (indexing started)", async () => { - // Arrange: initialize succeeds; fail soon after to enter error path with indexingStarted=true - vectorStore.initialize.mockResolvedValue(false) // existing collection - vectorStore.hasIndexedData.mockResolvedValue(false) // force full scan path - vectorStore.markIndexingIncomplete.mockRejectedValue(new Error("mark incomplete failure")) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + watcher.isProcessing = true + watcher.onDidStartBatchProcessing.mock.calls[0][0](["reading.ts"]) + await orchestrator.stopIndexing() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", ) + }) - // Act + it("idle cancel preserves state and message, while disable disposes idle resources", async () => { await orchestrator.startIndexing() - - // Assert: cleanup gated behind indexingStarted should have happened - expect(vectorStore.clearCollection).toHaveBeenCalledTimes(1) - expect(cacheManager.clearCacheFile).toHaveBeenCalledTimes(1) - - // Error state should be set - expect(stateManager.setSystemState).toHaveBeenCalled() - const lastCall = stateManager.setSystemState.mock.calls[stateManager.setSystemState.mock.calls.length - 1] - expect(lastCall[0]).toBe("Error") + state.setSystemState.mockClear() + await orchestrator.stopIndexing() + expect(state.setSystemState).not.toHaveBeenCalled() + expect(watcher.dispose).not.toHaveBeenCalled() + await orchestrator.stopWatcher() + expect(watcher.dispose).toHaveBeenCalledOnce() + expect(state.setSystemState).not.toHaveBeenCalled() }) -}) - -describe("CodeIndexOrchestrator - stopIndexing", () => { - const workspacePath = "/test/workspace" - let configManager: any - let stateManager: any - let cacheManager: any - let vectorStore: any - let scanner: any - let fileWatcher: any - - beforeEach(() => { - vi.clearAllMocks() - - configManager = { - isFeatureConfigured: true, - } - - let currentState = "Standby" - stateManager = { - get state() { - return currentState - }, - setSystemState: vi.fn().mockImplementation((state: string, _msg: string) => { - currentState = state - }), - reportFileQueueProgress: vi.fn(), - reportBlockIndexingProgress: vi.fn(), - } - - cacheManager = { - clearCacheFile: vi.fn().mockResolvedValue(undefined), - flush: vi.fn().mockResolvedValue(undefined), - } - - vectorStore = { - initialize: vi.fn().mockResolvedValue(false), - hasIndexedData: vi.fn().mockResolvedValue(false), - markIndexingIncomplete: vi.fn().mockResolvedValue(undefined), - markIndexingComplete: vi.fn().mockResolvedValue(undefined), - clearCollection: vi.fn().mockResolvedValue(undefined), + it.each(["scan", "gate", "worker"])("bounds stopping when %s never settles", async (pending) => { + const held = deferred() + if (pending === "scan") scanner.scanDirectory.mockReturnValue(held.promise) + if (pending === "gate") { + gate.settled.mockReturnValue(held.promise) + gate.hasPending.mockReturnValue(true) } - - scanner = { - scanDirectory: vi.fn(), - } - - fileWatcher = { - initialize: vi.fn().mockResolvedValue(undefined), - onDidStartBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onBatchProgressUpdate: vi.fn().mockReturnValue({ dispose: vi.fn() }), - onDidFinishBatchProcessing: vi.fn().mockReturnValue({ dispose: vi.fn() }), - dispose: vi.fn(), - } - }) - - it("should abort indexing when stopIndexing() is called", async () => { - // Make scanner hang until aborted - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - // Wait for abort signal - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) - }) - return { stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 } - }, + if (pending === "worker") worker.stopped = held.promise + const scan = orchestrator.startIndexing() + await drain() + watcher.isProcessing = true + const stop = orchestrator.stopIndexing() + expect(orchestrator.stopIndexing()).toBe(stop) + expect(orchestrator.stopWatcher()).toBe(stop) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(1) + await stop + expect(gate.abandon).toHaveBeenCalledOnce() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", ) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - // Start indexing (async, don't await) - const indexingPromise = orchestrator.startIndexing() - - // Give it a tick to begin - await new Promise((resolve) => setTimeout(resolve, 10)) - - // Stop indexing - orchestrator.stopIndexing() - - // Wait for indexing to complete - await indexingPromise - - // State should be Standby (not Error) - const setStateCalls = stateManager.setSystemState.mock.calls - const lastCall = setStateCalls[setStateCalls.length - 1] - expect(lastCall[0]).toBe("Standby") + expect(worker.logStopTimeout).toHaveBeenCalledTimes(pending === "worker" ? 1 : 0) + held.resolve() + await scan + expect(store.markIndexingComplete).toHaveBeenCalledTimes(pending === "scan" ? 0 : 1) }) - it("should set state to Standby after abort, not Error", async () => { - // Make scanner throw AbortError when signal is aborted - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) - }) - throw new DOMException("Indexing aborted", "AbortError") - }, - ) + it("waits for all scan, writes and physical worker exit before a fully settled stop", async () => { + const scanFinished = deferred() + const writesFinished = deferred() + const workerFinished = deferred() + scanner.scanDirectory.mockReturnValue(scanFinished.promise) + gate.settled.mockReturnValue(writesFinished.promise) + worker.stopped = workerFinished.promise + const scan = orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + scanFinished.resolve() + writesFinished.resolve() + await drain() + expect(state.state).toBe("Stopping") + workerFinished.resolve() + await stop + await scan + expect(state.setSystemState).toHaveBeenLastCalledWith("Standby", "embeddings:orchestrator.indexingStopped") + expect(gate.abandon).not.toHaveBeenCalled() + }) - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, + it("preserves incomplete-file status even when every stop wait settles", async () => { + const scanFinished = deferred() + scanner.scanDirectory.mockReturnValue(scanFinished.promise) + scanner.hasIncompleteFiles = true + void orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + scanFinished.resolve() + await stop + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", ) + }) - const indexingPromise = orchestrator.startIndexing() - await new Promise((resolve) => setTimeout(resolve, 10)) - - orchestrator.stopIndexing() - await indexingPromise - - // Should NOT have set Error state — abort is handled gracefully - const errorCalls = stateManager.setSystemState.mock.calls.filter((call: any[]) => call[0] === "Error") - expect(errorCalls).toHaveLength(0) - - // Should NOT have cleared collection on abort - expect(vectorStore.clearCollection).not.toHaveBeenCalled() + it("does not overwrite a newer attempt when the previous stop expires", async () => { + scanner.scanDirectory.mockReturnValue(deferred().promise) + void orchestrator.startIndexing() + await drain() + const stop = orchestrator.stopIndexing() + current = false + state.setSystemState("Indexing", "new attempt") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stop + expect(state.setSystemState).toHaveBeenLastCalledWith("Indexing", "new attempt") }) - it("should preserve partial index data after stop", async () => { - scanner.scanDirectory.mockImplementation( - async (_dir: string, _onError?: any, _onBlocksIndexed?: any, _onFileParsed?: any, signal?: AbortSignal) => { - await new Promise((resolve) => { - if (signal?.aborted) { - resolve() - return - } - signal?.addEventListener("abort", () => resolve()) + it.each(["scan", "watcher"])( + "latches fatal %s failure, preserves Error and rejects late callbacks", + async (source) => { + let lateProgress: any + if (source === "scan") { + scanner.scanDirectory.mockImplementation(async (_directory: string, _onError: any, onProgress: any) => { + lateProgress = onProgress + throw new ParserFatalError("parser-abort", "bad.ts") }) - return { stats: { processed: 5, skipped: 0 }, totalBlockCount: 5 } - }, - ) - - const orchestrator = new CodeIndexOrchestrator( - configManager, - stateManager, - workspacePath, - cacheManager, - vectorStore, - scanner, - fileWatcher, - ) - - const indexingPromise = orchestrator.startIndexing() - await new Promise((resolve) => setTimeout(resolve, 10)) - - orchestrator.stopIndexing() - await indexingPromise + } + await orchestrator.startIndexing() + if (source === "watcher") { + watcher.isProcessing = true + lateProgress = () => + watcher.onBatchProgressUpdate.mock.calls[0][0]({ processedInBatch: 1, totalInBatch: 1 }) + watcher.onParserFatal.mock.calls[0][0](new ParserFatalError("parse-timeout", "bad.ts")) + } + await orchestrator.stopIndexing() + lateProgress(100) + expect(state.state).toBe("Error") + expect(state.setSystemState.mock.calls.some(([next]: string[]) => next === "Stopping")).toBe(false) + expect(fatal).toHaveBeenCalledOnce() + expect(worker.stop).toHaveBeenCalledOnce() + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + expect(state.reportBlockIndexingProgress).not.toHaveBeenCalled() + expect(state.reportFileQueueProgress).not.toHaveBeenCalled() + await orchestrator.startIndexing() + expect(scanner.scanDirectory).toHaveBeenCalledOnce() + }, + ) - // Cache should NOT be cleared on user-initiated stop - expect(cacheManager.clearCacheFile).not.toHaveBeenCalled() - // Collection should NOT be cleared on user-initiated stop - expect(vectorStore.clearCollection).not.toHaveBeenCalled() + it("keeps ordinary scan errors and completed data after cancel", async () => { + scanner.scanDirectory.mockRejectedValue(new Error("network failed")) + await orchestrator.startIndexing() + await orchestrator.stopIndexing() + expect(state.state).toBe("Error") + expect(store.clearCollection).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() }) }) diff --git a/src/services/code-index/__tests__/parse-worker-client.real.spec.ts b/src/services/code-index/__tests__/parse-worker-client.real.spec.ts new file mode 100644 index 00000000000..c94df92652c --- /dev/null +++ b/src/services/code-index/__tests__/parse-worker-client.real.spec.ts @@ -0,0 +1,153 @@ +import { build } from "esbuild" +import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import * as path from "node:path" +import { Worker } from "node:worker_threads" +import { ParseWorkerClient } from "../parse-worker-client" +import { INDEX_STOP_LIMIT_MS, PARSE_TIMEOUT_MS } from "../constants" + +const options = { + content: "function example() { return 'a sufficiently long function body for indexing'; }", + fileHash: "hash", +} + +function within(promise: Promise, milliseconds: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Did not settle within ${milliseconds} ms`)), milliseconds) + promise.then( + (value) => { + clearTimeout(timer) + resolve(value) + }, + (error) => { + clearTimeout(timer) + reject(error) + }, + ) + }) +} + +async function probe(): Promise { + const started = performance.now() + await new Promise((resolve) => setTimeout(resolve, 20)) + const elapsed = performance.now() - started + expect(elapsed).toBeLessThan(500) + return elapsed +} + +describe("ParseWorkerClient real worker (T15)", () => { + let directory: string + const workers: Worker[] = [] + + beforeAll(async () => { + directory = await mkdtemp(path.join(tmpdir(), "roo-parse-worker-")) + await build({ + entryPoints: [path.resolve("services/code-index/parse-worker.ts")], + outfile: path.join(directory, "parse-worker.js"), + bundle: true, + platform: "node", + format: "cjs", + }) + await copyFile(path.resolve("services/code-index/fault-loop.wasm"), path.join(directory, "fault-loop.wasm")) + await copyFile( + path.resolve("node_modules/web-tree-sitter/tree-sitter.wasm"), + path.join(directory, "tree-sitter.wasm"), + ) + await copyFile( + path.resolve("node_modules/tree-sitter-wasms/out/tree-sitter-typescript.wasm"), + path.join(directory, "tree-sitter-typescript.wasm"), + ) + }) + + afterEach(async () => { + await Promise.all(workers.splice(0).map((worker) => worker.terminate())) + }) + + afterAll(async () => { + if (directory) await rm(directory, { recursive: true, force: true }) + }) + + function createWorker(fault?: string): Worker { + const environment = { ...process.env } + delete environment.ROO_PARSE_WORKER_FAULT + if (fault) environment.ROO_PARSE_WORKER_FAULT = fault + const worker = new Worker(path.join(directory, "parse-worker.js"), { env: environment, stdout: true }) + workers.push(worker) + return worker + } + + it("ships the exact unconditional WASM loop described by the source", async () => { + const bytes = await readFile(path.join(directory, "fault-loop.wasm")) + expect(bytes.toString("hex")).toBe( + "0061736d0100000001040160000003020100070801046c6f6f7000000a0901070003400c000b0b", + ) + expect(WebAssembly.validate(new Uint8Array(bytes))).toBe(true) + const source = await readFile(path.resolve("services/code-index/fault-loop.wat"), "utf8") + expect(source.trim().split(/\r?\n/)).toEqual([ + "(module", + ' (func (export "loop")', + " (loop", + " br 0)))", + ]) + }) + + it.each(["timeout", "stop"])( + "physically exits a real WASM hang on %s without blocking the main loop", + async (action) => { + const worker = createWorker("wasm-hang") + const entered = new Promise((resolve, reject) => { + worker.once("error", reject) + worker.stdout!.on("data", (chunk: Buffer) => { + if (chunk.toString().includes("parse worker entering wasm fault loop")) resolve() + }) + }) + const client = new ParseWorkerClient(() => worker) + let exitObserved = false + worker.once("exit", () => { + exitObserved = true + }) + const pending = client.parseFile("hang.ts", options).catch((error) => error) + await within(entered, 5000) + const hangProbe = await probe() + expect(exitObserved).toBe(false) + let stoppingAt: number + if (action === "stop") { + stoppingAt = performance.now() + expect(client.stop()).toBeUndefined() + } else { + expect(await within(pending, PARSE_TIMEOUT_MS)).toMatchObject({ category: "parse-timeout" }) + stoppingAt = performance.now() + } + const stoppingProbe = probe() + await within(client.stopped, INDEX_STOP_LIMIT_MS) + const exitElapsed = performance.now() - stoppingAt + expect(exitElapsed).toBeLessThan(INDEX_STOP_LIMIT_MS) + expect(exitObserved).toBe(true) + expect(await pending).toMatchObject( + action === "stop" ? { name: "AbortError" } : { category: "parse-timeout" }, + ) + console.log( + `T15 ${action}: physical exit ${exitElapsed.toFixed(1)} ms; probes ${hangProbe.toFixed(1)}/${(await stoppingProbe).toFixed(1)} ms`, + ) + }, + ) + + it("injects a fatal abort and permits recovery with a new worker without the fault", async () => { + const failed = new ParseWorkerClient(() => createWorker("abort")) + await expect(failed.parseFile("abort.ts", options)).rejects.toMatchObject({ + name: "ParserFatalError", + category: "parser-abort", + message: expect.stringContaining("Aborted()"), + }) + await within(failed.stopped, INDEX_STOP_LIMIT_MS) + const replacement = new ParseWorkerClient(() => createWorker()) + try { + const blocks = await within(replacement.parseFile("normal.ts", options), 5000) + expect(blocks.length).toBeGreaterThan(0) + expect(blocks[0].content).toContain("function example()") + } finally { + replacement.stop() + await within(replacement.stopped, INDEX_STOP_LIMIT_MS) + } + }) +}) diff --git a/src/services/code-index/__tests__/parse-worker-client.spec.ts b/src/services/code-index/__tests__/parse-worker-client.spec.ts index 2292a19773c..07cdcbb95c7 100644 --- a/src/services/code-index/__tests__/parse-worker-client.spec.ts +++ b/src/services/code-index/__tests__/parse-worker-client.spec.ts @@ -1,5 +1,4 @@ import { EventEmitter } from "node:events" -import { Worker } from "node:worker_threads" import { ParseWorkerClient, type ParseRequest } from "../parse-worker-client" class MockWorker extends EventEmitter { @@ -22,20 +21,59 @@ async function drain(): Promise { describe("ParseWorkerClient", () => { afterEach(() => vi.useRealTimers()) - it("interrupts an actual busy worker while the main event loop keeps running", async () => { - const worker = new Worker( - "require('node:worker_threads').parentPort.on('message', () => { while (true) {} })", - { eval: true }, - ) - const client = new ParseWorkerClient(() => worker, 200) - const exited = new Promise((resolve) => worker.once("exit", () => resolve())) - const started = performance.now() - const pending = client.parseFile("busy.ts", options).catch((error) => error) - await new Promise((resolve) => setTimeout(resolve, 20)) - expect(performance.now() - started).toBeLessThan(500) - expect(await pending).toMatchObject({ category: "parse-timeout" }) - await exited - expect(performance.now() - started).toBeLessThan(2000) + it("settles stopped immediately when stopped before creating a worker", async () => { + const factory = vi.fn() + const client = new ParseWorkerClient(factory) + expect(client.stop()).toBeUndefined() + await client.stopped + expect(factory).not.toHaveBeenCalled() + }) + + it.each(["resolved", "rejected", "pending"])( + "waits for physical exit with a %s termination promise and logs the parent's timeout once", + async (termination) => { + vi.useFakeTimers() + const log = vi.spyOn(console, "log").mockImplementation(() => {}) + try { + const worker = new MockWorker() + if (termination === "rejected") worker.terminate.mockRejectedValue(new Error("termination failed")) + if (termination === "pending") worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + let exited = false + void client.stopped.then(() => { + exited = true + }) + const pending = client.parseFile("active.ts", options).catch((error) => error) + await drain() + expect(client.stop()).toBeUndefined() + expect(await pending).toMatchObject({ name: "AbortError" }) + await vi.advanceTimersByTimeAsync(2000) + expect(exited).toBe(false) + client.logStopTimeout() + client.logStopTimeout() + expect(log).toHaveBeenCalledExactlyOnceWith("parse worker exit not observed within stop limit") + worker.emit("exit", 1) + await client.stopped + expect(exited).toBe(true) + expect(log).toHaveBeenLastCalledWith(expect.stringMatching(/^parse worker exited after \d+ ms$/)) + client.logStopTimeout() + expect(log).toHaveBeenCalledTimes(2) + } finally { + log.mockRestore() + } + }, + ) + + it("observes an unexpected physical exit even when terminate never settles", async () => { + const worker = new MockWorker() + worker.terminate.mockReturnValue(new Promise(() => {})) + const client = new ParseWorkerClient(() => worker) + const pending = client.parseFile("active.ts", options).catch((error) => error) + await drain() + worker.emit("exit", 1) + expect(await pending).toMatchObject({ name: "ParserFatalError" }) + await client.stopped + expect(worker.terminate).not.toHaveBeenCalled() }) it("dispatches FIFO and starts each deadline at dispatch, not enqueue", async () => { @@ -77,7 +115,7 @@ describe("ParseWorkerClient", () => { expect(await queued).toMatchObject({ name: "AbortError" }) await expect(client.parseFile("later.md", options)).rejects.toMatchObject({ name: "AbortError" }) expect(worker.requests).toHaveLength(1) - expect(worker.terminate).toHaveBeenCalledTimes(1) + expect(worker.terminate).toHaveBeenCalledTimes(failure === "exit" ? 0 : 1) }, ) diff --git a/src/services/code-index/__tests__/service-factory.spec.ts b/src/services/code-index/__tests__/service-factory.spec.ts index d7311a22747..229c8247771 100644 --- a/src/services/code-index/__tests__/service-factory.spec.ts +++ b/src/services/code-index/__tests__/service-factory.spec.ts @@ -6,6 +6,13 @@ import { OpenAICompatibleEmbedder } from "../embedders/openai-compatible" import { GeminiEmbedder } from "../embedders/gemini" import { QdrantVectorStore } from "../vector-store/qdrant-client" +import { DirectoryScanner, FileWatcher } from "../processors" +import { IndexWriteGate } from "../index-write-gate" +import { GenerationAllocator } from "../generation-allocator" + +vitest.mock("../generation-allocator", () => ({ GenerationAllocator: vitest.fn() })) +vitest.mock("../processors") + // Mock the embedders and vector store vitest.mock("../embedders/openai") vitest.mock("../embedders/ollama") @@ -47,6 +54,20 @@ describe("CodeIndexServiceFactory", () => { factory = new CodeIndexServiceFactory(mockConfigManager, "/test/workspace", mockCacheManager) }) + it("injects the same manager-owned allocator and gate into scanner and watcher", () => { + const gate = new IndexWriteGate() + const allocator = new GenerationAllocator(mockCacheManager) + factory = new CodeIndexServiceFactory(mockConfigManager, "/test/workspace", mockCacheManager, gate, allocator) + const parser = {} as any + const embedder = {} as any + const store = {} as any + const ignoreInstance = {} as any + factory.createDirectoryScanner(embedder, store, parser, ignoreInstance) + factory.createFileWatcher({} as any, embedder, store, mockCacheManager, ignoreInstance, undefined, parser) + expect(vitest.mocked(DirectoryScanner).mock.calls[0].slice(-2)).toEqual([gate, allocator]) + expect(vitest.mocked(FileWatcher).mock.calls[0].slice(-2)).toEqual([gate, allocator]) + }) + describe("createEmbedder", () => { it("should pass model ID to OpenAI embedder when using OpenAI provider", () => { // Arrange diff --git a/src/services/code-index/cache-manager.ts b/src/services/code-index/cache-manager.ts index 6cd95c871d5..a96e03c482b 100644 --- a/src/services/code-index/cache-manager.ts +++ b/src/services/code-index/cache-manager.ts @@ -4,106 +4,131 @@ import { ICacheManager } from "./interfaces/cache" import debounce from "lodash.debounce" import { safeWriteJson } from "../../utils/safeWriteJson" -/** - * Manages the cache for code indexing - */ export class CacheManager implements ICacheManager { - private cachePath: vscode.Uri + private readonly cachePath: vscode.Uri private fileHashes: Record = {} - private _debouncedSaveCache: () => void - - /** - * Creates a new cache manager - * @param context VS Code extension context - * @param workspacePath Path to the workspace - */ - constructor( - private context: vscode.ExtensionContext, - private workspacePath: string, - ) { + private openFiles: Record = {} + private watermark?: number + private saveTail: Promise = Promise.resolve() + private readonly debouncedSave + public loadSucceeded = false + public loadedAt = 0 + + constructor(context: vscode.ExtensionContext, workspacePath: string) { this.cachePath = vscode.Uri.joinPath( context.globalStorageUri, `roo-index-cache-${createHash("sha256").update(workspacePath).digest("hex")}.json`, ) - this._debouncedSaveCache = debounce(async () => { - await this._performSave() + this.debouncedSave = debounce(() => { + void this.saveNow().catch((error) => console.error("Failed to save cache:", error)) }, 1500) } - /** - * Initializes the cache manager by loading the cache file - */ async initialize(): Promise { + this.fileHashes = {} + this.openFiles = {} + this.watermark = undefined + this.loadSucceeded = false try { - const cacheData = await vscode.workspace.fs.readFile(this.cachePath) - this.fileHashes = JSON.parse(cacheData.toString()) - } catch (error) { + const buffer = await vscode.workspace.fs.readFile(this.cachePath) + const data = JSON.parse(buffer.toString()) + if (!data || typeof data !== "object" || Array.isArray(data)) throw new Error("Invalid indexing cache") + if (data.version === 2) { + if (!data.fileHashes || !data.openFiles) throw new Error("Invalid indexing cache records") + for (const [filePath, hash] of Object.entries(data.fileHashes)) { + if (typeof hash !== "string") throw new Error("Invalid indexing hash") + this.fileHashes[filePath] = hash + } + for (const [filePath, open] of Object.entries(data.openFiles)) { + if (open !== true) throw new Error("Invalid open-file record") + this.openFiles[filePath] = true + delete this.fileHashes[filePath] + } + if (Number.isSafeInteger(data.generationWatermark) && data.generationWatermark >= 0) { + this.watermark = data.generationWatermark + } + } else { + for (const [filePath, hash] of Object.entries(data)) { + if (typeof hash !== "string") throw new Error("Invalid legacy indexing cache") + this.fileHashes[filePath] = hash + } + } + this.loadSucceeded = true + } catch { this.fileHashes = {} + this.openFiles = {} + this.watermark = undefined } + this.loadedAt = Date.now() } - /** - * Saves the cache to disk - */ - private async _performSave(): Promise { - try { - await safeWriteJson(this.cachePath.fsPath, this.fileHashes) - } catch (error) { - console.error("Failed to save cache:", error) + get generationWatermark(): number | undefined { + return this.watermark + } + + async reserveGenerations(watermark: number): Promise { + this.watermark = watermark + await this.saveNow() + } + + async saveNow(): Promise { + this.debouncedSave.cancel?.() + const data = { + version: 2, + fileHashes: { ...this.fileHashes }, + openFiles: { ...this.openFiles }, + generationWatermark: this.watermark, } + // Serialize snapshots: an older debounce must never overwrite a durable invalidation. + const pending = this.saveTail.catch(() => {}).then(() => safeWriteJson(this.cachePath.fsPath, data)) + this.saveTail = pending + await pending } - /** - * Clears the cache file by writing an empty object to it - */ async clearCacheFile(): Promise { - try { - await safeWriteJson(this.cachePath.fsPath, {}) - this.fileHashes = {} - } catch (error) { - console.error("Failed to clear cache file:", error, this.cachePath) - } + this.fileHashes = {} + this.openFiles = {} + await this.saveNow() } - /** - * Gets the hash for a file path - * @param filePath Path to the file - * @returns The hash for the file or undefined if not found - */ - getHash(filePath: string): string | undefined { - return this.fileHashes[filePath] + async invalidateFile(filePath: string): Promise { + delete this.fileHashes[filePath] + this.openFiles[filePath] = true + await this.saveNow() } - /** - * Updates the hash for a file path - * @param filePath Path to the file - * @param hash New hash value - */ - updateHash(filePath: string, hash: string): void { + completeFile(filePath: string, hash: string): void { this.fileHashes[filePath] = hash - this._debouncedSaveCache() + delete this.openFiles[filePath] + this.debouncedSave() } - /** - * Deletes the hash for a file path - * @param filePath Path to the file - */ - deleteHash(filePath: string): void { + completeRemoval(filePath: string): void { delete this.fileHashes[filePath] - this._debouncedSaveCache() + delete this.openFiles[filePath] + this.debouncedSave() + } + + getOpenFiles(): Record { + return { ...this.openFiles } + } + + getTrackedPaths(): string[] { + return [...new Set([...Object.keys(this.fileHashes), ...Object.keys(this.openFiles)])] + } + + isTracked(filePath: string): boolean { + return Object.hasOwn(this.fileHashes, filePath) || Object.hasOwn(this.openFiles, filePath) + } + + getHash(filePath: string): string | undefined { + return this.fileHashes[filePath] } - /** - * Flushes any pending debounced cache writes to disk immediately. - */ async flush(): Promise { - await this._performSave() + await this.saveNow() } - /** - * Gets a copy of all file hashes - * @returns A copy of the file hashes record - */ getAllHashes(): Record { return { ...this.fileHashes } } diff --git a/src/services/code-index/embedders/__tests__/cancellation.spec.ts b/src/services/code-index/embedders/__tests__/cancellation.spec.ts new file mode 100644 index 00000000000..05aeab208cc --- /dev/null +++ b/src/services/code-index/embedders/__tests__/cancellation.spec.ts @@ -0,0 +1,245 @@ +import { OpenAiEmbedder } from "../openai" +import { OpenAICompatibleEmbedder } from "../openai-compatible" +import { OpenRouterEmbedder } from "../openrouter" +import { GeminiEmbedder } from "../gemini" +import { MistralEmbedder } from "../mistral" +import { VercelAiGatewayEmbedder } from "../vercel-ai-gateway" +import { CodeIndexOllamaEmbedder } from "../ollama" +import { BedrockEmbedder } from "../bedrock" +import type { IEmbedder } from "../../interfaces/embedder" +import { waitForRetry } from "../cancellation" + +const { request } = vi.hoisted(() => ({ request: vi.fn() })) + +vi.mock("openai", () => ({ + OpenAI: vi.fn().mockImplementation(() => ({ embeddings: { create: request } })), +})) +vi.mock("../../../../api/providers/openai-native", () => ({ + OpenAiNativeHandler: class { + constructor(protected options: object) {} + }, +})) +vi.mock("@aws-sdk/client-bedrock-runtime", () => ({ + BedrockRuntimeClient: vi.fn().mockImplementation(() => ({ send: request })), + InvokeModelCommand: vi.fn().mockImplementation((input) => ({ input })), +})) +vi.mock("@aws-sdk/credential-providers", () => ({ + fromIni: vi.fn(), + fromNodeProviderChain: vi.fn(), +})) +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) +vi.mock("../../../../shared/embeddingModels", () => ({ + getDefaultModelId: () => "test-model", + getModelQueryPrefix: () => "", +})) +vi.mock("../../constants", async (importOriginal) => ({ + ...(await importOriginal()), + MAX_BATCH_TOKENS: 4, +})) + +type Provider = { + name: string + create: () => IEmbedder + transport: "sdk" | "fetch" | "ollama" | "bedrock" +} + +const providers: Provider[] = [ + { name: "OpenAI", create: () => new OpenAiEmbedder({ openAiNativeApiKey: "test-key" }), transport: "sdk" }, + { + name: "OpenAI-compatible", + create: () => new OpenAICompatibleEmbedder("https://example.test/v1", "test-key"), + transport: "sdk", + }, + { name: "OpenRouter", create: () => new OpenRouterEmbedder("test-key"), transport: "sdk" }, + { name: "Gemini", create: () => new GeminiEmbedder("test-key"), transport: "sdk" }, + { name: "Mistral", create: () => new MistralEmbedder("test-key"), transport: "sdk" }, + { name: "Vercel", create: () => new VercelAiGatewayEmbedder("test-key"), transport: "sdk" }, + { name: "Ollama", create: () => new CodeIndexOllamaEmbedder({}), transport: "ollama" }, + { name: "Bedrock", create: () => new BedrockEmbedder("us-east-1"), transport: "bedrock" }, + { + name: "OpenAI-compatible direct fetch", + create: () => new OpenAICompatibleEmbedder("https://example.test/v1/embeddings", "test-key"), + transport: "fetch", + }, +] + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (reason: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +function successfulResponse(transport: Provider["transport"]) { + const data = { data: [{ embedding: [0.25, 0.5] }], embeddings: [[0.25, 0.5]] } + if (transport === "fetch" || transport === "ollama") { + return { ok: true, json: async () => data } + } + if (transport === "bedrock") { + return { body: new TextEncoder().encode(JSON.stringify({ embedding: [0.25, 0.5] })) } + } + return data +} + +beforeEach(() => { + vi.useFakeTimers() + request.mockReset() + vi.stubGlobal("fetch", request) + vi.spyOn(console, "warn").mockImplementation(() => {}) + vi.spyOn(console, "error").mockImplementation(() => {}) + for (const embedderClass of [OpenAICompatibleEmbedder, OpenRouterEmbedder]) { + Object.assign(Reflect.get(embedderClass, "globalRateLimitState"), { + isRateLimited: false, + rateLimitResetTime: 0, + consecutiveRateLimitErrors: 0, + lastRateLimitError: 0, + }) + } +}) + +afterEach(() => { + vi.unstubAllGlobals() + vi.clearAllMocks() + vi.useRealTimers() +}) + +describe.each(providers)("$name cancellation", ({ create, transport }) => { + it("forwards the caller signal to the transport", async () => { + const controller = new AbortController() + request.mockResolvedValue(successfulResponse(transport)) + await create().createEmbeddings(["first"], undefined, controller.signal) + const options = request.mock.calls[0][1] + if (transport === "bedrock") { + expect(options.abortSignal).toBe(controller.signal) + } else if (transport === "ollama") { + expect(options.signal.aborted).toBe(false) + controller.abort() + expect(options.signal.aborted).toBe(true) + expect(options.signal.reason).toBe(controller.signal.reason) + } else { + expect(options.signal).toBe(controller.signal) + } + }) + + it("refuses an already-aborted call without issuing a request", async () => { + const controller = new AbortController() + controller.abort() + await expect(create().createEmbeddings(["first"], undefined, controller.signal)).rejects.toBe( + controller.signal.reason, + ) + expect(request).not.toHaveBeenCalled() + }) + + it.each(["success", "failure", "cancellation-error"])( + "discards late %s and issues no queued batch or item", + async (outcome) => { + const controller = new AbortController() + const pending = deferred() + const started = deferred() + request.mockImplementation(() => { + started.resolve() + return pending.promise + }) + const result = create().createEmbeddings(["first123", "second12", "third123"], undefined, controller.signal) + await started.promise + controller.abort() + const requestOptions = request.mock.calls[0][1] + const forwardedSignal = transport === "bedrock" ? requestOptions.abortSignal : requestOptions.signal + expect(forwardedSignal.aborted).toBe(true) + expect(forwardedSignal.reason).toBe(controller.signal.reason) + if (outcome === "success") { + pending.resolve(successfulResponse(transport)) + } else { + pending.reject( + outcome === "failure" ? new Error("Late failure") : new DOMException("Late abort", "AbortError"), + ) + } + await expect(result).rejects.toBe(controller.signal.reason) + expect(request).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }, + ) +}) + +describe("retry cancellation", () => { + it.each(providers.filter(({ name }) => ["OpenAI", "OpenAI-compatible", "OpenRouter", "Bedrock"].includes(name)))( + "$name cancels backoff without another request", + async ({ create }) => { + const controller = new AbortController() + const error = Object.assign(new Error("Rate limited"), { status: 429, name: "ThrottlingException" }) + request.mockRejectedValue(error) + const result = create().createEmbeddings(["first"], undefined, controller.signal) + await vi.advanceTimersByTimeAsync(0) + expect(request).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(1) + controller.abort() + await expect(result).rejects.toBe(controller.signal.reason) + expect(vi.getTimerCount()).toBe(0) + await vi.runAllTimersAsync() + expect(request).toHaveBeenCalledTimes(1) + }, + ) + + it.each([OpenAICompatibleEmbedder, OpenRouterEmbedder])( + "cancels a request queued behind a global rate limit", + async (Embedder) => { + const embedder = + Embedder === OpenAICompatibleEmbedder + ? new OpenAICompatibleEmbedder("https://example.test/v1", "test-key") + : new OpenRouterEmbedder("test-key") + const firstController = new AbortController() + request.mockRejectedValue(Object.assign(new Error("Rate limited"), { status: 429 })) + const firstResult = embedder.createEmbeddings(["first"], undefined, firstController.signal) + await vi.advanceTimersByTimeAsync(0) + const queuedController = new AbortController() + const queuedResult = embedder.createEmbeddings(["second"], undefined, queuedController.signal) + await vi.advanceTimersByTimeAsync(0) + expect(vi.getTimerCount()).toBe(2) + queuedController.abort() + await expect(queuedResult).rejects.toBe(queuedController.signal.reason) + firstController.abort() + await expect(firstResult).rejects.toBe(firstController.signal.reason) + await vi.runAllTimersAsync() + expect(request).toHaveBeenCalledTimes(1) + }, + ) +}) + +describe("cancellable wait", () => { + it("refuses a previously aborted signal", () => { + const controller = new AbortController() + controller.abort() + expect(() => waitForRetry(100, controller.signal)).toThrow(controller.signal.reason) + expect(vi.getTimerCount()).toBe(0) + }) + + it("removes its abort listener when the wait completes", async () => { + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, "removeEventListener") + const result = waitForRetry(100, controller.signal) + await vi.advanceTimersByTimeAsync(100) + await result + expect(removeListener).toHaveBeenCalledWith("abort", expect.any(Function)) + expect(vi.getTimerCount()).toBe(0) + }) +}) + +it("Ollama keeps its timeout when a caller signal is supplied and cleans up after rejection", async () => { + const controller = new AbortController() + request.mockImplementation( + (_url, options) => + new Promise((_resolve, reject) => { + options.signal.addEventListener("abort", () => reject(options.signal.reason), { once: true }) + }), + ) + const result = new CodeIndexOllamaEmbedder({}).createEmbeddings(["first"], undefined, controller.signal) + const assertion = expect(result).rejects.toThrow("embeddings:validation.connectionFailed") + await vi.advanceTimersByTimeAsync(60_000) + await assertion + expect(controller.signal.aborted).toBe(false) + expect(request.mock.calls[0][1].signal.aborted).toBe(true) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/src/services/code-index/embedders/bedrock.ts b/src/services/code-index/embedders/bedrock.ts index 0c5d93d6f17..c6da90b049b 100644 --- a/src/services/code-index/embedders/bedrock.ts +++ b/src/services/code-index/embedders/bedrock.ts @@ -1,5 +1,6 @@ import { BedrockRuntimeClient, InvokeModelCommand, InvokeModelCommandInput } from "@aws-sdk/client-bedrock-runtime" import { fromIni, fromNodeProviderChain } from "@aws-sdk/credential-providers" +import { waitForRetry } from "./cancellation" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces" import { MAX_BATCH_TOKENS, @@ -53,7 +54,8 @@ export class BedrockEmbedder implements IEmbedder { * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId const allEmbeddings: number[][] = [] @@ -61,6 +63,7 @@ export class BedrockEmbedder implements IEmbedder { const remainingTexts = [...texts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -96,7 +99,7 @@ export class BedrockEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -115,9 +118,11 @@ export class BedrockEmbedder implements IEmbedder { private async _embedBatchWithRetries( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { try { + signal?.throwIfAborted() const embeddings: number[][] = [] let totalPromptTokens = 0 let totalTokens = 0 @@ -126,7 +131,8 @@ export class BedrockEmbedder implements IEmbedder { // Note: Amazon Titan models typically don't support batch embedding in a single request // So we process them individually for (const text of batchTexts) { - const embedding = await this._invokeEmbeddingModel(text, model) + signal?.throwIfAborted() + const embedding = await this._invokeEmbeddingModel(text, model, signal) embeddings.push(embedding.embedding) totalPromptTokens += embedding.inputTextTokenCount || 0 totalTokens += embedding.inputTextTokenCount || 0 @@ -140,6 +146,7 @@ export class BedrockEmbedder implements IEmbedder { }, } } catch (error: any) { + signal?.throwIfAborted() const hasMoreAttempts = attempts < MAX_RETRIES - 1 // Check if it's a rate limit error @@ -152,7 +159,7 @@ export class BedrockEmbedder implements IEmbedder { maxRetries: MAX_RETRIES, }), ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) + await waitForRetry(delayMs, signal) continue } @@ -176,6 +183,7 @@ export class BedrockEmbedder implements IEmbedder { private async _invokeEmbeddingModel( text: string, model: string, + signal?: AbortSignal, ): Promise<{ embedding: number[]; inputTextTokenCount?: number }> { let requestBody: any let modelId = model @@ -228,7 +236,9 @@ export class BedrockEmbedder implements IEmbedder { const command = new InvokeModelCommand(params) - const response = await this.bedrockClient.send(command) + signal?.throwIfAborted() + const response = await this.bedrockClient.send(command, ...(signal ? [{ abortSignal: signal }] : [])) + signal?.throwIfAborted() // Parse the response const responseBody = JSON.parse(new TextDecoder().decode(response.body)) diff --git a/src/services/code-index/embedders/cancellation.ts b/src/services/code-index/embedders/cancellation.ts new file mode 100644 index 00000000000..a5e0d61d9b0 --- /dev/null +++ b/src/services/code-index/embedders/cancellation.ts @@ -0,0 +1,15 @@ +export function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timeout) + signal?.removeEventListener("abort", onAbort) + reject(signal?.reason) + } + const timeout = setTimeout(() => { + signal?.removeEventListener("abort", onAbort) + resolve() + }, delayMs) + signal?.addEventListener("abort", onAbort, { once: true }) + }) +} diff --git a/src/services/code-index/embedders/gemini.ts b/src/services/code-index/embedders/gemini.ts index 6a34e753610..6b332a76218 100644 --- a/src/services/code-index/embedders/gemini.ts +++ b/src/services/code-index/embedders/gemini.ts @@ -66,10 +66,11 @@ export class GeminiEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } /** diff --git a/src/services/code-index/embedders/mistral.ts b/src/services/code-index/embedders/mistral.ts index 8348363bb1b..63e44b211f9 100644 --- a/src/services/code-index/embedders/mistral.ts +++ b/src/services/code-index/embedders/mistral.ts @@ -44,10 +44,11 @@ export class MistralEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } /** diff --git a/src/services/code-index/embedders/ollama.ts b/src/services/code-index/embedders/ollama.ts index acce5833356..40f839a2aa0 100644 --- a/src/services/code-index/embedders/ollama.ts +++ b/src/services/code-index/embedders/ollama.ts @@ -33,7 +33,8 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { * @param model - Optional model ID to override the default. * @returns A promise that resolves to an EmbeddingResponse containing the embeddings and usage data. */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId const url = `${this.baseUrl}/api/embed` // Endpoint as specified @@ -62,14 +63,11 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { }) : texts - try { - // Note: Standard Ollama API uses 'prompt' for single text, not 'input' for array. - // Implementing based on user's specific request structure. - - // Add timeout to prevent indefinite hanging - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDING_TIMEOUT_MS) + const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal + try { const response = await fetch(url, { method: "POST", headers: { @@ -79,9 +77,9 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { model: modelToUse, input: processedTexts, // Using 'input' as requested }), - signal: controller.signal, + signal: requestSignal, }) - clearTimeout(timeoutId) + signal?.throwIfAborted() if (!response.ok) { let errorBody = t("embeddings:ollama.couldNotReadErrorBody") @@ -100,6 +98,7 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { } const data = await response.json() + signal?.throwIfAborted() // Extract embeddings using 'embeddings' key as requested const embeddings = data.embeddings @@ -111,6 +110,7 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { embeddings: embeddings, } } catch (error: any) { + signal?.throwIfAborted() // Log the original error for debugging purposes console.error("Ollama embedding failed:", error) @@ -125,6 +125,8 @@ export class CodeIndexOllamaEmbedder implements IEmbedder { // Re-throw a more specific error for the caller throw new Error(t("embeddings:ollama.embeddingFailed", { message: error.message })) + } finally { + clearTimeout(timeoutId) } } diff --git a/src/services/code-index/embedders/openai-compatible.ts b/src/services/code-index/embedders/openai-compatible.ts index d07ae7d0be1..337b699f284 100644 --- a/src/services/code-index/embedders/openai-compatible.ts +++ b/src/services/code-index/embedders/openai-compatible.ts @@ -1,4 +1,5 @@ import { OpenAI } from "openai" +import { waitForRetry } from "./cancellation" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_BATCH_TOKENS, @@ -89,7 +90,8 @@ export class OpenAICompatibleEmbedder implements IEmbedder { * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -122,6 +124,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -157,7 +160,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -201,9 +204,11 @@ export class OpenAICompatibleEmbedder implements IEmbedder { url: string, batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise { const response = await fetch(url, { method: "POST", + ...(signal ? { signal } : {}), headers: { "Content-Type": "application/json", // Azure OpenAI uses 'api-key' header, while OpenAI uses 'Authorization' @@ -254,32 +259,39 @@ export class OpenAICompatibleEmbedder implements IEmbedder { private async _embedBatchWithRetries( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { // Use cached value for performance const isFullUrl = this.isFullUrl for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { // Check global rate limit before attempting request - await this.waitForGlobalRateLimit() + signal?.throwIfAborted() + await this.waitForGlobalRateLimit(signal) try { + signal?.throwIfAborted() let response: OpenAIEmbeddingResponse if (isFullUrl) { // Use direct HTTP request for full endpoint URLs - response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model) + response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model, signal) } else { // Use OpenAI SDK for base URLs - response = (await this.embeddingsClient.embeddings.create({ - input: batchTexts, - model: model, - // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 - // when processing numeric arrays, which breaks compatibility with models using larger dimensions. - // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. - encoding_format: "base64", - })) as OpenAIEmbeddingResponse + response = (await this.embeddingsClient.embeddings.create( + { + input: batchTexts, + model: model, + // OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256 + // when processing numeric arrays, which breaks compatibility with models using larger dimensions. + // By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves. + encoding_format: "base64", + }, + ...(signal ? [{ signal }] : []), + )) as OpenAIEmbeddingResponse } + signal?.throwIfAborted() // Convert base64 embeddings to float32 arrays const processedEmbeddings = response.data.map((item: EmbeddingItem) => { if (typeof item.embedding === "string") { @@ -309,6 +321,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { }, } } catch (error) { + signal?.throwIfAborted() const hasMoreAttempts = attempts < MAX_RETRIES - 1 // Check if it's a rate limit error @@ -330,7 +343,7 @@ export class OpenAICompatibleEmbedder implements IEmbedder { maxRetries: MAX_RETRIES, }), ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) + await waitForRetry(delayMs, signal) continue } } @@ -394,16 +407,18 @@ export class OpenAICompatibleEmbedder implements IEmbedder { /** * Waits if there's an active global rate limit */ - private async waitForGlobalRateLimit(): Promise { + private async waitForGlobalRateLimit(signal?: AbortSignal): Promise { + signal?.throwIfAborted() const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire() try { + signal?.throwIfAborted() const state = OpenAICompatibleEmbedder.globalRateLimitState if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { const waitTime = state.rateLimitResetTime - Date.now() // Silent wait - no logging to prevent flooding release() // Release mutex before waiting - await new Promise((resolve) => setTimeout(resolve, waitTime)) + await waitForRetry(waitTime, signal) return } diff --git a/src/services/code-index/embedders/openai.ts b/src/services/code-index/embedders/openai.ts index 3bf6587ffc2..15fa2c5e9d6 100644 --- a/src/services/code-index/embedders/openai.ts +++ b/src/services/code-index/embedders/openai.ts @@ -1,4 +1,5 @@ import { OpenAI } from "openai" +import { waitForRetry } from "./cancellation" import { OpenAiNativeHandler } from "../../../api/providers/openai-native" import { ApiHandlerOptions } from "../../../shared/api" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces" @@ -79,6 +80,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -146,6 +148,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { signal ? { signal } : undefined, ) + signal?.throwIfAborted() return { embeddings: response.data.map((item) => item.embedding), usage: { @@ -168,7 +171,7 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { maxRetries: MAX_RETRIES, }), ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) + await waitForRetry(delayMs, signal) continue } diff --git a/src/services/code-index/embedders/openrouter.ts b/src/services/code-index/embedders/openrouter.ts index f9ebc1d1b66..cd2f59c7fff 100644 --- a/src/services/code-index/embedders/openrouter.ts +++ b/src/services/code-index/embedders/openrouter.ts @@ -1,4 +1,5 @@ import { OpenAI } from "openai" +import { waitForRetry } from "./cancellation" import { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder" import { MAX_BATCH_TOKENS, @@ -93,7 +94,8 @@ export class OpenRouterEmbedder implements IEmbedder { * @param model Optional model identifier * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() const modelToUse = model || this.defaultModelId // Apply model-specific query prefix if required @@ -126,6 +128,7 @@ export class OpenRouterEmbedder implements IEmbedder { const remainingTexts = [...processedTexts] while (remainingTexts.length > 0) { + signal?.throwIfAborted() const currentBatch: string[] = [] let currentBatchTokens = 0 const processedIndices: number[] = [] @@ -161,7 +164,7 @@ export class OpenRouterEmbedder implements IEmbedder { } if (currentBatch.length > 0) { - const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse) + const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse, signal) allEmbeddings.push(...batchResult.embeddings) usage.promptTokens += batchResult.usage.promptTokens usage.totalTokens += batchResult.usage.totalTokens @@ -180,12 +183,15 @@ export class OpenRouterEmbedder implements IEmbedder { private async _embedBatchWithRetries( batchTexts: string[], model: string, + signal?: AbortSignal, ): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> { for (let attempts = 0; attempts < MAX_RETRIES; attempts++) { // Check global rate limit before attempting request - await this.waitForGlobalRateLimit() + signal?.throwIfAborted() + await this.waitForGlobalRateLimit(signal) try { + signal?.throwIfAborted() // Build the request parameters const requestParams: any = { input: batchTexts, @@ -207,8 +213,10 @@ export class OpenRouterEmbedder implements IEmbedder { const response = (await this.embeddingsClient.embeddings.create( requestParams, + ...(signal ? [{ signal }] : []), )) as OpenRouterEmbeddingResponse + signal?.throwIfAborted() // Convert base64 embeddings to float32 arrays const processedEmbeddings = response.data.map((item: EmbeddingItem) => { if (typeof item.embedding === "string") { @@ -238,6 +246,7 @@ export class OpenRouterEmbedder implements IEmbedder { }, } } catch (error) { + signal?.throwIfAborted() const hasMoreAttempts = attempts < MAX_RETRIES - 1 // Check if it's a rate limit error @@ -259,7 +268,7 @@ export class OpenRouterEmbedder implements IEmbedder { maxRetries: MAX_RETRIES, }), ) - await new Promise((resolve) => setTimeout(resolve, delayMs)) + await waitForRetry(delayMs, signal) continue } } @@ -329,11 +338,13 @@ export class OpenRouterEmbedder implements IEmbedder { /** * Waits if there's an active global rate limit */ - private async waitForGlobalRateLimit(): Promise { + private async waitForGlobalRateLimit(signal?: AbortSignal): Promise { + signal?.throwIfAborted() const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire() let mutexReleased = false try { + signal?.throwIfAborted() const state = OpenRouterEmbedder.globalRateLimitState if (state.isRateLimited && state.rateLimitResetTime > Date.now()) { @@ -341,7 +352,7 @@ export class OpenRouterEmbedder implements IEmbedder { // Silent wait - no logging to prevent flooding release() mutexReleased = true - await new Promise((resolve) => setTimeout(resolve, waitTime)) + await waitForRetry(waitTime, signal) return } diff --git a/src/services/code-index/embedders/vercel-ai-gateway.ts b/src/services/code-index/embedders/vercel-ai-gateway.ts index 7fb87e37a59..0cdb9ffc4c2 100644 --- a/src/services/code-index/embedders/vercel-ai-gateway.ts +++ b/src/services/code-index/embedders/vercel-ai-gateway.ts @@ -53,10 +53,11 @@ export class VercelAiGatewayEmbedder implements IEmbedder { * @param model Optional model identifier (uses constructor model if not provided) * @returns Promise resolving to embedding response */ - async createEmbeddings(texts: string[], model?: string): Promise { + async createEmbeddings(texts: string[], model?: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() // Use the provided model or fall back to the instance's model const modelToUse = model || this.modelId - return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse) + return this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse, ...(signal ? [signal] : [])) } /** diff --git a/src/services/code-index/fault-loop.wasm b/src/services/code-index/fault-loop.wasm new file mode 100644 index 0000000000000000000000000000000000000000..146022e9e9b2bf6dadf01099b0967ec3e4f4257b GIT binary patch literal 39 ucmZQbEY4+QU|?WmVN76PU}j=uVCP_D$;r + + constructor(private readonly finish: () => Promise) {} + + register(): void { + if (this.registrationComplete) throw new Error("File registration already complete") + this.outstanding++ + } + + async settled(success: boolean): Promise { + this.failed ||= !success + this.outstanding-- + await this.tryComplete() + } + + async registered(): Promise { + this.registrationComplete = true + await this.tryComplete() + } + + private async tryComplete(): Promise { + if (!this.registrationComplete || this.outstanding !== 0 || this.failed) return + this.finishing ??= this.finish().catch((error) => { + this.failed = true + throw error + }) + await this.finishing + } +} + +export class FileIndexWriter { + constructor( + private readonly cache: CacheManager, + private readonly store: IVectorStore, + private readonly gate: IndexWriteGate, + private readonly generations: GenerationAllocator, + private readonly workspacePath: string, + ) {} + + async remove(filePath: string, signal: AbortSignal): Promise { + const generation = await this.generations.allocate(signal) + await this.gate.run(signal, () => this.cache.invalidateFile(filePath)) + await this.gate.run(signal, () => this.store.deletePointsByFilePath(filePath, generation), [filePath]) + throwIfAborted(signal) + this.cache.completeRemoval(filePath) + } + + async replace( + filePath: string, + hash: string, + blocks: CodeBlock[], + embedder: IEmbedder, + signal: AbortSignal, + batchSize = BATCH_SEGMENT_THRESHOLD, + onComplete?: (blockCount: number) => void, + ): Promise { + const generation = await this.generations.allocate(signal) + await this.gate.run(signal, () => this.cache.invalidateFile(filePath)) + const selected = blocks.filter((block) => block.content.trim()) + const verify = async () => { + throwIfAborted(signal) + const counts = await this.store.countPointsForFile(filePath, generation) + throwIfAborted(signal) + if (counts.current !== selected.length || counts.other !== 0) { + throw new Error( + `Index verification failed for ${filePath}: expected ${selected.length}, current ${counts.current}, other ${counts.other}`, + ) + } + } + const completion = new FileCompletion(async () => { + await this.gate.run(signal, () => this.store.deletePointsByFilePath(filePath, generation), [filePath]) + await verify() + const waitedForPriorWrites = await this.gate.waitForPriorWrites(filePath, signal) + if (this.gate.needsReconciliation(filePath)) { + do { + await this.gate.waitForGrace(filePath, signal) + await verify() + } while (this.gate.graceDeadline(filePath) > Date.now()) + } else if (waitedForPriorWrites) { + await verify() + } + throwIfAborted(signal) + this.cache.completeFile(filePath, hash) + this.gate.reconciled(filePath) + onComplete?.(selected.length) + }) + const batches: CodeBlock[][] = [] + for (let offset = 0; offset < selected.length; offset += Math.max(1, batchSize)) { + completion.register() + batches.push(selected.slice(offset, offset + Math.max(1, batchSize))) + } + await completion.registered() + for (const batch of batches) { + try { + throwIfAborted(signal) + const { embeddings } = await embedder.createEmbeddings( + batch.map((block) => block.content.trim()), + undefined, + signal, + ) + throwIfAborted(signal) + if (embeddings.length !== batch.length) throw new Error(`Incomplete embeddings for ${filePath}`) + const points = batch.map((block, index) => ({ + id: uuidv5(`${block.segmentHash}:${generation}`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: embeddings[index], + payload: { + filePath: generateRelativeFilePath(filePath, this.workspacePath), + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + segmentHash: block.segmentHash, + generation, + }, + })) + await this.gate.run(signal, () => this.store.upsertPoints(points), [filePath]) + } catch (error) { + await completion.settled(false) + throw error + } + await completion.settled(true) + } + } +} diff --git a/src/services/code-index/generation-allocator.ts b/src/services/code-index/generation-allocator.ts new file mode 100644 index 00000000000..7729dbcd766 --- /dev/null +++ b/src/services/code-index/generation-allocator.ts @@ -0,0 +1,58 @@ +import { CacheManager } from "./cache-manager" +import { IVectorStore } from "./interfaces" +import { IndexWriteGate } from "./index-write-gate" +import { throwIfAborted } from "./parser-errors" + +const GENERATION_BLOCK_SIZE = 1024 + +export class GenerationAllocator { + private nextGeneration: number + private reservedThrough: number + private allocationTail: Promise = Promise.resolve() + private initialized = false + + constructor(private readonly cacheManager: CacheManager) { + this.reservedThrough = cacheManager.generationWatermark ?? 0 + this.nextGeneration = this.reservedThrough + 1 + } + + async initialize(vectorStore: IVectorStore, signal: AbortSignal, gate: IndexWriteGate): Promise { + if (this.initialized) return + throwIfAborted(signal) + if (this.cacheManager.generationWatermark === undefined) { + if (await vectorStore.hasIndexedData()) { + await gate.run(signal, () => vectorStore.clearCollection()) + } + throwIfAborted(signal) + await this.cacheManager.clearCacheFile() + } + await this.reserve(signal) + this.initialized = true + } + + private async reserve(signal?: AbortSignal): Promise { + throwIfAborted(signal) + const watermark = + Math.max(this.reservedThrough, this.cacheManager.generationWatermark ?? 0) + GENERATION_BLOCK_SIZE + if (!Number.isSafeInteger(watermark)) throw new Error("Index generation counter exhausted; clear index storage") + await this.cacheManager.reserveGenerations(watermark) + throwIfAborted(signal) + this.reservedThrough = watermark + } + + allocate(signal?: AbortSignal): Promise { + const allocation = this.allocationTail + .catch(() => {}) + .then(async () => { + throwIfAborted(signal) + if (this.nextGeneration > this.reservedThrough) await this.reserve(signal) + throwIfAborted(signal) + return this.nextGeneration++ + }) + this.allocationTail = allocation.then( + () => {}, + () => {}, + ) + return allocation + } +} diff --git a/src/services/code-index/index-write-gate.ts b/src/services/code-index/index-write-gate.ts index 6b674479510..c3951d980fa 100644 --- a/src/services/code-index/index-write-gate.ts +++ b/src/services/code-index/index-write-gate.ts @@ -1,22 +1,134 @@ -import { throwIfAborted } from "./parser-errors" +import { abortError, throwIfAborted } from "./parser-errors" +import { INDEX_STOP_LIMIT_MS } from "./constants" + +export const LATE_WRITE_GRACE_MS = 30_000 + +type IssuedWrite = { pending: Promise; files: string[]; abandoned?: boolean } export class IndexWriteGate { - private readonly issued = new Map>>() - private readonly uncertain = new Set() + private readonly issued = new Map>() + private readonly deadlines = new Map() + private readonly uncertainByAttempt = new WeakMap>() + private readonly owners = new WeakMap() - async ready(signal: AbortSignal): Promise { - throwIfAborted(signal) - const earlier = [...this.issued].filter(([owner]) => owner !== signal).flatMap(([, writes]) => [...writes]) - await Promise.allSettled(earlier) - throwIfAborted(signal) + private readonly stopDeadlines = new WeakMap() + private readonly observedOwners = new WeakSet() + + own(signal: AbortSignal, owner: AbortSignal): void { + this.owners.set(signal, owner) + } + + stopAt(signal: AbortSignal, expiresAt: number): void { + this.stopDeadlines.set(signal, expiresAt) + } + + graceDeadline(filePath: string): number { + return this.deadlines.get(filePath) ?? 0 + } + + recover(files: string[], startedAt = Date.now()): void { + for (const filePath of files) { + this.deadlines.set(filePath, Math.max(this.deadlines.get(filePath) ?? 0, startedAt + LATE_WRITE_GRACE_MS)) + } } needsReconciliation(filePath: string): boolean { - return this.uncertain.has(filePath) + return this.deadlines.has(filePath) } reconciled(filePath: string): void { - this.uncertain.delete(filePath) + this.deadlines.delete(filePath) + } + + hasPending(signal: AbortSignal): boolean { + return (this.issued.get(signal)?.size ?? 0) > 0 + } + + async settled(signal: AbortSignal): Promise { + while (this.hasPending(signal)) { + await Promise.allSettled([...this.issued.get(signal)!].map((write) => write.pending)) + } + } + + uncertainFiles(signal: AbortSignal): string[] { + return [...(this.uncertainByAttempt.get(signal) ?? [])] + } + + private markUncertain(signal: AbortSignal, files: string[], startedAt = Date.now()): void { + let uncertain = this.uncertainByAttempt.get(signal) + if (!uncertain) { + uncertain = new Set() + this.uncertainByAttempt.set(signal, uncertain) + } + for (const filePath of files) uncertain.add(filePath) + this.recover(files, startedAt) + } + + abandon(signal: AbortSignal, startedAt = Date.now()): void { + for (const write of this.issued.get(signal) ?? []) { + if (write.abandoned) continue + write.abandoned = true + this.markUncertain(signal, write.files, startedAt) + } + } + + async waitForPriorWrites(filePath: string, signal: AbortSignal): Promise { + const currentOwner = this.owners.get(signal) ?? signal + let waited = false + while (true) { + throwIfAborted(signal) + const pending: Promise[] = [] + let remaining = Infinity + for (const [owner, writes] of this.issued) { + if (owner === currentOwner || !owner.aborted) continue + const relevant = [...writes].filter((write) => !write.abandoned && write.files.includes(filePath)) + if (relevant.length === 0) continue + const stopDeadline = this.stopDeadlines.get(owner) + if (stopDeadline === undefined) throw new Error("Missing indexing stop deadline") + if (stopDeadline <= Date.now()) { + this.abandon(owner, stopDeadline) + continue + } + remaining = Math.min(remaining, stopDeadline - Date.now()) + pending.push(...relevant.map((write) => write.pending)) + } + if (pending.length === 0) return waited + waited = true + await new Promise((resolve, reject) => { + const finish = (error?: Error) => { + clearTimeout(timer) + signal.removeEventListener("abort", onAbort) + if (error) reject(error) + else resolve() + } + const onAbort = () => finish(abortError()) + const timer = setTimeout(() => finish(), remaining) + signal.addEventListener("abort", onAbort, { once: true }) + void Promise.allSettled(pending).then(() => finish()) + if (signal.aborted) onAbort() + }) + } + } + + async waitForGrace(filePath: string, signal: AbortSignal): Promise { + while (true) { + throwIfAborted(signal) + const remaining = (this.deadlines.get(filePath) ?? 0) - Date.now() + if (remaining <= 0) return + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer) + signal.removeEventListener("abort", onAbort) + reject(abortError()) + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + resolve() + }, remaining) + signal.addEventListener("abort", onAbort, { once: true }) + if (signal.aborted) onAbort() + }) + } } async run( @@ -24,28 +136,41 @@ export class IndexWriteGate { write: () => Result | Promise, files: string[] = [], ): Promise { - await this.ready(signal) throwIfAborted(signal) - let writes = this.issued.get(signal) + const owner = this.owners.get(signal) ?? signal + if (!this.observedOwners.has(owner)) { + this.observedOwners.add(owner) + const onAbort = () => { + if (!this.stopDeadlines.has(owner)) this.stopAt(owner, Date.now() + INDEX_STOP_LIMIT_MS) + } + if (owner.aborted) onAbort() + else owner.addEventListener("abort", onAbort, { once: true }) + } + let writes = this.issued.get(owner) if (!writes) { writes = new Set() - this.issued.set(signal, writes) + this.issued.set(owner, writes) } + let issued = false + let succeeded = false const pending = Promise.resolve().then(() => { throwIfAborted(signal) + issued = true return write() }) - writes.add(pending) + const record = { pending, files: [...files] } + writes.add(record) try { - return await pending + const result = await pending + succeeded = true + throwIfAborted(signal) + return result } catch (error) { - if (!(error instanceof Error && error.name === "AbortError")) { - for (const filePath of files) this.uncertain.add(filePath) - } + if (issued && !succeeded) this.markUncertain(owner, files) throw error } finally { - writes.delete(pending) - if (writes.size === 0) this.issued.delete(signal) + writes.delete(record) + if (writes.size === 0) this.issued.delete(owner) } } } diff --git a/src/services/code-index/interfaces/cache.ts b/src/services/code-index/interfaces/cache.ts index 01931a3a8b0..16963ed76a5 100644 --- a/src/services/code-index/interfaces/cache.ts +++ b/src/services/code-index/interfaces/cache.ts @@ -1,7 +1,14 @@ export interface ICacheManager { getHash(filePath: string): string | undefined - updateHash(filePath: string, hash: string): void - deleteHash(filePath: string): void + isTracked(filePath: string): boolean flush(): Promise getAllHashes(): Record + getOpenFiles(): Record + getTrackedPaths(): string[] + invalidateFile(filePath: string): Promise + completeFile(filePath: string, hash: string): void + completeRemoval(filePath: string): void + saveNow(): Promise + readonly generationWatermark: number | undefined + reserveGenerations(watermark: number): Promise } diff --git a/src/services/code-index/interfaces/file-processor.ts b/src/services/code-index/interfaces/file-processor.ts index e309eaf69b1..5271bb9fd40 100644 --- a/src/services/code-index/interfaces/file-processor.ts +++ b/src/services/code-index/interfaces/file-processor.ts @@ -28,6 +28,7 @@ export interface ICodeParser { * Interface for directory scanner */ export interface IDirectoryScanner { + readonly hasIncompleteFiles?: boolean /** * Scans a directory for code blocks * @param directoryPath Path to the directory to scan @@ -46,6 +47,7 @@ export interface IDirectoryScanner { skipped: number } totalBlockCount: number + incompleteFiles?: string[] }> } @@ -58,6 +60,7 @@ export interface IFileWatcher extends vscode.Disposable { */ initialize(signal?: AbortSignal): Promise readonly onParserFatal?: vscode.Event + readonly isProcessing?: boolean /** * Event emitted when a batch of files begins processing. diff --git a/src/services/code-index/interfaces/vector-store.ts b/src/services/code-index/interfaces/vector-store.ts index 7946563fd57..976d7105c63 100644 --- a/src/services/code-index/interfaces/vector-store.ts +++ b/src/services/code-index/interfaces/vector-store.ts @@ -36,16 +36,17 @@ export interface IVectorStore { ): Promise /** - * Deletes points by file path - * @param filePath Path of the file to delete points for + * Deletes legacy points and generations below the mandatory cutoff for this exact file. */ - deletePointsByFilePath(filePath: string): Promise + deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise /** - * Deletes points by multiple file paths - * @param filePaths Array of file paths to delete points for + * Applies the same generation fence to every exact file path. */ - deletePointsByMultipleFilePaths(filePaths: string[]): Promise + deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise + + /** Other includes legacy points without a generation. Both counts are exact. */ + countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> /** * Clears all points from the collection diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index dc7c702316c..20948b74302 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -15,6 +15,8 @@ import path from "path" import { t } from "../../i18n" import { IndexWriteGate } from "./index-write-gate" +import { GenerationAllocator } from "./generation-allocator" + export class CodeIndexManager { // --- Singleton Implementation --- private static instances = new Map() // Map workspace path to instance @@ -34,6 +36,9 @@ export class CodeIndexManager { private servicesStopped = false private disabledSinceFailure = false private readonly writeGate = new IndexWriteGate() + private generationAllocator?: GenerationAllocator + private currentAttempt = 0 + private readonly idleStop = Promise.resolve() public static getInstance(context: vscode.ExtensionContext, workspacePath?: string): CodeIndexManager | undefined { // Resolve the workspace folder to get both fsPath and the real URI @@ -116,7 +121,7 @@ export class CodeIndexManager { await this.context.workspaceState.update(this._workspaceEnabledKey(), enabled) if (!enabled) { this.disabledSinceFailure = true - this.stopIndexing() + void this.stopWatcher() } else if (this.disabledSinceFailure) { this.parserFailed = false this.disabledSinceFailure = false @@ -182,7 +187,7 @@ export class CodeIndexManager { // 2. Check if feature is enabled if (!this.isFeatureEnabled) { this.disabledSinceFailure = true - this.stopIndexing() + void this.stopWatcher() return { requiresRestart } } @@ -212,6 +217,7 @@ export class CodeIndexManager { if (!this._cacheManager) { this._cacheManager = new CacheManager(this.context, this.workspacePath) await this._cacheManager.initialize() + this.writeGate.recover(Object.keys(this._cacheManager.getOpenFiles()), this._cacheManager.loadedAt) } // 6. Determine if Core Services Need Recreation @@ -245,7 +251,12 @@ export class CodeIndexManager { return } - if (this.parserFailed || this.servicesStopped) { + if ( + this.parserFailed || + this.servicesStopped || + this._orchestrator?.state === "Indexed" || + this._orchestrator?.state === "Standby" + ) { this.parserFailed = false await this._recreateServices() } else if (this.getCurrentStatus().systemStatus === "Error") { @@ -263,24 +274,20 @@ export class CodeIndexManager { /** * Stops any in-progress indexing operation and the file watcher. */ - public stopIndexing(): void { - if (this._orchestrator) { - this._orchestrator.stopIndexing() - this.servicesStopped = true - } + public stopIndexing(): Promise { + if (!this._orchestrator) return this.idleStop + const stopped = this._orchestrator.stopIndexing() + this.servicesStopped = this._orchestrator.stopped + return stopped } /** * Stops the file watcher and potentially cleans up resources. */ - public stopWatcher(): void { - if (!this.isFeatureEnabled) { - return - } - if (this._orchestrator) { - this._orchestrator.stopWatcher() - this.servicesStopped = true - } + public stopWatcher(): Promise { + if (!this._orchestrator) return this.idleStop + this.servicesStopped = true + return this._orchestrator.stopWatcher() } /** @@ -305,7 +312,8 @@ export class CodeIndexManager { this._isRecoveringFromError = true try { - this._orchestrator?.stopIndexing() + this.currentAttempt++ + void this._orchestrator?.stopWatcher() // Clear error state this._stateManager.setSystemState("Standby", "") } catch (error) { @@ -328,7 +336,8 @@ export class CodeIndexManager { * Cleans up the manager instance. */ public dispose(): void { - this.stopIndexing() + this.currentAttempt++ + void this.stopWatcher() this._stateManager.dispose() } @@ -369,9 +378,12 @@ export class CodeIndexManager { * Used by both initialize() and handleSettingsChange(). */ private async _recreateServices(): Promise { + const attempt = ++this.currentAttempt + const isCurrentAttempt = () => this.currentAttempt === attempt + this.generationAllocator ??= new GenerationAllocator(this._cacheManager!) // Stop watcher if it exists if (this._orchestrator) { - this.stopWatcher() + void this.stopWatcher() } // Clear existing services to ensure clean state this._orchestrator = undefined @@ -383,6 +395,7 @@ export class CodeIndexManager { this.workspacePath, this._cacheManager!, this.writeGate, + this.generationAllocator, ) const ignoreInstance = ignore() @@ -408,6 +421,8 @@ export class CodeIndexManager { const rooIgnoreController = new RooIgnoreController(workspacePath) await rooIgnoreController.initialize() + if (!isCurrentAttempt()) return + // (Re)Create shared service instances const { embedder, vectorStore, scanner, fileWatcher, parser } = this._serviceFactory.createServices( this.context, @@ -418,7 +433,14 @@ export class CodeIndexManager { // Validate embedder configuration before proceeding const validationResult = await this._serviceFactory.validateEmbedder(embedder) + if (!isCurrentAttempt()) { + parser?.stop() + fileWatcher.dispose() + return + } if (!validationResult.valid) { + parser?.stop() + fileWatcher.dispose() const errorMessage = validationResult.error || "Embedder configuration validation failed" this._stateManager.setSystemState("Error", errorMessage) throw new Error(errorMessage) @@ -435,13 +457,16 @@ export class CodeIndexManager { fileWatcher, parser, (_error, message) => { + if (!isCurrentAttempt()) return this.parserFailed = true this.parserFailureMessage = message }, () => { - this.servicesStopped = true + if (isCurrentAttempt()) this.servicesStopped = true }, this.writeGate, + this.generationAllocator, + isCurrentAttempt, ) this.servicesStopped = false @@ -473,8 +498,7 @@ export class CodeIndexManager { // If feature is disabled, stop the service (including any active scan) if (!isFeatureEnabled) { this.disabledSinceFailure = true - this.stopIndexing() - this._stateManager.setSystemState("Standby", "Code indexing is disabled") + void this.stopWatcher() return } @@ -489,6 +513,10 @@ export class CodeIndexManager { if (!this._cacheManager) { this._cacheManager = new CacheManager(this.context, this.workspacePath) await this._cacheManager.initialize() + this.writeGate.recover( + Object.keys(this._cacheManager.getOpenFiles()), + this._cacheManager.loadedAt, + ) } // Recreate services with new configuration diff --git a/src/services/code-index/orchestrator.ts b/src/services/code-index/orchestrator.ts index 6d9fbdbe1d8..bab59b8f60d 100644 --- a/src/services/code-index/orchestrator.ts +++ b/src/services/code-index/orchestrator.ts @@ -8,15 +8,19 @@ import { CacheManager } from "./cache-manager" import { t } from "../../i18n" import { ParserFatalError, throwIfAborted } from "./parser-errors" import { IndexWriteGate } from "./index-write-gate" +import { GenerationAllocator } from "./generation-allocator" +import { INDEX_STOP_LIMIT_MS } from "./constants" -/** - * Manages the code indexing workflow, coordinating between different services and managers. - */ export class CodeIndexOrchestrator { private _fileWatcherSubscriptions: vscode.Disposable[] = [] - private _isProcessing: boolean = false + private _isProcessing = false private _abortController: AbortController | null = null - + private scanPromise?: Promise + private stopPromise?: Promise + private readonly idleStop = Promise.resolve() + private incompleteScan = false + private incompleteBatch = false + private watcherBatchPending = false public stopped = false private parserFailure?: ParserFatalError @@ -28,413 +32,243 @@ export class CodeIndexOrchestrator { private readonly vectorStore: IVectorStore, private readonly scanner: DirectoryScanner, private readonly fileWatcher: IFileWatcher, - private readonly parser?: { stop(): void }, + private readonly parser?: { stop(): void; readonly stopped?: Promise; logStopTimeout?(): void }, private readonly onFatal?: (error: ParserFatalError, message: string) => void, private readonly onStopped?: () => void, private readonly writeGate = new IndexWriteGate(), + private readonly generationAllocator = new GenerationAllocator(cacheManager), + private readonly isCurrentAttempt: () => boolean = () => true, ) {} + private isLive(signal: AbortSignal): boolean { + return !signal.aborted && this.isCurrentAttempt() + } + private failStop(error: ParserFatalError): void { - if (this.parserFailure || this._abortController?.signal.aborted) return + if (this.parserFailure || this._abortController?.signal.aborted || !this.isCurrentAttempt()) return this.parserFailure = error - this._abortController?.abort() - this.stopWatcher() const message = t("embeddings:orchestrator.parserFailed", { filePath: error.filePath, category: error.category, }) this.stateManager.setSystemState("Error", message) this.onFatal?.(error, message) + void this.stop(true) } - /** - * Starts the file watcher if not already running. - */ - private async _startWatcher(): Promise { - if (!this.configManager.isFeatureConfigured) { - throw new Error("Cannot start watcher: Service not configured.") - } - - this.stateManager.setSystemState("Indexing", "Initializing file watcher...") - - try { - const signal = this._abortController!.signal - await this.fileWatcher.initialize(signal) - throwIfAborted(signal) + private hasIncompleteFiles(): boolean { + return ( + this.incompleteScan || + this.incompleteBatch || + this.watcherBatchPending || + this.scanner.hasIncompleteFiles || + Object.keys(this.cacheManager.getOpenFiles()).length > 0 + ) + } - this._fileWatcherSubscriptions = [ - this.fileWatcher.onDidStartBatchProcessing((filePaths: string[]) => {}), - this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { - if (signal.aborted) return - if (totalInBatch > 0 && this.stateManager.state !== "Indexing") { - this.stateManager.setSystemState("Indexing", "Processing file changes...") - } - this.stateManager.reportFileQueueProgress( - processedInBatch, - totalInBatch, - currentFile ? path.basename(currentFile) : undefined, - ) - if (processedInBatch === totalInBatch) { - // Covers (N/N) and (0/0) - if (totalInBatch > 0) { - // Batch with items completed - this.stateManager.setSystemState("Indexed", "File changes processed. Index up-to-date.") - } else { - if (this.stateManager.state === "Indexing") { - // Only transition if it was "Indexing" - this.stateManager.setSystemState("Indexed", "Index up-to-date. File queue empty.") - } - } - } - }), - this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { - if (signal.aborted) return - if (summary.batchError) { - console.error(`[CodeIndexOrchestrator] Batch processing failed:`, summary.batchError) - } else { - const successCount = summary.processedFiles.filter( - (f: { status: string }) => f.status === "success", - ).length - const errorCount = summary.processedFiles.filter( - (f: { status: string }) => f.status === "error" || f.status === "local_error", - ).length - } - }), - ] - if (this.fileWatcher.onParserFatal) { - this._fileWatcherSubscriptions.push(this.fileWatcher.onParserFatal((error) => this.failStop(error))) - } - } catch (error) { - console.error("[CodeIndexOrchestrator] Failed to start file watcher:", error) - throw error + private async startWatcher(signal: AbortSignal): Promise { + await this.fileWatcher.initialize(signal) + throwIfAborted(signal) + if (!this.isCurrentAttempt()) return + this._fileWatcherSubscriptions = [ + this.fileWatcher.onDidStartBatchProcessing(() => { + if (!this.isLive(signal)) return + this.watcherBatchPending = true + this.stateManager.setSystemState("Indexing", "Processing file changes...") + }), + this.fileWatcher.onBatchProgressUpdate(({ processedInBatch, totalInBatch, currentFile }) => { + if (!this.isLive(signal)) return + this.stateManager.reportFileQueueProgress( + processedInBatch, + totalInBatch, + currentFile ? path.basename(currentFile) : undefined, + ) + }), + this.fileWatcher.onDidFinishBatchProcessing((summary: BatchProcessingSummary) => { + if (!this.isLive(signal)) return + this.watcherBatchPending = false + const incomplete = + !!summary.batchError || + summary.processedFiles.some((file) => file.status !== "success" && file.status !== "skipped") + // A later unrelated batch cannot complete files left unfinished by an earlier batch. + this.incompleteBatch ||= incomplete + if (this.hasIncompleteFiles()) { + this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStoppedPartial")) + } else { + this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) + } + }), + ] + if (this.fileWatcher.onParserFatal) { + this._fileWatcherSubscriptions.push(this.fileWatcher.onParserFatal((error) => this.failStop(error))) } } - /** - * Updates the status of a file in the state manager. - */ - - /** - * Initiates the indexing process (initial scan and starts watcher). - */ - public async startIndexing(): Promise { - if (this.stopped) return - // Check if workspace is available first - if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) { + public startIndexing(): Promise { + if (this.stopped || !this.isCurrentAttempt()) return Promise.resolve() + if (!vscode.workspace.workspaceFolders?.length) { this.stateManager.setSystemState("Error", t("embeddings:orchestrator.indexingRequiresWorkspace")) - console.warn("[CodeIndexOrchestrator] Start rejected: No workspace folder open.") - return + return Promise.resolve() } - if (!this.configManager.isFeatureConfigured) { this.stateManager.setSystemState("Standby", "Missing configuration. Save your settings to start indexing.") - console.warn("[CodeIndexOrchestrator] Start rejected: Missing configuration.") - return - } - - if ( - this._isProcessing || - (this.stateManager.state !== "Standby" && - this.stateManager.state !== "Error" && - this.stateManager.state !== "Indexed") - ) { - console.warn( - `[CodeIndexOrchestrator] Start rejected: Already processing or in state ${this.stateManager.state}.`, - ) - return + return Promise.resolve() } - + if (this._isProcessing) return this.scanPromise ?? Promise.resolve() + if (this._abortController) return Promise.resolve() this._isProcessing = true this._abortController = new AbortController() - const signal = this._abortController.signal this.stateManager.setSystemState("Indexing", "Initializing services...") + this.scanPromise = this.runScan(this._abortController.signal) + return this.scanPromise + } - // Track whether we successfully connected to Qdrant and started indexing - // This helps us decide whether to preserve cache on error - let indexingStarted = false - + private async runScan(signal: AbortSignal): Promise { try { const collectionCreated = await this.writeGate.run(signal, () => this.vectorStore.initialize()) throwIfAborted(signal) - - // Successfully connected to Qdrant - indexingStarted = true - - if (collectionCreated) { - await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) - } - - // Check if the collection already has indexed data - // If it does, we can skip the full scan and just start the watcher - const hasExistingData = await this.vectorStore.hasIndexedData() - + if (collectionCreated) await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) + await this.generationAllocator.initialize(this.vectorStore, signal, this.writeGate) throwIfAborted(signal) - - if (hasExistingData && !collectionCreated) { - // Collection exists with data - run incremental scan to catch any new/changed files - // This handles files added while workspace was closed or Qdrant was inactive - console.log( - "[CodeIndexOrchestrator] Collection already has indexed data. Running incremental scan for new/changed files...", - ) - this.stateManager.setSystemState("Indexing", "Checking for new or modified files...") - - // Mark as incomplete at the start of incremental scan - await this.writeGate.run(signal, () => this.vectorStore.markIndexingIncomplete()) - - let cumulativeBlocksIndexed = 0 - let cumulativeBlocksFoundSoFar = 0 - let batchErrors: Error[] = [] - - const handleFileParsed = (fileBlockCount: number) => { - if (signal.aborted) return - cumulativeBlocksFoundSoFar += fileBlockCount - this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) - } - - const handleBlocksIndexed = (indexedCount: number) => { - if (signal.aborted) return - cumulativeBlocksIndexed += indexedCount - this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) - } - - // Run incremental scan - scanner will skip unchanged files using cache - const result = await this.scanner.scanDirectory( - this.workspacePath, - (batchError: Error) => { - console.error( - `[CodeIndexOrchestrator] Error during incremental scan batch: ${batchError.message}`, - batchError, - ) - batchErrors.push(batchError) - }, - handleBlocksIndexed, - handleFileParsed, - signal, - ) - - if (signal.aborted) return - - if (!result) { - throw new Error("Incremental scan failed, is scanner initialized?") - } - - // If new files were found and indexed, log the results - if (cumulativeBlocksFoundSoFar > 0) { - console.log( - `[CodeIndexOrchestrator] Incremental scan completed: ${cumulativeBlocksIndexed} blocks indexed from new/changed files`, - ) - } else { - console.log("[CodeIndexOrchestrator] No new or changed files found") - } - - await this._startWatcher() - - // Mark indexing as complete after successful incremental scan - await this.writeGate.run(signal, () => this.vectorStore.markIndexingComplete()) - throwIfAborted(signal) - - this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) - } else { - // No existing data or collection was just created - do a full scan - this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") - - // Mark as incomplete at the start of full scan - await this.writeGate.run(signal, () => this.vectorStore.markIndexingIncomplete()) - - let cumulativeBlocksIndexed = 0 - let cumulativeBlocksFoundSoFar = 0 - let batchErrors: Error[] = [] - - const handleFileParsed = (fileBlockCount: number) => { - if (signal.aborted) return - cumulativeBlocksFoundSoFar += fileBlockCount - this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) - } - - const handleBlocksIndexed = (indexedCount: number) => { - if (signal.aborted) return - cumulativeBlocksIndexed += indexedCount - this.stateManager.reportBlockIndexingProgress(cumulativeBlocksIndexed, cumulativeBlocksFoundSoFar) - } - - const result = await this.scanner.scanDirectory( - this.workspacePath, - (batchError: Error) => { - console.error( - `[CodeIndexOrchestrator] Error during initial scan batch: ${batchError.message}`, - batchError, - ) - batchErrors.push(batchError) - }, - handleBlocksIndexed, - handleFileParsed, - signal, - ) - - if (signal.aborted) return - - if (!result) { - throw new Error("Scan failed, is scanner initialized?") - } - - const { stats } = result - - // Check if any blocks were actually indexed successfully - // If no blocks were indexed but blocks were found, it means all batches failed - if (cumulativeBlocksIndexed === 0 && cumulativeBlocksFoundSoFar > 0) { - if (batchErrors.length > 0) { - // Use the first batch error as it's likely representative of the main issue - const firstError = batchErrors[0] - throw new Error(`Indexing failed: ${firstError.message}`) - } else { - throw new Error(t("embeddings:orchestrator.indexingFailedNoBlocks")) - } - } - - // Check for partial failures - if a significant portion of blocks failed - const failureRate = (cumulativeBlocksFoundSoFar - cumulativeBlocksIndexed) / cumulativeBlocksFoundSoFar - if (batchErrors.length > 0 && failureRate > 0.1) { - // More than 10% of blocks failed to index - const firstError = batchErrors[0] - throw new Error( - `Indexing partially failed: Only ${cumulativeBlocksIndexed} of ${cumulativeBlocksFoundSoFar} blocks were indexed. ${firstError.message}`, - ) - } - - // CRITICAL: If there were ANY batch errors and NO blocks were successfully indexed, - // this is a complete failure regardless of the failure rate calculation - if (batchErrors.length > 0 && cumulativeBlocksIndexed === 0) { - const firstError = batchErrors[0] - throw new Error(`Indexing failed completely: ${firstError.message}`) - } - - // Final sanity check: If we found blocks but indexed none and somehow no errors were reported, - // this is still a failure - if (cumulativeBlocksFoundSoFar > 0 && cumulativeBlocksIndexed === 0) { - throw new Error(t("embeddings:orchestrator.indexingFailedCritical")) - } - - await this._startWatcher() - - // Mark indexing as complete after successful full scan - await this.writeGate.run(signal, () => this.vectorStore.markIndexingComplete()) - throwIfAborted(signal) - + if (!this.isCurrentAttempt()) return + await this.writeGate.run(signal, () => this.vectorStore.markIndexingIncomplete()) + let blocksIndexed = 0 + let blocksFound = 0 + const batchErrors: Error[] = [] + const result = await this.scanner.scanDirectory( + this.workspacePath, + (error) => { + if (this.isLive(signal)) batchErrors.push(error) + }, + (count) => { + if (!this.isLive(signal)) return + blocksIndexed += count + this.stateManager.reportBlockIndexingProgress(blocksIndexed, blocksFound) + }, + (count) => { + if (!this.isLive(signal)) return + blocksFound += count + this.stateManager.reportBlockIndexingProgress(blocksIndexed, blocksFound) + }, + signal, + ) + throwIfAborted(signal) + if (!this.isCurrentAttempt()) return + if (!result) throw new Error("Scan failed, is scanner initialized?") + this.incompleteScan = + batchErrors.length > 0 || !!result.incompleteFiles?.length || blocksIndexed < blocksFound + await this.startWatcher(signal) + throwIfAborted(signal) + if (!this.isCurrentAttempt()) return + if (this.hasIncompleteFiles()) { + this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStoppedPartial")) + return + } + await this.writeGate.run(signal, () => this.vectorStore.markIndexingComplete()) + if (this.isLive(signal)) { this.stateManager.setSystemState("Indexed", t("embeddings:orchestrator.fileWatcherStarted")) } - } catch (error: any) { + } catch (error) { + if (!this.isLive(signal)) return if (error instanceof ParserFatalError) { this.failStop(error) return } - if (this.parserFailure) return - // Handle abort gracefully — not an error, just a user-initiated stop - if (error?.name === "AbortError" || signal.aborted) { - console.log("[CodeIndexOrchestrator] Indexing aborted by user.") - if (!this.stopped) this.stopIndexing() + if (error instanceof Error && error.name === "AbortError") { + void this.stop(true) return } - - console.error("[CodeIndexOrchestrator] Error during indexing:", error) - if (indexingStarted) { - try { - await this.writeGate.run(signal, () => this.vectorStore.clearCollection()) - } catch (cleanupError) { - console.error("[CodeIndexOrchestrator] Failed to clean up after error:", cleanupError) - } - } - - // Only clear cache if indexing had started (Qdrant connection succeeded) - // If we never connected to Qdrant, preserve cache for incremental scan when it comes back - if (indexingStarted) { - // Indexing started but failed mid-way - clear cache to avoid cache-Qdrant mismatch - await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) - console.log( - "[CodeIndexOrchestrator] Indexing failed after starting. Clearing cache to avoid inconsistency.", - ) - } else { - // Never connected to Qdrant - preserve cache for future incremental scan - console.log( - "[CodeIndexOrchestrator] Failed to connect to Qdrant. Preserving cache for future incremental scan.", - ) - } - - if (signal.aborted) return this.stateManager.setSystemState( "Error", t("embeddings:orchestrator.failedDuringInitialScan", { - errorMessage: error.message || t("embeddings:orchestrator.unknownError"), + errorMessage: error instanceof Error ? error.message : t("embeddings:orchestrator.unknownError"), }), ) - this.stopWatcher() + void this.stop(true) } finally { this._isProcessing = false } } - /** - * Stops any in-progress indexing by aborting the scan and stopping the file watcher. - */ - public stopIndexing(): void { - this._abortController?.abort() - this.stopWatcher() - if (!this.parserFailure) - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.indexingStopped")) + public stopIndexing(): Promise { + return this.stop(false) + } + + public stopWatcher(): Promise { + return this.stop(true) } - /** - * Stops the file watcher and cleans up resources. - */ - public stopWatcher(): void { + private stop(force: boolean): Promise { + if (this.stopPromise) return this.stopPromise + const signal = this._abortController?.signal + const active = + this._isProcessing || + !!this.parserFailure || + this.fileWatcher.isProcessing || + (!!signal && this.writeGate.hasPending(signal)) + if (!active && !force) return this.idleStop + const preserveError = this.stateManager.state === "Error" || !!this.parserFailure + const expiresAt = Date.now() + INDEX_STOP_LIMIT_MS + if (signal) this.writeGate.stopAt(signal, expiresAt) + this.stopPromise = Promise.resolve().then(async () => { + let timer: ReturnType | undefined + let workerStopped = !this.parser?.stopped + const workerExit = this.parser?.stopped?.then(() => { + workerStopped = true + }) + const settled = Promise.allSettled([ + this.scanPromise, + signal ? this.writeGate.settled(signal) : undefined, + workerExit, + ]).then(() => false) + const expired = await Promise.race([ + settled, + new Promise((resolve) => { + timer = setTimeout(() => resolve(true), Math.max(0, expiresAt - Date.now())) + }), + ]) + clearTimeout(timer) + if (expired && signal) this.writeGate.abandon(signal, expiresAt) + if (expired && !workerStopped) this.parser?.logStopTimeout?.() + if (!active || !this.isCurrentAttempt() || preserveError) return + this.stateManager.setSystemState( + "Standby", + t( + expired || this.hasIncompleteFiles() + ? "embeddings:orchestrator.indexingStoppedPartial" + : "embeddings:orchestrator.indexingStopped", + ), + ) + }) + this.stopped = true this._abortController?.abort() this.parser?.stop() - this.stopped = true - this.onStopped?.() this.fileWatcher.dispose() - this._fileWatcherSubscriptions.forEach((sub) => sub.dispose()) + this._fileWatcherSubscriptions.forEach((subscription) => subscription.dispose()) this._fileWatcherSubscriptions = [] - - if (this.stateManager.state !== "Error" && this.stateManager.state !== "Stopping") { - this.stateManager.setSystemState("Standby", t("embeddings:orchestrator.fileWatcherStopped")) + this.onStopped?.() + if (active && !preserveError && this.isCurrentAttempt()) { + this.stateManager.setSystemState("Stopping", t("embeddings:orchestrator.indexingStopping")) } - this._isProcessing = false + return this.stopPromise } - /** - * Clears all index data by stopping the watcher, clearing the vector store, - * and resetting the cache file. - */ public async clearIndexData(): Promise { - this._isProcessing = true - + await this.stopWatcher() + const signal = new AbortController().signal try { - this.stopIndexing() - const cleanupSignal = new AbortController().signal - - try { - if (this.configManager.isFeatureConfigured) { - await this.writeGate.run(cleanupSignal, () => this.vectorStore.deleteCollection()) - } else { - console.warn("[CodeIndexOrchestrator] Service not configured, skipping vector collection clear.") - } - } catch (error: any) { - console.error("[CodeIndexOrchestrator] Failed to clear vector collection:", error) - this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${error.message}`) + if (this.configManager.isFeatureConfigured) { + await this.writeGate.run(signal, () => this.vectorStore.deleteCollection()) } - - await this.writeGate.run(cleanupSignal, () => this.cacheManager.clearCacheFile()) - - if (this.stateManager.state !== "Error") { - this.stateManager.setSystemState("Standby", "Index data cleared successfully.") + await this.writeGate.run(signal, () => this.cacheManager.clearCacheFile()) + if (this.isCurrentAttempt()) this.stateManager.setSystemState("Standby", "Index data cleared successfully.") + } catch (error) { + if (this.isCurrentAttempt()) { + this.stateManager.setSystemState("Error", `Failed to clear index data: ${String(error)}`) } - } finally { - this._isProcessing = false } } - /** - * Gets the current state of the indexing system. - */ public get state(): IndexingState { return this.stateManager.state } diff --git a/src/services/code-index/parse-worker-client.ts b/src/services/code-index/parse-worker-client.ts index 9cd6ee54f52..95ae0a26950 100644 --- a/src/services/code-index/parse-worker-client.ts +++ b/src/services/code-index/parse-worker-client.ts @@ -41,7 +41,14 @@ export class ParseWorkerClient implements ICodeParser { private active?: PendingParse private timer?: NodeJS.Timeout private nextId = 0 - private stopped = false + private isStopped = false + private resolveStopped!: () => void + readonly stopped = new Promise((resolve) => { + this.resolveStopped = resolve + }) + private stopStartedAt?: number + private exitObserved = false + private stopTimeoutLogged = false private readonly parser: CodeParser private idleFailure?: Pick @@ -70,7 +77,7 @@ export class ParseWorkerClient implements ICodeParser { this.idleFailure = undefined return new ParserFatalError(category, filePath, message) } - if (this.stopped) return abortError() + if (this.isStopped) return abortError() return undefined } @@ -104,13 +111,13 @@ export class ParseWorkerClient implements ICodeParser { } private dispatch(): void { - if (this.stopped || this.active || this.queue.length === 0) return + if (this.isStopped || this.active || this.queue.length === 0) return this.active = this.queue.shift()! try { if (!this.worker) { this.worker = this.workerFactory() this.worker.on("message", (reply) => { - if (this.stopped || reply.id !== this.active?.request.id) return + if (this.isStopped || reply.id !== this.active?.request.id) return if (reply.error?.fatal || (reply.error && isParserAbort(reply.error.message))) { this.fail("parser-abort", reply.error?.message) return @@ -129,7 +136,13 @@ export class ParseWorkerClient implements ICodeParser { this.dispatch() }) this.worker.on("error", (error) => this.fail("parser-abort", error.message)) - this.worker.on("exit", (code) => this.fail("parser-abort", `Parse worker exited (${code})`)) + this.worker.on("exit", (code) => { + this.exitObserved = true + this.fail("parser-abort", `Parse worker exited (${code})`) + console.log(`parse worker exited after ${Math.round(performance.now() - this.stopStartedAt!)} ms`) + this.worker = undefined + this.resolveStopped() + }) } this.timer = setTimeout(() => this.fail("parse-timeout"), this.timeoutMs) this.worker.postMessage(this.active.request) @@ -139,7 +152,7 @@ export class ParseWorkerClient implements ICodeParser { } private fail(category: ParserFatalError["category"], message?: string): void { - if (this.stopped) return + if (this.isStopped) return const failed = this.active this.active = undefined if (failed) this.settle(failed, new ParserFatalError(category, failed.request.filePath, message)) @@ -148,15 +161,27 @@ export class ParseWorkerClient implements ICodeParser { } stop(): void { - if (this.stopped) return - this.stopped = true + if (this.isStopped) return + this.isStopped = true + this.stopStartedAt = performance.now() clearTimeout(this.timer) if (this.active) this.settle(this.active, abortError()) this.active = undefined for (const pending of this.queue) this.settle(pending, abortError()) this.queue = [] - void this.worker?.terminate().catch(() => undefined) - this.worker = undefined + if (!this.worker) { + this.exitObserved = true + this.resolveStopped() + } else if (!this.exitObserved) { + // Termination acknowledgement alone is not evidence of physical exit. + void this.worker.terminate().catch(() => undefined) + } + } + + logStopTimeout(): void { + if (!this.isStopped || this.exitObserved || this.stopTimeoutLogged) return + this.stopTimeoutLogged = true + console.log("parse worker exit not observed within stop limit") } dispose(): void { diff --git a/src/services/code-index/parse-worker.ts b/src/services/code-index/parse-worker.ts index 79eba5b7bf0..9726798fcc6 100644 --- a/src/services/code-index/parse-worker.ts +++ b/src/services/code-index/parse-worker.ts @@ -1,12 +1,24 @@ import { parentPort } from "node:worker_threads" +import { readFile } from "node:fs/promises" +import * as path from "node:path" import { CodeParser } from "./processors/parser" import { isParserAbort } from "./parser-errors" import type { ParseRequest, ParseReply } from "./parse-worker-client" +let fault = process.env.ROO_PARSE_WORKER_FAULT const parser = new CodeParser(undefined, __dirname) parentPort!.on("message", async (request: ParseRequest) => { let reply: ParseReply try { + const nextFault = fault + fault = undefined + if (nextFault === "abort") throw new Error("Aborted() (injected parse worker fault)") + if (nextFault === "wasm-hang") { + const bytes = new Uint8Array(await readFile(path.join(__dirname, "fault-loop.wasm"))) + const { instance } = await WebAssembly.instantiate(bytes) + console.log("parse worker entering wasm fault loop") + ;(instance.exports.loop as () => void)() + } const blocks = await parser.parseFile(request.filePath, { content: request.content, fileHash: request.fileHash, diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 621333bccc6..a5c8df2bd0d 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -1,379 +1,1397 @@ -// npx vitest services/code-index/processors/__tests__/file-watcher.spec.ts - import * as vscode from "vscode" - +import path from "path" +import { createHash } from "crypto" import { FileWatcher } from "../file-watcher" +import { CodeIndexOrchestrator } from "../../orchestrator" +import { GenerationAllocator } from "../../generation-allocator" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" import { ParserFatalError } from "../../parser-errors" +import { CodeBlock } from "../../interfaces" +import { MAX_FILE_SIZE_BYTES, PARSE_TIMEOUT_MS, INDEX_STOP_LIMIT_MS } from "../../constants" +import { EventEmitter } from "node:events" +import { ParseWorkerClient, type ParseRequest } from "../../parse-worker-client" +import { CodeParser } from "../parser" +import { CacheManager } from "../../cache-manager" +import { safeWriteJson } from "../../../../utils/safeWriteJson" +import { FileCompletion } from "../../file-completion" -// Mock dependencies -vi.mock("../../cache-manager") -vi.mock("../../../core/ignore/RooIgnoreController", () => ({ - RooIgnoreController: vi.fn().mockImplementation(() => ({ - validateAccess: vi.fn().mockReturnValue(true), - })), -})) -vi.mock("ignore") -vi.mock("../parser", () => ({ - codeParser: { - parseFile: vi.fn().mockResolvedValue([]), - }, -})) -vi.mock("../../../glob/ignore-utils", () => ({ - isPathInIgnoredDirectory: vi.fn().mockReturnValue(false), -})) +vi.mock("../../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn() })) -// Mock vscode module +vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: vi.fn().mockImplementation(() => ({ validateAccess: () => true })), +})) +vi.mock("../../processors", () => ({})) +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) vi.mock("vscode", () => ({ workspace: { + workspaceFolders: [{ uri: { fsPath: "/mock/workspace" } }], createFileSystemWatcher: vi.fn(), - workspaceFolders: [ - { - uri: { - fsPath: "/mock/workspace", - }, - }, - ], - fs: { - stat: vi.fn().mockResolvedValue({ size: 1000 }), - readFile: vi.fn().mockResolvedValue(Buffer.from("test content")), - }, + fs: { stat: vi.fn(), readFile: vi.fn() }, }, RelativePattern: vi.fn().mockImplementation((base, pattern) => ({ base, pattern })), Uri: { - file: vi.fn().mockImplementation((path) => ({ fsPath: path })), + file: (filePath: string) => ({ fsPath: filePath }), + joinPath: (base: { fsPath: string }, relative: string) => ({ fsPath: path.join(base.fsPath, relative) }), + }, + EventEmitter: class { + private listeners = new Set<(value: unknown) => void>() + event = (listener: (value: unknown) => void) => { + this.listeners.add(listener) + return { dispose: () => this.listeners.delete(listener) } + } + fire = vi.fn((value: unknown) => { + for (const listener of this.listeners) listener(value) + }) + dispose() { + this.listeners.clear() + } }, - EventEmitter: vi.fn().mockImplementation(() => ({ - event: vi.fn(), - fire: vi.fn(), - dispose: vi.fn(), - })), - ExtensionContext: vi.fn(), })) -describe("FileWatcher", () => { - let fileWatcher: FileWatcher - let mockWatcher: any - let mockOnDidCreate: any - let mockOnDidChange: any - let mockOnDidDelete: any - let mockContext: any - let mockCacheManager: any - let mockEmbedder: any - let mockVectorStore: any - let mockIgnoreInstance: any +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((complete, fail) => { + resolve = complete + reject = fail + }) + return { promise, resolve, reject } +} - beforeEach(() => { - // Reset all mocks - vi.clearAllMocks() +async function drain() { + for (let iter = 0; iter < 150; iter++) await Promise.resolve() +} - // Create mock event handlers - mockOnDidCreate = vi.fn() - mockOnDidChange = vi.fn() - mockOnDidDelete = vi.fn() +class ControlledWorker extends EventEmitter { + requests: ParseRequest[] = [] + terminate = vi.fn(async () => 0) + postMessage(request: ParseRequest) { + this.requests.push(request) + } +} - // Create mock watcher - mockWatcher = { - onDidCreate: vi.fn().mockImplementation((handler) => { - mockOnDidCreate = handler - return { dispose: vi.fn() } - }), - onDidChange: vi.fn().mockImplementation((handler) => { - mockOnDidChange = handler - return { dispose: vi.fn() } - }), - onDidDelete: vi.fn().mockImplementation((handler) => { - mockOnDidDelete = handler - return { dispose: vi.fn() } - }), - dispose: vi.fn(), - } +const workspace = path.resolve("/mock/workspace") +const fileA = path.join(workspace, "a.ts") +const fileB = path.join(workspace, "b.ts") +const hash = (content: string) => createHash("sha256").update(content).digest("hex") - // Mock createFileSystemWatcher to return our mock watcher - vi.mocked(vscode.workspace.createFileSystemWatcher).mockReturnValue(mockWatcher) +function blocks(filePath = fileA, count = 1): CodeBlock[] { + return Array.from({ length: count }, (_, index) => ({ + file_path: filePath, + identifier: null, + type: "function", + start_line: 1, + end_line: 1, + content: `${path.basename(filePath)} block ${index}`, + fileHash: "hash", + segmentHash: `${filePath}:${index}`, + })) +} - // Create mock dependencies - mockContext = { - subscriptions: [], - } +describe("FileWatcher per-file completion", () => { + let cache: any + let store: any + let embedder: any + let parser: any + let ignore: any + let access: any + let gate: IndexWriteGate + let generations: GenerationAllocator + let watcher: FileWatcher + let watchers: FileWatcher[] + let handlers: Record void> + let disk: { hashes: Record; open: Record } + let hashes: Record + let open: Record + let points: Map + let contents: Map + let progress: ReturnType + let finished: ReturnType + let fatal: ReturnType + let state: any + let orchestrator: CodeIndexOrchestrator - mockCacheManager = { - getHash: vi.fn(), - updateHash: vi.fn(), - deleteHash: vi.fn(), - } + function makeWatcher() { + const created = new FileWatcher( + workspace, + {} as any, + cache, + embedder, + store, + ignore, + access, + 1, + parser, + gate, + generations, + ) + watchers.push(created) + return created + } - mockEmbedder = { - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), + function applyDelete(filePath: string, generation: number) { + const relative = path.relative(workspace, filePath) + for (const [pointId, point] of points) { + if ( + point.payload.filePath === relative && + (point.payload.generation === undefined || point.payload.generation < generation) + ) + points.delete(pointId) } + } - mockVectorStore = { - upsertPoints: vi.fn().mockResolvedValue(undefined), - deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), - deletePointsByMultipleFilePaths: vi.fn().mockResolvedValue(undefined), - } + async function start( + worker: { stop(): void; stopped: Promise } = { stop: vi.fn(), stopped: Promise.resolve() }, + isCurrentAttempt = () => true, + reuseState = false, + ) { + let status = "Standby" + if (!reuseState) + state = { + get state() { + return status + }, + setSystemState: vi.fn((next: string) => { + status = next + }), + reportBlockIndexingProgress: vi.fn(), + reportFileQueueProgress: vi.fn(), + } + orchestrator = new CodeIndexOrchestrator( + { isFeatureConfigured: true } as any, + state, + workspace, + cache, + store, + { + scanDirectory: vi.fn().mockResolvedValue({ stats: { processed: 0, skipped: 0 }, totalBlockCount: 0 }), + } as any, + watcher, + worker, + fatal, + undefined, + gate, + generations, + isCurrentAttempt, + ) + await orchestrator.startIndexing() + expect(state.state).toBe(Object.keys(cache.getOpenFiles()).length ? "Standby" : "Indexed") + } - mockIgnoreInstance = { - ignores: vi.fn().mockReturnValue(false), + async function useRealCache() { + let persisted: unknown = { version: 2, generationWatermark: 0, fileHashes: {}, openFiles: {} } + vi.mocked(safeWriteJson).mockImplementation(async (_filePath, value) => { + persisted = structuredClone(value) + }) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => + Buffer.from( + uri.fsPath.includes("roo-index-cache-") ? JSON.stringify(persisted) : contents.get(uri.fsPath)!, + ), + ) + const context = { globalStorageUri: { fsPath: path.join(workspace, "storage") } } as any + const reload = async () => { + const loaded = new CacheManager(context, workspace) + await loaded.initialize() + gate.recover(Object.keys(loaded.getOpenFiles()), loaded.loadedAt) + return loaded } + cache = await reload() + generations = new GenerationAllocator(cache) + watcher.dispose() + watcher = makeWatcher() + watcher.onBatchProgressUpdate(progress) + watcher.onDidFinishBatchProcessing(finished) + return reload + } - fileWatcher = new FileWatcher( - "/mock/workspace", - mockContext, - mockCacheManager, - mockEmbedder, - mockVectorStore, - mockIgnoreInstance, - undefined, - undefined, - { parseFile: vi.fn().mockResolvedValue([]) }, + async function queue(filePaths = [fileA], type = "change") { + for (const filePath of filePaths) handlers[type]({ fsPath: filePath }) + await vi.advanceTimersByTimeAsync(500) + await drain() + } + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(safeWriteJson).mockReset() + vi.useFakeTimers() + vi.setSystemTime(0) + watchers = [] + handlers = {} + hashes = {} + open = {} + disk = { hashes: {}, open: {} } + points = new Map() + contents = new Map([ + [fileA, "content a"], + [fileB, "content b"], + ]) + cache = { + generationWatermark: 0, + getHash: vi.fn((filePath) => hashes[filePath]), + getOpenFiles: vi.fn(() => ({ ...open })), + isTracked: vi.fn((filePath) => Object.hasOwn(hashes, filePath) || Object.hasOwn(open, filePath)), + getTrackedPaths: vi.fn(() => [...new Set([...Object.keys(hashes), ...Object.keys(open)])]), + reserveGenerations: vi.fn(async (watermark) => { + cache.generationWatermark = watermark + }), + invalidateFile: vi.fn(async (filePath) => { + delete hashes[filePath] + open[filePath] = true + disk = { hashes: { ...hashes }, open: { ...open } } + }), + completeFile: vi.fn((filePath, newHash) => { + hashes[filePath] = newHash + delete open[filePath] + }), + completeRemoval: vi.fn((filePath) => { + delete hashes[filePath] + delete open[filePath] + }), + clearCacheFile: vi.fn(), + } + store = { + initialize: vi.fn().mockResolvedValue(false), + markIndexingIncomplete: vi.fn(), + markIndexingComplete: vi.fn(), + upsertPoints: vi.fn(async (batch) => { + for (const point of batch) points.set(point.id, point) + }), + deletePointsByFilePath: vi.fn(async (filePath, generation) => applyDelete(filePath, generation)), + countPointsForFile: vi.fn(async (filePath, generation) => { + const selected = [...points.values()].filter( + (point) => point.payload.filePath === path.relative(workspace, filePath), + ) + return { + current: selected.filter((point) => point.payload.generation === generation).length, + other: selected.filter((point) => point.payload.generation !== generation).length, + } + }), + } + embedder = { createEmbeddings: vi.fn(async (texts: string[]) => ({ embeddings: texts.map(() => [1, 2, 3]) })) } + parser = { parseFile: vi.fn(async (filePath) => blocks(filePath)) } + ignore = { ignores: vi.fn(() => false) } + access = { validateAccess: vi.fn(() => true) } + gate = new IndexWriteGate() + generations = new GenerationAllocator(cache) + vi.mocked(vscode.workspace.fs.stat).mockResolvedValue({ size: 1000 } as any) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => + Buffer.from(contents.get(uri.fsPath) ?? "content"), + ) + vi.mocked(vscode.workspace.createFileSystemWatcher).mockImplementation( + () => + ({ + onDidCreate: (handler: any) => { + handlers.create = handler + }, + onDidChange: (handler: any) => { + handlers.change = handler + }, + onDidDelete: (handler: any) => { + handlers.delete = handler + }, + dispose: vi.fn(), + }) as any, ) + watcher = makeWatcher() + progress = vi.fn() + finished = vi.fn() + fatal = vi.fn() + watcher.onBatchProgressUpdate(progress) + watcher.onDidFinishBatchProcessing(finished) }) - describe("attempt ownership", () => { - function createWatcher(parser: any, signal?: AbortSignal) { - const watcher = new FileWatcher( - "/mock/workspace", - mockContext, - mockCacheManager, - mockEmbedder, - mockVectorStore, - mockIgnoreInstance, - { validateAccess: () => true } as any, - 1, - parser, + afterEach(() => { + for (const created of watchers) created.dispose() + vi.useRealTimers() + }) + + it.each(["cancel", "disable", "fatal"])( + "T5 retains A independently while B hangs on %s and skips A on retry", + async (stop) => { + const held = deferred() + parser.parseFile.mockImplementation(async (filePath: string) => + filePath === fileB ? held.promise : blocks(filePath), ) - return { watcher, initialize: () => watcher.initialize(signal) } + await start() + await queue([fileA, fileB]) + expect(hashes[fileA]).toBe(hash("content a")) + expect(hashes[fileB]).toBeUndefined() + expect(progress).toHaveBeenCalledWith({ processedInBatch: 1, totalInBatch: 2, currentFile: fileA }) + if (stop === "fatal") held.reject(new ParserFatalError("parser-abort", fileB)) + else if (stop === "disable") await orchestrator.stopWatcher() + else await orchestrator.stopIndexing() + await drain() + if (stop === "fatal") expect(state.state).toBe("Error") + else held.resolve(blocks(fileB)) + watcher = makeWatcher() + await start() + embedder.createEmbeddings.mockClear() + await queue([fileA]) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) + + it("T6/T18 waits for every batch and verification before hashing or reporting completion", async () => { + const held = deferred() + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + embedder.createEmbeddings + .mockResolvedValueOnce({ embeddings: [[1]] }) + .mockImplementationOnce(() => held.promise) + await start() + await queue() + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(progress.mock.calls.map(([value]) => value.processedInBatch)).toEqual([0]) + expect(finished).not.toHaveBeenCalled() + held.resolve({ embeddings: [[2]] }) + await drain() + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(hashes[fileA]).toBe(hash("content a")) + expect(state.state).toBe("Indexed") + }) + + it("T18 registers every batch before the first embedding can settle", async () => { + const registered = vi.spyOn(FileCompletion.prototype, "registered") + const register = vi.spyOn(FileCompletion.prototype, "register") + const held = deferred() + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + embedder.createEmbeddings + .mockImplementationOnce(async () => { + expect(register).toHaveBeenCalledTimes(3) + expect(registered).toHaveBeenCalledOnce() + const completion = registered.mock.contexts[0] as FileCompletion + expect(completion.registrationComplete).toBe(true) + expect(completion.outstanding).toBe(3) + return { embeddings: [[1]] } + }) + .mockImplementationOnce(() => held.promise) + try { + await start() + await queue() + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect((registered.mock.contexts[0] as FileCompletion).outstanding).toBe(2) + expect(hashes[fileA]).toBeUndefined() + expect(store.countPointsForFile).not.toHaveBeenCalled() + held.resolve({ embeddings: [[1]] }) + await drain() + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(hashes[fileA]).toBe(hash("content a")) + } finally { + registered.mockRestore() + register.mockRestore() } + }) - it("reports a fatal parser once, disposes events, and starts no more queued files", async () => { - const fatal = new ParserFatalError("parser-abort", "/mock/workspace/bad.ts") - const parser = { parseFile: vi.fn().mockRejectedValue(fatal) } - const { watcher, initialize } = createWatcher(parser) - await initialize() - const report = vi.spyOn((watcher as any).parserFatal, "fire") - await expect(watcher.processFile("/mock/workspace/bad.ts")).rejects.toBe(fatal) - expect(report).toHaveBeenCalledOnce() - expect(mockWatcher.dispose).toHaveBeenCalledOnce() - await mockOnDidChange({ fsPath: "/mock/workspace/next.ts" }) - await (watcher as any).triggerBatchProcessing() - expect(parser.parseFile).toHaveBeenCalledOnce() - expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + it("T6 refuses subsequent batches after a successful upsert is followed by cancellation", async () => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + await start() + const upsert = store.upsertPoints.getMockImplementation()! + store.upsertPoints.mockImplementationOnce(async (batch: any[]) => { + await upsert(batch) + void orchestrator.stopIndexing() }) + await queue() + expect(points.size).toBe(1) + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(gate.needsReconciliation(fileA)).toBe(false) + watcher = makeWatcher() + await start() + await queue() + expect(points.size).toBe(3) + expect(hashes[fileA]).toBe(hash("content a")) + }) - it("drops embeddings released after the attempt stops", async () => { - const parser = { - parseFile: vi.fn().mockResolvedValue([ - { - file_path: "/mock/workspace/file.ts", - content: "function test() {}", - start_line: 1, - end_line: 1, - }, - ]), - } - const controller = new AbortController() - const { watcher, initialize } = createWatcher(parser, controller.signal) - await initialize() - let release!: (value: any) => void - mockEmbedder.createEmbeddings.mockImplementation( - () => - new Promise((resolve) => { - release = resolve - }), - ) - const pending = (watcher as any) - .processBatch(new Map([["file", { uri: { fsPath: "/mock/workspace/file.ts" }, type: "change" }]])) - .catch((error: Error) => error) - await vi.waitFor(() => expect(mockEmbedder.createEmbeddings).toHaveBeenCalledOnce()) - controller.abort() - watcher.dispose() - release({ embeddings: [[1, 2, 3]] }) - await pending - expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() - expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() - expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + it("T6/T11 interrupted partial replacement has no durable hash; retry restores every block", async () => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + embedder.createEmbeddings + .mockResolvedValueOnce({ embeddings: [[1]] }) + .mockRejectedValueOnce(new Error("provider failed")) + await start() + await queue() + expect(hashes[fileA]).toBeUndefined() + expect(disk).toEqual({ hashes: {}, open: { [fileA]: true } }) + expect(points.size).toBe(1) + expect(finished.mock.calls[0][0].processedFiles[0].status).toBe("local_error") + expect(state.state).toBe("Standby") + watcher.dispose() + hashes = { ...disk.hashes } + open = { ...disk.open } + watcher = makeWatcher() + await start() + await queue() + expect(points.size).toBe(3) + expect(open[fileA]).toBeUndefined() + }) + + it.each(["success", "failure", "abort"])( + "dispose cancels the real embed signal and discards late %s", + async (outcome) => { + const held = deferred() + parser.parseFile.mockResolvedValue(blocks(fileA, 2)) + embedder.createEmbeddings.mockImplementation(() => held.promise) + await start() + await queue() + const signal = embedder.createEmbeddings.mock.calls[0][2] + await orchestrator.stopIndexing() + expect(signal.aborted).toBe(true) + if (outcome === "success") held.resolve({ embeddings: [[1]] }) + else held.reject(outcome === "abort" ? new DOMException("cancelled", "AbortError") : new Error("failed")) + await drain() + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(finished).not.toHaveBeenCalled() + expect(progress).toHaveBeenCalledTimes(1) + }, + ) + + it("reports fatal immediately without waiting for another file and suppresses queued events", async () => { + const held = deferred() + parser.parseFile.mockImplementation(async (filePath: string) => { + if (filePath === fileA) throw new ParserFatalError("parse-timeout", fileA) + return held.promise }) + await start() + await queue([fileA, fileB]) + expect(fatal).toHaveBeenCalledOnce() + expect(state.state).toBe("Error") + const calls = parser.parseFile.mock.calls.length + await queue([path.join(workspace, "later.ts")]) + expect(parser.parseFile).toHaveBeenCalledTimes(calls) + held.resolve(blocks(fileB)) + await drain() + expect(finished).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + }) - it.each(["fatal", "cancel", "disable"])( - "a replacement watcher processes scan-following changes after %s", - async () => { - const parser = { - parseFile: vi.fn().mockResolvedValue([ - { - file_path: "/mock/workspace/file.ts", - content: "function test() {}", - start_line: 1, - end_line: 1, - }, - ]), - } - const old = createWatcher(parser) - await old.initialize() - old.watcher.dispose() - const replacement = createWatcher(parser) - await replacement.initialize() - await mockOnDidChange({ fsPath: "/mock/workspace/file.ts" }) - await (replacement.watcher as any).triggerBatchProcessing() - expect(mockVectorStore.upsertPoints).toHaveBeenCalledOnce() - expect(mockCacheManager.updateHash).toHaveBeenCalledOnce() - expect((replacement.watcher as any)._onBatchProgressUpdate.fire).toHaveBeenCalled() - expect(mockVectorStore.deletePointsByFilePath.mock.invocationCallOrder[0]).toBeLessThan( - mockVectorStore.upsertPoints.mock.invocationCallOrder[0], - ) - replacement.watcher.dispose() - }, - ) + it("bounds concurrency and starts the next file as soon as one finishes", async () => { + const held = deferred() + const paths = Array.from({ length: 12 }, (_, index) => path.join(workspace, `${index}.ts`)) + parser.parseFile.mockImplementation(async (filePath: string) => (filePath === paths[0] ? [] : held.promise)) + await watcher.initialize() + await queue(paths) + expect(parser.parseFile).toHaveBeenCalledTimes(11) + expect(hashes[paths[0]]).toBeDefined() + expect(watcher.isProcessing).toBe(true) + watcher.dispose() + held.resolve([]) + await drain() }) - describe("file filtering", () => { - it("should ignore files in hidden directories on create events", async () => { - // Initialize the file watcher - await fileWatcher.initialize() + it("isProcessing is already true in the start callback so stop detects the batch", async () => { + await start() + watcher.onDidStartBatchProcessing(() => { + expect(watcher.isProcessing).toBe(true) + void orchestrator.stopIndexing() + }) + await queue() + expect(state.setSystemState).toHaveBeenCalledWith("Stopping", expect.any(String)) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) - // Spy on the vector store to see which files are actually processed - const processedFiles: string[] = [] - mockVectorStore.upsertPoints.mockImplementation(async (points: any[]) => { - points.forEach((point) => { - if (point.payload?.file_path) { - processedFiles.push(point.payload.file_path) - } - }) - }) + it("T14 uses fresh generations and distinct segment IDs on successive edits", async () => { + await start() + parser.parseFile.mockResolvedValueOnce(blocks(fileA, 3)).mockResolvedValueOnce(blocks(fileA, 1)) + await queue() + const first = [...points.values()] + expect(new Set(first.map((point) => point.id)).size).toBe(3) + contents.set(fileA, "edited") + await queue() + expect(points.size).toBe(1) + const current = [...points.values()][0] + expect(current.payload.generation).toBeGreaterThan(first[0].payload.generation) + expect(current.id).not.toBe(first[0].id) + expect(store.deletePointsByFilePath.mock.invocationCallOrder[0]).toBeGreaterThan( + store.upsertPoints.mock.invocationCallOrder[0], + ) + }) - // Simulate file creation events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.git/config", shouldProcess: false }, - { path: "/mock/workspace/.hidden/file.ts", shouldProcess: false }, - { path: "/mock/workspace/src/.next/static/file.js", shouldProcess: false }, - { path: "/mock/workspace/node_modules/package/index.js", shouldProcess: false }, - { path: "/mock/workspace/normal/file.js", shouldProcess: true }, - ] - - // Trigger file creation events - for (const { path } of testCases) { - await mockOnDidCreate({ fsPath: path }) - } + it.each([ + { current: 0, other: 0 }, + { current: 1, other: 1 }, + ])("rejects verification mismatch %j without false completed progress", async (counts) => { + store.countPointsForFile.mockResolvedValue(counts) + await start() + await queue() + expect(cache.completeFile).not.toHaveBeenCalled() + expect(progress.mock.calls.map(([value]) => value.processedInBatch)).toEqual([0]) + expect(finished.mock.calls[0][0].batchError).toBeInstanceOf(Error) + expect(state.state).toBe("Standby") + }) - // Wait for batch processing - await new Promise((resolve) => setTimeout(resolve, 600)) + it("T17 awaits durable invalidation before any embeddings or mutations", async () => { + const saved = deferred() + cache.invalidateFile.mockImplementationOnce(async (filePath: string) => { + delete hashes[filePath] + open[filePath] = true + await saved.promise + disk = { hashes: { ...hashes }, open: { ...open } } + }) + hashes[fileA] = "old hash" + await start() + await queue() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + saved.resolve() + await drain() + expect(disk).toEqual({ hashes: {}, open: { [fileA]: true } }) + expect(hashes[fileA]).toBe(hash("content a")) + }) - // Check that files in hidden directories were not processed - expect(processedFiles).not.toContain("src/.next/static/file.js") - expect(processedFiles).not.toContain(".git/config") - expect(processedFiles).not.toContain(".hidden/file.ts") + it("T17 fresh cache at first mutation dispatch sees awaited durable invalidation", async () => { + const reload = await useRealCache() + cache.completeFile(fileA, "previous revision") + await cache.flush() + await start() + const saved = deferred() + const write = vi.mocked(safeWriteJson).getMockImplementation()! + vi.mocked(safeWriteJson).mockImplementationOnce(async (filePath, value) => { + await saved.promise + await write(filePath, value) + }) + const dispatched = deferred() + let atDispatch: CacheManager | undefined + store.upsertPoints.mockImplementationOnce(async () => { + atDispatch = await reload() + await dispatched.promise }) + await queue() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect((await reload()).getHash(fileA)).toBe("previous revision") + saved.resolve() + await drain() + expect(store.upsertPoints).toHaveBeenCalledOnce() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(atDispatch).toBeInstanceOf(CacheManager) + expect(atDispatch!.getHash(fileA)).toBeUndefined() + expect(atDispatch!.getOpenFiles()).toEqual({ [fileA]: true }) + const stopped = orchestrator.stopWatcher() + dispatched.resolve() + await stopped + await drain() + }) - it("should ignore files in hidden directories on change events", async () => { - // Initialize the file watcher - await fileWatcher.initialize() - - // Track which files are processed - const processedFiles: string[] = [] - mockVectorStore.upsertPoints.mockImplementation(async (points: any[]) => { - points.forEach((point) => { - if (point.payload?.file_path) { - processedFiles.push(point.payload.file_path) - } - }) - }) + it("T17 save failure issues no mutation and reports an incomplete batch", async () => { + cache.invalidateFile.mockRejectedValue(new Error("disk full")) + await start() + await queue() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(finished.mock.calls[0][0].batchError).toBeInstanceOf(Error) + expect(state.state).toBe("Standby") + }) - // Simulate file change events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.vscode/settings.json", shouldProcess: false }, - { path: "/mock/workspace/src/.cache/data.json", shouldProcess: false }, - { path: "/mock/workspace/dist/bundle.js", shouldProcess: false }, - ] - - // Trigger file change events - for (const { path } of testCases) { - await mockOnDidChange({ fsPath: path }) - } + it.each(["delete", "ignore", "rooignore", "size"])("T21 cleans tracked open paths on %s", async (exclusion) => { + open[fileA] = true + points.set("legacy", { payload: { filePath: path.relative(workspace, fileA) } }) + if (exclusion === "ignore") ignore.ignores.mockReturnValue(true) + if (exclusion === "rooignore") access.validateAccess.mockReturnValue(false) + if (exclusion === "size") + vi.mocked(vscode.workspace.fs.stat).mockResolvedValue({ size: MAX_FILE_SIZE_BYTES + 1 } as any) + await start() + await queue([fileA], exclusion === "delete" ? "delete" : "change") + expect(points.size).toBe(0) + expect(cache.invalidateFile).toHaveBeenCalledWith(fileA) + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) + expect(cache.completeRemoval).toHaveBeenCalledWith(fileA) + expect(open[fileA]).toBeUndefined() + expect(parser.parseFile).not.toHaveBeenCalled() + }) - // Wait for batch processing - await new Promise((resolve) => setTimeout(resolve, 600)) + it.each(["create", "change", "delete"])("ignores untracked hidden paths on %s events", async (eventType) => { + await watcher.initialize() + const paths = [".git/config", ".hidden/file.ts", "src/.cache/file.ts", "node_modules/package/index.js"] + await queue( + paths.map((relative) => path.join(workspace, relative)), + eventType, + ) + expect(parser.parseFile).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(finished.mock.calls[0][0].processedFiles).toHaveLength(paths.length) + }) - // Check that files in hidden directories were not processed - expect(processedFiles).not.toContain(".vscode/settings.json") - expect(processedFiles).not.toContain("src/.cache/data.json") - }) + it("T21 invalidates a completed file before removal and preserves its open record on failure", async () => { + hashes[fileA] = hash("content a") + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("lost result")) + await start() + await queue([fileA], "delete") + expect(hashes[fileA]).toBeUndefined() + expect(disk).toEqual({ hashes: {}, open: { [fileA]: true } }) + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(finished.mock.calls[0][0].processedFiles[0].status).toBe("local_error") + watcher.dispose() + hashes = { ...disk.hashes } + open = { ...disk.open } + const retry = makeWatcher() + await retry.initialize() + const pending = retry.processFile(fileA) + await drain() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect((await pending).status).toBe("success") + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(hashes[fileA]).toBe(hash("content a")) + }) + + it.each(["delete", "ignore"])( + "T21 completed %s removal waits for the real cache save barrier", + async (eventType) => { + const reload = await useRealCache() + cache.completeFile(fileA, hash("content a")) + await cache.flush() + points.set("completed", { payload: { filePath: path.relative(workspace, fileA), generation: 0 } }) + await start() + const saved = deferred() + const write = vi.mocked(safeWriteJson).getMockImplementation()! + vi.mocked(safeWriteJson).mockImplementationOnce(async (filePath, value) => { + await saved.promise + await write(filePath, value) + }) + let atDispatch: CacheManager | undefined + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, generation: number) => { + atDispatch = await reload() + applyDelete(filePath, generation) + }) + if (eventType === "ignore") ignore.ignores.mockReturnValue(true) + await queue([fileA], eventType === "delete" ? "delete" : "change") + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()).toEqual({ [fileA]: true }) + expect((await reload()).getHash(fileA)).toBe(hash("content a")) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(points.size).toBe(1) + expect(finished).not.toHaveBeenCalled() + saved.resolve() + await drain() + expect(atDispatch!.getHash(fileA)).toBeUndefined() + expect(atDispatch!.getOpenFiles()).toEqual({ [fileA]: true }) + expect(store.deletePointsByFilePath).toHaveBeenCalledOnce() + expect(points.size).toBe(0) + expect(cache.getOpenFiles()).toEqual({}) + await cache.flush() + const completed = await reload() + expect(completed.getHash(fileA)).toBeUndefined() + expect(completed.getOpenFiles()).toEqual({}) + }, + ) - it("should ignore files in hidden directories on delete events", async () => { - // Initialize the file watcher - await fileWatcher.initialize() + it("T21 removal save failure issues no delete", async () => { + hashes[fileA] = "completed" + cache.invalidateFile.mockRejectedValueOnce(new Error("disk full")) + await start() + await queue([fileA], "delete") + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.completeRemoval).not.toHaveBeenCalled() + expect(finished.mock.calls[0][0].batchError).toBeInstanceOf(Error) + }) - // Track which files are deleted - const deletedFiles: string[] = [] - mockVectorStore.deletePointsByFilePath.mockImplementation(async (filePath: string) => { - deletedFiles.push(filePath) + it.each(["replacement", "removal", "exclusion"])( + "T10 late old %s cleanup cannot delete a recreated generation", + async (kind) => { + const held = deferred() + hashes[fileA] = "old" + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, generation: number) => { + await held.promise + applyDelete(filePath, generation) }) + await start() + if (kind === "exclusion") ignore.ignores.mockReturnValue(true) + await queue([fileA], kind === "removal" ? "delete" : "change") + expect(disk.open[fileA]).toBe(true) + const stopped = orchestrator.stopIndexing() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopped + ignore.ignores.mockReturnValue(false) + watcher = makeWatcher() + await start() + await queue() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(hashes[fileA]).toBe(hash("content a")) + const current = [...points.keys()] + held.resolve() + await drain() + expect([...points.keys()]).toEqual(current) + expect(hashes[fileA]).toBe(hash("content a")) + }, + ) - // Simulate file deletion events - const testCases = [ - { path: "/mock/workspace/src/file.ts", shouldProcess: true }, - { path: "/mock/workspace/.git/objects/abc123", shouldProcess: false }, - { path: "/mock/workspace/.DS_Store", shouldProcess: false }, - { path: "/mock/workspace/build/.cache/temp.js", shouldProcess: false }, - ] - - // Trigger file deletion events - for (const { path } of testCases) { - await mockOnDidDelete({ fsPath: path }) + it.each(["clean", "late", "cancel"])( + "T12 grace keeps progress incomplete until re-verification: %s", + async (outcome) => { + gate.recover([fileA], Date.now()) + await start() + await queue() + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(finished).not.toHaveBeenCalled() + expect(progress.mock.calls.map(([value]) => value.processedInBatch)).toEqual([0]) + expect(state.state).toBe("Indexing") + if (outcome === "late") { + const current = [...points.values()][0] + points.set("old write", { ...current, id: "old write", payload: { ...current.payload, generation: 0 } }) + } + if (outcome === "cancel") { + await orchestrator.stopIndexing() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(finished).not.toHaveBeenCalled() + expect(hashes[fileA]).toBeUndefined() + expect(state.state).toBe("Standby") + return } + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 501) + expect(hashes[fileA]).toBeUndefined() + await vi.advanceTimersByTimeAsync(1) + await drain() + if (outcome === "clean") { + expect(hashes[fileA]).toBe(hash("content a")) + expect(state.state).toBe("Indexed") + } else { + expect(hashes[fileA]).toBeUndefined() + expect(state.state).toBe("Standby") + await queue() + expect(points.size).toBe(1) + expect(hashes[fileA]).toBe(hash("content a")) + } + }, + ) - // Wait for batch processing - await new Promise((resolve) => setTimeout(resolve, 600)) + it("T12 late shared-key upsert before initial verification preserves current points but refuses completion", async () => { + const remote = deferred() + let lateWrite!: Promise + store.upsertPoints.mockImplementationOnce(async (batch: any[]) => { + lateWrite = remote.promise.then(() => { + for (const point of batch) points.set(point.id, point) + }) + throw new Error("local timeout") + }) + await start() + await queue() + const deadline = Date.now() + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + const cleanup = deferred() + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, generation: number) => { + applyDelete(filePath, generation) + await cleanup.promise + }) + await queue() + expect(store.countPointsForFile).not.toHaveBeenCalled() + const current = [...points.values()][0] + remote.resolve() + await lateWrite + expect(points.get(current.id)).toBe(current) + expect(new Set([...points.values()].map((point) => point.payload.segmentHash)).size).toBe(1) + expect(points.size).toBe(2) + cleanup.resolve() + await drain() + expect(store.countPointsForFile).toHaveBeenCalledOnce() + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(finished.mock.calls.at(-1)![0].batchError).toBeInstanceOf(Error) + expect(progress.mock.calls.every(([value]) => value.processedInBatch === 0)).toBe(true) + await queue() + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1) + expect(hashes[fileA]).toBeUndefined() + await vi.advanceTimersByTimeAsync(1) + expect(points.size).toBe(1) + expect(hashes[fileA]).toBe(hash("content a")) + }) + + it("T12 stop-expiry uncertainty starts grace at the exact watcher stop deadline", async () => { + const held = deferred() + store.upsertPoints.mockImplementationOnce(() => held.promise) + await start() + await queue() + const expiresAt = Date.now() + INDEX_STOP_LIMIT_MS + const stopped = orchestrator.stopIndexing() + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(gate.graceDeadline(fileA)).toBe(0) + await vi.advanceTimersByTimeAsync(1) + await stopped + expect(Date.now()).toBe(expiresAt) + const deadline = expiresAt + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + watcher = makeWatcher() + await start() + await queue() + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1) + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(1) + expect(hashes[fileA]).toBe(hash("content a")) + expect(state.state).toBe("Indexed") + held.resolve() + await drain() + expect(gate.needsReconciliation(fileA)).toBe(false) + }) - // Check that files in hidden directories were not processed - expect(deletedFiles).not.toContain(".git/objects/abc123") - expect(deletedFiles).not.toContain(".DS_Store") - expect(deletedFiles).not.toContain("build/.cache/temp.js") + it("T12 recovered open-file grace starts at real cache load completion, not later watcher startup", async () => { + const reload = await useRealCache() + await cache.invalidateFile(fileA) + await vi.advanceTimersByTimeAsync(1234) + const read = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! + const loaded = deferred() + vi.mocked(vscode.workspace.fs.readFile).mockImplementationOnce(async (uri) => { + await loaded.promise + return read(uri) }) + const reloading = reload() + await vi.advanceTimersByTimeAsync(1000) + loaded.resolve() + cache = await reloading + const loadedAt = cache.loadedAt + expect(loadedAt).toBe(2234) + await vi.advanceTimersByTimeAsync(5000) + generations = new GenerationAllocator(cache) + watcher.dispose() + watcher = makeWatcher() + await start() + const deadline = loadedAt + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + await queue() + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()).toEqual({ [fileA]: true }) + expect(finished).not.toHaveBeenCalled() + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(1) + expect(cache.getHash(fileA)).toBe(hash("content a")) + expect(cache.getOpenFiles()).toEqual({}) + expect(state.state).toBe("Indexed") + await cache.flush() + }) - it("should handle nested hidden directories correctly", async () => { - // Initialize the file watcher - await fileWatcher.initialize() - - // Track which files are processed - const processedFiles: string[] = [] - mockVectorStore.upsertPoints.mockImplementation(async (points: any[]) => { - points.forEach((point) => { - if (point.payload?.file_path) { - processedFiles.push(point.payload.file_path) - } - }) + it("T12 a locally rejected old upsert arrives remotely during retry grace and fails re-verification", async () => { + const remote = deferred() + let lateWrite!: Promise + store.upsertPoints.mockImplementationOnce(async (batch: any[]) => { + lateWrite = remote.promise.then(() => { + for (const point of batch) points.set(point.id, point) }) + throw new Error("local timeout") + }) + await start() + await queue() + expect(hashes[fileA]).toBeUndefined() + await queue() + const current = [...points.values()][0] + remote.resolve() + await lateWrite + expect(points.get(current.id)).toBe(current) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(hashes[fileA]).toBeUndefined() + expect(finished.mock.calls.at(-1)![0].batchError).toBeInstanceOf(Error) + await queue() + expect(points.size).toBe(1) + expect(hashes[fileA]).toBe(hash("content a")) + }) - // Test deeply nested hidden directories - const testCases = [ - { path: "/mock/workspace/src/components/Button.tsx", shouldProcess: true }, - { path: "/mock/workspace/src/.hidden/components/Button.tsx", shouldProcess: false }, - { path: "/mock/workspace/.hidden/src/components/Button.tsx", shouldProcess: false }, - { path: "/mock/workspace/src/components/.hidden/Button.tsx", shouldProcess: false }, - ] - - // Trigger file creation events - for (const { path } of testCases) { - await mockOnDidCreate({ fsPath: path }) + it.each(["failed cleanup", "partial upsert"])( + "T11 real cache reload keeps %s open and retry restores verified content", + async (failure) => { + const reload = await useRealCache() + cache.completeFile(fileA, "previous revision") + await cache.flush() + if (failure === "failed cleanup") + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + else { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + embedder.createEmbeddings + .mockResolvedValueOnce({ embeddings: [[1]] }) + .mockRejectedValueOnce(new Error("provider failed after first upsert")) } + await start() + await queue() + expect(points.size).toBe(1) + expect(cache.getHash(fileA)).toBeUndefined() + expect(finished.mock.calls[0][0].processedFiles[0].status).toBe("local_error") + await orchestrator.stopWatcher() + cache = await reload() + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + generations = new GenerationAllocator(cache) + watcher = makeWatcher() + await start() + await queue() + expect(cache.getHash(fileA)).toBeUndefined() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(points.size).toBe(failure === "partial upsert" ? 3 : 1) + expect(cache.getHash(fileA)).toBe(hash("content a")) + expect(cache.getOpenFiles()).toEqual({}) + expect(state.state).toBe("Indexed") + await cache.flush() + const completed = await reload() + expect(completed.getHash(fileA)).toBe(hash("content a")) + expect(completed.getOpenFiles()).toEqual({}) + }, + ) + + it.each(["delete", "ignore"])( + "T21 real cache reload replaces identical content after a lost %s result", + async (eventType) => { + let persisted: any = { version: 2, generationWatermark: 0, fileHashes: {}, openFiles: {} } + vi.mocked(safeWriteJson).mockImplementation(async (_filePath, value) => { + persisted = JSON.parse(JSON.stringify(value)) + }) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(async (uri) => { + return Buffer.from(uri.fsPath.includes("roo-index-cache-") ? JSON.stringify(persisted) : "content a") + }) + const context = { globalStorageUri: { fsPath: path.join(workspace, "storage") } } as any + cache = new CacheManager(context, workspace) + await cache.initialize() + cache.completeFile(fileA, hash("content a")) + await cache.flush() + points.set("original", { payload: { filePath: path.relative(workspace, fileA), generation: 0 } }) + generations = new GenerationAllocator(cache) + watcher = makeWatcher() + await start() + if (eventType === "ignore") ignore.ignores.mockReturnValue(true) + const lostResult = deferred() + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, generation: number) => { + applyDelete(filePath, generation) + await lostResult.promise + }) + await queue([fileA], eventType === "delete" ? "delete" : "change") + expect(points.size).toBe(0) + expect(persisted.fileHashes[fileA]).toBeUndefined() + expect(persisted.openFiles[fileA]).toBe(true) + watcher.dispose() + cache = new CacheManager(context, workspace) + await cache.initialize() + gate.recover(Object.keys(cache.getOpenFiles()), cache.loadedAt) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getTrackedPaths()).toContain(fileA) + ignore.ignores.mockReturnValue(false) + generations = new GenerationAllocator(cache) + await generations.initialize(store, new AbortController().signal, gate) + watcher = makeWatcher() + await start() + await queue() + expect(cache.getHash(fileA)).toBeUndefined() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + expect(state.state).toBe("Indexed") + expect(points.size).toBe(1) + expect(cache.getHash(fileA)).toBe(hash("content a")) + expect(cache.getOpenFiles()).toEqual({}) + lostResult.resolve() + await drain() + expect(cache.getHash(fileA)).toBe(hash("content a")) + await cache.flush() + }, + ) - // Wait for batch processing - await new Promise((resolve) => setTimeout(resolve, 600)) + it("T12 a rejected upsert starts grace and another uncertainty extends it", async () => { + store.upsertPoints.mockRejectedValueOnce(new Error("timeout")) + await start() + await queue() + expect(gate.needsReconciliation(fileA)).toBe(true) + await queue() + await vi.advanceTimersByTimeAsync(10_000) + gate.recover([fileA], Date.now()) + await vi.advanceTimersByTimeAsync(20_000) + expect(hashes[fileA]).toBeUndefined() + await vi.advanceTimersByTimeAsync(10_000) + expect(hashes[fileA]).toBe(hash("content a")) + }) - // Check that files in hidden directories were not processed - expect(processedFiles).not.toContain("src/.hidden/components/Button.tsx") - expect(processedFiles).not.toContain(".hidden/src/components/Button.tsx") - expect(processedFiles).not.toContain("src/components/.hidden/Button.tsx") + it("T24 after-window stale extras preserve current points and the next edit removes them", async () => { + gate.recover([fileA], Date.now()) + await start() + await queue([fileA, fileB]) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + const current = [...points.values()].find((point) => point.payload.filePath === "a.ts")! + const otherFile = [...points.values()].find((point) => point.payload.filePath === "b.ts")! + const late = deferred() + const oldWrite = late.promise.then(() => { + points.set("late", { ...current, id: "late", payload: { ...current.payload, generation: 0 } }) }) + late.resolve() + await oldWrite + expect(points.has(current.id)).toBe(true) + expect(points.has(otherFile.id)).toBe(true) + contents.set(fileA, "edit after late arrival") + await queue() + expect(points.size).toBe(2) + expect(points.has("late")).toBe(false) + expect(points.has(otherFile.id)).toBe(true) }) - describe("dispose", () => { - it("should dispose of the watcher when disposed", async () => { - await fileWatcher.initialize() - fileWatcher.dispose() + it("coalesces revisions and processes events arriving during a batch in the next batch", async () => { + const held = deferred() + parser.parseFile.mockImplementationOnce(() => held.promise) + await watcher.initialize() + await queue() + contents.set(fileA, "latest") + handlers.change({ fsPath: fileA }) + handlers.change({ fsPath: fileA }) + await vi.advanceTimersByTimeAsync(500) + expect(parser.parseFile).toHaveBeenCalledOnce() + held.resolve(blocks()) + await drain() + await vi.advanceTimersByTimeAsync(500) + expect(parser.parseFile).toHaveBeenCalledTimes(2) + expect(hashes[fileA]).toBe(hash("latest")) + }) - expect(mockWatcher.dispose).toHaveBeenCalled() - }) + it("T4 explicit retry starts a new scan and watcher; repeated fatal attempts remain bounded", async () => { + parser.parseFile.mockRejectedValue(new ParserFatalError("parser-abort", fileA)) + for (let attempt = 0; attempt < 2; attempt++) { + if (attempt) watcher = makeWatcher() + await start() + await queue() + expect(state.state).toBe("Error") + expect(fatal).toHaveBeenCalledTimes(attempt + 1) + await orchestrator.stopIndexing() + await queue() + expect(parser.parseFile).toHaveBeenCalledTimes(attempt + 1) + } + expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledTimes(2) + }) + + it("T5 completed A survives cancellation of B's deferred embedding and is skipped on retry", async () => { + const held = deferred() + embedder.createEmbeddings.mockImplementation(async (texts: string[]) => + texts[0].startsWith("b.ts") ? held.promise : { embeddings: [[1]] }, + ) + await start() + await queue([fileA, fileB]) + expect(hashes[fileA]).toBe(hash("content a")) + expect(hashes[fileB]).toBeUndefined() + await orchestrator.stopIndexing() + held.resolve({ embeddings: [[1]] }) + await drain() + expect([...points.values()].every((point) => point.payload.filePath === "a.ts")).toBe(true) + watcher = makeWatcher() + await start() + embedder.createEmbeddings.mockClear() + await queue([fileA]) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("T9 cancellation drops queued files as well as late in-flight embeddings", async () => { + const held = deferred() + embedder.createEmbeddings.mockImplementation(() => held.promise) + await start() + const paths = Array.from({ length: 12 }, (_, fileNumber) => path.join(workspace, `${fileNumber}.ts`)) + await queue(paths) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(10) + await orchestrator.stopIndexing() + for (const call of embedder.createEmbeddings.mock.calls) expect(call[2].aborted).toBe(true) + held.resolve({ embeddings: [[1]] }) + await drain() + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(10) + expect(parser.parseFile).toHaveBeenCalledTimes(10) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.completeFile).not.toHaveBeenCalled() + }) + + it.each(["abort", "hang"])("T1/T2/T8 real watcher worker %s fail-stops once with no cascade", async (failure) => { + const worker = new ControlledWorker() + const client = new ParseWorkerClient(() => worker) + parser = client + watcher.dispose() + watcher = makeWatcher() + await start(client) + await queue([fileA, fileB]) + expect(worker.requests.map((request) => request.filePath)).toEqual([fileA]) + const probe = vi.fn() + setTimeout(probe, 100) + await vi.advanceTimersByTimeAsync(100) + expect(probe).toHaveBeenCalledOnce() + if (failure === "hang") { + await vi.advanceTimersByTimeAsync(PARSE_TIMEOUT_MS - 101) + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(1) + } else { + worker.emit("message", { id: worker.requests[0].id, error: { fatal: true, message: "Aborted()" } }) + } + await drain() + expect(fatal).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + filePath: fileA, + category: failure === "hang" ? "parse-timeout" : "parser-abort", + }), + "embeddings:orchestrator.parserFailed", + ) + expect(state.state).toBe("Error") + const updates = state.setSystemState.mock.calls.length + await queue([fileB]) + worker.emit("message", { id: worker.requests[0].id, blocks: blocks() }) + worker.emit("exit", 1) + await orchestrator.stopIndexing() + await orchestrator.startIndexing() + expect(state.setSystemState).toHaveBeenCalledTimes(updates) + expect(worker.requests).toHaveLength(1) + expect(worker.terminate).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(cache.clearCacheFile).not.toHaveBeenCalled() + }) + + it.each(["cancel", "disable"])("T3/T7/T19 %s during real watcher hang is shared and bounded", async (action) => { + const worker = new ControlledWorker() + const client = new ParseWorkerClient(() => worker) + parser = client + watcher.dispose() + watcher = makeWatcher() + await start(client) + await queue([fileA, fileB]) + const stopped = action === "cancel" ? orchestrator.stopIndexing() : orchestrator.stopWatcher() + expect(orchestrator.stopIndexing()).toBe(stopped) + expect(orchestrator.stopWatcher()).toBe(stopped) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(1) + await stopped + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + await queue([fileB]) + expect(worker.requests).toHaveLength(1) + expect(worker.terminate).toHaveBeenCalledOnce() + worker.emit("exit", 1) + await client.stopped + }) + + it("T7 fully settled watcher stop reports stopped rather than partial", async () => { + const exit = deferred() + const worker = { stop: vi.fn(), stopped: exit.promise } + const ownership = vi.spyOn(gate, "own") + await start(worker) + await queue() + expect(hashes[fileA]).toBeDefined() + const write = deferred() + const pending = gate.run(ownership.mock.calls[0][1], () => write.promise).catch((error) => error) + await drain() + const stopped = orchestrator.stopIndexing() + expect(state.state).toBe("Stopping") + write.resolve() + await pending + await drain() + expect(state.state).toBe("Stopping") + exit.resolve() + await stopped + expect(state.setSystemState).toHaveBeenLastCalledWith("Standby", "embeddings:orchestrator.indexingStopped") + }) + + it("T8 idle cancel preserves watcher status and message; disable blocks later events", async () => { + await start() + state.setSystemState.mockClear() + await orchestrator.stopIndexing() + expect(state.setSystemState).not.toHaveBeenCalled() + await orchestrator.stopWatcher() + await queue() + expect(state.setSystemState).not.toHaveBeenCalled() + expect(parser.parseFile).not.toHaveBeenCalled() + }) + + it.each(["none", "late", "late-reject", "late-abort", "cancel-prior", "cancel-grace", "settled", "unrelated"])( + "T12/T19 actual watcher pre-expiry retry completion barrier: %s", + async (outcome) => { + const held = deferred() + let oldPoints: any[] = [] + store.upsertPoints.mockImplementationOnce(async (batch: any[]) => { + oldPoints = batch + if (outcome === "settled") for (const point of batch) points.set(point.id, point) + await held.promise + }) + let oldIsCurrent = true + await start(undefined, () => oldIsCurrent) + await queue() + expect(store.upsertPoints).toHaveBeenCalledOnce() + const old = orchestrator + const expiryAt = Date.now() + INDEX_STOP_LIMIT_MS + const stopped = old.stopIndexing() + expect(state.state).toBe("Stopping") + oldIsCurrent = false + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 502) + const verified: { at: number; counts: { current: number; other: number } }[] = [] + const count = store.countPointsForFile.getMockImplementation()! + store.countPointsForFile.mockImplementation(async (filePath: string, generation: number) => { + const counts = await count(filePath, generation) + if (filePath === fileA) verified.push({ at: Date.now(), counts }) + return counts + }) + watcher = makeWatcher() + const retryProgress = vi.fn() + const retryFinished = vi.fn() + watcher.onBatchProgressUpdate(retryProgress) + watcher.onDidFinishBatchProcessing(retryFinished) + await start(undefined, () => true, true) + state.reportFileQueueProgress.mockClear() + await queue(outcome === "unrelated" ? [fileA, fileB] : [fileA]) + const completedFiles = outcome === "unrelated" ? 1 : 0 + const currentIds = [...points.values()] + .filter((point) => point.payload.filePath === path.relative(workspace, fileA)) + .map((point) => point.id) + const unrelatedIds = [...points.values()] + .filter((point) => point.payload.filePath === path.relative(workspace, fileB)) + .map((point) => point.id) + const assertIncomplete = () => { + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(state.state).toBe("Indexing") + expect(watcher.isProcessing).toBe(true) + expect(retryFinished).not.toHaveBeenCalled() + expect(retryProgress.mock.calls.every(([value]) => value.processedInBatch <= completedFiles)).toBe(true) + expect( + state.reportFileQueueProgress.mock.calls.every( + ([processed]: number[]) => processed <= completedFiles, + ), + ).toBe(true) + if (outcome === "unrelated") { + expect(hashes[fileB]).toBe(hash("content b")) + expect( + [...points.values()] + .filter((point) => point.payload.filePath === path.relative(workspace, fileB)) + .map((point) => point.id), + ).toEqual(unrelatedIds) + expect(state.reportFileQueueProgress).toHaveBeenCalledWith(1, 2, "b.ts") + } + } + expect(verified).toEqual([{ at: expiryAt - 2, counts: { current: 1, other: 0 } }]) + expect(currentIds).toHaveLength(1) + expect(store.upsertPoints).toHaveBeenCalledTimes(outcome === "unrelated" ? 3 : 2) + expect(gate.graceDeadline(fileA)).toBe(0) + assertIncomplete() + if (outcome === "settled") { + await vi.advanceTimersByTimeAsync(1) + held.resolve() + await drain() + await stopped + expect(Date.now()).toBe(expiryAt - 1) + expect(hashes[fileA]).toBe(hash("content a")) + expect(gate.needsReconciliation(fileA)).toBe(false) + expect(verified.at(-1)).toEqual({ at: expiryAt - 1, counts: { current: 1, other: 0 } }) + expect(state.state).toBe("Indexed") + expect(retryFinished).toHaveBeenCalledOnce() + return + } + if (outcome === "cancel-prior") { + await orchestrator.stopIndexing() + await drain() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(retryFinished).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(2) + await stopped + return + } + await vi.advanceTimersByTimeAsync(1) + assertIncomplete() + expect(gate.graceDeadline(fileA)).toBe(0) + await vi.advanceTimersByTimeAsync(1) + await stopped + let deadline = expiryAt + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + assertIncomplete() + const stateUpdates = state.setSystemState.mock.calls.length + await vi.advanceTimersByTimeAsync(1000) + if (outcome === "late") { + for (const point of oldPoints) points.set(point.id, point) + held.resolve() + await drain() + expect(gate.graceDeadline(fileA)).toBe(deadline) + } + if (outcome === "late-reject" || outcome === "late-abort") { + held.reject( + outcome === "late-abort" + ? new DOMException("late cancellation rejection", "AbortError") + : new Error("late old write rejection"), + ) + await drain() + const originalDeadline = deadline + deadline = Date.now() + LATE_WRITE_GRACE_MS + expect(deadline).toBe(originalDeadline + 1000) + expect(gate.graceDeadline(fileA)).toBe(deadline) + await vi.advanceTimersByTimeAsync(originalDeadline - Date.now()) + assertIncomplete() + expect(verified).toEqual([{ at: expiryAt - 2, counts: { current: 1, other: 0 } }]) + } + if (outcome === "cancel-grace") { + await orchestrator.stopIndexing() + await drain() + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(retryFinished).not.toHaveBeenCalled() + return + } + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1) + assertIncomplete() + expect(state.setSystemState).toHaveBeenCalledTimes(stateUpdates) + await vi.advanceTimersByTimeAsync(1) + await drain() + expect(Date.now()).toBe(deadline) + expect(verified.at(-1)).toEqual({ + at: deadline, + counts: { current: 1, other: outcome === "late" ? 1 : 0 }, + }) + expect(retryFinished).toHaveBeenCalledOnce() + for (const pointId of currentIds) expect(points.has(pointId)).toBe(true) + if (outcome === "late") { + expect(hashes[fileA]).toBeUndefined() + expect(open[fileA]).toBe(true) + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(retryFinished.mock.calls[0][0].batchError).toBeInstanceOf(Error) + expect(retryProgress.mock.calls.every(([value]) => value.processedInBatch === 0)).toBe(true) + } else { + expect(hashes[fileA]).toBe(hash("content a")) + expect(state.state).toBe("Indexed") + expect(retryProgress).toHaveBeenLastCalledWith({ + processedInBatch: completedFiles + 1, + totalInBatch: completedFiles + 1, + currentFile: fileA, + }) + if (outcome === "unrelated") expect(hashes[fileB]).toBe(hash("content b")) + } + }, + ) + + it("T22 real parser oversized single-line watcher segments have distinct verified IDs", async () => { + const filePath = path.join(workspace, "long.md") + contents.set(filePath, "abcdefghij".repeat(500)) + parser = new CodeParser() + watcher.dispose() + watcher = makeWatcher() + await start() + await queue([filePath]) + const stored = [...points.values()] + expect(stored.length).toBeGreaterThan(1) + expect(new Set(stored.map((point) => point.id)).size).toBe(stored.length) + expect(new Set(stored.map((point) => point.payload.startLine)).size).toBe(1) + expect(store.countPointsForFile).toHaveBeenCalledWith(filePath, stored[0].payload.generation) + expect(hashes[filePath]).toBe(hash(contents.get(filePath)!)) + expect(state.state).toBe("Indexed") + }) + + it("tracks standalone processing and aborts it on dispose without initialize", async () => { + const held = deferred() + parser.parseFile.mockImplementation(() => held.promise) + const pending = watcher.processFile(fileA).catch((error) => error) + await drain() + expect(watcher.isProcessing).toBe(true) + watcher.dispose() + held.resolve(blocks()) + expect((await pending).name).toBe("AbortError") + expect(watcher.isProcessing).toBe(false) + expect(embedder.createEmbeddings).not.toHaveBeenCalled() }) }) diff --git a/src/services/code-index/processors/__tests__/scanner.recovery.spec.ts b/src/services/code-index/processors/__tests__/scanner.recovery.spec.ts new file mode 100644 index 00000000000..083657618ed --- /dev/null +++ b/src/services/code-index/processors/__tests__/scanner.recovery.spec.ts @@ -0,0 +1,1061 @@ +import * as path from "path" +import * as vscode from "vscode" +import { stat } from "fs/promises" +import { createHash } from "crypto" +import { EventEmitter } from "node:events" +import { DirectoryScanner } from "../scanner" +import { CodeIndexOrchestrator } from "../../orchestrator" +import { ParseWorkerClient, ParseRequest } from "../../parse-worker-client" +import { FileCompletion } from "../../file-completion" +import { CodeParser } from "../parser" +import { CacheManager } from "../../cache-manager" +import { GenerationAllocator } from "../../generation-allocator" +import { IndexWriteGate, LATE_WRITE_GRACE_MS } from "../../index-write-gate" +import { ParserFatalError } from "../../parser-errors" +import { CodeBlock } from "../../interfaces" +import { listFiles } from "../../../glob/list-files" +import { generateRelativeFilePath } from "../../shared/get-relative-path" +import { INDEX_STOP_LIMIT_MS, PARSE_TIMEOUT_MS, PARSING_CONCURRENCY } from "../../constants" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) + +vi.mock("../../../../utils/path", () => ({ getWorkspacePathForContext: (directory: string) => directory })) + +const storage = vi.hoisted(() => ({ snapshot: undefined as any, save: vi.fn() })) +vi.mock("../../../../utils/safeWriteJson", () => ({ + safeWriteJson: (...parameters: any[]) => storage.save(...parameters), +})) +vi.mock("fs/promises", () => ({ stat: vi.fn(), readFile: vi.fn() })) +vi.mock("vscode", () => ({ + workspace: { + getWorkspaceFolder: vi.fn(), + workspaceFolders: [{ uri: { fsPath: "scanner-recovery-workspace" } }], + fs: { readFile: vi.fn() }, + }, + Uri: { + file: (filePath: string) => ({ fsPath: filePath }), + joinPath: (base: { fsPath: string }, suffix: string) => ({ fsPath: `${base.fsPath}/${suffix}` }), + }, +})) +vi.mock("../../../../core/ignore/RooIgnoreController", () => ({ + RooIgnoreController: class { + async initialize() {} + filterPaths(paths: string[]) { + return paths + } + }, +})) +vi.mock("../../../glob/list-files", () => ({ listFiles: vi.fn() })) + +function deferred() { + let resolve!: (value: Value) => void + let reject!: (error: Error) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +class ControlledScannerWorker extends EventEmitter { + requests: ParseRequest[] = [] + postMessage(request: ParseRequest) { + this.requests.push(request) + } + terminate = vi.fn().mockResolvedValue(1) +} + +const workspacePath = path.resolve("scanner-recovery-workspace") +const fileA = path.join(workspacePath, "a.ts") +const fileB = path.join(workspacePath, "b.ts") +const contentHash = createHash("sha256").update("current content").digest("hex") +const context = { globalStorageUri: { fsPath: "cache" } } as vscode.ExtensionContext + +function blocks(filePath: string, count = 1): CodeBlock[] { + return Array.from({ length: count }, (_, blockNumber) => ({ + file_path: filePath, + content: `function block${blockNumber}() { return ${blockNumber} }`, + start_line: 1, + end_line: 1, + identifier: null, + type: "function", + fileHash: contentHash, + segmentHash: `${filePath}:${blockNumber}`, + })) +} + +describe("DirectoryScanner durable per-file recovery", () => { + let cache: CacheManager + let gate: IndexWriteGate + let generations: GenerationAllocator + let points: Map + let store: any + let embedder: any + let parser: any + let ignore: any + let scanner: DirectoryScanner + let listedPaths: string[] + + const drain = () => vi.advanceTimersByTimeAsync(0) + const createScanner = () => new DirectoryScanner(embedder, store, parser, cache, ignore, 1, gate, generations) + const createOrchestrator = (isCurrent = () => true, sharedState?: any) => { + const state = sharedState ?? { + state: "Standby", + setSystemState: vi.fn(function (this: { state: string }, next: string) { + this.state = next + }), + reportBlockIndexingProgress: vi.fn(), + reportFileQueueProgress: vi.fn(), + } + const watcher = { + isProcessing: false, + processFile: vi.fn(), + initialize: vi.fn(), + dispose: vi.fn(), + onDidStartBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + onBatchProgressUpdate: vi.fn(() => ({ dispose: vi.fn() })), + onDidFinishBatchProcessing: vi.fn(() => ({ dispose: vi.fn() })), + } + Object.assign(store, { + initialize: vi.fn().mockResolvedValue(false), + hasIndexedData: vi.fn().mockResolvedValue(false), + markIndexingIncomplete: vi.fn(), + markIndexingComplete: vi.fn(), + }) + const fatal = vi.fn() + const orchestrator = new CodeIndexOrchestrator( + { isFeatureConfigured: true } as any, + state, + workspacePath, + cache, + store, + createScanner(), + watcher, + parser.stop ? parser : undefined, + fatal, + undefined, + gate, + generations, + isCurrent, + ) + return { orchestrator, state, watcher, fatal } + } + const filePoints = (filePath: string) => + [...points.values()].filter( + (point) => point.payload.filePath === generateRelativeFilePath(filePath, workspacePath), + ) + const applyDelete = (filePath: string, beforeGeneration: number) => { + for (const point of filePoints(filePath)) { + if (point.payload.generation === undefined || point.payload.generation < beforeGeneration) { + points.delete(point.id) + } + } + } + const seed = (filePath: string, generation = 0) => { + const point = { + id: `old:${filePath}:${generation}`, + vector: [1], + payload: { filePath: generateRelativeFilePath(filePath, workspacePath), generation }, + } + points.set(point.id, point) + return point + } + const reload = async () => { + cache = new CacheManager(context, workspacePath) + await cache.initialize() + generations = new GenerationAllocator(cache) + gate.recover(Object.keys(cache.getOpenFiles()), cache.loadedAt) + scanner = createScanner() + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(new Date("2026-10-31T00:00:00Z")) + storage.snapshot = undefined + storage.save.mockReset().mockImplementation(async (_filePath: string, data: any) => { + storage.snapshot = structuredClone(data) + }) + vi.mocked(stat) + .mockReset() + .mockResolvedValue({ size: 100 } as any) + vi.mocked(vscode.workspace.getWorkspaceFolder).mockReturnValue({ uri: { fsPath: workspacePath } } as any) + vi.mocked(vscode.workspace.fs.readFile) + .mockReset() + .mockImplementation(async (uri) => { + if (uri.fsPath.startsWith("cache/")) return Buffer.from(JSON.stringify(storage.snapshot)) + return Buffer.from("current content") + }) + listedPaths = [fileA] + vi.mocked(listFiles).mockImplementation(async () => [[...listedPaths], false]) + cache = new CacheManager(context, workspacePath) + gate = new IndexWriteGate() + generations = new GenerationAllocator(cache) + points = new Map() + store = { + upsertPoints: vi.fn().mockImplementation(async (newPoints: any[]) => { + for (const point of newPoints) points.set(point.id, point) + }), + deletePointsByFilePath: vi.fn().mockImplementation(async (filePath: string, cutoff: number) => { + applyDelete(filePath, cutoff) + }), + countPointsForFile: vi.fn().mockImplementation(async (filePath: string, generation: number) => { + const stored = filePoints(filePath) + return { + current: stored.filter((point) => point.payload.generation === generation).length, + other: stored.filter((point) => point.payload.generation !== generation).length, + } + }), + } + embedder = { + createEmbeddings: vi + .fn() + .mockImplementation(async (texts: string[]) => ({ embeddings: texts.map(() => [1]) })), + } + parser = { parseFile: vi.fn().mockImplementation(async (filePath: string) => blocks(filePath)) } + ignore = { ignores: vi.fn().mockReturnValue(false) } + scanner = createScanner() + }) + + afterEach(() => { + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it.each(["cancel", "disable", "fatal"])("T5 preserves completed A while B is in flight on %s", async (stop) => { + listedPaths = [fileA, fileB] + const held = deferred() + parser.parseFile.mockImplementation(async (filePath: string) => + filePath === fileB ? held.promise : blocks(filePath), + ) + const controller = new AbortController() + const progress = vi.fn() + const pending = scanner + .scanDirectory(workspacePath, undefined, progress, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(cache.getHash(fileA)).toBe(contentHash) + expect(cache.getHash(fileB)).toBeUndefined() + expect(progress).toHaveBeenCalledExactlyOnceWith(1) + expect(scanner.hasIncompleteFiles).toBe(true) + if (stop === "fatal") held.reject(new ParserFatalError("parser-abort", fileB)) + else { + controller.abort() + held.resolve(blocks(fileB)) + } + await pending + parser.parseFile.mockImplementation(async (filePath: string) => blocks(filePath)) + embedder.createEmbeddings.mockClear() + await scanner.scanDirectory(workspacePath) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(scanner.hasIncompleteFiles).toBe(false) + }) + + it("marks discovered, queued and pre-invalidation files incomplete", async () => { + listedPaths = Array.from({ length: PARSING_CONCURRENCY + 2 }, (_, fileNumber) => + path.join(workspacePath, `${fileNumber}.ts`), + ) + const held = deferred() + parser.parseFile.mockReturnValue(held.promise) + const controller = new AbortController() + const pending = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(parser.parseFile).toHaveBeenCalledTimes(PARSING_CONCURRENCY) + expect(scanner.hasIncompleteFiles).toBe(true) + expect(storage.snapshot).toBeUndefined() + controller.abort() + held.resolve([]) + expect(await pending).toMatchObject({ name: "AbortError" }) + expect(scanner.hasIncompleteFiles).toBe(true) + expect(parser.parseFile).toHaveBeenCalledTimes(PARSING_CONCURRENCY) + }) + + it("fatal parsing immediately stops queued dispatch without waiting for running files", async () => { + listedPaths = Array.from({ length: PARSING_CONCURRENCY + 2 }, (_, fileNumber) => + path.join(workspacePath, `${fileNumber}.ts`), + ) + const held = deferred() + const fatal = new ParserFatalError("parse-timeout", listedPaths[0]) + parser.parseFile.mockImplementation(async (filePath: string) => { + if (filePath === listedPaths[0]) throw fatal + return held.promise + }) + const pending = scanner.scanDirectory(workspacePath).catch((error) => error) + await drain() + expect(await pending).toBe(fatal) + held.resolve([]) + await drain() + expect(parser.parseFile).toHaveBeenCalledTimes(PARSING_CONCURRENCY) + expect(store.upsertPoints).not.toHaveBeenCalled() + }) + + it.each(["success", "failure", "abort"])("T6/T18 final batch gates completion: %s", async (outcome) => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings + .mockImplementationOnce(async () => ({ embeddings: [[1]] })) + .mockReturnValueOnce(held.promise) + const controller = new AbortController() + const progress = vi.fn() + const pending = scanner + .scanDirectory(workspacePath, vi.fn(), progress, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(store.upsertPoints).toHaveBeenCalledTimes(1) + expect(cache.getHash(fileA)).toBeUndefined() + expect(progress).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + if (outcome === "failure") held.reject(new Error("embedding failed")) + else { + if (outcome === "abort") controller.abort() + held.resolve({ embeddings: [[1]] }) + } + await pending + if (outcome === "success") { + expect(cache.getHash(fileA)).toBe(contentHash) + expect(progress).toHaveBeenCalledExactlyOnceWith(3) + } else { + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + await scanner.scanDirectory(workspacePath) + expect(filePoints(fileA)).toHaveLength(3) + expect(cache.getHash(fileA)).toBe(contentHash) + } + }) + + it.each(["replacement", "removal", "exclusion"])( + "T17 awaits durable invalidation before %s; failed save issues no mutation", + async (operation) => { + cache.completeFile(fileA, "previous hash") + seed(fileA) + if (operation === "removal") listedPaths = [] + if (operation === "exclusion") ignore.ignores.mockReturnValue(true) + const held = deferred() + const save = storage.save.getMockImplementation()! + storage.save.mockImplementation(async (filePath: string, data: any) => { + await save(filePath, data) + if (data.openFiles[fileA]) await held.promise + }) + const pending = scanner.scanDirectory(workspacePath, vi.fn()) + await drain() + expect(storage.snapshot.fileHashes[fileA]).toBeUndefined() + expect(storage.snapshot.openFiles[fileA]).toBe(true) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + held.reject(new Error("disk full")) + const result = await pending + expect(result.incompleteFiles).toEqual([fileA]) + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + }, + ) + + it("T11 failed cleanup reloads open without hash; retry restores complete content", async () => { + seed(fileA) + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + const result = await scanner.scanDirectory(workspacePath, vi.fn()) + expect(result.incompleteFiles).toEqual([fileA]) + await reload() + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await pending + expect(filePoints(fileA)).toHaveLength(1) + expect(cache.getHash(fileA)).toBe(contentHash) + expect(cache.getOpenFiles()).toEqual({}) + }) + + it.each(["deleted", "excluded", "oversized"])( + "T20/T21 removes an interrupted open file when %s", + async (selection) => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockResolvedValueOnce({ embeddings: [[1]] }).mockReturnValueOnce(held.promise) + const controller = new AbortController() + const interrupted = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(filePoints(fileA)).toHaveLength(1) + controller.abort() + held.resolve({ embeddings: [[1]] }) + await interrupted + await reload() + if (selection === "deleted") listedPaths = [] + if (selection === "excluded") ignore.ignores.mockReturnValue(true) + if (selection === "oversized") vi.mocked(stat).mockResolvedValue({ size: 2 * 1024 * 1024 } as any) + store.deletePointsByFilePath.mockRejectedValueOnce(new Error("delete failed")) + const failed = await scanner.scanDirectory(workspacePath, vi.fn()) + expect(failed.incompleteFiles).toEqual([fileA]) + expect(cache.getOpenFiles()[fileA]).toBe(true) + const result = await scanner.scanDirectory(workspacePath) + expect(result.incompleteFiles).toEqual([]) + expect(filePoints(fileA)).toEqual([]) + expect(cache.getTrackedPaths()).toEqual([]) + expect(store.deletePointsByFilePath).toHaveBeenCalledWith(fileA, expect.any(Number)) + }, + ) + + it.each(["deleted", "excluded"])( + "T20/T21 lost completed-file removal result forces identical-content replacement: %s", + async (selection) => { + await scanner.scanDirectory(workspacePath) + if (selection === "deleted") listedPaths = [] + else ignore.ignores.mockReturnValue(true) + const held = deferred() + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, cutoff: number) => { + applyDelete(filePath, cutoff) + await held.promise + }) + const controller = new AbortController() + const removal = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(filePoints(fileA)).toEqual([]) + expect(storage.snapshot.fileHashes[fileA]).toBeUndefined() + expect(storage.snapshot.openFiles[fileA]).toBe(true) + controller.abort() + await reload() + listedPaths = [fileA] + ignore.ignores.mockReturnValue(false) + store.countPointsForFile.mockClear() + const retry = scanner.scanDirectory(workspacePath) + await drain() + expect(store.countPointsForFile).toHaveBeenCalledOnce() + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(cache.getHash(fileA)).toBeUndefined() + await vi.advanceTimersByTimeAsync(1) + const deadline = Date.now() + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + expect(store.countPointsForFile).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + await retry + expect(Date.now()).toBe(deadline) + expect(store.countPointsForFile).toHaveBeenCalledTimes(2) + expect(cache.getHash(fileA)).toBe(contentHash) + expect(filePoints(fileA)).toHaveLength(1) + expect(cache.getOpenFiles()).toEqual({}) + expect(store.countPointsForFile).toHaveBeenLastCalledWith(fileA, filePoints(fileA)[0].payload.generation) + held.resolve() + await removal + expect(cache.getHash(fileA)).toBe(contentHash) + }, + ) + + it.each(["cleanup", "removal", "exclusion"])( + "T10/T20/T21 delayed old %s cannot delete a verified retry", + async (operation) => { + if (operation !== "cleanup") { + await scanner.scanDirectory(workspacePath) + if (operation === "removal") listedPaths = [] + else ignore.ignores.mockReturnValue(true) + } + const held = deferred() + store.deletePointsByFilePath.mockImplementationOnce(async (filePath: string, cutoff: number) => { + await held.promise + applyDelete(filePath, cutoff) + }) + const controller = new AbortController() + const old = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + controller.abort() + gate.abandon(controller.signal) + listedPaths = [fileA] + ignore.ignores.mockReturnValue(false) + const retry = createScanner().scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await retry + const currentIds = filePoints(fileA).map((point) => point.id) + expect(cache.getHash(fileA)).toBe(contentHash) + held.resolve() + await old + expect(filePoints(fileA).map((point) => point.id)).toEqual(currentIds) + }, + ) + + it.each([ + { current: 0, other: 0 }, + { current: 1, other: 1 }, + ])("T10 verification refuses mismatched counts %o", async (counts) => { + store.countPointsForFile.mockResolvedValue(counts) + const result = await scanner.scanDirectory(workspacePath, vi.fn()) + expect(result.incompleteFiles).toEqual([fileA]) + expect(cache.getHash(fileA)).toBeUndefined() + }) + + it.each(["none", "inside", "after", "cancel"])("T12/T24 uncertain old upsert arrival: %s", async (arrival) => { + let delayedPoints: any[] = [] + store.upsertPoints.mockImplementationOnce(async (newPoints: any[]) => { + delayedPoints = newPoints + throw new Error("local timeout") + }) + await scanner.scanDirectory(workspacePath, vi.fn()) + const progress = vi.fn() + const controller = new AbortController() + let finished = false + const retry = scanner + .scanDirectory(workspacePath, vi.fn(), progress, undefined, controller.signal) + .then((result) => { + finished = true + return result + }) + .catch((error) => error) + await drain() + const currentIds = filePoints(fileA).map((point) => point.id) + expect(currentIds).toHaveLength(1) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + expect(progress).not.toHaveBeenCalled() + expect(finished).toBe(false) + if (arrival === "inside") for (const point of delayedPoints) points.set(point.id, point) + if (arrival === "cancel") controller.abort() + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 1) + if (arrival !== "cancel") expect(finished).toBe(false) + await vi.advanceTimersByTimeAsync(1) + const result = await retry + if (arrival === "cancel") { + expect(result).toMatchObject({ name: "AbortError" }) + expect(scanner.hasIncompleteFiles).toBe(true) + expect(cache.getHash(fileA)).toBeUndefined() + } else if (arrival === "inside") { + expect(result.incompleteFiles).toEqual([fileA]) + expect(cache.getHash(fileA)).toBeUndefined() + for (const pointId of currentIds) expect(points.has(pointId)).toBe(true) + await scanner.scanDirectory(workspacePath) + expect(filePoints(fileA)).toHaveLength(1) + } else { + expect(cache.getHash(fileA)).toBe(contentHash) + expect(progress).toHaveBeenCalledExactlyOnceWith(1) + if (arrival === "after") { + listedPaths = [fileB] + // Keep A selected so only an explicit re-index removes its late extras. + listedPaths.push(fileA) + await scanner.scanDirectory(workspacePath) + const otherIds = filePoints(fileB).map((point) => point.id) + for (const point of delayedPoints) points.set(point.id, point) + for (const pointId of currentIds) expect(points.has(pointId)).toBe(true) + expect(filePoints(fileA)).toHaveLength(2) + expect(filePoints(fileB).map((point) => point.id)).toEqual(otherIds) + await cache.invalidateFile(fileA) + await scanner.scanDirectory(workspacePath) + expect(filePoints(fileA)).toHaveLength(1) + expect(filePoints(fileB).map((point) => point.id)).toEqual(otherIds) + } + } + }) + + it("T12 a second uncertainty extends the grace deadline", async () => { + gate.recover([fileA]) + const pending = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(10_000) + gate.recover([fileA]) + await vi.advanceTimersByTimeAsync(20_000) + expect(cache.getHash(fileA)).toBeUndefined() + await vi.advanceTimersByTimeAsync(10_000) + await pending + expect(cache.getHash(fileA)).toBe(contentHash) + }) + + it("keeps oversized tracked files incomplete while unrelated parsing is in flight", async () => { + listedPaths = [fileA, fileB] + cache.completeFile(fileA, contentHash) + seed(fileA) + vi.mocked(stat).mockImplementation( + async (filePath) => + ({ + size: filePath === fileA ? 2 * 1024 * 1024 : 100, + }) as any, + ) + const held = deferred() + parser.parseFile.mockReturnValue(held.promise) + const controller = new AbortController() + const pending = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(scanner.hasIncompleteFiles).toBe(true) + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + controller.abort() + held.resolve([]) + await pending + expect(scanner.hasIncompleteFiles).toBe(true) + }) + + it("T11 partial upsert survives reload as open and is fully replaced", async () => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings + .mockImplementationOnce(async () => ({ embeddings: [[1]] })) + .mockReturnValueOnce(held.promise) + const controller = new AbortController() + const interrupted = scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(filePoints(fileA)).toHaveLength(1) + controller.abort() + held.resolve({ embeddings: [[1]] }) + await interrupted + await reload() + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + parser.parseFile.mockResolvedValue(blocks(fileA, 2)) + const retry = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await retry + expect(filePoints(fileA)).toHaveLength(2) + expect(cache.getHash(fileA)).toBe(contentHash) + expect(cache.getOpenFiles()).toEqual({}) + }) + + it("T6 refuses queued writes after abort and retries without hashing partial content", async () => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const controller = new AbortController() + const upsert = store.upsertPoints.getMockImplementation()! + store.upsertPoints.mockImplementationOnce(async (newPoints: any[]) => { + await upsert(newPoints) + controller.abort() + }) + const result = await scanner + .scanDirectory(workspacePath, undefined, undefined, undefined, controller.signal) + .catch((error) => error) + expect(result).toMatchObject({ name: "AbortError" }) + expect(store.upsertPoints).toHaveBeenCalledTimes(1) + expect(embedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(cache.getHash(fileA)).toBeUndefined() + const retry = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await retry + expect(filePoints(fileA)).toHaveLength(3) + expect(cache.getHash(fileA)).toBe(contentHash) + }) + + it("T5 completion does not wait for an unrelated embedding request", async () => { + listedPaths = [fileA, fileB] + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockImplementation(async (texts: string[]) => { + return texts[0].includes("held") ? held.promise : { embeddings: [[1]] } + }) + parser.parseFile.mockImplementation(async (filePath: string) => { + const selected = blocks(filePath) + if (filePath === fileB) selected[0].content = "held embedding" + return selected + }) + const controller = new AbortController() + const progress = vi.fn() + const pending = scanner + .scanDirectory(workspacePath, undefined, progress, undefined, controller.signal) + .catch((error) => error) + await drain() + expect(cache.getHash(fileA)).toBe(contentHash) + expect(cache.getHash(fileB)).toBeUndefined() + expect(progress).toHaveBeenCalledExactlyOnceWith(1) + controller.abort() + held.resolve({ embeddings: [[1]] }) + await pending + expect(filePoints(fileB)).toEqual([]) + }) + + it("T2 real scanner/client timeout reaches orchestrator Error without blocking a scheduled probe", async () => { + listedPaths = [fileA, fileB] + const worker = new ControlledScannerWorker() + parser = new ParseWorkerClient(() => worker) + const { orchestrator, state, fatal, watcher } = createOrchestrator() + const pending = orchestrator.startIndexing() + await drain() + expect(worker.requests.map((request) => request.filePath)).toEqual([fileA]) + const probe = vi.fn() + setTimeout(probe, 100) + await vi.advanceTimersByTimeAsync(100) + expect(probe).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(PARSE_TIMEOUT_MS - 101) + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(1) + await pending + expect(fatal).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ filePath: fileA, category: "parse-timeout" }), + "embeddings:orchestrator.parserFailed", + ) + expect(state.state).toBe("Error") + expect(worker.terminate).toHaveBeenCalledOnce() + expect(worker.requests).toHaveLength(1) + expect(watcher.initialize).not.toHaveBeenCalled() + expect(store.upsertPoints).not.toHaveBeenCalled() + worker.emit("exit", 1) + await orchestrator.stopIndexing() + }) + + it.each(["cancel", "disable"])("T3 real scanner/client hang %s is shared and bounded", async (action) => { + listedPaths = [fileA, fileB] + const worker = new ControlledScannerWorker() + const client = new ParseWorkerClient(() => worker) + parser = client + const { orchestrator, state, watcher, fatal } = createOrchestrator() + const pending = orchestrator.startIndexing() + await drain() + expect(worker.requests).toHaveLength(1) + const stopped = action === "cancel" ? orchestrator.stopIndexing() : orchestrator.stopWatcher() + expect(orchestrator.stopIndexing()).toBe(stopped) + expect(orchestrator.stopWatcher()).toBe(stopped) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 1) + expect(state.state).toBe("Stopping") + await vi.advanceTimersByTimeAsync(1) + await stopped + await pending + expect(state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + worker.emit("message", { id: worker.requests[0].id, blocks: blocks(fileA) }) + await orchestrator.startIndexing() + await expect(client.parseFile(fileB)).rejects.toMatchObject({ name: "AbortError" }) + expect(worker.requests).toHaveLength(1) + expect(worker.terminate).toHaveBeenCalledOnce() + expect(watcher.initialize).not.toHaveBeenCalled() + expect(watcher.dispose).toHaveBeenCalledOnce() + expect(fatal).not.toHaveBeenCalled() + expect(embedder.createEmbeddings).not.toHaveBeenCalled() + worker.emit("exit", 1) + await client.stopped + }) + + it.each(["failure", "cancellation-error"])( + "T9 scanner discards late embedding %s after bounded stop", + async (outcome) => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockReturnValueOnce(held.promise) + const { orchestrator, state } = createOrchestrator() + const pending = orchestrator.startIndexing() + await drain() + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + const signal = embedder.createEmbeddings.mock.calls[0][2] as AbortSignal + const stopped = orchestrator.stopIndexing() + expect(signal.aborted).toBe(true) + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS) + await stopped + const updates = state.setSystemState.mock.calls.length + const progressUpdates = state.reportBlockIndexingProgress.mock.calls.length + held.reject( + outcome === "failure" + ? new Error("late provider failure") + : new DOMException("cancelled", "AbortError"), + ) + await pending + expect(state.setSystemState).toHaveBeenCalledTimes(updates) + expect(state.reportBlockIndexingProgress).toHaveBeenCalledTimes(progressUpdates) + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(store.upsertPoints).not.toHaveBeenCalled() + expect(store.deletePointsByFilePath).not.toHaveBeenCalled() + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + }, + ) + + it("T12 late old upsert before initial verification refuses completion and preserves current IDs", async () => { + let delayedPoints: any[] = [] + store.upsertPoints.mockImplementationOnce(async (newPoints: any[]) => { + delayedPoints = newPoints + throw new Error("local timeout") + }) + await scanner.scanDirectory(workspacePath, vi.fn()) + const rejectedAt = Date.now() + expect(gate.graceDeadline(fileA)).toBe(rejectedAt + LATE_WRITE_GRACE_MS) + const count = store.countPointsForFile.getMockImplementation()! + let currentIds: string[] = [] + store.countPointsForFile.mockImplementationOnce(async (filePath: string, generation: number) => { + currentIds = filePoints(fileA).map((point) => point.id) + for (const point of delayedPoints) points.set(point.id, point) + return count(filePath, generation) + }) + const progress = vi.fn() + const result = await scanner.scanDirectory(workspacePath, vi.fn(), progress) + expect(Date.now()).toBe(rejectedAt) + expect(result.incompleteFiles).toEqual([fileA]) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + expect(progress).not.toHaveBeenCalled() + expect(currentIds).toHaveLength(1) + expect(filePoints(fileA)).toHaveLength(2) + for (const pointId of currentIds) expect(points.has(pointId)).toBe(true) + const retry = scanner.scanDirectory(workspacePath) + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS) + await retry + expect(filePoints(fileA)).toHaveLength(1) + expect(cache.getHash(fileA)).toBe(contentHash) + }) + + it("T12 recovered open-file grace starts at cache load completion, not later scanner dispatch", async () => { + await cache.invalidateFile(fileA) + const read = vi.mocked(vscode.workspace.fs.readFile).getMockImplementation()! + const held = deferred() + vi.mocked(vscode.workspace.fs.readFile).mockImplementationOnce(() => held.promise) + const fresh = new CacheManager(context, workspacePath) + const loading = fresh.initialize() + await vi.advanceTimersByTimeAsync(5000) + held.resolve(Buffer.from(JSON.stringify(storage.snapshot))) + await loading + gate.recover(Object.keys(fresh.getOpenFiles()), fresh.loadedAt) + const loadedAt = Date.now() + expect(fresh.loadedAt).toBe(loadedAt) + cache = fresh + generations = new GenerationAllocator(cache) + await vi.advanceTimersByTimeAsync(7000) + vi.mocked(vscode.workspace.fs.readFile).mockImplementation(read) + const { orchestrator, state } = createOrchestrator() + const pending = orchestrator.startIndexing() + await drain() + expect(gate.graceDeadline(fileA)).toBe(loadedAt + LATE_WRITE_GRACE_MS) + expect(cache.getHash(fileA)).toBeUndefined() + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(LATE_WRITE_GRACE_MS - 7001) + expect(cache.getHash(fileA)).toBeUndefined() + expect(state.state).toBe("Indexing") + await vi.advanceTimersByTimeAsync(1) + await pending + expect(cache.getHash(fileA)).toBe(contentHash) + expect(state.state).toBe("Indexed") + }) + + it.each(["replacement", "removal", "exclusion"])( + "T17 fresh cache reload at first %s mutation is durably open", + async (operation) => { + cache.completeFile(fileA, "previous hash") + await cache.flush() + seed(fileA) + if (operation === "removal") listedPaths = [] + if (operation === "exclusion") ignore.ignores.mockReturnValue(true) + const mutation = operation === "replacement" ? store.upsertPoints : store.deletePointsByFilePath + const apply = mutation.getMockImplementation()! + let observed = false + mutation.mockImplementationOnce(async (...parameters: any[]) => { + expect(store.upsertPoints.mock.calls.length + store.deletePointsByFilePath.mock.calls.length).toBe(1) + const fresh = new CacheManager(context, workspacePath) + await fresh.initialize() + expect(fresh.loadSucceeded).toBe(true) + expect(fresh.getHash(fileA)).toBeUndefined() + expect(fresh.getOpenFiles()[fileA]).toBe(true) + observed = true + await apply(...parameters) + }) + await scanner.scanDirectory(workspacePath) + expect(observed).toBe(true) + expect(cache.getOpenFiles()).toEqual({}) + }, + ) + + it("T18 all batches are registered before first embedding dispatch", async () => { + parser.parseFile.mockResolvedValue(blocks(fileA, 3)) + const register = vi.spyOn(FileCompletion.prototype, "register") + const registered = vi.spyOn(FileCompletion.prototype, "registered") + const held = deferred<{ embeddings: number[][] }>() + embedder.createEmbeddings.mockImplementationOnce(() => { + expect(register).toHaveBeenCalledTimes(3) + expect(registered).toHaveBeenCalledOnce() + const completion = registered.mock.contexts[0] as FileCompletion + expect(completion.registrationComplete).toBe(true) + expect(completion.outstanding).toBe(3) + return held.promise + }) + const pending = scanner.scanDirectory(workspacePath) + await drain() + expect(embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(cache.getHash(fileA)).toBeUndefined() + held.resolve({ embeddings: [[1]] }) + await pending + expect(store.upsertPoints).toHaveBeenCalledTimes(3) + expect(cache.getHash(fileA)).toBe(contentHash) + }) + + it.each(["none", "late", "late-reject", "late-abort", "cancel-prior", "cancel-grace", "settled", "unrelated"])( + "T12/T19 real scanner pre-expiry retry completion barrier: %s", + async (outcome) => { + const held = deferred() + let oldPoints: any[] = [] + store.upsertPoints.mockImplementationOnce(async (newPoints: any[]) => { + oldPoints = newPoints + if (outcome === "settled") for (const point of newPoints) points.set(point.id, point) + await held.promise + }) + let oldIsCurrent = true + const old = createOrchestrator(() => oldIsCurrent) + const oldPending = old.orchestrator.startIndexing() + await drain() + expect(store.upsertPoints).toHaveBeenCalledOnce() + const expiryAt = Date.now() + INDEX_STOP_LIMIT_MS + const stopped = old.orchestrator.stopIndexing() + expect(old.state.state).toBe("Stopping") + oldIsCurrent = false + await vi.advanceTimersByTimeAsync(INDEX_STOP_LIMIT_MS - 2) + const verified: { at: number; counts: { current: number; other: number } }[] = [] + const count = store.countPointsForFile.getMockImplementation()! + store.countPointsForFile.mockImplementation(async (filePath: string, generation: number) => { + const counts = await count(filePath, generation) + if (filePath === fileA) verified.push({ at: Date.now(), counts }) + return counts + }) + if (outcome === "unrelated") listedPaths = [fileA, fileB] + old.state.reportBlockIndexingProgress.mockClear() + const retry = createOrchestrator(() => true, old.state) + let finishedAt: number | undefined + const pending = retry.orchestrator.startIndexing().then(() => { + finishedAt = Date.now() + }) + await drain() + const completedBlocks = outcome === "unrelated" ? 1 : 0 + const currentIds = filePoints(fileA).map((point) => point.id) + const unrelatedIds = filePoints(fileB).map((point) => point.id) + const assertIncomplete = () => { + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + expect(retry.state.state).toBe("Indexing") + expect(finishedAt).toBeUndefined() + expect(store.markIndexingComplete).not.toHaveBeenCalled() + expect( + retry.state.reportBlockIndexingProgress.mock.calls.every( + ([indexed]: number[]) => indexed <= completedBlocks, + ), + ).toBe(true) + if (outcome === "unrelated") { + expect(cache.getHash(fileB)).toBe(contentHash) + expect(filePoints(fileB).map((point) => point.id)).toEqual(unrelatedIds) + expect(retry.state.reportBlockIndexingProgress).toHaveBeenCalledWith(1, 2) + } + } + expect(verified).toEqual([{ at: expiryAt - 2, counts: { current: 1, other: 0 } }]) + expect(currentIds).toHaveLength(1) + expect(store.upsertPoints).toHaveBeenCalledTimes(outcome === "unrelated" ? 3 : 2) + expect(gate.graceDeadline(fileA)).toBe(0) + assertIncomplete() + if (outcome === "settled") { + await vi.advanceTimersByTimeAsync(1) + held.resolve() + await drain() + await pending + await stopped + await oldPending + expect(finishedAt).toBe(expiryAt - 1) + expect(gate.needsReconciliation(fileA)).toBe(false) + expect(verified.at(-1)).toEqual({ at: expiryAt - 1, counts: { current: 1, other: 0 } }) + expect(cache.getHash(fileA)).toBe(contentHash) + expect(retry.state.state).toBe("Indexed") + return + } + if (outcome === "cancel-prior") { + await retry.orchestrator.stopIndexing() + await pending + expect(retry.state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + await vi.advanceTimersByTimeAsync(2) + await stopped + return + } + await vi.advanceTimersByTimeAsync(1) + assertIncomplete() + expect(gate.graceDeadline(fileA)).toBe(0) + await vi.advanceTimersByTimeAsync(1) + await stopped + let deadline = expiryAt + LATE_WRITE_GRACE_MS + expect(gate.graceDeadline(fileA)).toBe(deadline) + assertIncomplete() + const stateUpdates = retry.state.setSystemState.mock.calls.length + await vi.advanceTimersByTimeAsync(1000) + if (outcome === "late") { + for (const point of oldPoints) points.set(point.id, point) + held.resolve() + await oldPending + await drain() + expect(gate.graceDeadline(fileA)).toBe(deadline) + } + if (outcome === "late-reject" || outcome === "late-abort") { + held.reject( + outcome === "late-abort" + ? new DOMException("late cancellation rejection", "AbortError") + : new Error("late old write rejection"), + ) + await oldPending + await drain() + const originalDeadline = deadline + deadline = Date.now() + LATE_WRITE_GRACE_MS + expect(deadline).toBe(originalDeadline + 1000) + expect(gate.graceDeadline(fileA)).toBe(deadline) + await vi.advanceTimersByTimeAsync(originalDeadline - Date.now()) + assertIncomplete() + expect(verified).toEqual([{ at: expiryAt - 2, counts: { current: 1, other: 0 } }]) + } + if (outcome === "cancel-grace") { + await retry.orchestrator.stopIndexing() + await pending + expect(retry.state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + return + } + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1) + assertIncomplete() + expect(retry.state.setSystemState).toHaveBeenCalledTimes(stateUpdates) + await vi.advanceTimersByTimeAsync(1) + await pending + expect(finishedAt).toBe(deadline) + expect(verified.at(-1)).toEqual({ + at: deadline, + counts: { current: 1, other: outcome === "late" ? 1 : 0 }, + }) + for (const pointId of currentIds) expect(points.has(pointId)).toBe(true) + if (outcome === "late") { + expect(cache.getHash(fileA)).toBeUndefined() + expect(cache.getOpenFiles()[fileA]).toBe(true) + expect(retry.state.setSystemState).toHaveBeenLastCalledWith( + "Standby", + "embeddings:orchestrator.indexingStoppedPartial", + ) + expect(store.markIndexingComplete).not.toHaveBeenCalled() + } else { + expect(cache.getHash(fileA)).toBe(contentHash) + expect(retry.state.state).toBe("Indexed") + expect(store.markIndexingComplete).toHaveBeenCalledOnce() + if (outcome === "unrelated") { + expect(cache.getHash(fileB)).toBe(contentHash) + expect(filePoints(fileB).map((point) => point.id)).toEqual(unrelatedIds) + } + } + }, + ) + + it("T22 real parser single-line segments have distinct generation IDs and verified counts", async () => { + const markdownPath = path.join(workspacePath, "long.md") + listedPaths = [markdownPath] + vi.mocked(vscode.workspace.fs.readFile).mockResolvedValue(Buffer.from("abcdefghij".repeat(500))) + parser = new CodeParser() + scanner = createScanner() + const errors = vi.fn() + const result = await scanner.scanDirectory(workspacePath, errors) + expect(errors.mock.calls).toEqual([]) + const stored = filePoints(markdownPath) + expect(stored.length).toBeGreaterThan(1) + expect(new Set(stored.map((point) => point.id)).size).toBe(stored.length) + expect(new Set(stored.map((point) => point.payload.startLine)).size).toBe(1) + expect(result.totalBlockCount).toBe(stored.length) + expect(result.incompleteFiles).toEqual([]) + expect(store.countPointsForFile).toHaveBeenCalledWith(markdownPath, stored[0].payload.generation) + expect(cache.getHash(markdownPath)).toBeDefined() + }) +}) diff --git a/src/services/code-index/processors/__tests__/scanner.spec.ts b/src/services/code-index/processors/__tests__/scanner.spec.ts index 8ed63c43602..5f40860be32 100644 --- a/src/services/code-index/processors/__tests__/scanner.spec.ts +++ b/src/services/code-index/processors/__tests__/scanner.spec.ts @@ -68,10 +68,14 @@ describe("DirectoryScanner", () => { let mockStats: any beforeEach(async () => { + vi.useFakeTimers() mockEmbedder = { - createEmbeddings: vi.fn().mockResolvedValue({ embeddings: [[0.1, 0.2, 0.3]] }), + createEmbeddings: vi.fn().mockImplementation(async (texts: string[]) => ({ + embeddings: texts.map(() => [0.1, 0.2, 0.3]), + })), embedderInfo: { name: "mock-embedder", dimensions: 384 }, } + const storedPoints = new Map() mockVectorStore = { upsertPoints: vi.fn().mockResolvedValue(undefined), deletePointsByFilePath: vi.fn().mockResolvedValue(undefined), @@ -81,17 +85,28 @@ describe("DirectoryScanner", () => { clearCollection: vi.fn().mockResolvedValue(undefined), deleteCollection: vi.fn().mockResolvedValue(undefined), collectionExists: vi.fn().mockResolvedValue(true), + countPointsForFile: vi.fn().mockImplementation(async (_filePath: string, generation: number) => ({ + current: [...storedPoints.values()].filter((point) => point.payload.generation === generation).length, + other: 0, + })), } + mockVectorStore.upsertPoints.mockImplementation(async (points: any[]) => { + for (const point of points) storedPoints.set(point.id, point) + }) mockCodeParser = { parseFile: vi.fn().mockResolvedValue([]), } mockCacheManager = { getHash: vi.fn().mockReturnValue(undefined), getAllHashes: vi.fn().mockReturnValue({}), - updateHash: vi.fn().mockResolvedValue(undefined), - deleteHash: vi.fn().mockResolvedValue(undefined), initialize: vi.fn().mockResolvedValue(undefined), clearCacheFile: vi.fn().mockResolvedValue(undefined), + generationWatermark: 1024, + reserveGenerations: vi.fn().mockResolvedValue(undefined), + getTrackedPaths: vi.fn().mockImplementation(() => Object.keys(mockCacheManager.getAllHashes())), + invalidateFile: vi.fn().mockResolvedValue(undefined), + completeFile: vi.fn(), + completeRemoval: vi.fn(), } mockIgnoreInstance = { ignores: vi.fn().mockReturnValue(false), @@ -144,6 +159,10 @@ describe("DirectoryScanner", () => { vi.mocked(listFiles).mockResolvedValue([["test/file1.js", "test/file2.js"], false]) }) + afterEach(() => { + vi.useRealTimers() + }) + describe("scanDirectory", () => { it("should skip files larger than MAX_FILE_SIZE_BYTES", async () => { const { listFiles } = await import("../../../glob/list-files") @@ -215,8 +234,9 @@ describe("DirectoryScanner", () => { ;(mockCacheManager.getAllHashes as any).mockReturnValue({ "old/file.js": "old-hash" }) await scanner.scanDirectory("/test") - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("old/file.js") - expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("old/file.js") + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("old/file.js", expect.any(Number)) + expect(mockCacheManager.invalidateFile).toHaveBeenCalledWith("old/file.js") + expect(mockCacheManager.completeRemoval).toHaveBeenCalledWith("old/file.js") }) it("should filter out files in hidden directories", async () => { @@ -426,17 +446,19 @@ describe("DirectoryScanner", () => { const pending = scanner .scanDirectory("/test", vi.fn(), undefined, undefined, controller.signal) .catch((error) => error) - await vi.waitFor(() => expect(mockVectorStore.upsertPoints).toHaveBeenCalledTimes(3)) - expect(mockCacheManager.updateHash).not.toHaveBeenCalled() - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledOnce() - expect(mockVectorStore.deletePointsByFilePath.mock.invocationCallOrder[0]).toBeLessThan( + await vi.advanceTimersByTimeAsync(0) + expect(mockVectorStore.upsertPoints).toHaveBeenCalledOnce() + expect(mockCacheManager.completeFile).not.toHaveBeenCalled() + expect(mockVectorStore.deletePointsByFilePath).not.toHaveBeenCalled() + expect(mockCacheManager.invalidateFile.mock.invocationCallOrder[0]).toBeLessThan( mockVectorStore.upsertPoints.mock.invocationCallOrder[0], ) + mockVectorStore.countPointsForFile.mockResolvedValue({ current: 3, other: 0 }) if (outcome === "abort") controller.abort() release() await pending - if (outcome === "success") expect(mockCacheManager.updateHash).toHaveBeenCalledOnce() - else expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + if (outcome === "success") expect(mockCacheManager.completeFile).toHaveBeenCalledOnce() + else expect(mockCacheManager.completeFile).not.toHaveBeenCalled() }, ) @@ -466,12 +488,13 @@ describe("DirectoryScanner", () => { const pending = scanner .scanDirectory("/test", undefined, undefined, undefined, controller.signal) .catch((error) => error) - await vi.waitFor(() => expect(mockEmbedder.createEmbeddings).toHaveBeenCalledOnce()) + await vi.advanceTimersByTimeAsync(0) + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledOnce() controller.abort() release({ embeddings: [[1, 2, 3]] }) await pending expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() - expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + expect(mockCacheManager.completeFile).not.toHaveBeenCalled() }) it("reconciles an uncertain unchanged file before replacement upserts", async () => { @@ -492,11 +515,18 @@ describe("DirectoryScanner", () => { 1, gate, ) - await scanner.scanDirectory("/test") - expect(mockCodeParser.parseFile).toHaveBeenCalledOnce() - expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("test/file.js") - expect(mockCacheManager.deleteHash).toHaveBeenCalledWith("test/file.js") - expect(gate.needsReconciliation("test/file.js")).toBe(false) + vi.useFakeTimers() + try { + const pending = scanner.scanDirectory("/test") + await vi.advanceTimersByTimeAsync(30_000) + await pending + expect(mockCodeParser.parseFile).toHaveBeenCalledOnce() + expect(mockVectorStore.deletePointsByFilePath).toHaveBeenCalledWith("test/file.js", expect.any(Number)) + expect(mockCacheManager.invalidateFile).toHaveBeenCalledWith("test/file.js") + expect(gate.needsReconciliation("test/file.js")).toBe(false) + } finally { + vi.useRealTimers() + } }) it("propagates fatal parser errors without treating them as per-file errors", async () => { @@ -507,7 +537,7 @@ describe("DirectoryScanner", () => { const onError = vi.fn() await expect(scanner.scanDirectory("/test", onError)).rejects.toBe(fatal) expect(onError).not.toHaveBeenCalled() - expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + expect(mockCacheManager.completeFile).not.toHaveBeenCalled() }) it("should stop processing files when signal is aborted", async () => { diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index af61b752638..0dfeb62a262 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -1,50 +1,36 @@ import * as vscode from "vscode" -import { - QDRANT_CODE_BLOCK_NAMESPACE, - MAX_FILE_SIZE_BYTES, - BATCH_SEGMENT_THRESHOLD, - MAX_BATCH_RETRIES, - INITIAL_RETRY_DELAY_MS, -} from "../constants" +import { MAX_FILE_SIZE_BYTES, BATCH_SEGMENT_THRESHOLD } from "../constants" import { createHash } from "crypto" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" -import { v5 as uuidv5 } from "uuid" import { Ignore } from "ignore" import { scannerExtensions } from "../shared/supported-extensions" -import { - IFileWatcher, - FileProcessingResult, - IEmbedder, - IVectorStore, - PointStruct, - BatchProcessingSummary, -} from "../interfaces" +import { IFileWatcher, FileProcessingResult, IEmbedder, IVectorStore, BatchProcessingSummary } from "../interfaces" import { ParseWorkerClient } from "../parse-worker-client" import type { ICodeParser } from "../interfaces" import { IndexWriteGate } from "../index-write-gate" +import { GenerationAllocator } from "../generation-allocator" +import { FileIndexWriter } from "../file-completion" import { ParserFatalError, throwIfAborted } from "../parser-errors" import { CacheManager } from "../cache-manager" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { generateRelativeFilePath } from "../shared/get-relative-path" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" -import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" -/** - * Implementation of the file watcher interface - */ +type FileEvent = { uri: vscode.Uri; type: "create" | "change" | "delete" } + export class FileWatcher implements IFileWatcher { - private ignoreInstance?: Ignore private fileWatcher?: vscode.FileSystemWatcher - private ignoreController: RooIgnoreController - private accumulatedEvents: Map = new Map() + private readonly ignoreController: RooIgnoreController + private readonly accumulatedEvents = new Map() private batchProcessDebounceTimer?: NodeJS.Timeout private readonly BATCH_DEBOUNCE_DELAY_MS = 500 private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10 private readonly batchSegmentThreshold: number - + private readonly writer?: FileIndexWriter private readonly localAbortController = new AbortController() private signal = this.localAbortController.signal private processingBatch = false + private activeFiles = 0 private disposed = false private fatalReported = false private readonly parserFatal = new vscode.EventEmitter() @@ -56,48 +42,27 @@ export class FileWatcher implements IFileWatcher { currentFile?: string }>() private readonly _onDidFinishBatchProcessing = new vscode.EventEmitter() - - /** - * Event emitted when a batch of files begins processing - */ public readonly onDidStartBatchProcessing = this._onDidStartBatchProcessing.event - - /** - * Event emitted to report progress during batch processing - */ public readonly onBatchProgressUpdate = this._onBatchProgressUpdate.event - - /** - * Event emitted when a batch of files has finished processing - */ public readonly onDidFinishBatchProcessing = this._onDidFinishBatchProcessing.event - /** - * Creates a new file watcher - * @param workspacePath Path to the workspace - * @param context VS Code extension context - * @param embedder Optional embedder - * @param vectorStore Optional vector store - * @param cacheManager Cache manager - */ constructor( - private workspacePath: string, - private context: vscode.ExtensionContext, + private readonly workspacePath: string, + private readonly context: vscode.ExtensionContext, private readonly cacheManager: CacheManager, - private embedder?: IEmbedder, - private vectorStore?: IVectorStore, - ignoreInstance?: Ignore, + private readonly embedder?: IEmbedder, + private readonly vectorStore?: IVectorStore, + private readonly ignoreInstance?: Ignore, ignoreController?: RooIgnoreController, batchSegmentThreshold?: number, private readonly parser: ICodeParser = new ParseWorkerClient(), private readonly writeGate = new IndexWriteGate(), + private readonly generations = new GenerationAllocator(cacheManager), ) { this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) - if (ignoreInstance) { - this.ignoreInstance = ignoreInstance + if (vectorStore) { + this.writer = new FileIndexWriter(cacheManager, vectorStore, writeGate, generations, workspacePath) } - // Get the configurable batch size from VSCode settings, fallback to default - // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -106,107 +71,67 @@ export class FileWatcher implements IFileWatcher { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { - // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } } - /** - * Initializes the file watcher - */ + public get isProcessing(): boolean { + return this.processingBatch || this.activeFiles > 0 + } + async initialize(signal?: AbortSignal): Promise { if (this.disposed) throw new DOMException("Indexing aborted", "AbortError") - this.signal = signal ?? this.signal + this.signal = signal + ? AbortSignal.any([signal, this.localAbortController.signal]) + : this.localAbortController.signal + if (signal) this.writeGate.own(this.signal, signal) throwIfAborted(this.signal) - // Create file watcher const filePattern = new vscode.RelativePattern( this.workspacePath, - `**/*{${scannerExtensions.map((e) => e.substring(1)).join(",")}}`, + `**/*{${scannerExtensions.map((extension) => extension.substring(1)).join(",")}}`, ) this.fileWatcher = vscode.workspace.createFileSystemWatcher(filePattern) - - // Register event handlers - this.fileWatcher.onDidCreate(this.handleFileCreated.bind(this)) - this.fileWatcher.onDidChange(this.handleFileChanged.bind(this)) - this.fileWatcher.onDidDelete(this.handleFileDeleted.bind(this)) + this.fileWatcher.onDidCreate((uri) => this.handleFileEvent(uri, "create")) + this.fileWatcher.onDidChange((uri) => this.handleFileEvent(uri, "change")) + this.fileWatcher.onDidDelete((uri) => this.handleFileEvent(uri, "delete")) } - /** - * Disposes the file watcher - */ dispose(): void { if (this.disposed) return this.disposed = true this.localAbortController.abort() this.parserFatal.dispose() this.fileWatcher?.dispose() - if (this.batchProcessDebounceTimer) { - clearTimeout(this.batchProcessDebounceTimer) - } + clearTimeout(this.batchProcessDebounceTimer) this._onDidStartBatchProcessing.dispose() this._onBatchProgressUpdate.dispose() this._onDidFinishBatchProcessing.dispose() this.accumulatedEvents.clear() } - /** - * Handles file creation events - * @param uri URI of the created file - */ - private async handleFileCreated(uri: vscode.Uri): Promise { - if (this.disposed || this.signal.aborted) return - this.accumulatedEvents.set(uri.fsPath, { uri, type: "create" }) - this.scheduleBatchProcessing() - } - - /** - * Handles file change events - * @param uri URI of the changed file - */ - private async handleFileChanged(uri: vscode.Uri): Promise { - if (this.disposed || this.signal.aborted) return - this.accumulatedEvents.set(uri.fsPath, { uri, type: "change" }) - this.scheduleBatchProcessing() - } - - /** - * Handles file deletion events - * @param uri URI of the deleted file - */ - private async handleFileDeleted(uri: vscode.Uri): Promise { + private handleFileEvent(uri: vscode.Uri, type: FileEvent["type"]): void { if (this.disposed || this.signal.aborted) return - this.accumulatedEvents.set(uri.fsPath, { uri, type: "delete" }) + this.accumulatedEvents.set(uri.fsPath, { uri, type }) this.scheduleBatchProcessing() } - /** - * Schedules batch processing with debounce - */ private scheduleBatchProcessing(): void { if (this.disposed || this.signal.aborted) return - if (this.batchProcessDebounceTimer) { - clearTimeout(this.batchProcessDebounceTimer) - } - this.batchProcessDebounceTimer = setTimeout(() => this.triggerBatchProcessing(), this.BATCH_DEBOUNCE_DELAY_MS) + clearTimeout(this.batchProcessDebounceTimer) + this.batchProcessDebounceTimer = setTimeout( + () => void this.triggerBatchProcessing(), + this.BATCH_DEBOUNCE_DELAY_MS, + ) } - /** - * Triggers processing of accumulated events - */ private async triggerBatchProcessing(): Promise { - if (this.disposed || this.signal.aborted || this.processingBatch || this.accumulatedEvents.size === 0) { - return - } - + if (this.disposed || this.signal.aborted || this.processingBatch || this.accumulatedEvents.size === 0) return const eventsToProcess = new Map(this.accumulatedEvents) this.accumulatedEvents.clear() - - const filePathsInBatch = Array.from(eventsToProcess.keys()) - this._onDidStartBatchProcessing.fire(filePathsInBatch) - this.processingBatch = true try { + this._onDidStartBatchProcessing.fire([...eventsToProcess.keys()]) await this.processBatch(eventsToProcess) } catch (error) { if (error instanceof ParserFatalError) this.reportFatal(error) @@ -219,221 +144,45 @@ export class FileWatcher implements IFileWatcher { } } - private async _processFilesAndPrepareUpserts( - filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }>, - batchResults: FileProcessingResult[], - processedCountInBatch: number, - totalFilesInBatch: number, - pathsToExplicitlyDelete: string[], - ): Promise<{ - pointsForBatchUpsert: PointStruct[] - successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }> - processedCount: number - }> { - const pointsForBatchUpsert: PointStruct[] = [] - const successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }> = [] - const filesToProcessConcurrently = [...filesToUpsertDetails] - - for (let i = 0; i < filesToProcessConcurrently.length; i += this.FILE_PROCESSING_CONCURRENCY_LIMIT) { - const chunkToProcess = filesToProcessConcurrently.slice(i, i + this.FILE_PROCESSING_CONCURRENCY_LIMIT) - - const chunkProcessingPromises = chunkToProcess.map(async (fileDetail) => { - this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: fileDetail.path, - }) - try { - const result = await this.processFile(fileDetail.path) - return { path: fileDetail.path, result: result, error: undefined } - } catch (e) { - if (e instanceof ParserFatalError || (e instanceof Error && e.name === "AbortError")) throw e - const error = e as Error - console.error(`[FileWatcher] Unhandled exception processing file ${fileDetail.path}:`, e) - return { path: fileDetail.path, result: undefined, error: error } - } - }) - - const settledChunkResults = await Promise.allSettled(chunkProcessingPromises) - const fatal = settledChunkResults.find( - (result) => result.status === "rejected" && result.reason instanceof ParserFatalError, - ) - if (fatal?.status === "rejected") throw fatal.reason - throwIfAborted(this.signal) - if (this.disposed) - return { - pointsForBatchUpsert: [], - successfullyProcessedForUpsert: [], - processedCount: processedCountInBatch, - } - - for (const settledResult of settledChunkResults) { - let resultPath: string | undefined - - if (settledResult.status === "fulfilled") { - const { path, result, error: directError } = settledResult.value - resultPath = path - - if (directError) { - batchResults.push({ path, status: "error", error: directError }) - } else if (result) { - if (result.status === "skipped" || result.status === "local_error") { - batchResults.push(result) - } else if (result.status === "processed_for_batching" && result.pointsToUpsert) { - pointsForBatchUpsert.push(...result.pointsToUpsert) - if (result.path && result.newHash) { - successfullyProcessedForUpsert.push({ path: result.path, newHash: result.newHash }) - } else if (result.path && !result.newHash) { - successfullyProcessedForUpsert.push({ path: result.path }) - } - } else { - batchResults.push({ - path, - status: "error", - error: new Error( - `Unexpected result status from processFile: ${result.status} for file ${path}`, - ), - }) - } - } else { - batchResults.push({ - path, - status: "error", - error: new Error(`Fulfilled promise with no result or error for file ${path}`), - }) - } - } else { - const error = settledResult.reason as Error - const rejectedPath = (settledResult.reason as any)?.path || "unknown" - console.error("[FileWatcher] A file processing promise was rejected:", settledResult.reason) - batchResults.push({ - path: rejectedPath, - status: "error", - error: error, - }) - } - - if (!pathsToExplicitlyDelete.includes(resultPath || "")) { - processedCountInBatch++ - } - if (processedCountInBatch < totalFilesInBatch) + private async processBatch(eventsToProcess: Map): Promise { + throwIfAborted(this.signal) + const events = [...eventsToProcess.values()] + const results: FileProcessingResult[] = [] + let nextEvent = 0 + let completed = 0 + this._onBatchProgressUpdate.fire({ processedInBatch: 0, totalInBatch: events.length }) + const processNext = async () => { + while (nextEvent < events.length) { + throwIfAborted(this.signal) + const event = events[nextEvent++] + const result = + event.type === "delete" + ? await this.removeFile(event.uri.fsPath) + : await this.processFile(event.uri.fsPath) + throwIfAborted(this.signal) + results.push(result) + if (result.status === "success" || result.status === "skipped") { + completed++ this._onBatchProgressUpdate.fire({ - processedInBatch: processedCountInBatch, - totalInBatch: totalFilesInBatch, - currentFile: resultPath, + processedInBatch: completed, + totalInBatch: events.length, + currentFile: result.path, }) + } } } - - return { - pointsForBatchUpsert, - successfullyProcessedForUpsert, - processedCount: processedCountInBatch, - } - } - - private async processBatch( - eventsToProcess: Map, - ): Promise { - throwIfAborted(this.signal) - await this.writeGate.ready(this.signal) - const batchResults: FileProcessingResult[] = [] - let processedCountInBatch = 0 - const totalFilesInBatch = eventsToProcess.size - const overallBatchError: Error | undefined = undefined - - // Initial progress update - this._onBatchProgressUpdate.fire({ - processedInBatch: 0, - totalInBatch: totalFilesInBatch, - currentFile: undefined, - }) - - // Categorize events - const pathsToExplicitlyDelete: string[] = [] - const filesToUpsertDetails: Array<{ path: string; uri: vscode.Uri; originalType: "create" | "change" }> = [] - - for (const event of eventsToProcess.values()) { - if (event.type === "delete") { - pathsToExplicitlyDelete.push(event.uri.fsPath) - } else { - filesToUpsertDetails.push({ - path: event.uri.fsPath, - uri: event.uri, - originalType: event.type, - }) - } - } - - // Phase 2: Process files and prepare upserts - const { - pointsForBatchUpsert, - successfullyProcessedForUpsert, - processedCount: upsertCount, - } = await this._processFilesAndPrepareUpserts( - filesToUpsertDetails, - batchResults, - processedCountInBatch, - totalFilesInBatch, - pathsToExplicitlyDelete, + // Each worker completes its file before taking another; no file waits for a batch-wide flush. + await Promise.all( + Array.from({ length: Math.min(events.length, this.FILE_PROCESSING_CONCURRENCY_LIMIT) }, processNext), ) - processedCountInBatch = upsertCount - - throwIfAborted(this.signal) - if (this.disposed) return - const replacementPaths = successfullyProcessedForUpsert.map((file) => file.path) - const cleanupPaths = [...new Set([...pathsToExplicitlyDelete, ...replacementPaths])] - for (const filePath of cleanupPaths) { - if (this.vectorStore) { - await this.writeGate.run(this.signal, () => this.vectorStore!.deletePointsByFilePath(filePath), [ - filePath, - ]) - } - await this.writeGate.run(this.signal, () => this.cacheManager.deleteHash(filePath), [filePath]) - } - if (this.vectorStore) { - for ( - let pointOffset = 0; - pointOffset < pointsForBatchUpsert.length; - pointOffset += this.batchSegmentThreshold - ) { - const points = pointsForBatchUpsert.slice(pointOffset, pointOffset + this.batchSegmentThreshold) - await this.writeGate.run(this.signal, () => this.vectorStore!.upsertPoints(points), replacementPaths) - } - } - for (const file of successfullyProcessedForUpsert) { - if (file.newHash) - await this.writeGate.run(this.signal, () => this.cacheManager.updateHash(file.path, file.newHash!), [ - file.path, - ]) - this.writeGate.reconciled(file.path) - batchResults.push({ path: file.path, status: "success" }) - } throwIfAborted(this.signal) - // Finalize + const errors = results.filter((result) => result.status === "local_error") this._onDidFinishBatchProcessing.fire({ - processedFiles: batchResults, - batchError: overallBatchError, - }) - this._onBatchProgressUpdate.fire({ - processedInBatch: totalFilesInBatch, - totalInBatch: totalFilesInBatch, + processedFiles: results, + batchError: errors.length ? new Error(`${errors.length} file changes could not be indexed`) : undefined, }) - - if (this.accumulatedEvents.size === 0) { - this._onBatchProgressUpdate.fire({ - processedInBatch: 0, - totalInBatch: 0, - currentFile: undefined, - }) - } } - /** - * Processes a file - * @param filePath Path to the file to process - * @returns Promise resolving to processing result - */ private reportFatal(error: ParserFatalError): void { if (this.fatalReported || this.disposed || this.signal.aborted) return this.fatalReported = true @@ -441,108 +190,60 @@ export class FileWatcher implements IFileWatcher { this.dispose() } - async processFile(filePath: string): Promise { + private async removeFile(filePath: string): Promise { try { throwIfAborted(this.signal) - await this.writeGate.ready(this.signal) - // Get relative path for ignore checks - const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) - - // Check if file is in an ignored directory - // Use relative path to avoid matching parent directories outside the workspace - if (isPathInIgnoredDirectory(relativeFilePath)) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is in an ignored directory", - } + if (!this.cacheManager.isTracked(filePath)) { + return { path: filePath, status: "skipped", reason: "File is not tracked" } } + if (!this.writer) throw new Error("Vector store is not configured") + await this.writer.remove(filePath, this.signal) + return { path: filePath, status: "success" } + } catch (error) { + if (this.signal.aborted || (error instanceof Error && error.name === "AbortError")) throw error + return { path: filePath, status: "local_error", error: error as Error } + } + } - // Check if file should be ignored + async processFile(filePath: string): Promise { + this.activeFiles++ + try { + throwIfAborted(this.signal) + const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) if ( + isPathInIgnoredDirectory(relativeFilePath) || !this.ignoreController.validateAccess(filePath) || - (this.ignoreInstance && this.ignoreInstance.ignores(relativeFilePath)) + this.ignoreInstance?.ignores(relativeFilePath) ) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is ignored by .rooignore or .gitignore", - } + return await this.removeFile(filePath) } - - // Check file size const fileStat = await vscode.workspace.fs.stat(vscode.Uri.file(filePath)) + throwIfAborted(this.signal) if (fileStat.size > MAX_FILE_SIZE_BYTES) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is too large", - } + return await this.removeFile(filePath) } - - // Read file content const fileContent = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) - const content = fileContent.toString() - - // Calculate hash + throwIfAborted(this.signal) + const content = Buffer.from(fileContent).toString("utf8") const newHash = createHash("sha256").update(content).digest("hex") - - // Check if file has changed if (this.cacheManager.getHash(filePath) === newHash && !this.writeGate.needsReconciliation(filePath)) { - return { - path: filePath, - status: "skipped" as const, - reason: "File has not changed", - } + return { path: filePath, status: "skipped", reason: "File has not changed" } } - - // Parse file + if (!this.writer || !this.embedder) throw new Error("Indexing services are not configured") const blocks = await this.parser.parseFile(filePath, { content, fileHash: newHash, signal: this.signal }) throwIfAborted(this.signal) - - // Prepare points for batch processing - let pointsToUpsert: PointStruct[] = [] - if (this.embedder && blocks.length > 0) { - const texts = blocks.map((block) => block.content) - const { embeddings } = await this.embedder.createEmbeddings(texts, undefined, this.signal) - - throwIfAborted(this.signal) - - pointsToUpsert = blocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) - const stableName = `${normalizedAbsolutePath}:${block.start_line}` - const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, this.workspacePath), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - }, - } - }) - } - - return { - path: filePath, - status: "processed_for_batching" as const, - newHash, - pointsToUpsert, - } + await this.writer.replace(filePath, newHash, blocks, this.embedder, this.signal, this.batchSegmentThreshold) + throwIfAborted(this.signal) + return { path: filePath, status: "success" } } catch (error) { if (error instanceof ParserFatalError) { this.reportFatal(error) throw error } - if (error instanceof Error && error.name === "AbortError") throw error - return { - path: filePath, - status: "local_error" as const, - error: error as Error, - } + if (this.signal.aborted || (error instanceof Error && error.name === "AbortError")) throw error + return { path: filePath, status: "local_error", error: error as Error } + } finally { + this.activeFiles-- } } } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index a809147bb4b..bf3960403f1 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -3,36 +3,30 @@ import { Ignore } from "ignore" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { stat } from "fs/promises" import * as path from "path" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { generateRelativeFilePath } from "../shared/get-relative-path" import { getWorkspacePathForContext } from "../../../utils/path" import { scannerExtensions } from "../shared/supported-extensions" import * as vscode from "vscode" -import { CodeBlock, ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" +import { ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces" import { createHash } from "crypto" -import { v5 as uuidv5 } from "uuid" -import pLimit from "p-limit" -import { Mutex } from "async-mutex" import { CacheManager } from "../cache-manager" -import { t } from "../../../i18n" import { - QDRANT_CODE_BLOCK_NAMESPACE, MAX_FILE_SIZE_BYTES, MAX_LIST_FILES_LIMIT_CODE_INDEX, BATCH_SEGMENT_THRESHOLD, - MAX_BATCH_RETRIES, - INITIAL_RETRY_DELAY_MS, PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, - MAX_PENDING_BATCHES, } from "../constants" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" -import { sanitizeErrorMessage } from "../shared/validation-helpers" import { Package } from "../../../shared/package" import { IndexWriteGate } from "../index-write-gate" import { ParserFatalError, throwIfAborted } from "../parser-errors" +import { GenerationAllocator } from "../generation-allocator" +import { FileIndexWriter } from "../file-completion" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number + private incompleteFiles = new Set() constructor( private readonly embedder: IEmbedder, @@ -42,9 +36,8 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, private readonly writeGate = new IndexWriteGate(), + private readonly generations = new GenerationAllocator(cacheManager), ) { - // Get the configurable batch size from VSCode settings, fallback to default - // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { this.batchSegmentThreshold = batchSegmentThreshold } else { @@ -53,460 +46,155 @@ export class DirectoryScanner implements IDirectoryScanner { .getConfiguration(Package.name) .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) } catch { - // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } } - /** - * Recursively scans a directory for code blocks in supported files. - * @param directoryPath The directory to scan - * @param rooIgnoreController Optional RooIgnoreController instance for filtering - * @param context VS Code ExtensionContext for cache storage - * @param onError Optional error handler callback - * @returns Promise<{codeBlocks: CodeBlock[], stats: {processed: number, skipped: number}}> Array of parsed code blocks and processing stats - */ + public get hasIncompleteFiles(): boolean { + return this.incompleteFiles.size > 0 + } + public async scanDirectory( directory: string, onError?: (error: Error) => void, onBlocksIndexed?: (indexedCount: number) => void, onFileParsed?: (fileBlockCount: number) => void, - signal?: AbortSignal, - ): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> { - signal ??= new AbortController().signal - await this.writeGate.ready(signal) - const fileCompletions = new Map() - const failedFiles = new Set() - let fatalError: ParserFatalError | undefined - const directoryPath = directory - // Capture workspace context at scan start - const scanWorkspace = getWorkspacePathForContext(directoryPath) - - // Get all files recursively (handles .gitignore automatically) - const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) - - // Filter out directories (marked with trailing '/') - const filePaths = allPaths.filter((p) => !p.endsWith("/")) - - // Initialize RooIgnoreController if not provided - const ignoreController = new RooIgnoreController(directoryPath) - + signal = new AbortController().signal, + ): Promise<{ + stats: { processed: number; skipped: number } + totalBlockCount: number + incompleteFiles?: string[] + }> { + throwIfAborted(signal) + const incompleteFiles = new Set() + this.incompleteFiles = incompleteFiles + const scanWorkspace = getWorkspacePathForContext(directory) + const [allPaths] = await listFiles(directory, true, MAX_LIST_FILES_LIMIT_CODE_INDEX) + throwIfAborted(signal) + const ignoreController = new RooIgnoreController(directory) await ignoreController.initialize() + throwIfAborted(signal) + const allowedPaths = ignoreController.filterPaths(allPaths.filter((filePath) => !filePath.endsWith("/"))) + const selectedPaths = new Set( + allowedPaths.filter((filePath) => { + const relativePath = generateRelativeFilePath(filePath, scanWorkspace) + return ( + !isPathInIgnoredDirectory(relativePath) && + scannerExtensions.includes(path.extname(filePath).toLowerCase()) && + !this.ignoreInstance.ignores(relativePath) + ) + }), + ) + // Discovery precedes dispatch so stopping during a read or parse still reports partial work. + for (const filePath of selectedPaths) incompleteFiles.add(filePath) + const trackedPaths = new Set(this.cacheManager.getTrackedPaths()) + for (const filePath of trackedPaths) { + if (!selectedPaths.has(filePath)) incompleteFiles.add(filePath) + } - // Filter paths using .rooignore - const allowedPaths = ignoreController.filterPaths(filePaths) - - // Filter by supported extensions, ignore patterns, and excluded directories - const supportedPaths = allowedPaths.filter((filePath) => { - const ext = path.extname(filePath).toLowerCase() - const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace) - - // Check if file is in an ignored directory using the shared helper - // Use relative path to avoid matching parent directories outside the workspace - if (isPathInIgnoredDirectory(relativeFilePath)) { - return false - } - - return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath) - }) - - // Initialize tracking variables - const processedFiles = new Set() + const writer = new FileIndexWriter( + this.cacheManager, + this.qdrantClient, + this.writeGate, + this.generations, + scanWorkspace, + ) let processedCount = 0 let skippedCount = 0 - - // Initialize parallel processing tools - const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing - const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing - const mutex = new Mutex() - - // Shared batch accumulators (protected by mutex) - let currentBatchBlocks: CodeBlock[] = [] - let currentBatchTexts: string[] = [] - let currentBatchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[] = [] - const activeBatchPromises = new Set>() - let pendingBatchCount = 0 - - // Initialize block counter let totalBlockCount = 0 - - // Process all files in parallel with concurrency control - const parsePromises = supportedPaths.map((filePath) => - parseLimiter(async () => { - // Check abort signal before processing each file - if (signal?.aborted || fatalError) return - - try { - // Check file size - const stats = await stat(filePath) - if (stats.size > MAX_FILE_SIZE_BYTES) { - skippedCount++ // Skip large files - return - } - - // Read file content - const content = await vscode.workspace.fs - .readFile(vscode.Uri.file(filePath)) - .then((buffer) => Buffer.from(buffer).toString("utf-8")) - - // Calculate current hash - const currentFileHash = createHash("sha256").update(content).digest("hex") - processedFiles.add(filePath) - - // Check against cache - const cachedFileHash = this.cacheManager.getHash(filePath) - const isNewFile = !cachedFileHash - if (cachedFileHash === currentFileHash && !this.writeGate.needsReconciliation(filePath)) { - // File is unchanged - skippedCount++ - return - } - - // File is new or changed - parse it using the injected parser function - const blocks = await this.codeParser.parseFile(filePath, { - content, - fileHash: currentFileHash, - signal, - }) - throwIfAborted(signal) - if (fatalError) throw fatalError - // Cleanup must precede every replacement batch, not just the batch carrying the hash. - if (this.qdrantClient) { - await this.writeGate.run(signal!, () => this.qdrantClient.deletePointsByFilePath(filePath), [ - filePath, - ]) - } - await this.writeGate.run(signal!, () => this.cacheManager.deleteHash(filePath), [filePath]) - fileCompletions.set(filePath, currentFileHash) - const fileBlockCount = blocks.length - onFileParsed?.(fileBlockCount) - processedCount++ - - // Process embeddings if configured - if (this.embedder && this.qdrantClient && blocks.length > 0) { - // Add to batch accumulators - let addedBlocksFromFile = false - for (const block of blocks) { - const trimmedContent = block.content.trim() - if (trimmedContent) { - const release = await mutex.acquire() - try { - currentBatchBlocks.push(block) - currentBatchTexts.push(trimmedContent) - addedBlocksFromFile = true - - // Check if batch threshold is met - // Check abort signal before dispatching batch - if (signal?.aborted) { - throw new DOMException("Indexing aborted", "AbortError") - } - - if (currentBatchBlocks.length >= this.batchSegmentThreshold) { - // Wait if we've reached the maximum pending batches - while (pendingBatchCount >= MAX_PENDING_BATCHES) { - if (signal?.aborted) { - throw new DOMException("Indexing aborted", "AbortError") - } - await Promise.race(activeBatchPromises) - } - - // Copy current batch data and clear accumulators - const batchBlocks = [...currentBatchBlocks] - const batchTexts = [...currentBatchTexts] - const batchFileInfos = [...currentBatchFileInfos] - currentBatchBlocks = [] - currentBatchTexts = [] - currentBatchFileInfos = [] - - // Increment pending batch count - pendingBatchCount++ - - // Queue batch processing - const batchPromise = batchLimiter(() => - this.processBatch( - batchBlocks, - batchTexts, - batchFileInfos, - scanWorkspace, - onError, - onBlocksIndexed, - signal, - failedFiles, - ), - ) - activeBatchPromises.add(batchPromise) - - // Clean up completed promises to prevent memory accumulation - batchPromise.finally(() => { - activeBatchPromises.delete(batchPromise) - pendingBatchCount-- - }) - } - } finally { - release() - } - } - } - - // Add file info once per file (outside the block loop) - if (addedBlocksFromFile) { - const release = await mutex.acquire() - try { - totalBlockCount += fileBlockCount - currentBatchFileInfos.push({ - filePath, - fileHash: currentFileHash, - isNew: isNewFile, - }) - } finally { - release() - } - } - } else { - // Only update hash if not being processed in a batch - throwIfAborted(signal) - } - } catch (error) { - if (error instanceof ParserFatalError) { - fatalError = error - throw error - } - // Re-throw AbortError — it's not a file processing error, just a user-initiated stop - if (error instanceof DOMException && error.name === "AbortError") { - throw error - } - console.error(`Error processing file ${filePath} in workspace ${scanWorkspace}:`, error) - if (onError) { - onError( - error instanceof Error - ? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`) - : new Error( - t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) + - ` (Workspace: ${scanWorkspace})`, - ), - ) - } - } - }), - ) - - // Wait for all parsing to complete - await Promise.all(parsePromises) - - // Check abort signal before processing remaining batch - if (signal?.aborted) { - return { - stats: { - processed: processedCount, - skipped: skippedCount, - }, - totalBlockCount, + let fatalError: ParserFatalError | undefined + const checkActive = () => { + if (fatalError) throw fatalError + throwIfAborted(signal) + } + const reportError = (error: unknown, filePath: string) => { + if (error instanceof ParserFatalError) { + fatalError ??= error + throw fatalError } + checkActive() + if (error instanceof Error && error.name === "AbortError") throw error + const message = error instanceof Error ? error.message : String(error) + onError?.(new Error(`${message} (Workspace: ${scanWorkspace}, File: ${filePath})`)) } - - // Process any remaining items in batch - if (currentBatchBlocks.length > 0) { - const release = await mutex.acquire() + const processFile = async (filePath: string) => { try { - // Copy current batch data and clear accumulators - const batchBlocks = [...currentBatchBlocks] - const batchTexts = [...currentBatchTexts] - const batchFileInfos = [...currentBatchFileInfos] - currentBatchBlocks = [] - currentBatchTexts = [] - currentBatchFileInfos = [] - - // Increment pending batch count for final batch - pendingBatchCount++ - - // Queue final batch processing - const batchPromise = batchLimiter(() => - this.processBatch( - batchBlocks, - batchTexts, - batchFileInfos, - scanWorkspace, - onError, - onBlocksIndexed, - signal, - failedFiles, - ), + checkActive() + const stats = await stat(filePath) + checkActive() + if (stats.size > MAX_FILE_SIZE_BYTES) { + selectedPaths.delete(filePath) + if (!trackedPaths.has(filePath)) incompleteFiles.delete(filePath) + skippedCount++ + return + } + const buffer = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) + checkActive() + const content = Buffer.from(buffer).toString("utf-8") + const hash = createHash("sha256").update(content).digest("hex") + if (this.cacheManager.getHash(filePath) === hash && !this.writeGate.needsReconciliation(filePath)) { + incompleteFiles.delete(filePath) + skippedCount++ + return + } + const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: hash, signal }) + checkActive() + processedCount++ + const selectedBlockCount = blocks.filter((block) => block.content.trim()).length + onFileParsed?.(selectedBlockCount) + if (!this.embedder || !this.qdrantClient) return + totalBlockCount += selectedBlockCount + await writer.replace( + filePath, + hash, + blocks, + this.embedder, + signal, + this.batchSegmentThreshold, + (count) => { + checkActive() + incompleteFiles.delete(filePath) + onBlocksIndexed?.(count) + }, ) - activeBatchPromises.add(batchPromise) - - // Clean up completed promises to prevent memory accumulation - batchPromise.finally(() => { - activeBatchPromises.delete(batchPromise) - pendingBatchCount-- - }) - } finally { - release() - } - } - - // Wait for all batch processing to complete - await Promise.all(activeBatchPromises) - - throwIfAborted(signal) - for (const [filePath, fileHash] of fileCompletions) { - if (failedFiles.has(filePath)) continue - await this.writeGate.run(signal, () => this.cacheManager.updateHash(filePath, fileHash), [filePath]) - this.writeGate.reconciled(filePath) - } - - // Check abort signal before handling deleted files - if (signal?.aborted) { - return { - stats: { - processed: processedCount, - skipped: skippedCount, - }, - totalBlockCount, + } catch (error) { + reportError(error, filePath) } } - - // Handle deleted files - const oldHashes = this.cacheManager.getAllHashes() - for (const cachedFilePath of Object.keys(oldHashes)) { - if (!processedFiles.has(cachedFilePath)) { - // File was deleted or is no longer supported/indexed - if (this.qdrantClient) { - try { - await this.writeGate.run( - signal, - () => this.qdrantClient.deletePointsByFilePath(cachedFilePath), - [cachedFilePath], - ) - await this.writeGate.run(signal, () => this.cacheManager.deleteHash(cachedFilePath), [ - cachedFilePath, - ]) - } catch (error: any) { - const errorMessage = error instanceof Error ? error.message : String(error) - - console.error( - `[DirectoryScanner] Failed to delete points for ${cachedFilePath} in workspace ${scanWorkspace}:`, - error, - ) - - if (onError) { - onError( - error instanceof Error - ? new Error( - `${error.message} (Workspace: ${scanWorkspace}, File: ${cachedFilePath})`, - ) - : new Error( - t("embeddings:scanner.unknownErrorDeletingPoints", { - filePath: cachedFilePath, - }) + ` (Workspace: ${scanWorkspace})`, - ), - ) - } - // Log error and continue processing instead of re-throwing - console.error(`Failed to delete points for removed file: ${cachedFilePath}`, errorMessage) - } + const processPaths = async (paths: string[], process: (filePath: string) => Promise) => { + let nextPath = 0 + const worker = async () => { + while (nextPath < paths.length) { + checkActive() + const filePath = paths[nextPath++] + await process(filePath) } } + const concurrency = Math.min(PARSING_CONCURRENCY, BATCH_PROCESSING_CONCURRENCY, paths.length) + await Promise.all(Array.from({ length: concurrency }, worker)) } + await processPaths([...selectedPaths], processFile) + checkActive() - return { - stats: { - processed: processedCount, - skipped: skippedCount, - }, - totalBlockCount, - } - } - - private async processBatch( - batchBlocks: CodeBlock[], - batchTexts: string[], - batchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[], - scanWorkspace: string, - onError?: (error: Error) => void, - onBlocksIndexed?: (indexedCount: number) => void, - signal = new AbortController().signal, - failedFiles = new Set(), - ): Promise { - if (batchBlocks.length === 0) return - - let attempts = 0 - let success = false - let lastError: Error | null = null - - while (attempts < MAX_BATCH_RETRIES && !success) { - attempts++ + const removedPaths = this.cacheManager.getTrackedPaths().filter((filePath) => !selectedPaths.has(filePath)) + for (const filePath of removedPaths) incompleteFiles.add(filePath) + await processPaths(removedPaths, async (filePath) => { try { - throwIfAborted(signal) - - // Create embeddings for batch - const { embeddings } = await this.embedder.createEmbeddings(batchTexts, undefined, signal) - throwIfAborted(signal) - - // Prepare points for Qdrant - const points = batchBlocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, scanWorkspace) - - // Use segmentHash for unique ID generation to handle multiple segments from same line - const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, scanWorkspace), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - segmentHash: block.segmentHash, - }, - } - }) - - // Upsert points to Qdrant - await this.writeGate.run(signal, () => this.qdrantClient.upsertPoints(points), [ - ...new Set(batchBlocks.map((block) => block.file_path)), - ]) - throwIfAborted(signal) - onBlocksIndexed?.(batchBlocks.length) - - // Update hashes for successfully processed files in this batch - success = true + checkActive() + if (!this.qdrantClient) return + await writer.remove(filePath, signal) + checkActive() + incompleteFiles.delete(filePath) } catch (error) { - if (signal.aborted || (error instanceof Error && error.name === "AbortError")) { - for (const block of batchBlocks) failedFiles.add(block.file_path) - return - } - lastError = error as Error - console.error( - `[DirectoryScanner] Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}:`, - error, - ) - - if (attempts < MAX_BATCH_RETRIES) { - const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1) - await new Promise((resolve) => setTimeout(resolve, delay)) - } - } - } - - if (!success && lastError) { - for (const block of batchBlocks) failedFiles.add(block.file_path) - console.error(`[DirectoryScanner] Failed to process batch after ${MAX_BATCH_RETRIES} attempts`) - if (onError) { - // Preserve the original error message from embedders which now have detailed i18n messages - const errorMessage = lastError.message || "Unknown error" - - // For other errors, provide context - onError( - new Error( - t("embeddings:scanner.failedToProcessBatchWithError", { - maxRetries: MAX_BATCH_RETRIES, - errorMessage, - }), - ), - ) + reportError(error, filePath) } + }) + checkActive() + return { + stats: { processed: processedCount, skipped: skippedCount }, + totalBlockCount, + incompleteFiles: [...incompleteFiles], } } } diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index 406f400880e..21f04389a94 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -23,6 +23,7 @@ import { DirectoryScanner, FileWatcher } from "./processors" import { ParseWorkerClient } from "./parse-worker-client" import { IndexWriteGate } from "./index-write-gate" import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" +import { GenerationAllocator } from "./generation-allocator" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" import { BATCH_SEGMENT_THRESHOLD } from "./constants" @@ -36,6 +37,7 @@ export class CodeIndexServiceFactory { private readonly workspacePath: string, private readonly cacheManager: CacheManager, private readonly writeGate = new IndexWriteGate(), + private readonly generationAllocator = new GenerationAllocator(cacheManager), ) {} /** @@ -194,6 +196,7 @@ export class CodeIndexServiceFactory { ignoreInstance, batchSize, this.writeGate, + this.generationAllocator, ) } @@ -230,6 +233,7 @@ export class CodeIndexServiceFactory { batchSize, parser, this.writeGate, + this.generationAllocator, ) } diff --git a/src/services/code-index/vector-store/README.integration.md b/src/services/code-index/vector-store/README.integration.md new file mode 100644 index 00000000000..50453fbeaa8 --- /dev/null +++ b/src/services/code-index/vector-store/README.integration.md @@ -0,0 +1,56 @@ +# T23: real Qdrant verification + +## Required MANUAL acceptance + +T23 is required before acceptance and must run against real Qdrant; mocks and health +checks do not satisfy it. The PR author runs the explicit integration suite below +against the proposed changes before acceptance and records the command, Qdrant version, +actual output, and exit code. The reviewer checks those results before acceptance. +A failed or unrun T23 blocks acceptance; historical results do not replace this run. +This is a manual acceptance requirement, not a CI job. + +Run from the existing checkout. Docker commands run at repository root: + +```shell +docker run --detach --rm --name roo-vector-t23 -p 127.0.0.1:16333:6333 qdrant/qdrant:v1.17.0 +``` + +Once Qdrant is ready, run from `src`: + +```shell +npx vitest run --config services/code-index/vector-store/vitest.integration.config.ts +``` + +Stop only the container created for this run: + +```shell +docker stop roo-vector-t23 +``` + +The harness uses unique workspace-derived collections and deletes them after each case. +It holds actual delete HTTP requests in a local proxy until after a retry's writes and +verification, then forwards the original request to Qdrant. No Qdrant calls are mocked. +Only translation is stubbed. Tests assert legacy cleanup, exact current/other counts, +exact file matching, generation index creation, delayed cleanup/removal fencing, and +visibility of a late old upsert without loss of current points. + +`QDRANT_INTEGRATION_URL` overrides the default `http://127.0.0.1:16333`; the delay proxy +requires an HTTP endpoint. An unavailable server fails the suite, never skips it. +The `.integration.ts` suffix is intentionally outside default Vitest unit discovery; +only the explicit integration configuration includes these tests. + +## Observed compatibility limitation + +Verified on 2026-10-30 with the installed client 1.14.0: + +- `npx vitest run --config services/code-index/vector-store/vitest.integration.config.ts` + against the Docker `qdrant/qdrant:v1.17.0` image reported `4 passed`. + The client emitted its version-compatibility warning; that check was not disabled. +- The same command against `qdrant/qdrant:v1.14.1` failed intermittently: after an + awaited fenced delete, exact counts sometimes reported `current: 2, other: 1`. + A diagnostic scroll showed the retained point had no `generation` field. + This is a failed real-server premise, not acceptance evidence for that version. + No retries or weakened assertions conceal the failure. + +The local late-upsert measurement is only one completed local request, not evidence +that arbitrary abandoned writes settle within the late-write grace period. diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.integration.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.integration.ts new file mode 100644 index 00000000000..3e25b5e0629 --- /dev/null +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.integration.ts @@ -0,0 +1,147 @@ +import { QdrantClient } from "@qdrant/js-client-rest" +import { createHash, randomUUID } from "node:crypto" +import { createServer, request as httpRequest, Server } from "node:http" +import type { AddressInfo } from "node:net" +import path from "node:path" +import { QdrantVectorStore } from "../qdrant-client" + +vi.mock("../../../../i18n", () => ({ t: (key: string) => key })) + +function deferred() { + let resolve!: () => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +const qdrantUrl = process.env.QDRANT_INTEGRATION_URL ?? "http://127.0.0.1:16333" +const client = new QdrantClient({ url: qdrantUrl }) +let store: QdrantVectorStore +let collectionName: string +let workspacePath: string +let proxy: Server +let delayedStore: QdrantVectorStore +let heldDelete: { arrived: ReturnType; release: ReturnType } | undefined + +beforeEach(async () => { + workspacePath = path.resolve(`qdrant-t23-${randomUUID()}`) + collectionName = `ws-${createHash("sha256").update(workspacePath).digest("hex").substring(0, 16)}` + store = new QdrantVectorStore(workspacePath, qdrantUrl, 2) + await store.initialize() + proxy = createServer(async (incoming, outgoing) => { + if (incoming.url?.includes("/points/delete") && heldDelete) { + const held = heldDelete + held.arrived.resolve() + await held.release.promise + } + const upstream = httpRequest( + new URL(incoming.url!, qdrantUrl), + { + method: incoming.method, + headers: { ...incoming.headers, host: new URL(qdrantUrl).host }, + }, + (response) => { + outgoing.writeHead(response.statusCode!, response.headers) + response.pipe(outgoing) + }, + ) + upstream.on("error", (error) => { + outgoing.writeHead(502) + outgoing.end(error.message) + }) + incoming.pipe(upstream) + }) + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)) + const proxyPort = (proxy.address() as AddressInfo).port + delayedStore = new QdrantVectorStore(workspacePath, `http://127.0.0.1:${proxyPort}`, 2) +}) + +afterEach(async () => { + heldDelete?.release.resolve() + heldDelete = undefined + if (proxy) await new Promise((resolve, reject) => proxy.close((error) => (error ? reject(error) : resolve()))) + await client.deleteCollection(collectionName) +}) + +function point(filePath: string, generation?: number) { + return { + id: randomUUID(), + vector: [1, 0], + payload: { filePath, ...(generation === undefined ? {} : { generation }) }, + } +} + +describe("T23 real Qdrant generation fencing", () => { + it("creates the integer index and exact counts include legacy, older, and newer points", async () => { + const info = await client.getCollection(collectionName) + expect(info.payload_schema?.generation?.data_type).toBe("integer") + expect(info.payload_schema?.filePath?.data_type).toBe("keyword") + expect(info.payload_schema?.type?.data_type).toBe("keyword") + for (let segment = 0; segment <= 4; segment++) { + expect(info.payload_schema?.[`pathSegments.${segment}`]?.data_type).toBe("keyword") + } + await store.upsertPoints([ + point("file.ts"), + point("file.ts", 1), + point("file.ts", 2), + point("file.ts", 2), + point("file.ts", 3), + point(path.join("file.ts", "child"), 1), + point("unrelated.ts", 2), + ]) + expect(await store.countPointsForFile(path.join(workspacePath, "file.ts"), 2)).toEqual({ current: 2, other: 3 }) + await store.deletePointsByFilePath("file.ts", 2) + expect(await store.countPointsForFile("file.ts", 2)).toEqual({ current: 2, other: 1 }) + expect(await store.countPointsForFile(path.join("file.ts", "child"), 1)).toEqual({ current: 1, other: 0 }) + expect(await store.countPointsForFile("unrelated.ts", 2)).toEqual({ current: 1, other: 0 }) + expect(await store.countPointsForFile("missing.ts", 2)).toEqual({ current: 0, other: 0 }) + await store.deletePointsByMultipleFilePaths([path.join("file.ts", "child"), "unrelated.ts"], 3) + expect(await store.countPointsForFile(path.join("file.ts", "child"), 1)).toEqual({ current: 0, other: 0 }) + expect(await store.countPointsForFile("unrelated.ts", 2)).toEqual({ current: 0, other: 0 }) + }) + + it.each(["cleanup", "removal"])( + "a delayed old %s executes after retry verification without removing new points", + async () => { + await store.upsertPoints([point("file.ts"), point("file.ts", 1)]) + heldDelete = { arrived: deferred(), release: deferred() } + const pendingDelete = delayedStore.deletePointsByFilePath("file.ts", 2) + await heldDelete.arrived.promise + try { + await store.upsertPoints([point("file.ts", 3), point("file.ts", 3)]) + await store.deletePointsByFilePath("file.ts", 3) + expect(await store.countPointsForFile("file.ts", 3)).toEqual({ current: 2, other: 0 }) + // These older points arrive after the request was issued, but before its remote execution. + await store.upsertPoints([point("file.ts"), point("file.ts", 1), point("unrelated.ts", 1)]) + } finally { + heldDelete.release.resolve() + await pendingDelete + } + const counts = await store.countPointsForFile("file.ts", 3) + if (counts.other !== 0) { + console.log( + "T23 unexpected retained points", + JSON.stringify(await client.scroll(collectionName, { with_payload: true })), + ) + } + expect(counts).toEqual({ current: 2, other: 0 }) + expect(await store.countPointsForFile("unrelated.ts", 1)).toEqual({ current: 1, other: 0 }) + }, + ) + + it("detects a late old upsert without overwriting current generation IDs", async () => { + const currentPoint = point("file.ts", 3) + await store.upsertPoints([currentPoint]) + await store.deletePointsByFilePath("file.ts", 3) + expect(await store.countPointsForFile("file.ts", 3)).toEqual({ current: 1, other: 0 }) + const startedAt = performance.now() + await store.upsertPoints([point("file.ts", 1)]) + expect(await store.countPointsForFile("file.ts", 3)).toEqual({ current: 1, other: 1 }) + const retained = await client.retrieve(collectionName, { ids: [currentPoint.id], with_payload: true }) + expect(retained[0].payload?.generation).toBe(3) + console.log(`T23 local late-upsert visibility: ${(performance.now() - startedAt).toFixed(2)} ms`) + await store.deletePointsByFilePath("file.ts", 3) + expect(await store.countPointsForFile("file.ts", 3)).toEqual({ current: 1, other: 0 }) + }) +}) diff --git a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts index 44ba0c80868..9b9e6f98b45 100644 --- a/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts +++ b/src/services/code-index/vector-store/__tests__/qdrant-client.spec.ts @@ -38,6 +38,7 @@ const mockQdrantClientInstance = { upsert: vitest.fn(), query: vitest.fn(), delete: vitest.fn(), + count: vitest.fn(), } const mockCreateHashInstance = { @@ -75,7 +76,94 @@ describe("QdrantVectorStore", () => { vi.spyOn(vectorStore, "collectionExists").mockResolvedValue(true) const failure = new Error("delete request timed out") mockQdrantClientInstance.delete.mockRejectedValueOnce(failure) - await expect(vectorStore.deletePointsByFilePath("/test/workspace/file.ts")).rejects.toBe(failure) + await expect(vectorStore.deletePointsByFilePath("/test/workspace/file.ts", 17)).rejects.toBe(failure) + }) + + describe("generation fencing and exact counts", () => { + it("matches exact relative paths and only lower or legacy generations", async () => { + mockQdrantClientInstance.delete.mockResolvedValue({}) + await vectorStore.deletePointsByMultipleFilePaths(["/test/workspace/file.ts", "other.ts"], 17) + expect(mockQdrantClientInstance.delete).toHaveBeenCalledWith(expectedCollectionName, { + filter: { + must: [ + { key: "filePath", match: { any: ["file.ts", "other.ts"] } }, + { should: [{ key: "generation", range: { lt: 17 } }, { is_empty: { key: "generation" } }] }, + ], + }, + wait: true, + }) + expect(mockQdrantClientInstance.getCollection).not.toHaveBeenCalled() + }) + + it.each([undefined, NaN, Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( + "refuses an unsafe or missing cutoff %s before issuing a delete", + async (cutoff) => { + await expect(vectorStore.deletePointsByFilePath("file.ts", cutoff as number)).rejects.toThrow("cutoff") + expect(mockQdrantClientInstance.delete).not.toHaveBeenCalled() + }, + ) + + it("does not issue an empty multi-file deletion", async () => { + await vectorStore.deletePointsByMultipleFilePaths([], 17) + expect(mockQdrantClientInstance.delete).not.toHaveBeenCalled() + }) + + it("awaits remote deletion completion", async () => { + let finishDelete!: () => void + mockQdrantClientInstance.delete.mockReturnValueOnce( + new Promise((resolve) => { + finishDelete = resolve + }), + ) + let finished = false + const pending = vectorStore.deletePointsByFilePath("file.ts", 17).then(() => { + finished = true + }) + await Promise.resolve() + expect(finished).toBe(false) + finishDelete() + await pending + expect(finished).toBe(true) + }) + + it("counts the current generation separately from all others, including legacy", async () => { + mockQdrantClientInstance.count.mockResolvedValueOnce({ count: 2 }).mockResolvedValueOnce({ count: 3 }) + await expect(vectorStore.countPointsForFile("/test/workspace/file.ts", 17)).resolves.toEqual({ + current: 2, + other: 3, + }) + const fileCondition = { key: "filePath", match: { value: "file.ts" } } + const generationCondition = { key: "generation", match: { value: 17 } } + expect(mockQdrantClientInstance.count.mock.calls).toEqual([ + [expectedCollectionName, { filter: { must: [fileCondition, generationCondition] }, exact: true }], + [ + expectedCollectionName, + { filter: { must: [fileCondition], must_not: [generationCondition] }, exact: true }, + ], + ]) + }) + + it.each([0, 1])("propagates failure of count query %s instead of reporting completion", async (failedQuery) => { + const failure = new Error("count unavailable") + mockQdrantClientInstance.count.mockReset().mockResolvedValue({ count: 0 }) + if (failedQuery === 1) mockQdrantClientInstance.count.mockResolvedValueOnce({ count: 2 }) + mockQdrantClientInstance.count.mockRejectedValueOnce(failure) + await expect(vectorStore.countPointsForFile("file.ts", 17)).rejects.toBe(failure) + }) + + it("preserves caller-provided generation-scoped IDs and payloads", async () => { + mockQdrantClientInstance.upsert.mockResolvedValue({}) + const point = { + id: "generation-scoped-id", + vector: [1, 0], + payload: { filePath: "file.ts", generation: 17 }, + } + await vectorStore.upsertPoints([point]) + expect(mockQdrantClientInstance.upsert).toHaveBeenCalledWith(expectedCollectionName, { + points: [{ ...point, payload: { ...point.payload, pathSegments: { "0": "file.ts" } } }], + wait: true, + }) + }) }) it("should correctly initialize QdrantClient and collectionName in constructor", () => { @@ -553,14 +641,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) }) it("should not create a new collection if one exists with matching vectorSize and return false", async () => { // Mock getCollection to return existing collection info with matching vector size @@ -587,14 +677,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) }) it("should recreate collection if it exists but vectorSize mismatches and return true", async () => { const differentVectorSize = 768 @@ -644,14 +736,16 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) for (let i = 0; i <= 4; i++) { expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() // Restore console.warn }) it("should log warning for non-404 errors but still create collection", async () => { @@ -665,7 +759,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.deleteCollection).not.toHaveBeenCalled() - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) expect(console.warn).toHaveBeenCalledWith( expect.stringContaining(`Warning during getCollectionInfo for "${expectedCollectionName}"`), genericError.message, @@ -693,43 +787,34 @@ describe("QdrantVectorStore", () => { expect(console.error).toHaveBeenCalledTimes(1) // Only the outer try/catch ;(console.error as any).mockRestore() }) - it("should log but not fail if payload index creation errors occur", async () => { - // Mock successful collection creation - mockQdrantClientInstance.getCollection.mockRejectedValue({ - response: { status: 404 }, - message: "Not found", + it.each(["filePath", "generation"])("fails initialization when the %s index fails", async (fieldName) => { + mockQdrantClientInstance.getCollection.mockResolvedValue({ + config: { params: { vectors: { size: mockVectorSize } } }, + }) + mockQdrantClientInstance.createPayloadIndex.mockImplementation(async (_collection, request) => { + if (request.field_name === fieldName) throw new Error(`${fieldName} index failed`) + return {} }) - mockQdrantClientInstance.createCollection.mockResolvedValue(true as any) - - // Mock payload index creation to fail - const indexError = new Error("Index creation failed") - mockQdrantClientInstance.createPayloadIndex.mockRejectedValue(indexError) - vitest.spyOn(console, "warn").mockImplementation(() => {}) // Suppress console.warn - - const result = await vectorStore.initialize() - - // Should still return true since main collection setup succeeded - expect(result).toBe(true) - expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - - // Verify all payload index creations were attempted (6: type + 5 pathSegments) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) - // Verify warnings were logged for each failed index (now 6) - expect(console.warn).toHaveBeenCalledTimes(6) - // Verify warning for 'type' index - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Could not create payload index for type`), - indexError.message, - ) - for (let i = 0; i <= 4; i++) { - expect(console.warn).toHaveBeenCalledWith( - expect.stringContaining(`Could not create payload index for pathSegments.${i}`), - indexError.message, - ) - } + await expect(vectorStore.initialize()).rejects.toThrow(`${fieldName} index failed`) + }) - ;(console.warn as any).mockRestore() + it("creates awaited filePath and generation indexes on an existing collection", async () => { + mockQdrantClientInstance.getCollection.mockResolvedValue({ + config: { params: { vectors: { size: mockVectorSize } } }, + }) + mockQdrantClientInstance.createPayloadIndex.mockResolvedValue({}) + await vectorStore.initialize() + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { + field_name: "filePath", + field_schema: "keyword", + wait: true, + }) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledWith(expectedCollectionName, { + field_name: "generation", + field_schema: "integer", + wait: true, + }) }) it("should throw vectorDimensionMismatch error when deleteCollection fails during recreation", async () => { @@ -850,7 +935,7 @@ describe("QdrantVectorStore", () => { expect(mockQdrantClientInstance.getCollection).toHaveBeenCalledTimes(2) expect(mockQdrantClientInstance.deleteCollection).toHaveBeenCalledTimes(1) expect(mockQdrantClientInstance.createCollection).toHaveBeenCalledTimes(1) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() }) @@ -947,7 +1032,7 @@ describe("QdrantVectorStore", () => { on_disk: true, }, }) - expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(6) + expect(mockQdrantClientInstance.createPayloadIndex).toHaveBeenCalledTimes(8) ;(console.warn as any).mockRestore() }) diff --git a/src/services/code-index/vector-store/qdrant-client.ts b/src/services/code-index/vector-store/qdrant-client.ts index 06a3caae715..f69d6b51a11 100644 --- a/src/services/code-index/vector-store/qdrant-client.ts +++ b/src/services/code-index/vector-store/qdrant-client.ts @@ -296,11 +296,24 @@ export class QdrantVectorStore implements IVectorStore { * Creates payload indexes for the collection, handling errors gracefully. */ private async _createPayloadIndexes(): Promise { + // Completion verification depends on these indexes; initialization must fail if creation fails. + await this.client.createPayloadIndex(this.collectionName, { + field_name: "filePath", + field_schema: "keyword", + wait: true, + }) + await this.client.createPayloadIndex(this.collectionName, { + field_name: "generation", + field_schema: "integer", + wait: true, + }) + // Create index for the 'type' field to enable metadata filtering try { await this.client.createPayloadIndex(this.collectionName, { field_name: "type", field_schema: "keyword", + wait: true, }) } catch (indexError: any) { const errorMessage = (indexError?.message || "").toLowerCase() @@ -318,6 +331,7 @@ export class QdrantVectorStore implements IVectorStore { await this.client.createPayloadIndex(this.collectionName, { field_name: `pathSegments.${i}`, field_schema: "keyword", + wait: true, }) } catch (indexError: any) { const errorMessage = (indexError?.message || "").toLowerCase() @@ -467,77 +481,50 @@ export class QdrantVectorStore implements IVectorStore { } } - /** - * Deletes points by file path - * @param filePath Path of the file to delete points for - */ - async deletePointsByFilePath(filePath: string): Promise { - return this.deletePointsByMultipleFilePaths([filePath]) + private relativeFilePath(filePath: string): string { + return path.normalize(path.isAbsolute(filePath) ? path.relative(this.workspacePath, filePath) : filePath) } - async deletePointsByMultipleFilePaths(filePaths: string[]): Promise { - if (filePaths.length === 0) { - return - } - - try { - // First check if the collection exists - const collectionExists = await this.collectionExists() - if (!collectionExists) { - console.warn( - `[QdrantVectorStore] Skipping deletion - collection "${this.collectionName}" does not exist`, - ) - return - } - - const workspaceRoot = this.workspacePath - - // Build filters using pathSegments to match the indexed fields - const filters = filePaths.map((filePath) => { - // IMPORTANT: Use the relative path to match what's stored in upsertPoints - // upsertPoints stores the relative filePath, not the absolute path - const relativePath = path.isAbsolute(filePath) ? path.relative(workspaceRoot, filePath) : filePath - - // Normalize the relative path - const normalizedRelativePath = path.normalize(relativePath) - - // Split the path into segments like we do in upsertPoints - const segments = normalizedRelativePath.split(path.sep).filter(Boolean) - - // Create a filter that matches all segments of the path - // This ensures we only delete points that match the exact file path - const mustConditions = segments.map((segment, index) => ({ - key: `pathSegments.${index}`, - match: { value: segment }, - })) + async deletePointsByFilePath(filePath: string, beforeGeneration: number): Promise { + return this.deletePointsByMultipleFilePaths([filePath], beforeGeneration) + } - return { must: mustConditions } - }) + async deletePointsByMultipleFilePaths(filePaths: string[], beforeGeneration: number): Promise { + if (!Number.isSafeInteger(beforeGeneration) || beforeGeneration < 0) { + throw new Error("A non-negative safe integer generation cutoff is required") + } + if (filePaths.length === 0) return - // Use 'should' to match any of the file paths (OR condition) - const filter = filters.length === 1 ? filters[0] : { should: filters } + await this.client.delete(this.collectionName, { + filter: { + must: [ + { key: "filePath", match: { any: filePaths.map((filePath) => this.relativeFilePath(filePath)) } }, + { + should: [ + { key: "generation", range: { lt: beforeGeneration } }, + { is_empty: { key: "generation" } }, + ], + }, + ], + }, + wait: true, + }) + } - await this.client.delete(this.collectionName, { - filter, - wait: true, - }) - } catch (error: any) { - // Extract more detailed error information - const errorMessage = error?.message || String(error) - const errorStatus = error?.status || error?.response?.status || error?.statusCode - const errorDetails = error?.response?.data || error?.data || "" - - console.error(`[QdrantVectorStore] Failed to delete points by file paths:`, { - error: errorMessage, - status: errorStatus, - details: errorDetails, - collection: this.collectionName, - fileCount: filePaths.length, - // Include first few file paths for debugging (avoid logging too many) - samplePaths: filePaths.slice(0, 3), - }) - throw error - } + async countPointsForFile(filePath: string, generation: number): Promise<{ current: number; other: number }> { + const fileCondition = { key: "filePath", match: { value: this.relativeFilePath(filePath) } } + const generationCondition = { key: "generation", match: { value: generation } } + const [current, other] = await Promise.all([ + this.client.count(this.collectionName, { + filter: { must: [fileCondition, generationCondition] }, + exact: true, + }), + this.client.count(this.collectionName, { + filter: { must: [fileCondition], must_not: [generationCondition] }, + exact: true, + }), + ]) + return { current: current.count, other: other.count } } /** diff --git a/src/services/code-index/vector-store/vitest.integration.config.ts b/src/services/code-index/vector-store/vitest.integration.config.ts new file mode 100644 index 00000000000..17477704e23 --- /dev/null +++ b/src/services/code-index/vector-store/vitest.integration.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + globals: true, + include: ["services/code-index/vector-store/__tests__/*.integration.ts"], + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}) diff --git a/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts b/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts index 80013a0119b..3ba877d1ef6 100644 --- a/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts +++ b/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts @@ -810,7 +810,7 @@ describe("useSelectedModel", () => { }) it.each([ - ["gpt-6-astra", 271789, true], + ["gpt-6-astra", 266789, true], ["grok-4.5", 199783, false], ["grok-4.6", 199784, false], ["gemini-3.7-flash", 935783, true], From 78b3a1099d2dbcebd87a6d761f3a18563edd0be1 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 1 Nov 2026 21:19:21 -0800 Subject: [PATCH 3/4] fix(code-index): wait for in-flight Clear Index before recreating services --- .../code-index/__tests__/manager.spec.ts | 35 +++++++++++++++++++ src/services/code-index/manager.ts | 7 +++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index c02a2ac9d9a..60235b3f5f0 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -305,6 +305,41 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { internal._orchestrator = undefined }) + it.each(["resolves", "rejects"])("retry waits for an in-flight clear that %s", async (outcome) => { + const internal = manager as any + internal._configManager = { isFeatureEnabled: true, isFeatureConfigured: true } + internal._cacheManager = {} + internal._searchService = {} + let settle!: () => void + const pendingClear = new Promise((resolve, reject) => { + settle = outcome === "resolves" ? resolve : () => reject(new Error("delete failed")) + }) + const old = { clearIndexData: vi.fn(() => pendingClear), stopWatcher: vi.fn() } + internal._orchestrator = old + const createServices = vi.fn(() => ({ + embedder: {}, + vectorStore: {}, + scanner: {}, + fileWatcher: { dispose: vi.fn() }, + })) + MockedCodeIndexServiceFactory.mockImplementation( + () => ({ createServices, validateEmbedder: vi.fn().mockResolvedValue({ valid: true }) }) as any, + ) + const clear = manager.clearIndexData().catch(() => {}) + internal.servicesStopped = true + const retry = manager.startIndexing() + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(createServices).not.toHaveBeenCalled() + expect(internal._orchestrator).toBeUndefined() + settle() + await clear + await retry + expect(createServices).toHaveBeenCalledOnce() + expect(internal._orchestrator).not.toBe(old) + expect(internal._orchestrator.startIndexing).toHaveBeenCalledOnce() + internal._orchestrator = undefined + }) + it("preserves the shared allocator across recreation and fences an old attempt's callbacks", async () => { const internal = manager as any internal._cacheManager = {} diff --git a/src/services/code-index/manager.ts b/src/services/code-index/manager.ts index 20948b74302..6753121b28c 100644 --- a/src/services/code-index/manager.ts +++ b/src/services/code-index/manager.ts @@ -39,6 +39,7 @@ export class CodeIndexManager { private generationAllocator?: GenerationAllocator private currentAttempt = 0 private readonly idleStop = Promise.resolve() + private clearing: Promise = Promise.resolve() public static getInstance(context: vscode.ExtensionContext, workspacePath?: string): CodeIndexManager | undefined { // Resolve the workspace folder to get both fsPath and the real URI @@ -350,7 +351,9 @@ export class CodeIndexManager { return } this.assertInitialized() - await this._orchestrator!.clearIndexData() + const clearing = this._orchestrator!.clearIndexData() + this.clearing = clearing.catch(() => {}) + await clearing } // --- Private Helpers --- @@ -420,6 +423,8 @@ export class CodeIndexManager { // Create RooIgnoreController instance const rooIgnoreController = new RooIgnoreController(workspacePath) await rooIgnoreController.initialize() + // Clear publishes Standby before deleteCollection() settles; a retry must not write concurrently. + await this.clearing if (!isCurrentAttempt()) return From 9922e834b1c81c51a00ea9d5071a92d9250dad1f Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 1 Nov 2026 21:59:59 -0800 Subject: [PATCH 4/4] test(code-index): make clear-index retry synchronization deterministic --- src/services/code-index/__tests__/manager.spec.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/services/code-index/__tests__/manager.spec.ts b/src/services/code-index/__tests__/manager.spec.ts index 60235b3f5f0..7c454901d93 100644 --- a/src/services/code-index/__tests__/manager.spec.ts +++ b/src/services/code-index/__tests__/manager.spec.ts @@ -326,9 +326,22 @@ describe("CodeIndexManager - handleSettingsChange regression", () => { () => ({ createServices, validateEmbedder: vi.fn().mockResolvedValue({ valid: true }) }) as any, ) const clear = manager.clearIndexData().catch(() => {}) + // Signal when the retry actually awaits the clear, instead of guessing with a sleep. + const realClearing = internal.clearing + let signalAwaiting!: () => void + const retryAwaitingClear = new Promise((resolve) => (signalAwaiting = resolve)) + internal.clearing = { + then: (onFulfilled: any, onRejected: any) => { + signalAwaiting() + return realClearing.then(onFulfilled, onRejected) + }, + } internal.servicesStopped = true const retry = manager.startIndexing() - await new Promise((resolve) => setTimeout(resolve, 20)) + let retryFinished = false + await Promise.race([retryAwaitingClear, retry.then(() => (retryFinished = true))]) + expect(retryFinished).toBe(false) + for (let tick = 0; tick < 10; tick++) await Promise.resolve() expect(createServices).not.toHaveBeenCalled() expect(internal._orchestrator).toBeUndefined() settle()