diff --git a/CLAUDE.md b/CLAUDE.md index 5a40e78..320e017 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,7 +103,7 @@ tree. `ci`) enforces a monotonic floor and blocks new .js under `COVERED_DIRS` (`src/shared`, `src/preload`, `src/main/{ipc,services,analysis,errors,utils,workers,transports,core}`, `src/renderer/{services,hooks,phases,store,utils,components}` today (every `.js` under `src/` is - now opted in — 493/493 as of 2026-11-09, and `GLOBAL_DIRECTIVE_FLOOR` matches at 493) — see + now opted in — 494/494 as of 2026-11-11, and `GLOBAL_DIRECTIVE_FLOOR` matches at 494) — see `scripts/verify-checkjs-ratchet.js` for the authoritative list) from landing without the directive. When adding a file to a covered dir: put `// @ts-check` on line 1 and fix the errors before commit. When opting in a new file elsewhere: bump `GLOBAL_DIRECTIVE_FLOOR` in diff --git a/scripts/verify-checkjs-ratchet.js b/scripts/verify-checkjs-ratchet.js index d80eb91..b7db8e0 100644 --- a/scripts/verify-checkjs-ratchet.js +++ b/scripts/verify-checkjs-ratchet.js @@ -43,7 +43,7 @@ const COVERED_DIRS = [ 'src/renderer/utils', 'src/renderer/components' ]; -const GLOBAL_DIRECTIVE_FLOOR = 493; +const GLOBAL_DIRECTIVE_FLOOR = 494; // ──────────────────────────────────────────────────────────────────────── const DIRECTIVE_SCAN_LINES = 5; diff --git a/src/main/analysis/documentAnalysis.js b/src/main/analysis/documentAnalysis.js index f50880d..588a723 100644 --- a/src/main/analysis/documentAnalysis.js +++ b/src/main/analysis/documentAnalysis.js @@ -50,9 +50,10 @@ const { const { analyzeTextWithLlama, normalizeCategoryToSmartFolders } = require('./documentLlm'); const { getOrganizationGuidance } = require('./organizationGuidance'); const { getAnalysisLimits } = require('./analysisLimits'); -const { normalizeAnalysisResult } = require('./utils'); +const { normalizeAnalysisResult, dropPromptEchoes } = require('./utils'); const { normalizeExtractedTextForStorage } = require('./analysisTextUtils'); const { + FILENAME_ONLY_CONFIDENCE, getIntelligentCategory, getIntelligentKeywords, safeSuggestedName, @@ -733,7 +734,8 @@ async function analyzeDocumentFile(filePath, smartFolders = [], options = {}) { const intelligentKeywords = getIntelligentKeywords(fileName, fileExtension); let purpose = 'Office document (content extraction failed)'; - const confidence = 70; + // The contents could not be read: a filename guess, flagged for review. + const confidence = FILENAME_ONLY_CONFIDENCE; // A reader this build lacks, or a password, is not a capability a // model install restores: stamp a method the watcher never // retries (analysisProvenance.format_unsupported). @@ -848,7 +850,17 @@ async function analyzeDocumentFile(filePath, smartFolders = [], options = {}) { ? listedKeywords : getIntelligentKeywords(fileName, fileExtension); - const category = normalizeCategoryToSmartFolders('archive', smartFolders); + // File an archive by what it holds: a zip of .stl parts belongs with + // 3D prints, not under the generic 'archive' (which only ever reached + // Uncategorized). Taken only when the entries' dominant type lands on + // one of the user's own folders. + const contentFolder = archiveInfo.dominantExtension + ? findSmartFolderName( + getIntelligentCategory(fileName, `.${archiveInfo.dominantExtension}`, smartFolders), + smartFolders + ) + : null; + const category = contentFolder || normalizeCategoryToSmartFolders('archive', smartFolders); return { purpose: archiveInfo.summary || 'Archive file', @@ -856,7 +868,9 @@ async function analyzeDocumentFile(filePath, smartFolders = [], options = {}) { category, date: fileDate, keywords, - confidence: 70, + // Only the entry names were read. A folder matched on them is a fair + // guess; the generic 'archive' bucket is not one. + confidence: contentFolder ? 70 : FILENAME_ONLY_CONFIDENCE, suggestedName: safeSuggestedName(fileName, fileExtension), extractionMethod: 'archive' }; @@ -1032,7 +1046,7 @@ async function analyzeDocumentFile(filePath, smartFolders = [], options = {}) { }); const normalized = normalizeAnalysisResult( { - ...analysis, + ...dropPromptEchoes(analysis, smartFolders), llmSuggestedName: analysis.suggestedName || null, contentLength: extractedTextLength, extractionMethod: 'content', @@ -1226,6 +1240,7 @@ async function tryExtractArchiveMetadata(filePath) { const listing = await describeArchive(filePath); if (listing) { info.keywords = deriveKeywordsFromFilenames(listing.entryNames); + info.dominantExtension = dominantEntryExtension(listing.entryNames); info.summary = listing.summary; if (typeof listing.totalEntries === 'number') info.totalEntries = listing.totalEntries; info.processedEntries = listing.entryNames.length; @@ -1251,6 +1266,7 @@ async function tryExtractArchiveMetadata(filePath) { const sampledEntries = allEntries.slice(0, 50); const names = sampledEntries.map((/** @type {any} */ e) => e.entryName); info.keywords = deriveKeywordsFromFilenames(names); + info.dominantExtension = dominantEntryExtension(names); info.totalEntries = totalEntries; info.processedEntries = sampledEntries.length; info.summary = `ZIP archive with ${totalEntries} entries`; @@ -1267,6 +1283,53 @@ async function tryExtractArchiveMetadata(filePath) { return info; } +/** + * The extension that at least half of an archive's file entries share, or + * null when no single type dominates. Folder entries are ignored. + * @param {string[]} names + * @returns {string|null} + */ +function dominantEntryExtension(names) { + /** @type {Record} */ + const counts = {}; + let files = 0; + for (const name of names || []) { + const base = String(name).split(/[\\/]/).pop() || ''; + if (!base) continue; // folder entry ("parts/") + files += 1; + const dot = base.lastIndexOf('.'); + if (dot <= 0 || dot === base.length - 1) continue; + const ext = base.slice(dot + 1).toLowerCase(); + counts[ext] = (counts[ext] || 0) + 1; + } + let best = null; + let bestCount = 0; + for (const [ext, count] of Object.entries(counts)) { + if (count > bestCount) { + best = ext; + bestCount = count; + } + } + return best && bestCount * 2 >= files ? best : null; +} + +/** + * The user's folder whose name equals `candidate` (case-insensitive), or null. + * Uncategorized does not count: it is the absence of a match. + * @param {string|null|undefined} candidate + * @param {any[]} smartFolders + * @returns {string|null} + */ +function findSmartFolderName(candidate, smartFolders) { + if (typeof candidate !== 'string' || !candidate) return null; + const wanted = candidate.trim().toLowerCase(); + if (!wanted || wanted === 'uncategorized') return null; + const folder = (smartFolders || []).find( + (f) => typeof f?.name === 'string' && f.name.trim().toLowerCase() === wanted + ); + return folder ? folder.name : null; +} + /** @param {string[]} names */ function deriveKeywordsFromFilenames(names) { /** @type {Record} */ diff --git a/src/main/analysis/fallbackUtils.js b/src/main/analysis/fallbackUtils.js index ef7d071..bb707ae 100644 --- a/src/main/analysis/fallbackUtils.js +++ b/src/main/analysis/fallbackUtils.js @@ -22,6 +22,47 @@ const FolderMatchingService = require('../services/FolderMatchingService'); const { createLogger } = require('../../shared/logger'); const logger = createLogger('FallbackUtils'); +/** + * Confidence for a result built from the file name alone (nothing inside the + * file was read). Below auto-organize's default threshold (75%) and in the + * history view's warning band (50-69%), so these files ask for a look instead + * of reading as settled. They were reported at 65-75, above real content + * analyses that scored lower. + */ +const FILENAME_ONLY_CONFIDENCE = 50; + +/** + * Lowercased words of a file name, space-joined and space-padded so a phrase + * can be found as whole words: "AR+15_foregrip-stls.zip" -> " ar 15 foregrip stls zip ". + * @param {string} fileName + * @returns {string} + */ +function toWordString(fileName) { + const words = String(fileName || '') + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter(Boolean); + return ` ${words.join(' ')} `; +} + +/** + * Whether `phrase` appears in the file name as whole words, allowing a plural + * on its last word ("invoice" matches "invoices"). Substring matching filed + * "network" under a "Work" folder and "barcode" under "code". + * @param {string} wordString - From toWordString + * @param {string} phrase - Lowercased word or phrase + * @returns {boolean} + */ +function nameHasWords(wordString, phrase) { + const words = String(phrase || '') + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter(Boolean); + if (words.length === 0) return false; + const joined = words.join(' '); + return [joined, `${joined}s`, `${joined}es`].some((form) => wordString.includes(` ${form} `)); +} + const normalizeCategoryToSmartFolders = typeof FolderMatchingService?.matchCategoryToFolder === 'function' ? FolderMatchingService.matchCategoryToFolder.bind(FolderMatchingService) @@ -67,6 +108,9 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { const extNoDot = lowerExtension.replace(/^\./, ''); if (smartFolders && smartFolders.length > 0) { + // Folder words are matched against the name's whole words (see + // nameHasWords); the generic pattern table further down is unchanged. + const nameWords = toWordString(fileName); const validFolders = smartFolders.filter( (f) => f && f.name && typeof f.name === 'string' && f.name.trim().length > 0 ); @@ -75,11 +119,11 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { for (const folder of validFolders) { const folderNameLower = folder.name.toLowerCase(); let score = 0; - if (lowerFileName.includes(folderNameLower)) score += 10; + if (nameHasWords(nameWords, folderNameLower)) score += 10; const folderWords = folderNameLower .split(/[\s_-]+/) .filter((/** @type {string} */ w) => w.length > 2); - for (const word of folderWords) if (lowerFileName.includes(word)) score += 8; + for (const word of folderWords) if (nameHasWords(nameWords, word)) score += 8; // SEMANTIC EXTENSION MATCHING: Understand that "3D printing" folder means .stl files belong there // This checks folder NAME for semantic concepts (e.g., "3D Prints" folder → .stl files) @@ -95,7 +139,7 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { const descWords = descLower .split(/[\s,.-]+/) .filter((/** @type {string} */ w) => w.length > 3); - for (const word of descWords) if (lowerFileName.includes(word)) score += 6; + for (const word of descWords) if (nameHasWords(nameWords, word)) score += 6; // SEMANTIC EXTENSION MATCHING: Understand that "models for my Ender 3" means .stl files // This checks folder DESCRIPTION for semantic concepts @@ -118,7 +162,7 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { if (Array.isArray(folder.semanticTags)) { for (const tag of folder.semanticTags) { const tagLower = String(tag).toLowerCase(); - if (lowerFileName.includes(tagLower)) score += 5; + if (nameHasWords(nameWords, tagLower)) score += 5; // Extension match in semantic tags (e.g., semanticTags: ["stl", "3d-model"]) if (extNoDot && (tagLower === lowerExtension || tagLower === extNoDot)) { score += 10; // High score for explicit extension in semantic tags @@ -129,7 +173,7 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { if (Array.isArray(folder.keywords)) { for (const kw of folder.keywords) { const kwLower = String(kw).toLowerCase(); - if (lowerFileName.includes(kwLower)) score += 4; + if (nameHasWords(nameWords, kwLower)) score += 4; // Extension match in keywords (e.g., keywords: [".stl", "3mf", "gcode"]) if (extNoDot && (kwLower === lowerExtension || kwLower === extNoDot)) { score += 12; // High score for explicit extension in keywords @@ -142,18 +186,18 @@ function getIntelligentCategory(fileName, extension, smartFolders = []) { if (Array.isArray(pathParts)) { const parts = pathParts.filter((p) => p && p.length > 2); for (const part of parts) { - if (lowerFileName.includes(part)) score += 3; + if (nameHasWords(nameWords, part)) score += 3; } } } if (folder.category) { for (const word of folder.category.toLowerCase().split(/[\s_-]+/)) { - if (word.length > 2 && lowerFileName.includes(word)) score += 2; + if (word.length > 2 && nameHasWords(nameWords, word)) score += 2; } } if (Array.isArray(folder.relatedFolders)) { for (const relatedName of folder.relatedFolders) - if (lowerFileName.includes(String(relatedName).toLowerCase())) score += 1; + if (nameHasWords(nameWords, String(relatedName))) score += 1; } if (folder.confidenceScore && folder.confidenceScore > 0.8) score *= 1.2; if (score > bestScore) { @@ -633,10 +677,13 @@ function createFallbackAnalysis(params) { fileExtension, reason, smartFolders = [], - confidence = 65, + confidence: requestedConfidence = FILENAME_ONLY_CONFIDENCE, type = 'document', options = {} } = params; + // Nothing but the name was read, so the result never claims more than a + // filename guess is worth, whatever the caller asked for. + const confidence = Math.min(requestedConfidence, FILENAME_ONLY_CONFIDENCE); const { extractionMethod = 'filename_fallback', @@ -698,6 +745,7 @@ function createFallbackAnalysis(params) { } module.exports = { + FILENAME_ONLY_CONFIDENCE, getIntelligentCategory, getIntelligentKeywords, safeSuggestedName, diff --git a/src/main/analysis/imageAnalysis.js b/src/main/analysis/imageAnalysis.js index 2cbda8d..2e2cc7a 100644 --- a/src/main/analysis/imageAnalysis.js +++ b/src/main/analysis/imageAnalysis.js @@ -29,7 +29,7 @@ const { formatOrganizationGuidanceBlock } = require('./organizationGuidance'); const { recognizeIfAvailable } = require('../utils/tesseractUtils'); -const { normalizeAnalysisResult } = require('./utils'); +const { normalizeAnalysisResult, dropPromptEchoes } = require('./utils'); const { normalizeExtractedTextForStorage } = require('./analysisTextUtils'); const { getIntelligentCategory: getIntelligentImageCategory, @@ -2268,7 +2268,7 @@ async function runImageAnalysis(filePath, smartFolders, options) { const normalizedCategory = categoryMatch.folder; const normalized = normalizeAnalysisResult( { - ...analysis, + ...dropPromptEchoes(analysis, smartFolders), category: normalizedCategory || analysis.category, suggestedNewFolder: categoryMatch.suggestedNewFolder || analysis.suggestedNewFolder || undefined, diff --git a/src/main/analysis/mediaAnalysis.js b/src/main/analysis/mediaAnalysis.js index 512ca29..aa70d9f 100644 --- a/src/main/analysis/mediaAnalysis.js +++ b/src/main/analysis/mediaAnalysis.js @@ -28,7 +28,7 @@ const crypto = require('crypto'); const { createLogger } = require('../../shared/logger'); const { ERROR_CODES } = require('../../shared/errorCodes'); const { attachErrorCode } = require('../../shared/errorHandlingUtils'); -const { normalizeAnalysisResult } = require('./utils'); +const { normalizeAnalysisResult, dropPromptEchoes } = require('./utils'); const { isMemoryBudgetExceeded } = require('../../shared/analysisProvenance'); const { analyzeVideoWithCloud, resolveCloudFrameProvider } = require('./cloudMedia'); @@ -686,17 +686,20 @@ async function analyzeMediaFile(filePath, dependencies = {}, options = {}) { ? 'whisper-transcription' : 'no-speech'; + // Fields the model copied out of its prompt are cleared (and the + // confidence capped) before anything below reads them. + const answer = dropPromptEchoes(llmResult, smartFolders); const normalized = normalizeAnalysisResult( { - ...llmResult, + ...answer, type: kind, - category: llmResult?.category || baseFallback.category, - keywords: llmResult?.keywords || baseFallback.keywords, - confidence: llmResult?.confidence ?? baseFallback.confidence, - llmSuggestedName: llmResult?.suggestedName || null, - suggestedName: llmResult?.suggestedName, + category: answer?.category || baseFallback.category, + keywords: answer?.keywords || baseFallback.keywords, + confidence: answer?.confidence ?? baseFallback.confidence, + llmSuggestedName: answer?.suggestedName || null, + suggestedName: answer?.suggestedName, purpose: - llmResult?.purpose || + answer?.purpose || (usedCloudFrames ? 'Spoken content transcribed locally; video frames analyzed by the selected cloud provider' : hasUsableTranscript diff --git a/src/main/analysis/semanticFolderMatcher.js b/src/main/analysis/semanticFolderMatcher.js index ae3edfc..1e2a94b 100644 --- a/src/main/analysis/semanticFolderMatcher.js +++ b/src/main/analysis/semanticFolderMatcher.js @@ -220,9 +220,17 @@ function normalizeLlmConfidence(raw) { * @param {number|null} params.topScore - top folder cosine similarity in [0, 1], or null * @param {number|null} params.runnerUpScore - second-best folder cosine in [0, 1], or null * @param {boolean} params.llmCategoryUnmatchable - LLM picked a label that maps to no folder + * @param {boolean} [params.signalsDisagree] - The LLM's folder was kept although the + * embedding's best folder is a different one * @returns {{confidence: number, cap: number, reason: string|null, original: number, spread: number|null}} */ -function calibrateConfidence({ rawConfidence, topScore, runnerUpScore, llmCategoryUnmatchable }) { +function calibrateConfidence({ + rawConfidence, + topScore, + runnerUpScore, + llmCategoryUnmatchable, + signalsDisagree = false +}) { const original = typeof rawConfidence === 'number' && Number.isFinite(rawConfidence) ? Math.max(0, Math.min(100, Math.round(rawConfidence))) @@ -273,6 +281,10 @@ function calibrateConfidence({ rawConfidence, topScore, runnerUpScore, llmCatego setCap(CAL.UNMATCHABLE_CATEGORY_CAP ?? 75, 'unmatchable_category'); } + if (signalsDisagree) { + setCap(CAL.DISAGREEMENT_CAP ?? 65, 'llm_embedding_disagree'); + } + const confidence = Math.min(original, cap); return { confidence, @@ -522,6 +534,9 @@ async function applySemanticFolderMatching(params) { // Hoisted so the calibration block at the bottom of this function can // read it regardless of which override branch fired. let llmCategoryUnmatchable = false; + // Set when the LLM's folder is kept although the embedding confidently + // prefers another one; the calibration below caps the claim. + let signalsDisagree = false; // Process candidates and potentially override category if (Array.isArray(liveCandidates) && liveCandidates.length > 0) { @@ -595,6 +610,12 @@ async function applySemanticFolderMatching(params) { } } + signalsDisagree = + !shouldOverride && + categoryMatchesFolder && + !vectorAgrees && + top.score >= THRESHOLDS.FOLDER_MATCH_CONFIDENCE; + if (shouldOverride) { logger.info('[FolderMatcher] Embedding override - folder match exceeds LLM confidence', { type, @@ -683,7 +704,8 @@ async function applySemanticFolderMatching(params) { rawConfidence: preCalibrationPercent, topScore: typeof topCandidate?.score === 'number' ? topCandidate.score : null, runnerUpScore: typeof runnerUpCandidate?.score === 'number' ? runnerUpCandidate.score : null, - llmCategoryUnmatchable + llmCategoryUnmatchable, + signalsDisagree }); const confidencePercent = calibration.confidence; if (calibration.reason) { diff --git a/src/main/analysis/utils.js b/src/main/analysis/utils.js index ec50fb7..fa5cdf8 100644 --- a/src/main/analysis/utils.js +++ b/src/main/analysis/utils.js @@ -1,5 +1,6 @@ // @ts-check /** @typedef {import('../../shared/analysisSchema').AnalysisResult} AnalysisResult */ +const { ANALYSIS_SCHEMA_PROMPT } = require('../../shared/analysisSchema'); // Coerce a non-numeric confidence shape into a number on the caller's // existing scale. Models routinely return numeric strings ("0.85"), // percent strings ("85%"), or categorical tokens ("high"/"medium"/"low") @@ -87,6 +88,95 @@ function normalizeAnalysisResult(raw, fallback = {}) { return { ...result, ...normalized }; } +// Filler words left out when comparing a model's answer with the prompt. +const ECHO_STOPWORDS = new Set( + 'a an and are as at be by e g for from in into is it its of on or that the this to with'.split( + ' ' + ) +); +const ECHO_FIELDS = ['summary', 'purpose', 'project', 'reasoning', 'type', 'entity']; +// A value is an echo when nearly all of its words come from one source text +// AND it reproduces a good part of that text. The second test keeps a real +// answer that merely shares vocabulary ("A logo for a company or +// organization") from reading as a copy of a longer instruction. +const ECHO_CONTAINMENT = 0.8; +const ECHO_SOURCE_COVERAGE = 0.4; +const ECHO_MIN_WORDS = 4; +// Same ceiling as a filename-only guess (fallbackUtils FILENAME_ONLY_CONFIDENCE). +const ECHO_CONFIDENCE_CAP = 50; + +/** @param {unknown} text @returns {string[]} */ +function echoWords(text) { + return String(text ?? '') + .toLowerCase() + .replace(/['’]s\b/g, '') + .split(/[^\p{L}\p{N}]+/u) + .filter((w) => w && !ECHO_STOPWORDS.has(w)); +} + +/** + * Drop answers a model copied out of its own prompt instead of writing. + * + * Small vision models in particular fill a field with the schema's + * instruction for it ("A concise summary of the document's content.") or + * with a smart folder's description ("Art, graphic design, images not + * related to other categories, logos"). Kept, those read as real summaries, + * feed the embedding, and the category picked alongside them looked certain + * (one such answer filed a banner under Research at 90%). Copied fields are + * cleared; when any was copied, the confidence is capped like a filename + * guess so the file is flagged for review. Same idea as + * normalizeImageContentType, applied to the free-text fields. + * + * @param {Record} result - Model output (mutated copy returned) + * @param {Array<{ description?: string }>} [smartFolders] + * @returns {Record} + */ +function dropPromptEchoes(result, smartFolders = []) { + if (!result || typeof result !== 'object') return result; + const sources = [ + // The instructions without their "(e.g., ...)" examples: an example like + // 'sunset_beach' is a legitimate thing for a picture to show. + ...Object.values(ANALYSIS_SCHEMA_PROMPT) + .filter((v) => typeof v === 'string') + .map((v) => v.replace(/\(e\.g\.[^)]*\)/gi, ' ')), + ...(Array.isArray(smartFolders) ? smartFolders : []) + .map((f) => f?.description) + .filter((d) => typeof d === 'string' && d.trim()) + ] + .map((text) => new Set(echoWords(text))) + .filter((source) => source.size > 0); + + /** @param {unknown} value */ + const isEcho = (value) => { + if (typeof value !== 'string') return false; + // Compared without "(e.g., ...)" on both sides: a copied instruction + // carries its examples along. + const words = new Set(echoWords(value.replace(/\(e\.g\.[^)]*\)/gi, ' '))); + if (words.size < ECHO_MIN_WORDS) return false; + return sources.some((source) => { + let shared = 0; + for (const word of words) if (source.has(word)) shared += 1; + return ( + shared / words.size >= ECHO_CONTAINMENT && shared / source.size >= ECHO_SOURCE_COVERAGE + ); + }); + }; + + const echoed = ECHO_FIELDS.filter((field) => isEcho(result[field])); + if (echoed.length === 0) return result; + + const cleaned = { ...result }; + for (const field of echoed) cleaned[field] = null; + const confidence = _coerceConfidence(cleaned.confidence); + if (confidence !== null) { + // Callers use both scales; cap on the one the value is on. + cleaned.confidence = + confidence > 1 ? Math.min(confidence, ECHO_CONFIDENCE_CAP) : Math.min(confidence, 0.5); + } + cleaned.promptEchoFields = echoed; + return cleaned; +} + /** * Read the STRATOSORT_ANALYSIS_EMBEDDING_QUEUE_WAIT env flag in a single * canonical way. Three analysis modules (image, media, semanticFolderMatcher) @@ -105,4 +195,8 @@ function isPipelineEmbeddingQueueWaitEnabled() { ); } -module.exports = { normalizeAnalysisResult, isPipelineEmbeddingQueueWaitEnabled }; +module.exports = { + normalizeAnalysisResult, + dropPromptEchoes, + isPipelineEmbeddingQueueWaitEnabled +}; diff --git a/src/main/ipc/analysisHistory.js b/src/main/ipc/analysisHistory.js index 08680ed..6241d1f 100644 --- a/src/main/ipc/analysisHistory.js +++ b/src/main/ipc/analysisHistory.js @@ -556,6 +556,30 @@ function registerAnalysisHistoryIpc(servicesOrParams) { } }) ); + + // Lean list of every analyzed file still filed as Uncategorized, so the + // Organize screen can spot themes across past runs, not just the open one. + safeHandle( + ipcMain, + IPC_CHANNELS.ANALYSIS_HISTORY.GET_UNCATEGORIZED, + createHandler({ + logger, + context, + schema: schemas?.uncategorizedOptions, + serviceName: 'analysisHistory', + getService: getHistoryService, + fallbackResponse: { items: [] }, + handler: async (event, options = {}, service) => { + try { + const items = await service.getUncategorizedItems(options || {}); + return { items: Array.isArray(items) ? items : [] }; + } catch (error) { + logger.error('Failed to get uncategorized analysis entries:', error); + return { items: [] }; + } + } + }) + ); } module.exports = registerAnalysisHistoryIpc; diff --git a/src/main/ipc/smartFolders.js b/src/main/ipc/smartFolders.js index 364eea3..d53e962 100644 --- a/src/main/ipc/smartFolders.js +++ b/src/main/ipc/smartFolders.js @@ -1412,6 +1412,35 @@ async function remapKnowledgeUnderRenamedFolder({ } } +/** + * Tell the suggestion service what a folder is now for, so later files follow + * the new wording and Settings can show the change. Never fails the edit. + * @param {{ logger: LoggerLike, getServiceIntegration?: () => any, folder: Record, previous: Record }} params + */ +async function rememberFolderDescriptionChange({ + logger, + getServiceIntegration, + folder, + previous +}) { + try { + const service = + typeof getServiceIntegration === 'function' + ? getServiceIntegration()?.suggestionService + : null; + if (typeof service?.recordFolderDescriptionChange !== 'function') return; + await service.recordFolderDescriptionChange({ + folderId: folder.id, + name: folder.name, + previousName: previous.name, + description: folder.description, + previousDescription: previous.description + }); + } catch (error) { + logger.warn('[SMART-FOLDERS] Could not record the description change:', error?.message); + } +} + /** * @param {SmartFoldersHandlerContext} ctx * @param {IpcMainInvokeEvent} event @@ -1643,9 +1672,17 @@ async function handleSmartFoldersEdit( }; } } + const descriptionChanged = + typeof updatedFolder.description === 'string' && + updatedFolder.description.trim() !== String(originalFolder.description || '').trim(); try { customFolders[folderIndex] = { ...customFolders[folderIndex], + // Keywords were derived from the old description. Once the user says + // what the folder is for, their wording alone shapes its matching. + ...(descriptionChanged && !Array.isArray(updatedFolder.keywords) + ? { keywords: [], semanticTags: [] } + : {}), ...updatedFolder, updatedAt: new Date().toISOString() }; @@ -1670,6 +1707,12 @@ async function handleSmartFoldersEdit( scanPaths: pathChanged ? [customFolders[folderIndex].path] : [], getSmartFolderWatcher }); + await rememberFolderDescriptionChange({ + logger, + getServiceIntegration, + folder: customFolders[folderIndex], + previous: originalFolder + }); logger.info('[SMART-FOLDERS] Edited Smart Folder:', folderId); return { success: true, diff --git a/src/main/ipc/validationSchemas.js b/src/main/ipc/validationSchemas.js index 3118a90..a5b87d9 100644 --- a/src/main/ipc/validationSchemas.js +++ b/src/main/ipc/validationSchemas.js @@ -1494,6 +1494,13 @@ if (!z) { offset: z.number().int().min(0).optional() }); + /** + * Uncategorized-files request (analysis-history:get-uncategorized) + */ + const uncategorizedOptionsSchema = z.object({ + limit: z.number().int().min(1).max(2000).optional() + }); + const embeddingsFullRebuildSchema = z .object({ modelOverride: z.string().min(1).max(500).optional() @@ -1678,6 +1685,7 @@ if (!z) { analysisExportFormat: analysisExportFormatSchema, transcriptExport: transcriptExportSchema, timelineOptions: timelineOptionsSchema, + uncategorizedOptions: uncategorizedOptionsSchema, // Backup backupPath: backupPathSchema, diff --git a/src/main/services/analysisHistory/AnalysisHistoryServiceCore.js b/src/main/services/analysisHistory/AnalysisHistoryServiceCore.js index bf708a6..9dc7ea0 100644 --- a/src/main/services/analysisHistory/AnalysisHistoryServiceCore.js +++ b/src/main/services/analysisHistory/AnalysisHistoryServiceCore.js @@ -67,6 +67,7 @@ const { getRecentAnalysis: getRecentAnalysisHelper, getAnalysisByDateRange: getAnalysisByDateRangeHelper, buildTimelineItems, + buildUncategorizedItems, getCategories: getCategoriesHelper, getTags: getTagsHelper } = require('./queries'); @@ -1158,6 +1159,11 @@ class AnalysisHistoryServiceCore { return buildTimelineItems(this.analysisHistory, options); } + async getUncategorizedItems(options = {}) { + await this.initialize(); + return buildUncategorizedItems(this.analysisHistory, options); + } + async getCategories() { await this.initialize(); return getCategoriesHelper(this.analysisIndex); diff --git a/src/main/services/analysisHistory/queries.js b/src/main/services/analysisHistory/queries.js index a8bb762..c21fb0b 100644 --- a/src/main/services/analysisHistory/queries.js +++ b/src/main/services/analysisHistory/queries.js @@ -15,6 +15,7 @@ const { toLocalCalendarDay } = require('../../../shared/contentDateSanity'); const { normalizePathForIndex } = require('../../../shared/pathSanitization'); +const { isDefaultCategoryName } = require('../../../shared/categoryFolderMatching'); // Lookup normalizer must match the write side in `indexManager.js`, which uses // `normalizePathForIndex` (NFC unicode + win32/darwin lowercase + forward-slash @@ -456,6 +457,48 @@ function buildTimelineItems(analysisHistory, options = {}) { return { items: page, total, hasMore: offset + page.length < total }; } +/** + * Every analyzed file whose category is still the default one, newest first. + * Scans the entries rather than `categoryIndex`: a move rewrites an entry's + * category without re-keying that index. Items are lean (no extracted text) + * and shaped like the Organize screen's files. + * @param {any} analysisHistory + * @param {{limit?: number}} [options] + * @returns {Array<{path: string, name: string, extension: string, analysis: {category: string, tags: string[], entity?: string, project?: string}}>} + */ +function buildUncategorizedItems(analysisHistory, options = {}) { + const entries = analysisHistory?.entries; + if (!entries || typeof entries !== 'object') return []; + const limit = Number.isInteger(options.limit) && options.limit > 0 ? options.limit : 500; + + const matched = []; + for (const entry of Object.values(entries)) { + const analysis = /** @type {any} */ (entry)?.analysis; + if (!analysis || !isDefaultCategoryName(analysis.category)) continue; + const path = + typeof entry.organization?.actual === 'string' && entry.organization.actual + ? entry.organization.actual + : entry.originalPath; + if (typeof path !== 'string' || !path) continue; + matched.push({ + path, + name: typeof entry.fileName === 'string' ? entry.fileName : '', + extension: typeof entry.fileExtension === 'string' ? entry.fileExtension : '', + timestamp: typeof entry.timestamp === 'string' ? entry.timestamp : '', + analysis: { + category: analysis.category, + tags: Array.isArray(analysis.tags) + ? analysis.tags.filter((/** @type {any} */ t) => typeof t === 'string').slice(0, 20) + : [], + ...(typeof analysis.entity === 'string' ? { entity: analysis.entity } : {}), + ...(typeof analysis.project === 'string' ? { project: analysis.project } : {}) + } + }); + } + matched.sort((a, b) => (a.timestamp < b.timestamp ? 1 : a.timestamp > b.timestamp ? -1 : 0)); + return matched.slice(0, limit).map(({ timestamp: _timestamp, ...item }) => item); +} + /** * Get all available categories with counts * Performance: Uses index directly, no iteration over entries @@ -489,6 +532,7 @@ module.exports = { getRecentAnalysis, getAnalysisByDateRange, buildTimelineItems, + buildUncategorizedItems, getCategories, getTags }; diff --git a/src/main/services/organization/OrganizationSuggestionServiceCore.js b/src/main/services/organization/OrganizationSuggestionServiceCore.js index b36f553..c9f4546 100644 --- a/src/main/services/organization/OrganizationSuggestionServiceCore.js +++ b/src/main/services/organization/OrganizationSuggestionServiceCore.js @@ -40,6 +40,9 @@ const { const { PatternMatcher } = require('./patternMatcher'); const { FeedbackMemoryStore, getMetrics: getFeedbackMetrics } = require('./feedbackMemoryStore'); + +/** Earlier wordings kept per folder for the Settings history. */ +const FOLDER_DESCRIPTION_HISTORY_LIMIT = 5; const { buildMemoryEntry, applyMemoryRuleAdjustments, @@ -1678,6 +1681,55 @@ class OrganizationSuggestionServiceCore { return result; } + /** + * A user rewording what a folder is for is the plainest statement of what + * belongs there. Keep it as one memory per folder, so matching files lean + * toward that folder and Settings shows what was changed and when. + * @param {{ folderId?: string, name: string, previousName?: string, description?: string, previousDescription?: string }} change + * @returns {Promise} the stored memory, or null when nothing changed + */ + async recordFolderDescriptionChange({ + folderId, + name, + previousName, + description, + previousDescription + }) { + const next = String(description || '').trim(); + const previous = String(previousDescription || '').trim(); + if (!name || !next) return null; + + await this._ensureFeedbackMemoryLoaded(); + const autoKey = `folder-description:${folderId || name.toLowerCase()}`; + const existing = (await this.feedbackMemoryStore.list()).find( + (entry) => entry?.autoKey === autoKey + ); + const renamed = Boolean(previousName) && previousName !== name; + if (next === previous && !(existing && renamed)) return null; + + const text = normalizeText(`"${name}" is for: ${next}`); + const metadata = { + source: 'folder_description', + targetFolder: name, + scope: { type: 'folder', value: name } + }; + const entry = existing + ? await this.updateFeedbackMemory(existing.id, text, metadata) + : await this.addFeedbackMemory(text, metadata); + if (!entry) return null; + + const history = Array.isArray(existing?.history) ? [...existing.history] : []; + if (previous && previous !== next) { + history.unshift({ description: previous, changedAt: new Date().toISOString() }); + } + return this.feedbackMemoryStore.update( + entry.id, + // A description is prose, not a "X goes to Y" rule: keep none parsed from it. + { autoKey, rules: [], history: history.slice(0, FOLDER_DESCRIPTION_HISTORY_LIMIT) }, + {} + ); + } + async addFeedbackMemory(/** @type {any} */ text, metadata = {}) { await this._ensureFeedbackMemoryLoaded(); const entry = buildMemoryEntry(text, metadata); diff --git a/src/preload/preload.js b/src/preload/preload.js index ae80c12..a91e30a 100644 --- a/src/preload/preload.js +++ b/src/preload/preload.js @@ -139,7 +139,8 @@ const IPC_CHANNELS = { CLEAR: 'analysis-history:clear', EXPORT: 'analysis-history:export', EXPORT_TRANSCRIPT: 'analysis-history:export-transcript', - GET_TIMELINE: 'analysis-history:get-timeline' + GET_TIMELINE: 'analysis-history:get-timeline', + GET_UNCATEGORIZED: 'analysis-history:get-uncategorized' }, // EMBEDDINGS @@ -1914,7 +1915,9 @@ contextBridge.exposeInMainWorld('electronAPI', { format }), getTimeline: (/** @type {any} */ options) => - secureIPC.safeInvoke(IPC_CHANNELS.ANALYSIS_HISTORY.GET_TIMELINE, options || {}) + secureIPC.safeInvoke(IPC_CHANNELS.ANALYSIS_HISTORY.GET_TIMELINE, options || {}), + getUncategorized: (/** @type {any} */ options) => + secureIPC.safeInvoke(IPC_CHANNELS.ANALYSIS_HISTORY.GET_UNCATEGORIZED, options || {}) }, // AI Provider Configuration diff --git a/src/renderer/components/SettingsPanel.jsx b/src/renderer/components/SettingsPanel.jsx index 70eefbc..c91a536 100644 --- a/src/renderer/components/SettingsPanel.jsx +++ b/src/renderer/components/SettingsPanel.jsx @@ -69,6 +69,7 @@ import { lockAppScroll, unlockAppScroll } from '../utils/scrollLock'; import { isTopOverlay, registerOverlay, unregisterOverlay } from '../utils/overlayStack'; import { toErrorMessage } from '../utils/errorMapping'; import AutoOrganizeSection from './settings/AutoOrganizeSection'; +import FeedbackMemoryPanel from './organize/FeedbackMemoryPanel'; import BackgroundModeSection from './settings/BackgroundModeSection'; import NotificationSettingsSection from './settings/NotificationSettingsSection'; import LlamaConfigSection from './settings/LlamaConfigSection'; @@ -1755,6 +1756,7 @@ const SettingsPanel = React.memo(function SettingsPanel() { > + {effectiveAdvancedMode && ( + + What StratoSort has learned from your corrections and folder descriptions. Future + suggestions follow these; edit or delete any that are wrong. + +