diff --git a/Backlog.md b/Backlog.md index 5670710..3fb7606 100644 --- a/Backlog.md +++ b/Backlog.md @@ -457,14 +457,14 @@ Goal: Understand what files and folders represent rather than relying only on ex ## Detection signals - [x] File extension -- [ ] MIME/content signature +- [x] MIME/content signature (light idle magic-byte; never hydrates) - [x] Folder structure - [x] Git metadata - [x] Media metadata - [x] Known application structures (`node_modules`, `bin`, `obj`, `.vs`) - [x] File age (old installers flagged in preview) - [ ] File relationships -- [ ] Index metadata +- [x] Index metadata --- diff --git a/docs/Documentation.md b/docs/Documentation.md index ac88452..1b08594 100644 --- a/docs/Documentation.md +++ b/docs/Documentation.md @@ -343,7 +343,22 @@ Classification **suggests** moves. Nothing moves until you Preview and Queue. Build output names (`node_modules`, `bin`, `obj`, `.vs`, and similar) are never moved. Online-only cloud items are skipped. Old installers (older than one year) still propose a move and show a warning. Destinations may sit *inside* the source folder (Downloads → Downloads\Software). Destinations are remembered in preferences. -Never auto-reorganizes. No MIME/content sniffing (that would hydrate cloud files). +Organize uses the **indexed category** when one exists (including ZIP contents that are mostly photos, and **Classify as…** overrides). Extension heuristics fill in the rest. + +Never auto-reorganizes. A light magic-byte peek for unknown extensions runs only during idle maintenance on local files — never on online-only cloud items. + +--- + +## Classification + +The index stores a category on each entry (Photos, Video, Documents, Archive, and so on) plus a short reason. + +- Details shows a **Category** column (tooltip has the reason) +- Search accepts `category:photos` or the category dropdown +- Storage analysis has **By category** next to **By file type** +- Context menu **Classify as…** writes a user override that later scans do not replace + +ZIP/7z files stay Archive until their contents are indexed; idle maintenance then promotes them when a category dominates. --- @@ -416,7 +431,7 @@ Left open on purpose: - Multi-PC search, sharing, encrypted vaults - Robocopy as a second transfer engine - Scheduled profiles and folder-watcher triggers -- MIME/EXIF classification +- Full EXIF / ffprobe classification - Duplicate “backup copy” auto-tagging --- diff --git a/src/Explorer.Analysis/AnalysisService.cs b/src/Explorer.Analysis/AnalysisService.cs index 234eaa9..74ea51d 100644 --- a/src/Explorer.Analysis/AnalysisService.cs +++ b/src/Explorer.Analysis/AnalysisService.cs @@ -76,6 +76,16 @@ public sealed class AnalysisService ct => _store.Analysis.UsageByExtensionAsync(sourceId, pathRelPrefix, take, ct), cancellationToken); + public Task> UsageByCategoryAsync( + long? sourceId, + string? pathRelPrefix, + int take = AppConstants.AnalysisTopN, + CancellationToken cancellationToken = default) + => CachedAsync( + $"categories:{sourceId}:{pathRelPrefix}:{take}", + ct => _store.Analysis.UsageByCategoryAsync(sourceId, pathRelPrefix, take, ct), + cancellationToken); + public Task> UsageBySourceAsync(CancellationToken cancellationToken = default) => CachedAsync("sources", ct => _store.Analysis.UsageBySourceAsync(ct), cancellationToken); diff --git a/src/Explorer.App/MainWindow.xaml b/src/Explorer.App/MainWindow.xaml index 0062eb2..50f65a9 100644 --- a/src/Explorer.App/MainWindow.xaml +++ b/src/Explorer.App/MainWindow.xaml @@ -44,6 +44,17 @@ + + + + + + + + + + + @@ -537,6 +548,13 @@ + + + + + + + @@ -661,6 +679,13 @@ + + + + + + + @@ -768,6 +793,11 @@ + + diff --git a/src/Explorer.Application/BackgroundMaintenance.cs b/src/Explorer.Application/BackgroundMaintenance.cs index eb6a57d..718fc0b 100644 --- a/src/Explorer.Application/BackgroundMaintenance.cs +++ b/src/Explorer.Application/BackgroundMaintenance.cs @@ -36,6 +36,27 @@ public interface IHistoryMaintenance Task TryCaptureAsync(CancellationToken cancellationToken = default); } +public interface IIdleClassifyWork +{ + bool IsPaused { get; } + string? CurrentPath { get; } + void Pause(); + void Resume(); + /// Returns true when any classification work was performed. + Task ProcessPendingAsync(CancellationToken cancellationToken = default); +} + +public sealed class NullIdleClassifyWork : IIdleClassifyWork +{ + public static NullIdleClassifyWork Instance { get; } = new(); + public bool IsPaused => true; + public string? CurrentPath => null; + public void Pause() { } + public void Resume() { } + public Task ProcessPendingAsync(CancellationToken cancellationToken = default) + => Task.FromResult(false); +} + public interface IForegroundWorkSignal { bool HasForegroundWork(); diff --git a/src/Explorer.Application/BrowseService.cs b/src/Explorer.Application/BrowseService.cs index ea4db64..a8e61aa 100644 --- a/src/Explorer.Application/BrowseService.cs +++ b/src/Explorer.Application/BrowseService.cs @@ -395,7 +395,11 @@ public sealed class BrowseService Cloud = entry.CloudAvailability is { } availability ? new CloudPresence(null, availability, entry.SizeBytes, entry.AllocatedSizeBytes, availability == CloudAvailability.OnlineOnly) : null, - Hydration = ItemHydrationFlags.All + Hydration = ItemHydrationFlags.All, + IndexEntryId = entry.Id, + Category = entry.Category, + CategoryReason = entry.CategoryReason, + CategorySource = entry.CategorySource }; private async IAsyncEnumerable ListLiveProgressiveAsync( @@ -609,7 +613,11 @@ public sealed class BrowseService fileId: item.FileId ?? entry.FileId, cloud: cloud, indexedChildCount: entry.ChildFileCount + entry.ChildDirCount, - hydration: item.Hydration | ItemHydrationFlags.Index); + hydration: item.Hydration | ItemHydrationFlags.Index, + indexEntryId: entry.Id, + category: entry.Category, + categoryReason: entry.CategoryReason, + categorySource: entry.CategorySource); return Annotate(hydrated, sizeFromIndex: item.IsDirectory && entry.AggregateSize > 0, preferences, probeAccess: false); } @@ -785,7 +793,11 @@ public sealed class BrowseService Attributes = c.Attributes, FileId = c.FileId, ReparseTag = c.ReparseTag, - AllocatedSizeBytes = c.AllocatedSizeBytes + AllocatedSizeBytes = c.AllocatedSizeBytes, + IndexEntryId = c.Id, + Category = c.Category, + CategoryReason = c.CategoryReason, + CategorySource = c.CategorySource }).ToList(); var hint = isArchiveFile && items.Count == 0 diff --git a/src/Explorer.Application/EntryClassificationService.cs b/src/Explorer.Application/EntryClassificationService.cs new file mode 100644 index 0000000..c7ae2f9 --- /dev/null +++ b/src/Explorer.Application/EntryClassificationService.cs @@ -0,0 +1,173 @@ +using Explorer.Domain; +using Explorer.Domain.Abstractions; +using Microsoft.Extensions.Logging; + +namespace Explorer.Application; + +public sealed class EntryClassificationService : IIdleClassifyWork +{ + private readonly IIndexStore _store; + private readonly IFileSystemEnumerator _enumerator; + private readonly IHydrationGuard _hydration; + private readonly ILogger _logger; + private readonly IHostActivitySink _activity; + private volatile bool _paused = true; + private volatile string? _currentPath; + + public EntryClassificationService( + IIndexStore store, + IFileSystemEnumerator enumerator, + IHydrationGuard hydration, + ILogger logger, + IHostActivitySink? activity = null) + { + _store = store; + _enumerator = enumerator; + _hydration = hydration; + _logger = logger; + _activity = activity ?? NullHostActivitySink.Instance; + } + + public bool IsPaused => _paused; + public string? CurrentPath => _currentPath; + + public void Pause() => _paused = true; + public void Resume() => _paused = false; + + public async Task ProcessPendingAsync(CancellationToken cancellationToken = default) + { + if (_paused) + { + return false; + } + + var backfill = await _store.Entries.BackfillCheapCategoriesAsync(80, cancellationToken) + .ConfigureAwait(false); + if (backfill > 0) + { + _activity.Record("Classify", "Backfilled " + backfill + " extension categories"); + return true; + } + + var archives = await _store.Entries.GetArchiveIdsNeedingContentClassifyAsync(20, cancellationToken) + .ConfigureAwait(false); + if (archives.Count > 0) + { + foreach (var id in archives) + { + if (_paused || cancellationToken.IsCancellationRequested) + { + break; + } + + await _store.Entries.RefreshCategoryFromChildrenAsync(id, cancellationToken) + .ConfigureAwait(false); + } + + _activity.Record("Classify", "Updated " + archives.Count + " archive(s) from contents"); + return true; + } + + return await TrySignatureClassifyAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task TrySignatureClassifyAsync(CancellationToken cancellationToken) + { + var unknowns = await _store.Entries.GetUnknownFilesForSignatureAsync(12, cancellationToken) + .ConfigureAwait(false); + if (unknowns.Count == 0) + { + return false; + } + + var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false); + var byId = sources.ToDictionary(s => s.Id); + var changed = 0; + foreach (var entry in unknowns) + { + if (_paused || cancellationToken.IsCancellationRequested) + { + break; + } + + if (!byId.TryGetValue(entry.SourceId, out var source) || string.IsNullOrEmpty(source.LastRootPath)) + { + continue; + } + + var full = PathRules.Combine(source.LastRootPath, entry.PathRel); + _currentPath = full; + try + { + var item = _enumerator.GetItem(full); + if (item is null) + { + MarkSignatureSkip(entry, "File was not found."); + await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false); + continue; + } + + if (_hydration.WouldHydrateOnRead(item) + || await _hydration.WouldHydrateOnReadAsync(full, cancellationToken).ConfigureAwait(false)) + { + MarkSignatureSkip(entry, "Skipped online-only cloud file."); + await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false); + continue; + } + + await using var stream = new FileStream( + PathRules.ToExtended(full), + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + bufferSize: 64, + FileOptions.SequentialScan | FileOptions.Asynchronous); + var buffer = new byte[16]; + var read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken) + .ConfigureAwait(false); + var hit = FileSignatureClassifier.TryClassify(buffer.AsSpan(0, read)); + if (hit is null) + { + entry.CategorySource = CategorySources.Mime; + entry.CategoryConfidence = 10; + entry.CategoryReason = "No known file signature."; + entry.CategoryUtc = DateTimeOffset.UtcNow; + await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false); + continue; + } + + EntryCategoryAssigner.Apply(entry, hit, CategorySources.Mime, 70); + await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false); + changed++; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug(ex, "Signature classify failed for {Path}", full); + try + { + MarkSignatureSkip(entry, "Could not read file signature."); + await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false); + } + catch (Exception markEx) when (markEx is not OperationCanceledException) + { + _logger.LogDebug(markEx, "Could not mark signature skip for {Path}", full); + } + } + } + + _currentPath = null; + if (changed > 0) + { + _activity.Record("Classify", "Signature-classified " + changed + " file(s)"); + } + + return changed > 0 || unknowns.Count > 0; + } + + private static void MarkSignatureSkip(IndexEntry entry, string reason) + => EntryCategoryAssigner.Apply( + entry, + new FileClassification(entry.Category, reason), + CategorySources.Mime, + 5); +} diff --git a/src/Explorer.Application/ReorganizePlanner.cs b/src/Explorer.Application/ReorganizePlanner.cs index 6441edc..8f8f8a4 100644 --- a/src/Explorer.Application/ReorganizePlanner.cs +++ b/src/Explorer.Application/ReorganizePlanner.cs @@ -15,7 +15,8 @@ public sealed class ReorganizePlanner Func isRepoRoot, Func? wouldHydrate = null, Func? pathExists = null, - DateTimeOffset? now = null) + DateTimeOffset? now = null, + IReadOnlyDictionary? indexedByName = null) { if (string.IsNullOrWhiteSpace(sourceRoot)) { @@ -63,13 +64,16 @@ public sealed class ReorganizePlanner var childCategories = item.IsDirectory && !FileClassifier.SkipDescent(item.Name) ? ChildCategories(item, enumerator, isRepoRoot) : null; - var classification = FileClassifier.Classify( - item.Name, - item.FullPath, - item.IsDirectory, - item.Attributes, - item.IsDirectory && isRepoRoot(item.FullPath), - childCategories); + var classification = PreferIndexed( + item, + indexedByName, + FileClassifier.Classify( + item.Name, + item.FullPath, + item.IsDirectory, + item.Attributes, + item.IsDirectory && isRepoRoot(item.FullPath), + childCategories)); if (FileClassifier.ShouldLeave(classification.Category)) { @@ -143,6 +147,28 @@ public sealed class ReorganizePlanner }; } + private static FileClassification PreferIndexed( + FileSystemItem item, + IReadOnlyDictionary? indexedByName, + FileClassification fallback) + { + if (indexedByName is null + || !indexedByName.TryGetValue(item.Name, out var stored)) + { + return fallback; + } + + if (CategorySources.IsUser(stored.CategorySource) + || stored.Category is not FileCategory.Unknown) + { + return new FileClassification( + stored.Category, + stored.CategoryReason ?? "Indexed category."); + } + + return fallback; + } + private static IReadOnlyList ChildCategories( FileSystemItem folder, IFileSystemEnumerator enumerator, diff --git a/src/Explorer.Application/WorkbenchHost.cs b/src/Explorer.Application/WorkbenchHost.cs index 01d5f4c..39c7925 100644 --- a/src/Explorer.Application/WorkbenchHost.cs +++ b/src/Explorer.Application/WorkbenchHost.cs @@ -57,4 +57,53 @@ public sealed class LocalIndexMutations : IIndexMutations => _store.Hashes.EnqueueSizeCollisionsAsync(sourceId, cancellationToken); public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default) => _store.Relations.UpsertAsync(relation, cancellationToken); + + public async Task SetUserCategoryByPathAsync( + string fullPath, + FileCategory category, + string? reason = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(fullPath)) + { + return; + } + + var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false); + Source? best = null; + var bestLen = -1; + var path = PathRules.FromExtended(fullPath).TrimEnd('\\'); + foreach (var source in sources) + { + if (string.IsNullOrEmpty(source.LastRootPath)) + { + continue; + } + + var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\'); + if (path.Equals(root, StringComparison.OrdinalIgnoreCase) + || path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase)) + { + if (root.Length > bestLen) + { + best = source; + bestLen = root.Length; + } + } + } + + if (best?.LastRootPath is null) + { + return; + } + + var rel = PathRules.MakeRelative(best.LastRootPath, fullPath) ?? ""; + var entry = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + return; + } + + await _store.Entries.SetUserCategoryAsync(entry.Id, category, reason, cancellationToken).ConfigureAwait(false); + } } diff --git a/src/Explorer.Contracts/IWorkbenchHost.cs b/src/Explorer.Contracts/IWorkbenchHost.cs index 2e492f0..e10bca0 100644 --- a/src/Explorer.Contracts/IWorkbenchHost.cs +++ b/src/Explorer.Contracts/IWorkbenchHost.cs @@ -83,4 +83,6 @@ public interface IIndexMutations Task MarkRenameBatchUndoneAsync(long id, CancellationToken cancellationToken = default); Task EnqueueHashCollisionsAsync(long? sourceId, CancellationToken cancellationToken = default); Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default); + Task SetUserCategoryByPathAsync(string fullPath, FileCategory category, string? reason = null, CancellationToken cancellationToken = default) + => Task.CompletedTask; } diff --git a/src/Explorer.Domain/Abstractions/IIndexStore.cs b/src/Explorer.Domain/Abstractions/IIndexStore.cs index e106d9c..a3d20ab 100644 --- a/src/Explorer.Domain/Abstractions/IIndexStore.cs +++ b/src/Explorer.Domain/Abstractions/IIndexStore.cs @@ -59,6 +59,11 @@ public interface IEntryStore Task DeleteExpiredTombstonesAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken = default); Task RenameSubtreePathAsync(long sourceId, string oldPathRel, string newPathRel, CancellationToken cancellationToken = default); Task CountPresentAsync(long sourceId, CancellationToken cancellationToken = default); + Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default); + Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default); + Task BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default); + Task> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default); + Task> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default); } public interface IExcludeStore @@ -109,6 +114,7 @@ public sealed class SearchRequest public bool DirectChildrenOnly { get; init; } public bool IncludeOffline { get; init; } = true; public bool IncludeDeleted { get; init; } + public FileCategory? Category { get; init; } public int Skip { get; init; } public int Take { get; init; } = 500; } @@ -121,6 +127,7 @@ public interface IAnalysisStore Task> LargestDirectoriesAsync(long? sourceId, long? parentId, int take, CancellationToken cancellationToken = default); Task> LargestFilesAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default); Task> UsageByExtensionAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default); + Task> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default); Task> UsageBySourceAsync(CancellationToken cancellationToken = default); Task> ChildrenBySizeAsync(long parentId, int take, CancellationToken cancellationToken = default); } @@ -132,6 +139,13 @@ public sealed class ExtensionUsage public long FileCount { get; init; } } +public sealed class CategoryUsage +{ + public required string Category { get; init; } + public long TotalSize { get; init; } + public long FileCount { get; init; } +} + public sealed class SourceUsage { public long SourceId { get; init; } diff --git a/src/Explorer.Domain/Entities.cs b/src/Explorer.Domain/Entities.cs index 1bb73bf..3d36f09 100644 --- a/src/Explorer.Domain/Entities.cs +++ b/src/Explorer.Domain/Entities.cs @@ -68,6 +68,11 @@ public sealed class IndexEntry public long ScanGeneration { get; set; } public long? AllocatedSizeBytes { get; set; } public CloudAvailability? CloudAvailability { get; set; } + public FileCategory Category { get; set; } = FileCategory.Unknown; + public string? CategoryReason { get; set; } + public int CategoryConfidence { get; set; } + public string CategorySource { get; set; } = CategorySources.None; + public DateTimeOffset? CategoryUtc { get; set; } } public sealed class ExcludeRule @@ -150,6 +155,10 @@ public sealed class FileSystemItem public bool AvailableToImport { get; init; } public ItemHydrationFlags Hydration { get; init; } = ItemHydrationFlags.All; public int IndexedChildCount { get; init; } + public long? IndexEntryId { get; init; } + public FileCategory Category { get; init; } + public string? CategoryReason { get; init; } + public string? CategorySource { get; init; } public bool IsReparsePoint => (Attributes & AttributeFlags.ReparsePoint) != 0; public FileSystemItem Overlay( @@ -168,7 +177,11 @@ public sealed class FileSystemItem long? capacityBytes = null, bool? availableToImport = null, ItemHydrationFlags? hydration = null, - int? indexedChildCount = null) + int? indexedChildCount = null, + long? indexEntryId = null, + FileCategory? category = null, + string? categoryReason = null, + string? categorySource = null) => new() { FullPath = FullPath, @@ -189,7 +202,11 @@ public sealed class FileSystemItem CapacityBytes = capacityBytes ?? CapacityBytes, AvailableToImport = availableToImport ?? AvailableToImport, Hydration = hydration ?? Hydration, - IndexedChildCount = indexedChildCount ?? IndexedChildCount + IndexedChildCount = indexedChildCount ?? IndexedChildCount, + IndexEntryId = indexEntryId ?? IndexEntryId, + Category = category ?? Category, + CategoryReason = categoryReason ?? CategoryReason, + CategorySource = categorySource ?? CategorySource }; } diff --git a/src/Explorer.Domain/EntryCategoryAssigner.cs b/src/Explorer.Domain/EntryCategoryAssigner.cs new file mode 100644 index 0000000..42cbd5b --- /dev/null +++ b/src/Explorer.Domain/EntryCategoryAssigner.cs @@ -0,0 +1,129 @@ +namespace Explorer.Domain; + +/// Where an category came from. +public static class CategorySources +{ + public const string None = "none"; + public const string Extension = "extension"; + public const string Folder = "folder"; + public const string Children = "children"; + public const string ArchiveContents = "archive_contents"; + public const string Mime = "mime"; + public const string User = "user"; + + public static bool IsUser(string? source) + => string.Equals(source, User, StringComparison.OrdinalIgnoreCase); + + public static bool IsAutomatic(string? source) + => !IsUser(source); +} + +public static class EntryCategoryAssigner +{ + public static void ApplyAutomatic( + IndexEntry entry, + string? rootPath = null, + bool isRepoRoot = false, + IReadOnlyList? childCategories = null) + { + if (CategorySources.IsUser(entry.CategorySource)) + { + return; + } + + var full = string.IsNullOrWhiteSpace(rootPath) + ? (string.IsNullOrEmpty(entry.PathRel) ? entry.Name : entry.PathRel) + : PathRules.Combine(rootPath, entry.PathRel); + var result = FileClassifier.Classify( + entry.Name, + full, + entry.IsDirectory, + entry.Attributes, + isRepoRoot, + childCategories); + Apply(entry, result, InferSource(entry, result, childCategories), Confidence(result, childCategories)); + } + + public static void ApplyUser(IndexEntry entry, FileCategory category, string? reason = null) + => Apply( + entry, + new FileClassification(category, reason ?? "Set by user."), + CategorySources.User, + 100); + + public static void Apply( + IndexEntry entry, + FileClassification classification, + string source, + int confidence) + { + entry.Category = classification.Category; + entry.CategoryReason = classification.Reason; + entry.CategorySource = source; + entry.CategoryConfidence = Math.Clamp(confidence, 0, 100); + entry.CategoryUtc = DateTimeOffset.UtcNow; + } + + public static FileCategory ParseCategory(string? value) + => Enum.TryParse(value, ignoreCase: true, out var cat) ? cat : FileCategory.Unknown; + + public static string InferSource( + IndexEntry entry, + FileClassification classification, + IReadOnlyList? childCategories) + { + if (childCategories is { Count: > 0 } && classification.Category != FileCategory.Unknown) + { + return entry.IsDirectory + ? CategorySources.Children + : CategorySources.ArchiveContents; + } + + if (entry.IsDirectory) + { + return classification.Category == FileCategory.Unknown + ? CategorySources.None + : CategorySources.Folder; + } + + if (ArchiveFormats.IsArchive(entry.Name)) + { + return CategorySources.Extension; + } + + return classification.Category == FileCategory.Unknown + ? CategorySources.None + : CategorySources.Extension; + } + + public static int Confidence(FileClassification classification, IReadOnlyList? childCategories) + { + if (classification.Category == FileCategory.Unknown) + { + return 0; + } + + if (childCategories is { Count: > 0 }) + { + var known = childCategories.Count(c => c is not FileCategory.Unknown); + if (known == 0) + { + return 40; + } + + var majority = childCategories + .Where(c => c == classification.Category) + .Count(); + return Math.Clamp(40 + majority * 60 / known, 40, 95); + } + + return classification.Category switch + { + FileCategory.SystemData or FileCategory.BuildOutput => 95, + FileCategory.Archive or FileCategory.Photos or FileCategory.Video + or FileCategory.Audio or FileCategory.Documents => 85, + FileCategory.Installer or FileCategory.Backup or FileCategory.CodeRepository => 80, + _ => 50 + }; + } +} diff --git a/src/Explorer.Domain/FileClassifier.cs b/src/Explorer.Domain/FileClassifier.cs index ca35be6..de10b4d 100644 --- a/src/Explorer.Domain/FileClassifier.cs +++ b/src/Explorer.Domain/FileClassifier.cs @@ -87,6 +87,13 @@ public static class FileClassifier { if (ArchiveFormats.IsArchive(name)) { + if (TryMajority(childCategories, out var archiveMajority)) + { + return new FileClassification( + archiveMajority, + "Archive contents are mostly " + OrganizeDestinations.Label(archiveMajority).ToLowerInvariant() + "."); + } + return new FileClassification(FileCategory.Archive, "Archive file."); } @@ -147,24 +154,62 @@ public static class FileClassifier return new FileClassification(FileCategory.Backup, "Backup folder name."); } - if (childCategories is { Count: > 0 }) + if (TryMajority(childCategories, out var folderMajority)) { - var known = childCategories.Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData).ToList(); - if (known.Count > 0) - { - var majority = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First(); - if (majority.Count() * 10 >= known.Count * 7) - { - return new FileClassification(majority.Key, "Folder contents are mostly " + OrganizeDestinations.Label(majority.Key).ToLowerInvariant() + "."); - } - } + return new FileClassification( + folderMajority, + "Folder contents are mostly " + OrganizeDestinations.Label(folderMajority).ToLowerInvariant() + "."); } return new FileClassification(FileCategory.Unknown, "No matching signal."); } + private static bool TryMajority(IReadOnlyList? childCategories, out FileCategory majority) + { + majority = FileCategory.Unknown; + if (childCategories is not { Count: > 0 }) + { + return false; + } + + var known = childCategories + .Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData) + .ToList(); + if (known.Count == 0) + { + return false; + } + + var top = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First(); + if (top.Count() * 10 < known.Count * 7) + { + return false; + } + + majority = top.Key; + return true; + } + public static bool ShouldLeave(FileCategory category) => category is FileCategory.Unknown or FileCategory.SystemData or FileCategory.BuildOutput or FileCategory.Backup; public static bool SkipDescent(string name) => BuildNames.Contains(name); + + public static bool IsPhotoFile(string name) + { + var ext = NameNormalizer.Extension(name); + return ext is not null && Photos.Contains(ext); + } + + public static bool IsVideoFile(string name) + { + var ext = NameNormalizer.Extension(name); + return ext is not null && Videos.Contains(ext); + } + + public static bool IsAudioFile(string name) + { + var ext = NameNormalizer.Extension(name); + return ext is not null && Audio.Contains(ext); + } } diff --git a/src/Explorer.Domain/FileSignatureClassifier.cs b/src/Explorer.Domain/FileSignatureClassifier.cs new file mode 100644 index 0000000..b74ed76 --- /dev/null +++ b/src/Explorer.Domain/FileSignatureClassifier.cs @@ -0,0 +1,84 @@ +namespace Explorer.Domain; + +/// Light magic-byte hints for files with no useful extension category. +public static class FileSignatureClassifier +{ + public static FileClassification? TryClassify(ReadOnlySpan header) + { + if (header.Length < 4) + { + return null; + } + + if (header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF) + { + return new FileClassification(FileCategory.Photos, "JPEG signature."); + } + + if (header.Length >= 8 + && header[0] == 0x89 && header[1] == 0x50 && header[2] == 0x4E && header[3] == 0x47) + { + return new FileClassification(FileCategory.Photos, "PNG signature."); + } + + if (header[0] == 0x47 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x38) + { + return new FileClassification(FileCategory.Photos, "GIF signature."); + } + + if (header[0] == 0x25 && header[1] == 0x50 && header[2] == 0x44 && header[3] == 0x46) + { + return new FileClassification(FileCategory.Documents, "PDF signature."); + } + + if (header[0] == 0x50 && header[1] == 0x4B && (header[2] == 0x03 || header[2] == 0x05 || header[2] == 0x07)) + { + return new FileClassification(FileCategory.Archive, "ZIP-family signature."); + } + + if (header.Length >= 12 + && header[0] == 0x52 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x46) + { + var form = System.Text.Encoding.ASCII.GetString(header.Slice(8, 4)); + if (form is "WAVE" or "AVI ") + { + return new FileClassification( + form == "WAVE" ? FileCategory.Audio : FileCategory.Video, + "RIFF " + form.Trim() + " signature."); + } + } + + if (header.Length >= 12 + && header[4] == 0x66 && header[5] == 0x74 && header[6] == 0x79 && header[7] == 0x70) + { + return new FileClassification(FileCategory.Video, "ISO BMFF (ftyp) signature."); + } + + if (header[0] == 0x49 && header[1] == 0x44 && header[2] == 0x33) + { + return new FileClassification(FileCategory.Audio, "ID3 signature."); + } + + if (header[0] == 0x7F && header[1] == 0x45 && header[2] == 0x4C && header[3] == 0x46) + { + return new FileClassification(FileCategory.Installer, "ELF binary signature."); + } + + if (header[0] == 0x4D && header[1] == 0x5A) + { + return new FileClassification(FileCategory.Installer, "PE/MZ signature."); + } + + if (header[0] == 0x37 && header[1] == 0x7A && header[2] == 0xBC && header[3] == 0xAF) + { + return new FileClassification(FileCategory.Archive, "7z signature."); + } + + if (header[0] == 0x52 && header[1] == 0x61 && header[2] == 0x72 && header[3] == 0x21) + { + return new FileClassification(FileCategory.Archive, "RAR signature."); + } + + return null; + } +} diff --git a/src/Explorer.Domain/OrganizeDestinations.cs b/src/Explorer.Domain/OrganizeDestinations.cs index 9458069..9a250a2 100644 --- a/src/Explorer.Domain/OrganizeDestinations.cs +++ b/src/Explorer.Domain/OrganizeDestinations.cs @@ -48,6 +48,51 @@ public sealed class OrganizeDestinations _ => "Unknown" }; + public static bool TryParse(string? value, out FileCategory category) + { + category = FileCategory.Unknown; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var key = value.Trim().Replace(" ", "", StringComparison.Ordinal).Replace("_", "", StringComparison.Ordinal); + if (Enum.TryParse(key, ignoreCase: true, out category) && category != FileCategory.Unknown) + { + return true; + } + + category = key.ToLowerInvariant() switch + { + "photo" or "photos" or "image" or "images" or "picture" or "pictures" => FileCategory.Photos, + "video" or "videos" or "movie" or "movies" => FileCategory.Video, + "audio" or "music" or "sound" => FileCategory.Audio, + "doc" or "docs" or "document" or "documents" => FileCategory.Documents, + "code" or "repo" or "repository" or "git" or "development" => FileCategory.CodeRepository, + "installer" or "installers" or "setup" => FileCategory.Installer, + "backup" or "backups" => FileCategory.Backup, + "archive" or "archives" or "zip" => FileCategory.Archive, + "system" or "systemdata" => FileCategory.SystemData, + "build" or "buildoutput" => FileCategory.BuildOutput, + "unknown" => FileCategory.Unknown, + _ => (FileCategory)(-1) + }; + return (int)category >= 0; + } + + public static readonly FileCategory[] UserChoices = + [ + FileCategory.Photos, + FileCategory.Video, + FileCategory.Audio, + FileCategory.Documents, + FileCategory.Installer, + FileCategory.Archive, + FileCategory.Backup, + FileCategory.CodeRepository, + FileCategory.Unknown + ]; + private static string? EmptyToNull(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); } diff --git a/src/Explorer.FileOperations/ReorganizeService.cs b/src/Explorer.FileOperations/ReorganizeService.cs index b10a726..29a0569 100644 --- a/src/Explorer.FileOperations/ReorganizeService.cs +++ b/src/Explorer.FileOperations/ReorganizeService.cs @@ -13,6 +13,7 @@ public sealed class ReorganizeService private readonly IHydrationGuard _hydration; private readonly IGitStatusProvider _git; private readonly UiPreferencesStore _preferences; + private readonly IIndexStore _store; public ReorganizeService( ReorganizePlanner planner, @@ -21,7 +22,8 @@ public sealed class ReorganizeService IFileSystemEnumerator enumerator, IHydrationGuard hydration, IGitStatusProvider git, - UiPreferencesStore preferences) + UiPreferencesStore preferences, + IIndexStore store) { _planner = planner; _ops = ops; @@ -30,6 +32,7 @@ public sealed class ReorganizeService _hydration = hydration; _git = git; _preferences = preferences; + _store = store; } public OrganizeDestinations LoadDestinations() @@ -63,14 +66,24 @@ public sealed class ReorganizeService } public OperationPlan Preview(string sourceRoot, OrganizeDestinations destinations) - => _planner.Build( + => PreviewAsync(sourceRoot, destinations).GetAwaiter().GetResult(); + + public async Task PreviewAsync( + string sourceRoot, + OrganizeDestinations destinations, + CancellationToken cancellationToken = default) + { + var indexed = await LoadIndexedChildrenAsync(sourceRoot, cancellationToken).ConfigureAwait(false); + return _planner.Build( sourceRoot, destinations, _enumerator, path => _volumes.IsPathReachable(path), path => _git.IsRepoRoot(path), item => _hydration.WouldHydrateOnRead(item), - RenameBatchService.PathExists); + RenameBatchService.PathExists, + indexedByName: indexed); + } public async Task EnqueueAsync( string sourceRoot, @@ -78,7 +91,7 @@ public sealed class ReorganizeService OperationPlan? plan = null, CancellationToken cancellationToken = default) { - plan ??= Preview(sourceRoot, destinations); + plan ??= await PreviewAsync(sourceRoot, destinations, cancellationToken).ConfigureAwait(false); if (!plan.CanEnqueue) { return plan; @@ -97,6 +110,62 @@ public sealed class ReorganizeService return plan; } + private async Task> LoadIndexedChildrenAsync( + string sourceRoot, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(sourceRoot)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + try + { + var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false); + Source? best = null; + var bestLen = -1; + foreach (var source in sources) + { + if (string.IsNullOrEmpty(source.LastRootPath)) + { + continue; + } + + var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\'); + var path = PathRules.FromExtended(sourceRoot).TrimEnd('\\'); + if (path.Equals(root, StringComparison.OrdinalIgnoreCase) + || path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase)) + { + if (root.Length > bestLen) + { + best = source; + bestLen = root.Length; + } + } + } + + if (best?.LastRootPath is null) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var rel = PathRules.MakeRelative(best.LastRootPath, sourceRoot) ?? ""; + var dir = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false); + if (dir is null) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var children = await _store.Entries.GetChildrenAsync(best.Id, dir.Id, EntryStatus.Present, cancellationToken) + .ConfigureAwait(false); + return children.ToDictionary(c => c.Name, StringComparer.OrdinalIgnoreCase); + } + catch (Exception) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + } + private static string Known(Environment.SpecialFolder folder) => Environment.GetFolderPath(folder); diff --git a/src/Explorer.Hosting.Client/Ipc/WorkbenchPipeClient.cs b/src/Explorer.Hosting.Client/Ipc/WorkbenchPipeClient.cs index 5934b86..867119c 100644 --- a/src/Explorer.Hosting.Client/Ipc/WorkbenchPipeClient.cs +++ b/src/Explorer.Hosting.Client/Ipc/WorkbenchPipeClient.cs @@ -579,6 +579,17 @@ public sealed class WorkbenchPipeClient : IWorkbenchHost, ICloudOverlay, IHostCo => _client.CallAsync("Mutations.EnqueueHashCollisions", cancellationToken, n: sourceId ?? 0); public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default) => _client.CallAsync("Mutations.UpsertRelation", cancellationToken, payload: Json(relation)); + public Task SetUserCategoryByPathAsync( + string fullPath, + FileCategory category, + string? reason = null, + CancellationToken cancellationToken = default) + => _client.CallAsync( + "Mutations.SetUserCategory", + cancellationToken, + s: fullPath, + dest: reason, + payload: category.ToString()); private static string Json(T value) => JsonSerializer.Serialize(value, WorkbenchIpc.Json); } } diff --git a/src/Explorer.Hosting/BackgroundMaintenanceCoordinator.cs b/src/Explorer.Hosting/BackgroundMaintenanceCoordinator.cs index 64e8208..01b00e4 100644 --- a/src/Explorer.Hosting/BackgroundMaintenanceCoordinator.cs +++ b/src/Explorer.Hosting/BackgroundMaintenanceCoordinator.cs @@ -14,6 +14,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg private readonly IForegroundWorkSignal _foreground; private readonly IIdleIndexWork _indexing; private readonly IIdleHashWork _hash; + private readonly IIdleClassifyWork _classify; private readonly IHistoryMaintenance _history; private readonly IIndexStore _store; private readonly IVolumeService _volumes; @@ -44,7 +45,8 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg IIndexStore store, IVolumeService volumes, ILogger logger, - IHostActivitySink? activity = null) + IHostActivitySink? activity = null, + IIdleClassifyWork? classify = null) { _idle = idle; _power = power; @@ -52,6 +54,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg _foreground = foreground; _indexing = indexing; _hash = hash; + _classify = classify ?? NullIdleClassifyWork.Instance; _history = history; _store = store; _volumes = volumes; @@ -71,6 +74,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _hash.Pause(); + _classify.Pause(); using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1)); while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) { @@ -115,10 +119,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg { _idleScanQueued.Clear(); _idleVerified.Clear(); - if (!_hash.IsPaused) - { - _hash.Pause(); - } + PauseBackgroundWorkers(); if (_lastAllowed) { @@ -147,10 +148,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg if (_indexing.IsBusy || _indexing.HasIdleWork) { - if (!_hash.IsPaused) - { - _hash.Pause(); - } + PauseBackgroundWorkers(); // Keep the idle-maintenance caption only while idle work is still queued/running. // Watcher/user jobs must not keep showing a stale "checking External" line. @@ -165,10 +163,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg var verify = BackgroundMaintenancePlanner.NextLocalVerify(sources, _idleVerified); if (verify is not null && verify.LastRootPath is not null && _volumes.IsPathReachable(verify.LastRootPath)) { - if (!_hash.IsPaused) - { - _hash.Pause(); - } + PauseBackgroundWorkers(); _idleVerified.Add(verify.Id); _indexing.EnqueueIdleVerify(verify.Id, ""); @@ -183,10 +178,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg var scan = BackgroundMaintenancePlanner.NextLocalScan(sources, DateTimeOffset.UtcNow, _idleScanQueued); if (scan is not null && scan.LastRootPath is not null && _volumes.IsPathReachable(scan.LastRootPath)) { - if (!_hash.IsPaused) - { - _hash.Pause(); - } + PauseBackgroundWorkers(); _idleScanQueued.Add(scan.Id); _indexing.EnqueueIdleFullScan(scan.Id); @@ -196,6 +188,16 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg return; } + PauseHash(); + _classify.Resume(); + if (await _classify.ProcessPendingAsync(cancellationToken).ConfigureAwait(false)) + { + _workMessage = "Idle maintenance · classifying"; + Publish(decision, _workMessage); + return; + } + + _classify.Pause(); _hash.Resume(); if (hashPending) { @@ -232,6 +234,23 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg Publish(decision, _workMessage); } + private void PauseBackgroundWorkers() + { + PauseHash(); + if (!_classify.IsPaused) + { + _classify.Pause(); + } + } + + private void PauseHash() + { + if (!_hash.IsPaused) + { + _hash.Pause(); + } + } + private string PauseMessage(BackgroundWorkDecision decision) { if (decision.SkipReason == MaintenanceSkipReason.ForegroundOperations) diff --git a/src/Explorer.Hosting/ExplorerHostServices.cs b/src/Explorer.Hosting/ExplorerHostServices.cs index 808928d..9177ef8 100644 --- a/src/Explorer.Hosting/ExplorerHostServices.cs +++ b/src/Explorer.Hosting/ExplorerHostServices.cs @@ -100,8 +100,10 @@ public static class ExplorerHostServices sp.GetRequiredService())); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); diff --git a/src/Explorer.Hosting/Ipc/WorkbenchPipeServer.cs b/src/Explorer.Hosting/Ipc/WorkbenchPipeServer.cs index 9a2b5e0..49248a0 100644 --- a/src/Explorer.Hosting/Ipc/WorkbenchPipeServer.cs +++ b/src/Explorer.Hosting/Ipc/WorkbenchPipeServer.cs @@ -462,6 +462,16 @@ public sealed class WorkbenchPipeServer : BackgroundService case "Mutations.UpsertRelation": await Workbench.Mutations.UpsertRelationAsync(Read(request.Payload)).ConfigureAwait(false); return reply; + case "Mutations.SetUserCategory": + if (OrganizeDestinations.TryParse(request.Payload, out var userCategory) + || Enum.TryParse(request.Payload, true, out userCategory)) + { + await Workbench.Mutations.SetUserCategoryByPathAsync( + request.S ?? "", + userCategory, + request.Dest).ConfigureAwait(false); + } + return reply; case "Cloud.Places": reply.Payload = JsonSerializer.Serialize(_overlay!.GetPlaces(), WorkbenchIpc.Json); return reply; diff --git a/src/Explorer.Indexing/FolderReconciler.cs b/src/Explorer.Indexing/FolderReconciler.cs index 36d4bf4..c385e9f 100644 --- a/src/Explorer.Indexing/FolderReconciler.cs +++ b/src/Explorer.Indexing/FolderReconciler.cs @@ -190,6 +190,9 @@ public sealed class FolderReconciler } }, cancellationToken).ConfigureAwait(false); + await _store.Entries.RefreshCategoryFromChildrenAsync(parent.Id, cancellationToken) + .ConfigureAwait(false); + foreach (var path in archivesToTomb) { await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken) @@ -202,6 +205,8 @@ public sealed class FolderReconciler { await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken) .ConfigureAwait(false); + await _store.Entries.RefreshCategoryFromChildrenAsync(archive.Id, cancellationToken) + .ConfigureAwait(false); } } diff --git a/src/Explorer.Presentation/ViewModels/AnalysisViewModel.cs b/src/Explorer.Presentation/ViewModels/AnalysisViewModel.cs index d447d0e..382dacc 100644 --- a/src/Explorer.Presentation/ViewModels/AnalysisViewModel.cs +++ b/src/Explorer.Presentation/ViewModels/AnalysisViewModel.cs @@ -69,6 +69,7 @@ public sealed partial class AnalysisViewModel : ObservableObject public const string PageFolders = "Biggest folders"; public const string PageFiles = "Biggest files"; public const string PageTypes = "By file type"; + public const string PageCategories = "By category"; public const string PageSources = "By source"; private readonly AnalysisService _analysis; @@ -93,7 +94,7 @@ public sealed partial class AnalysisViewModel : ObservableObject Rows = []; } - public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageSources]; + public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageCategories, PageSources]; public ObservableCollection Scopes { get; } public ObservableCollection TreeRoots { get; } public ObservableCollection VisibleNodes { get; } @@ -461,6 +462,25 @@ public sealed partial class AnalysisViewModel : ObservableObject 0, (string?)null)).ToList(); } + else if (page == PageCategories) + { + var cats = await _analysis.UsageByCategoryAsync(sourceId, null, AppConstants.AnalysisTopN, cancellationToken) + .ConfigureAwait(false); + items = cats.Select(t => ( + OrganizeDestinations.TryParse(t.Category, out var cat) + ? OrganizeDestinations.Label(cat) + : string.IsNullOrEmpty(t.Category) ? "Unknown" : t.Category, + t.TotalSize, + (string?)null, + (string?)null, + (string?)null, + (long?)null, + sourceId, + false, + (int)t.FileCount, + 0, + (string?)null)).ToList(); + } else { var usage = await _analysis.UsageBySourceAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Explorer.Presentation/ViewModels/FolderItemViewModel.cs b/src/Explorer.Presentation/ViewModels/FolderItemViewModel.cs index 55fdaed..ca9f477 100644 --- a/src/Explorer.Presentation/ViewModels/FolderItemViewModel.cs +++ b/src/Explorer.Presentation/ViewModels/FolderItemViewModel.cs @@ -85,6 +85,27 @@ public sealed partial class FolderItemViewModel : ObservableObject }; public bool HasCloudStatus => !string.IsNullOrEmpty(CloudLabel); + public string CategoryLabel + => Item.Category == FileCategory.Unknown + && (string.IsNullOrEmpty(Item.CategorySource) || Item.CategorySource == CategorySources.None) + ? "" + : OrganizeDestinations.Label(Item.Category); + + public string CategoryTooltip + { + get + { + var label = CategoryLabel; + if (string.IsNullOrEmpty(label)) + { + return ""; + } + + var reason = Item.CategoryReason; + return string.IsNullOrEmpty(reason) ? label : label + " · " + reason; + } + } + [ObservableProperty] private string _gitLabel = ""; public bool HasGitLabel => !string.IsNullOrEmpty(GitLabel); @@ -122,18 +143,8 @@ public sealed partial class FolderItemViewModel : ObservableObject internal static class MediaKinds { - private static readonly HashSet Images = new(StringComparer.OrdinalIgnoreCase) - { - ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tif", ".tiff", ".ico", ".jfif" - }; - - private static readonly HashSet Videos = new(StringComparer.OrdinalIgnoreCase) - { - ".mp4", ".mkv", ".avi", ".mov", ".wmv", ".webm", ".m4v", ".mpg", ".mpeg" - }; - - public static bool IsImage(string name) => Images.Contains(Path.GetExtension(name)); - public static bool IsVideo(string name) => Videos.Contains(Path.GetExtension(name)); + public static bool IsImage(string name) => FileClassifier.IsPhotoFile(name); + public static bool IsVideo(string name) => FileClassifier.IsVideoFile(name); } file static class ItemExt diff --git a/src/Explorer.Presentation/ViewModels/MainViewModel.cs b/src/Explorer.Presentation/ViewModels/MainViewModel.cs index d1f7e8b..088cd68 100644 --- a/src/Explorer.Presentation/ViewModels/MainViewModel.cs +++ b/src/Explorer.Presentation/ViewModels/MainViewModel.cs @@ -34,6 +34,7 @@ public sealed partial class MainViewModel : ObservableObject private readonly ConversionPlanner _conversionPlanner; private readonly IFileSystemEnumerator _enumerator; private readonly IMediaConversionProvider _conversion; + private readonly IIndexMutations _mutations; private readonly IThumbnailService? _thumbnails; private readonly IHostConnection? _host; private readonly IBackgroundMaintenance? _maintenance; @@ -65,6 +66,7 @@ public sealed partial class MainViewModel : ObservableObject [ObservableProperty] private bool _showTags; [ObservableProperty] private bool _showRunProfile; [ObservableProperty] private bool _showOrganizeFolder; + [ObservableProperty] private bool _showClassifyAs; [ObservableProperty] private bool _canUndoRenameBatch; [ObservableProperty] private bool _showExtractArchive; [ObservableProperty] private bool _showCompress; @@ -141,6 +143,7 @@ public sealed partial class MainViewModel : ObservableObject _conversionPlanner = conversionPlanner; _enumerator = enumerator; _conversion = conversion; + _mutations = mutations; _thumbnails = thumbnails; _host = hostConnection; _maintenance = maintenance; @@ -737,6 +740,36 @@ public sealed partial class MainViewModel : ObservableObject public ReorganizeViewModel CreateReorganizeViewModel() => new(_reorganize, OrganizeSourcePath()); + [RelayCommand] + public async Task ClassifyAsAsync(string? category) + { + if (!OrganizeDestinations.TryParse(category, out var parsed) + && !Enum.TryParse(category, true, out parsed)) + { + Footer = "Unknown category."; + return; + } + + var items = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList(); + if (items.Count == 0) + { + Footer = "Select a file or folder to classify."; + return; + } + + var n = 0; + foreach (var item in items) + { + await _mutations.SetUserCategoryByPathAsync(item.FullPath, parsed, "Set by user.").ConfigureAwait(true); + n++; + } + + await ActivePane.RefreshAsync().ConfigureAwait(true); + Footer = n == 1 + ? "Classified as " + OrganizeDestinations.Label(parsed) + "." + : "Classified " + n + " items as " + OrganizeDestinations.Label(parsed) + "."; + } + public GitChangesViewModel CreateGitChangesViewModel() => new(_gitCommands, _workspace, _hydration); @@ -984,6 +1017,7 @@ public sealed partial class MainViewModel : ObservableObject ShowRunProfile = ShowBatchRename; ShowOrganizeFolder = OrganizeSourcePath() is not null; var real = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList(); + ShowClassifyAs = real.Count > 0; ShowExtractArchive = real.Count > 0 && real.All(i => !i.IsDirectory && ArchiveFormats.IsArchive(i.Item.Name)); ShowCompress = real.Count > 0; ShowConvert = real.Any(i => i.IsDirectory || ConversionFormats.IsConvertible(i.Item.Name)); diff --git a/src/Explorer.Presentation/ViewModels/ReorganizeViewModel.cs b/src/Explorer.Presentation/ViewModels/ReorganizeViewModel.cs index b7817ce..a2f01e1 100644 --- a/src/Explorer.Presentation/ViewModels/ReorganizeViewModel.cs +++ b/src/Explorer.Presentation/ViewModels/ReorganizeViewModel.cs @@ -44,7 +44,7 @@ public sealed partial class ReorganizeViewModel : ObservableObject { Status = "Analyzing…"; CanQueue = false; - var plan = await Task.Run(() => _organize.Preview(SourcePath.Trim(), CurrentDestinations())).ConfigureAwait(true); + var plan = await _organize.PreviewAsync(SourcePath.Trim(), CurrentDestinations()).ConfigureAwait(true); ApplyPlan(plan); } diff --git a/src/Explorer.Presentation/ViewModels/SearchViewModel.cs b/src/Explorer.Presentation/ViewModels/SearchViewModel.cs index 3fefc97..448e6a3 100644 --- a/src/Explorer.Presentation/ViewModels/SearchViewModel.cs +++ b/src/Explorer.Presentation/ViewModels/SearchViewModel.cs @@ -9,6 +9,7 @@ using Explorer.Search; namespace Explorer.Presentation.ViewModels; public sealed record SearchScopeChoice(SearchScopeKind Kind, string Label); +public sealed record CategoryChoice(string Value, string Label); public sealed partial class SearchViewModel : ObservableObject { @@ -20,6 +21,7 @@ public sealed partial class SearchViewModel : ObservableObject [ObservableProperty] private string _text = ""; [ObservableProperty] private SearchScopeKind _scope = SearchScopeKind.AllKnown; [ObservableProperty] private string? _extension; + [ObservableProperty] private string _category = ""; [ObservableProperty] private bool _foldersOnly; [ObservableProperty] private bool _filesOnly; [ObservableProperty] private string? _minSizeText; @@ -46,6 +48,12 @@ public sealed partial class SearchViewModel : ObservableObject new(SearchScopeKind.OfflineMedia, "Offline media") ]; + public CategoryChoice[] CategoryChoices { get; } = + [ + new("", "Any category"), + .. OrganizeDestinations.UserChoices.Select(c => new CategoryChoice(c.ToString(), OrganizeDestinations.Label(c))) + ]; + public void OpenWithoutSearch() { IsOpen = true; @@ -103,6 +111,7 @@ public sealed partial class SearchViewModel : ObservableObject Scope = scope, Text = Text, Extension = Extension, + Category = OrganizeDestinations.TryParse(Category, out var cat) ? cat : null, IsDirectory = DirectoryFilter(FoldersOnly, FilesOnly), MinSize = ParseSize(MinSizeText), MaxSize = ParseSize(MaxSizeText), @@ -139,7 +148,11 @@ public sealed partial class SearchViewModel : ObservableObject FileId = entry.FileId, ReparseTag = entry.ReparseTag, FreeSpaceBytes = space.FreeBytes, - CapacityBytes = space.CapacityBytes + CapacityBytes = space.CapacityBytes, + IndexEntryId = entry.Id, + Category = entry.Category, + CategoryReason = entry.CategoryReason, + CategorySource = entry.CategorySource }, entry.IsDirectory)); } diff --git a/src/Explorer.Search/Explorer.Search.csproj b/src/Explorer.Search/Explorer.Search.csproj index 64ae0f5..9938935 100644 --- a/src/Explorer.Search/Explorer.Search.csproj +++ b/src/Explorer.Search/Explorer.Search.csproj @@ -9,4 +9,7 @@ + + + diff --git a/src/Explorer.Search/SearchService.cs b/src/Explorer.Search/SearchService.cs index 7664c6b..ed9bd20 100644 --- a/src/Explorer.Search/SearchService.cs +++ b/src/Explorer.Search/SearchService.cs @@ -15,6 +15,7 @@ public sealed class SearchQuery public DateTimeOffset? CreatedBefore { get; init; } public DateTimeOffset? ModifiedAfter { get; init; } public DateTimeOffset? ModifiedBefore { get; init; } + public FileCategory? Category { get; init; } public IReadOnlyList? SourceIds { get; init; } public string? PathRelPrefix { get; init; } public bool IncludeOffline { get; init; } = true; @@ -33,9 +34,19 @@ public sealed class SearchService var text = query.Text?.Trim(); string? extension = query.Extension; string? name = text; - if (!string.IsNullOrEmpty(text) && text.StartsWith("*.", StringComparison.Ordinal) && !text.Contains(' ', StringComparison.Ordinal)) + var category = query.Category; + if (!string.IsNullOrEmpty(text)) { - extension = text[2..]; + (name, var hinted) = SplitCategoryHint(text); + if (hinted is not null) + { + category = hinted; + } + } + + if (!string.IsNullOrEmpty(name) && name.StartsWith("*.", StringComparison.Ordinal) && !name.Contains(' ', StringComparison.Ordinal)) + { + extension = name[2..]; name = null; } @@ -43,6 +54,7 @@ public sealed class SearchService { Name = name, Extension = extension, + Category = category, IsDirectory = query.IsDirectory, MinSize = query.MinSize, MaxSize = query.MaxSize, @@ -62,4 +74,25 @@ public sealed class SearchService return _store.Search.SearchAsync(request, cancellationToken); } + + internal static (string? Name, FileCategory? Category) SplitCategoryHint(string text) + { + FileCategory? category = null; + var name = System.Text.RegularExpressions.Regex.Replace( + text, + @"\b(?:category|cat):([A-Za-z]+)\b", + match => + { + if (OrganizeDestinations.TryParse(match.Groups[1].Value, out var parsed)) + { + category = parsed; + return ""; + } + + return match.Value; + }, + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + name = name.Trim(); + return (string.IsNullOrEmpty(name) ? null : name, category); + } } diff --git a/src/Explorer.Storage.Sqlite/AnalysisHistoryHashStores.cs b/src/Explorer.Storage.Sqlite/AnalysisHistoryHashStores.cs index a5585d6..0fe096d 100644 --- a/src/Explorer.Storage.Sqlite/AnalysisHistoryHashStores.cs +++ b/src/Explorer.Storage.Sqlite/AnalysisHistoryHashStores.cs @@ -91,6 +91,24 @@ internal sealed class AnalysisStore : IAnalysisStore return rows.AsList(); } + public async Task> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default) + { + await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false); + var sql = """ + SELECT ifnull(category, 'Unknown') AS Category, SUM(size_bytes) AS TotalSize, COUNT(*) AS FileCount + FROM entries + WHERE is_dir=0 AND status=0 + """; + if (sourceId is not null) sql += " AND source_id=@sourceId"; + if (!string.IsNullOrEmpty(pathRelPrefix)) + sql += " AND (path_rel=@pathRelPrefix OR path_rel LIKE @like ESCAPE '\\')"; + sql += " GROUP BY category ORDER BY TotalSize DESC LIMIT @take"; + var like = string.IsNullOrEmpty(pathRelPrefix) ? null : pathRelPrefix.Replace("\\", "\\\\") + "\\\\%"; + var rows = await conn.QueryAsync(new CommandDefinition( + sql, new { sourceId, pathRelPrefix, like, take }, cancellationToken: cancellationToken)).ConfigureAwait(false); + return rows.AsList(); + } + public async Task> UsageBySourceAsync(CancellationToken cancellationToken = default) { await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Explorer.Storage.Sqlite/EntryStore.cs b/src/Explorer.Storage.Sqlite/EntryStore.cs index 6956332..e9bebbd 100644 --- a/src/Explorer.Storage.Sqlite/EntryStore.cs +++ b/src/Explorer.Storage.Sqlite/EntryStore.cs @@ -88,14 +88,41 @@ internal sealed class EntryStore : IEntryStore internal static async Task UpsertCore(SqliteConnection conn, IndexEntry entry) { - var existing = await SqliteExec.ScalarAsync( + var existingId = await SqliteExec.ScalarAsync( conn, "SELECT id FROM entries WHERE source_id=@SourceId AND ifnull(parent_id,-1)=ifnull(@ParentId,-1) AND name_norm=@NameNorm", new { entry.SourceId, entry.ParentId, entry.NameNorm }).ConfigureAwait(false); - if (existing is > 0) + if (existingId is > 0) { - entry.Id = existing.Value; + entry.Id = existingId.Value; + var existingSource = await SqliteExec.ScalarAsync( + conn, + "SELECT ifnull(category_source, 'none') FROM entries WHERE id=@Id", + new { entry.Id }).ConfigureAwait(false); + if (CategorySources.IsUser(existingSource)) + { + entry.Category = EntryCategoryAssigner.ParseCategory( + await SqliteExec.ScalarAsync(conn, "SELECT category FROM entries WHERE id=@Id", new { entry.Id }) + .ConfigureAwait(false)); + entry.CategoryReason = await SqliteExec.ScalarAsync( + conn, "SELECT category_reason FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false); + entry.CategoryConfidence = await SqliteExec.ScalarAsync( + conn, "SELECT ifnull(category_confidence, 0) FROM entries WHERE id=@Id", new { entry.Id }) + .ConfigureAwait(false) is { } conf + ? (int)conf + : 0; + entry.CategorySource = CategorySources.User; + var utc = await SqliteExec.ScalarAsync( + conn, "SELECT category_utc FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false); + entry.CategoryUtc = string.IsNullOrEmpty(utc) ? null : DateTimeOffset.Parse(utc); + } + else if (string.IsNullOrWhiteSpace(entry.CategorySource) + || entry.CategorySource == CategorySources.None) + { + EntryCategoryAssigner.ApplyAutomatic(entry); + } + await SqliteExec.ExecuteAsync(conn, """ UPDATE entries SET name=@Name, extension=@Extension, is_dir=@IsDir, size_bytes=@SizeBytes, @@ -103,21 +130,32 @@ internal sealed class EntryStore : IEntryStore last_indexed_utc=@LastIndexedUtc, attributes=@Attributes, file_id=@FileId, parent_file_id=@ParentFileId, reparse_tag=@ReparseTag, status=@Status, deleted_utc=@DeletedUtc, path_rel=@PathRel, scan_generation=@ScanGeneration, - allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability + allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability, + category=@Category, category_reason=@CategoryReason, + category_confidence=@CategoryConfidence, category_source=@CategorySource, + category_utc=@CategoryUtc WHERE id=@Id """, ToArgs(entry)).ConfigureAwait(false); return entry.Id; } + if (string.IsNullOrWhiteSpace(entry.CategorySource) + || entry.CategorySource == CategorySources.None) + { + EntryCategoryAssigner.ApplyAutomatic(entry); + } + var id = await SqliteExec.InsertAsync(conn, """ INSERT INTO entries (source_id, parent_id, name, name_norm, extension, is_dir, size_bytes, aggregate_size, child_file_count, child_dir_count, created_utc, modified_utc, last_seen_utc, last_indexed_utc, attributes, file_id, parent_file_id, reparse_tag, status, deleted_utc, path_rel, content_hash, hash_state, scan_generation, - allocated_size, cloud_availability) + allocated_size, cloud_availability, + category, category_reason, category_confidence, category_source, category_utc) VALUES (@SourceId, @ParentId, @Name, @NameNorm, @Extension, @IsDir, @SizeBytes, @AggregateSize, @ChildFileCount, @ChildDirCount, @CreatedUtc, @ModifiedUtc, @LastSeenUtc, @LastIndexedUtc, @Attributes, @FileId, @ParentFileId, @ReparseTag, @Status, @DeletedUtc, @PathRel, @ContentHash, @HashState, @ScanGeneration, - @AllocatedSizeBytes, @CloudAvailability) + @AllocatedSizeBytes, @CloudAvailability, + @Category, @CategoryReason, @CategoryConfidence, @CategorySource, @CategoryUtc) """, ToArgs(entry)).ConfigureAwait(false); entry.Id = id; return id; @@ -262,6 +300,121 @@ internal sealed class EntryStore : IEntryStore new { sourceId }).ConfigureAwait(false); } + public Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default) + => _store.WriteAsync(async conn => + { + var row = await conn.QuerySingleOrDefaultAsync( + "SELECT * FROM entries WHERE id=@id", new { id = folderOrArchiveId }).ConfigureAwait(false); + if (row is null) + { + return; + } + + var entry = row.ToModel(); + if (CategorySources.IsUser(entry.CategorySource)) + { + return; + } + + var children = (await conn.QueryAsync( + "SELECT ifnull(category, 'Unknown') FROM entries WHERE parent_id=@id AND status=0", + new { id = folderOrArchiveId }).ConfigureAwait(false)).ToList(); + if (children.Count == 0) + { + return; + } + + var cats = children.Select(EntryCategoryAssigner.ParseCategory).ToList(); + EntryCategoryAssigner.ApplyAutomatic(entry, childCategories: cats); + await SqliteExec.ExecuteAsync(conn, """ + UPDATE entries SET + category=@Category, category_reason=@CategoryReason, + category_confidence=@CategoryConfidence, category_source=@CategorySource, + category_utc=@CategoryUtc + WHERE id=@Id + """, ToArgs(entry)).ConfigureAwait(false); + }, cancellationToken); + + public Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default) + => _store.WriteAsync(async conn => + { + var row = await conn.QuerySingleOrDefaultAsync( + "SELECT * FROM entries WHERE id=@id", new { id = entryId }).ConfigureAwait(false); + if (row is null) + { + return; + } + + var entry = row.ToModel(); + EntryCategoryAssigner.ApplyUser(entry, category, reason); + await SqliteExec.ExecuteAsync(conn, """ + UPDATE entries SET + category=@Category, category_reason=@CategoryReason, + category_confidence=@CategoryConfidence, category_source=@CategorySource, + category_utc=@CategoryUtc + WHERE id=@Id + """, ToArgs(entry)).ConfigureAwait(false); + }, cancellationToken); + + public Task BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default) + => _store.WriteAsync(async conn => + { + take = Math.Clamp(take, 1, 500); + var rows = (await conn.QueryAsync(new CommandDefinition(""" + SELECT * FROM entries + WHERE status=0 AND ifnull(category_source, 'none') IN ('none', '') + AND category_utc IS NULL + ORDER BY id + LIMIT @take + """, new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false)).ToList(); + foreach (var row in rows) + { + var entry = row.ToModel(); + EntryCategoryAssigner.ApplyAutomatic(entry); + await SqliteExec.ExecuteAsync(conn, """ + UPDATE entries SET + category=@Category, category_reason=@CategoryReason, + category_confidence=@CategoryConfidence, category_source=@CategorySource, + category_utc=@CategoryUtc + WHERE id=@Id + """, ToArgs(entry)).ConfigureAwait(false); + } + + return rows.Count; + }, cancellationToken); + + public async Task> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default) + { + take = Math.Clamp(take, 1, 100); + await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false); + var ids = await conn.QueryAsync(new CommandDefinition(""" + SELECT a.id + FROM entries a + WHERE a.status=0 AND a.is_dir=0 AND a.category='Archive' + AND ifnull(a.category_source, 'none') IN ('extension', 'none') + AND EXISTS ( + SELECT 1 FROM entries c WHERE c.parent_id=a.id AND c.status=0 LIMIT 1) + ORDER BY a.id + LIMIT @take + """, new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false); + return ids.ToList(); + } + + public async Task> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default) + { + take = Math.Clamp(take, 1, 50); + await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false); + var rows = await conn.QueryAsync(new CommandDefinition(""" + SELECT * FROM entries + WHERE status=0 AND is_dir=0 AND category='Unknown' + AND ifnull(category_source, 'none') IN ('none', '') + AND ifnull(cloud_availability, 0) != 1 + ORDER BY id + LIMIT @take + """, new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false); + return rows.Select(r => r.ToModel()).ToList(); + } + private static string EscapeLike(string? value) { if (string.IsNullOrEmpty(value)) @@ -302,7 +455,12 @@ internal sealed class EntryStore : IEntryStore HashState = (int)e.HashState, e.ScanGeneration, e.AllocatedSizeBytes, - CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null + CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null, + Category = e.Category.ToString(), + e.CategoryReason, + e.CategoryConfidence, + e.CategorySource, + CategoryUtc = e.CategoryUtc?.ToString("O") }; } @@ -335,6 +493,11 @@ internal sealed class EntryRow public long scan_generation { get; set; } public long? allocated_size { get; set; } public int? cloud_availability { get; set; } + public string? category { get; set; } + public string? category_reason { get; set; } + public int category_confidence { get; set; } + public string? category_source { get; set; } + public string? category_utc { get; set; } public IndexEntry ToModel() => new() { @@ -364,7 +527,12 @@ internal sealed class EntryRow HashState = (HashState)hash_state, ScanGeneration = scan_generation, AllocatedSizeBytes = allocated_size, - CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null + CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null, + Category = EntryCategoryAssigner.ParseCategory(category), + CategoryReason = category_reason, + CategoryConfidence = category_confidence, + CategorySource = string.IsNullOrWhiteSpace(category_source) ? CategorySources.None : category_source, + CategoryUtc = Parse(category_utc) }; private static DateTimeOffset? Parse(string? v) diff --git a/src/Explorer.Storage.Sqlite/SchemaScript.cs b/src/Explorer.Storage.Sqlite/SchemaScript.cs index dcd18fe..fa3ff2e 100644 --- a/src/Explorer.Storage.Sqlite/SchemaScript.cs +++ b/src/Explorer.Storage.Sqlite/SchemaScript.cs @@ -63,11 +63,17 @@ internal static class SchemaScript hash_state INTEGER NOT NULL DEFAULT 0, scan_generation INTEGER NOT NULL DEFAULT 0, allocated_size INTEGER, - cloud_availability INTEGER + cloud_availability INTEGER, + category TEXT NOT NULL DEFAULT 'Unknown', + category_reason TEXT, + category_confidence INTEGER NOT NULL DEFAULT 0, + category_source TEXT NOT NULL DEFAULT 'none', + category_utc TEXT ) """, "CREATE UNIQUE INDEX ix_entries_identity ON entries(source_id, ifnull(parent_id, -1), name_norm)", "CREATE INDEX ix_entries_parent ON entries(source_id, parent_id, status)", + "CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0", "CREATE INDEX ix_entries_ext_size ON entries(source_id, extension, size_bytes) WHERE is_dir = 0", "CREATE INDEX ix_entries_size ON entries(source_id, size_bytes) WHERE is_dir = 0 AND status = 0", "CREATE INDEX ix_entries_modified ON entries(source_id, modified_utc)", diff --git a/src/Explorer.Storage.Sqlite/SearchStore.cs b/src/Explorer.Storage.Sqlite/SearchStore.cs index a39a71a..fec3dfa 100644 --- a/src/Explorer.Storage.Sqlite/SearchStore.cs +++ b/src/Explorer.Storage.Sqlite/SearchStore.cs @@ -63,6 +63,12 @@ internal sealed class SearchStore : ISearchStore args.Add("Ext", NameNormalizer.Normalize(request.Extension.TrimStart('.'))); } + if (request.Category is { } category) + { + sql.Append(" AND e.category = @Category"); + args.Add("Category", category.ToString()); + } + if (request.MinSize is not null) { sql.Append(" AND e.size_bytes >= @MinSize"); diff --git a/src/Explorer.Storage.Sqlite/SqliteIndexStore.cs b/src/Explorer.Storage.Sqlite/SqliteIndexStore.cs index 3d37aa9..12354ae 100644 --- a/src/Explorer.Storage.Sqlite/SqliteIndexStore.cs +++ b/src/Explorer.Storage.Sqlite/SqliteIndexStore.cs @@ -606,6 +606,25 @@ public sealed class SqliteIndexStore : IIndexStore, IAsyncDisposable SetUserVersion(conn, 10); _logger.LogInformation("Migrated SQLite schema to v10 (hash queue state index)"); + version = 10; + } + + if (version < 11) + { + EnsureColumn(conn, "entries", "category", "TEXT NOT NULL DEFAULT 'Unknown'"); + EnsureColumn(conn, "entries", "category_reason", "TEXT"); + EnsureColumn(conn, "entries", "category_confidence", "INTEGER NOT NULL DEFAULT 0"); + EnsureColumn(conn, "entries", "category_source", "TEXT NOT NULL DEFAULT 'none'"); + EnsureColumn(conn, "entries", "category_utc", "TEXT"); + using (var idx = conn.CreateCommand()) + { + idx.CommandText = + "CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0"; + idx.ExecuteNonQuery(); + } + + SetUserVersion(conn, 11); + _logger.LogInformation("Migrated SQLite schema to v11 (entry categories)"); } } diff --git a/tests/Explorer.Analysis.Tests/AnalysisTests.cs b/tests/Explorer.Analysis.Tests/AnalysisTests.cs index d177321..1b5a42f 100644 --- a/tests/Explorer.Analysis.Tests/AnalysisTests.cs +++ b/tests/Explorer.Analysis.Tests/AnalysisTests.cs @@ -39,6 +39,8 @@ public class AnalysisTests Assert.Equal("a.mkv", files[0].Name); var types = await analysis.UsageByExtensionAsync(source.Id, null); Assert.Contains(types, t => t.Extension == "mkv" && t.TotalSize == 200); + var cats = await analysis.UsageByCategoryAsync(source.Id, null); + Assert.Contains(cats, c => c.Category == "Video" && c.TotalSize == 200); var drill = await analysis.DrilldownAsync(root.Id); Assert.Contains(drill, e => e.Name == "Movies"); Assert.Equal("Movies", dirs[0].PathRel); diff --git a/tests/Explorer.Application.Tests/EntryClassificationServiceTests.cs b/tests/Explorer.Application.Tests/EntryClassificationServiceTests.cs new file mode 100644 index 0000000..ce16688 --- /dev/null +++ b/tests/Explorer.Application.Tests/EntryClassificationServiceTests.cs @@ -0,0 +1,160 @@ +using Explorer.Application; +using Explorer.Domain; +using Explorer.Domain.Abstractions; +using Explorer.Storage.Sqlite; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Explorer.Application.Tests; + +public class EntryClassificationServiceTests +{ + [Fact] + public async Task Archive_children_override_zip_as_photos() + { + await using var store = await OpenStore(); + var source = await AddSource(store, @"C:\media"); + var zip = await Upsert(store, new IndexEntry + { + SourceId = source.Id, + Name = "photos.zip", + NameNorm = "photos.zip", + Extension = "zip", + PathRel = "photos.zip", + LastSeenUtc = DateTimeOffset.UtcNow + }); + await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg")); + await Upsert(store, Child(source.Id, zip.Id, "b.jpg", "jpg")); + await Upsert(store, Child(source.Id, zip.Id, "c.jpg", "jpg")); + + var service = Create(store, new MemoryEnumerator()); + service.Resume(); + Assert.True(await service.ProcessPendingAsync()); + var loaded = await store.Entries.GetAsync(zip.Id); + Assert.Equal(FileCategory.Photos, loaded!.Category); + Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource); + Assert.False(await service.ProcessPendingAsync()); + } + + [Fact] + public async Task Signature_classifies_jpeg_without_extension() + { + var root = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var file = Path.Combine(root, "pic.bin"); + File.WriteAllBytes(file, [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01]); + await using var store = await OpenStore(); + var source = await AddSource(store, root); + var entry = await Upsert(store, new IndexEntry + { + SourceId = source.Id, + Name = "pic.bin", + NameNorm = "pic.bin", + Extension = "bin", + PathRel = "pic.bin", + LastSeenUtc = DateTimeOffset.UtcNow + }); + Assert.Equal(FileCategory.Unknown, entry.Category); + + var enumerator = new MemoryEnumerator(); + enumerator.Add(new FileSystemItem { FullPath = file, Name = "pic.bin" }); + var service = Create(store, enumerator); + service.Resume(); + Assert.True(await service.ProcessPendingAsync()); + var loaded = await store.Entries.GetAsync(entry.Id); + Assert.Equal(FileCategory.Photos, loaded!.Category); + Assert.Equal(CategorySources.Mime, loaded.CategorySource); + } + + [Fact] + public async Task Pause_skips_work() + { + await using var store = await OpenStore(); + var source = await AddSource(store, @"C:\media"); + var zip = await Upsert(store, new IndexEntry + { + SourceId = source.Id, + Name = "photos.zip", + NameNorm = "photos.zip", + Extension = "zip", + PathRel = "photos.zip", + LastSeenUtc = DateTimeOffset.UtcNow + }); + await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg")); + var service = Create(store, new MemoryEnumerator()); + Assert.True(service.IsPaused); + Assert.False(await service.ProcessPendingAsync()); + var loaded = await store.Entries.GetAsync(zip.Id); + Assert.Equal(FileCategory.Archive, loaded!.Category); + } + + private static EntryClassificationService Create(IIndexStore store, IFileSystemEnumerator enumerator) + => new( + store, + enumerator, + new NeverHydrate(), + NullLogger.Instance); + + private static async Task OpenStore() + { + var db = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N"), "index.db"); + var store = new SqliteIndexStore(db, NullLogger.Instance); + await store.OpenAsync(); + return store; + } + + private static async Task AddSource(IIndexStore store, string root) + { + var source = new Source + { + StableKey = Guid.NewGuid().ToString("N"), + Kind = SourceKind.NtfsLocal, + DisplayName = "Test", + LastRootPath = root, + Status = SourceStatus.Online + }; + source.Id = await store.Sources.UpsertAsync(source); + return source; + } + + private static async Task Upsert(IIndexStore store, IndexEntry entry) + { + entry.Id = await store.Entries.UpsertAsync(entry); + return (await store.Entries.GetAsync(entry.Id))!; + } + + private static IndexEntry Child(long sourceId, long parentId, string name, string ext) + => new() + { + SourceId = sourceId, + ParentId = parentId, + Name = name, + NameNorm = name, + Extension = ext, + PathRel = name, + LastSeenUtc = DateTimeOffset.UtcNow + }; + + private sealed class NeverHydrate : IHydrationGuard + { + public bool WouldHydrateOnRead(FileSystemItem item) => false; + public bool WouldHydrateOnRead(int attributes, CloudAvailability? availability) => false; + public Task WouldHydrateOnReadAsync(string path, CancellationToken cancellationToken = default) + => Task.FromResult(false); + } + + private sealed class MemoryEnumerator : IFileSystemEnumerator + { + private readonly Dictionary _items = new(StringComparer.OrdinalIgnoreCase); + + public void Add(FileSystemItem item) => _items[item.FullPath] = item; + + public IEnumerable EnumerateChildren(string directoryPath) => []; + public FileSystemItem? GetItem(string path) + => _items.TryGetValue(path, out var item) ? item : null; + public IReadOnlyList EnumerateChildrenSafe(string directoryPath, out string? error) + { + error = null; + return []; + } + } +} diff --git a/tests/Explorer.Application.Tests/ReorganizePlannerTests.cs b/tests/Explorer.Application.Tests/ReorganizePlannerTests.cs index 3d3343e..89157da 100644 --- a/tests/Explorer.Application.Tests/ReorganizePlannerTests.cs +++ b/tests/Explorer.Application.Tests/ReorganizePlannerTests.cs @@ -132,6 +132,40 @@ public class ReorganizePlannerTests Assert.Null(fs.GetItem(@"C:\Pictures\photo.jpg")); } + [Fact] + public void Prefers_indexed_archive_contents_over_extension() + { + var fs = Tree() + .Dir(@"C:\Downloads") + .File(@"C:\Downloads\photos.zip", 4, T0) + .Dir(@"C:\Pictures") + .Dir(@"C:\Archive"); + + var indexed = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["photos.zip"] = new IndexEntry + { + Name = "photos.zip", + NameNorm = "photos.zip", + PathRel = "photos.zip", + Category = FileCategory.Photos, + CategorySource = CategorySources.ArchiveContents, + CategoryReason = "Archive contents are mostly photos." + } + }; + var plan = new ReorganizePlanner().Build( + @"C:\Downloads", + Map(), + fs, + _ => true, + _ => false, + now: T0, + pathExists: _ => false, + indexedByName: indexed); + Assert.Contains(plan.Operations, o => o.SourcePath.EndsWith("photos.zip") && o.DestinationPath == @"C:\Pictures\photos.zip"); + Assert.DoesNotContain(plan.Operations, o => o.DestinationPath == @"C:\Archive\photos.zip"); + } + private static OperationPlan Build( IFileSystemEnumerator fs, Action? configure = null, diff --git a/tests/Explorer.Domain.Tests/DomainTests.cs b/tests/Explorer.Domain.Tests/DomainTests.cs index 5cdb587..e112d17 100644 --- a/tests/Explorer.Domain.Tests/DomainTests.cs +++ b/tests/Explorer.Domain.Tests/DomainTests.cs @@ -484,6 +484,108 @@ public class FileClassifierTests => FileClassifier.Classify(name, @"C:\Downloads\" + name, isDirectory, isDirectory ? AttributeFlags.Directory : 0, isRepoRoot).Category; } +public class EntryCategoryAssignerTests +{ + [Fact] + public void ApplyAutomatic_sets_extension_category_on_files() + { + var entry = new IndexEntry + { + Name = "clip.mkv", + NameNorm = "clip.mkv", + PathRel = "clip.mkv", + Extension = "mkv" + }; + EntryCategoryAssigner.ApplyAutomatic(entry); + Assert.Equal(FileCategory.Video, entry.Category); + Assert.Equal(CategorySources.Extension, entry.CategorySource); + Assert.True(entry.CategoryConfidence >= 80); + } + + [Fact] + public void User_category_is_not_overwritten() + { + var entry = new IndexEntry + { + Name = "clip.mkv", + NameNorm = "clip.mkv", + PathRel = "clip.mkv", + Extension = "mkv" + }; + EntryCategoryAssigner.ApplyUser(entry, FileCategory.Documents, "Manual"); + EntryCategoryAssigner.ApplyAutomatic(entry); + Assert.Equal(FileCategory.Documents, entry.Category); + Assert.Equal(CategorySources.User, entry.CategorySource); + } + + [Fact] + public void Children_majority_sets_folder_or_archive_source() + { + var folder = new IndexEntry + { + Name = "Vacation", + NameNorm = "vacation", + PathRel = "Vacation", + IsDirectory = true, + Attributes = AttributeFlags.Directory + }; + EntryCategoryAssigner.ApplyAutomatic( + folder, + childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos, FileCategory.Documents]); + Assert.Equal(FileCategory.Photos, folder.Category); + Assert.Equal(CategorySources.Children, folder.CategorySource); + + var zip = new IndexEntry + { + Name = "photos.zip", + NameNorm = "photos.zip", + PathRel = "photos.zip", + Extension = "zip" + }; + EntryCategoryAssigner.ApplyAutomatic( + zip, + childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos]); + Assert.Equal(FileCategory.Photos, zip.Category); + Assert.Equal(CategorySources.ArchiveContents, zip.CategorySource); + } +} + +public class OrganizeDestinationsTests +{ + [Fact] + public void Parses_category_aliases() + { + Assert.True(OrganizeDestinations.TryParse("photos", out var photos)); + Assert.Equal(FileCategory.Photos, photos); + Assert.True(OrganizeDestinations.TryParse("zip", out var zip)); + Assert.Equal(FileCategory.Archive, zip); + Assert.True(OrganizeDestinations.TryParse("unknown", out var unknown)); + Assert.Equal(FileCategory.Unknown, unknown); + Assert.False(OrganizeDestinations.TryParse("not-a-category", out _)); + } +} + +public class FileSignatureClassifierTests +{ + [Fact] + public void Recognizes_common_headers() + { + Assert.Equal(FileCategory.Photos, Hit([0xFF, 0xD8, 0xFF, 0xE0])); + Assert.Equal(FileCategory.Photos, Hit([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A])); + Assert.Equal(FileCategory.Documents, Hit([0x25, 0x50, 0x44, 0x46])); + Assert.Equal(FileCategory.Archive, Hit([0x50, 0x4B, 0x03, 0x04])); + Assert.Equal(FileCategory.Archive, Hit([0x37, 0x7A, 0xBC, 0xAF])); + Assert.Equal(FileCategory.Installer, Hit([0x4D, 0x5A, 0x90, 0x00])); + Assert.Equal(FileCategory.Audio, Hit("RIFF....WAVE"u8.ToArray())); + Assert.Equal(FileCategory.Video, Hit("RIFF....AVI "u8.ToArray())); + Assert.Null(FileSignatureClassifier.TryClassify([0x00, 0x01, 0x02, 0x03])); + Assert.Null(FileSignatureClassifier.TryClassify([0xFF, 0xD8])); + } + + private static FileCategory Hit(byte[] header) + => FileSignatureClassifier.TryClassify(header)!.Category; +} + public class GitChangeTests { [Fact] diff --git a/tests/Explorer.Hosting.Tests/BackgroundMaintenanceCoordinatorTests.cs b/tests/Explorer.Hosting.Tests/BackgroundMaintenanceCoordinatorTests.cs index 504ecb8..0d0670e 100644 --- a/tests/Explorer.Hosting.Tests/BackgroundMaintenanceCoordinatorTests.cs +++ b/tests/Explorer.Hosting.Tests/BackgroundMaintenanceCoordinatorTests.cs @@ -149,6 +149,44 @@ public class BackgroundMaintenanceCoordinatorTests Assert.Empty(indexing.IdleScans); } + [Fact] + public async Task Idle_classifies_before_hashing() + { + var hash = new FakeHash { Pending = true }; + var indexing = new FakeIndexing(); + var classify = new FakeClassify { Work = true }; + var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify); + await coordinator.TickAsync(CancellationToken.None); + Assert.Equal(1, classify.Calls); + Assert.True(hash.IsPaused); + Assert.Contains("classifying", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Idle_hashes_when_classify_is_idle() + { + var hash = new FakeHash { Pending = true }; + var indexing = new FakeIndexing(); + var classify = new FakeClassify(); + var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify); + await coordinator.TickAsync(CancellationToken.None); + Assert.Equal(1, classify.Calls); + Assert.False(hash.IsPaused); + Assert.Contains("hashing", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Activity_pauses_classify() + { + var hash = new FakeHash(); + var indexing = new FakeIndexing(); + var classify = new FakeClassify { Paused = false }; + var coordinator = Create(hash, indexing, idle: TimeSpan.Zero, classify: classify); + await coordinator.TickAsync(CancellationToken.None); + Assert.True(classify.IsPaused); + Assert.Equal(0, classify.Calls); + } + private static Source StaleLocal() => new() { @@ -168,8 +206,9 @@ public class BackgroundMaintenanceCoordinatorTests bool enabled = true, bool ac = true, bool foreground = false, - IReadOnlyList? sources = null) - => Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources); + IReadOnlyList? sources = null, + FakeClassify? classify = null) + => Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources, classify); private static BackgroundMaintenanceCoordinator Create( FakeHash hash, @@ -178,7 +217,8 @@ public class BackgroundMaintenanceCoordinatorTests bool enabled = true, bool ac = true, bool foreground = false, - IReadOnlyList? sources = null) + IReadOnlyList? sources = null, + FakeClassify? classify = null) { var dir = Path.Combine(Path.GetTempPath(), "ew-maint", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); @@ -199,7 +239,8 @@ public class BackgroundMaintenanceCoordinatorTests new FakeHistory(), new FakeStore(sources ?? []), new FakeVolumes(), - NullLogger.Instance); + NullLogger.Instance, + classify: classify); } private sealed class FakeIdle(TimeSpan idle) : IUserIdleMonitor @@ -233,14 +274,31 @@ public class BackgroundMaintenanceCoordinatorTests private sealed class FakeHash : IIdleHashWork { + public bool Pending { get; set; } public bool Paused { get; set; } = true; public bool IsPaused => Paused; public string? CurrentPath => null; public void Pause() => Paused = true; public void Resume() => Paused = false; public void BeginUserRequested() { } - public Task HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(false); - public Task CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(0L); + public Task HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending); + public Task CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending ? 1L : 0L); + } + + private sealed class FakeClassify : IIdleClassifyWork + { + public bool Work { get; set; } + public int Calls { get; private set; } + public bool Paused { get; set; } = true; + public bool IsPaused => Paused; + public string? CurrentPath => null; + public void Pause() => Paused = true; + public void Resume() => Paused = false; + public Task ProcessPendingAsync(CancellationToken cancellationToken = default) + { + Calls++; + return Task.FromResult(Work); + } } private sealed class FakeHistory : IHistoryMaintenance diff --git a/tests/Explorer.Hosting.Tests/CoreRegistrationTests.cs b/tests/Explorer.Hosting.Tests/CoreRegistrationTests.cs index 4353033..9d655de 100644 --- a/tests/Explorer.Hosting.Tests/CoreRegistrationTests.cs +++ b/tests/Explorer.Hosting.Tests/CoreRegistrationTests.cs @@ -36,6 +36,7 @@ public class CoreRegistrationTests Assert.NotNull(sp.GetService()); Assert.NotNull(sp.GetService()); Assert.NotNull(sp.GetService()); + Assert.NotNull(sp.GetService()); Assert.IsType(sp.GetService()); } finally diff --git a/tests/Explorer.Search.Tests/SearchServiceTests.cs b/tests/Explorer.Search.Tests/SearchServiceTests.cs index 0d0e18f..1232ace 100644 --- a/tests/Explorer.Search.Tests/SearchServiceTests.cs +++ b/tests/Explorer.Search.Tests/SearchServiceTests.cs @@ -44,4 +44,37 @@ public class SearchServiceTests Assert.Contains(both, e => e.Name == "Folder"); Assert.Contains(both, e => e.Name == "notes.txt"); } + + [Fact] + public async Task Category_hint_and_filter() + { + var db = Path.Combine(Path.GetTempPath(), "ew-search", Guid.NewGuid().ToString("N"), "index.db"); + await using var store = new SqliteIndexStore(db, NullLogger.Instance); + await store.OpenAsync(); + var source = new Source { StableKey = "k", DisplayName = "d", Kind = SourceKind.NtfsLocal, LastRootPath = @"C:\d", Status = SourceStatus.Online }; + source.Id = await store.Sources.UpsertAsync(source); + var root = new IndexEntry { SourceId = source.Id, Name = "d", NameNorm = "d", IsDirectory = true, PathRel = "", LastSeenUtc = DateTimeOffset.UtcNow }; + root.Id = await store.Entries.UpsertAsync(root); + await store.Entries.UpsertAsync(new IndexEntry + { + SourceId = source.Id, ParentId = root.Id, Name = "clip.mkv", NameNorm = "clip.mkv", Extension = "mkv", + PathRel = "clip.mkv", LastSeenUtc = DateTimeOffset.UtcNow + }); + await store.Entries.UpsertAsync(new IndexEntry + { + SourceId = source.Id, ParentId = root.Id, Name = "notes.pdf", NameNorm = "notes.pdf", Extension = "pdf", + PathRel = "notes.pdf", LastSeenUtc = DateTimeOffset.UtcNow + }); + + var search = new SearchService(store); + var hinted = await search.SearchAsync(new SearchQuery { Text = "category:video", SourceIds = [source.Id] }); + Assert.Single(hinted); + Assert.Equal("clip.mkv", hinted[0].Name); + var filtered = await search.SearchAsync(new SearchQuery { Category = FileCategory.Documents, SourceIds = [source.Id] }); + Assert.Single(filtered); + Assert.Equal("notes.pdf", filtered[0].Name); + var split = SearchService.SplitCategoryHint("vacation category:photos"); + Assert.Equal("vacation", split.Name); + Assert.Equal(FileCategory.Photos, split.Category); + } } diff --git a/tests/Explorer.Storage.Tests/StorageTests.cs b/tests/Explorer.Storage.Tests/StorageTests.cs index 1470c3a..13d9c2e 100644 --- a/tests/Explorer.Storage.Tests/StorageTests.cs +++ b/tests/Explorer.Storage.Tests/StorageTests.cs @@ -114,6 +114,135 @@ public class EntryAndSearchTests Assert.Contains(byDate, e => e.Name == "Movie.mkv"); } + [Fact] + public async Task Upsert_assigns_and_preserves_categories() + { + await using var store = await Stores.Open(); + var source = await store.AddSourceAsync(@"C:\media"); + var root = new IndexEntry + { + SourceId = source.Id, + Name = "media", + NameNorm = "media", + IsDirectory = true, + PathRel = "", + LastSeenUtc = DateTimeOffset.UtcNow + }; + root.Id = await store.Entries.UpsertAsync(root); + var photo = new IndexEntry + { + SourceId = source.Id, + ParentId = root.Id, + Name = "shot.jpg", + NameNorm = "shot.jpg", + Extension = "jpg", + PathRel = "shot.jpg", + LastSeenUtc = DateTimeOffset.UtcNow + }; + photo.Id = await store.Entries.UpsertAsync(photo); + var loaded = await store.Entries.GetAsync(photo.Id); + Assert.Equal(FileCategory.Photos, loaded!.Category); + Assert.Equal(CategorySources.Extension, loaded.CategorySource); + + await store.Entries.SetUserCategoryAsync(photo.Id, FileCategory.Documents, "Manual"); + photo.SizeBytes = 100; + await store.Entries.UpsertAsync(photo); + loaded = await store.Entries.GetAsync(photo.Id); + Assert.Equal(FileCategory.Documents, loaded!.Category); + Assert.Equal(CategorySources.User, loaded.CategorySource); + + var other = new IndexEntry + { + SourceId = source.Id, + ParentId = root.Id, + Name = "clip.mkv", + NameNorm = "clip.mkv", + Extension = "mkv", + PathRel = "clip.mkv", + LastSeenUtc = DateTimeOffset.UtcNow + }; + other.Id = await store.Entries.UpsertAsync(other); + await store.Entries.RefreshCategoryFromChildrenAsync(root.Id); + var folder = await store.Entries.GetAsync(root.Id); + Assert.Equal(FileCategory.Unknown, folder!.Category); + } + + [Fact] + public async Task Archive_contents_and_signature_queues() + { + await using var store = await Stores.Open(); + var source = await store.AddSourceAsync(@"C:\media"); + var zip = new IndexEntry + { + SourceId = source.Id, + Name = "photos.zip", + NameNorm = "photos.zip", + Extension = "zip", + PathRel = "photos.zip", + LastSeenUtc = DateTimeOffset.UtcNow + }; + zip.Id = await store.Entries.UpsertAsync(zip); + Assert.Equal(FileCategory.Archive, (await store.Entries.GetAsync(zip.Id))!.Category); + + for (var i = 0; i < 3; i++) + { + await store.Entries.UpsertAsync(new IndexEntry + { + SourceId = source.Id, + ParentId = zip.Id, + Name = $"shot{i}.jpg", + NameNorm = $"shot{i}.jpg", + Extension = "jpg", + PathRel = $"photos.zip/shot{i}.jpg", + LastSeenUtc = DateTimeOffset.UtcNow + }); + } + + var needing = await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10); + Assert.Contains(zip.Id, needing); + await store.Entries.RefreshCategoryFromChildrenAsync(zip.Id); + var loaded = await store.Entries.GetAsync(zip.Id); + Assert.Equal(FileCategory.Photos, loaded!.Category); + Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource); + Assert.Empty(await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10)); + + var mystery = new IndexEntry + { + SourceId = source.Id, + Name = "mystery.bin", + NameNorm = "mystery.bin", + Extension = "bin", + PathRel = "mystery.bin", + LastSeenUtc = DateTimeOffset.UtcNow + }; + mystery.Id = await store.Entries.UpsertAsync(mystery); + var unknowns = await store.Entries.GetUnknownFilesForSignatureAsync(10); + Assert.Contains(unknowns, e => e.Id == mystery.Id); + Assert.Equal(0, await store.Entries.BackfillCheapCategoriesAsync(50)); + } + + [Fact] + public async Task User_category_by_path_is_preserved() + { + await using var store = await Stores.Open(); + var source = await store.AddSourceAsync(@"C:\media"); + var photo = new IndexEntry + { + SourceId = source.Id, + Name = "shot.jpg", + NameNorm = "shot.jpg", + Extension = "jpg", + PathRel = "shot.jpg", + LastSeenUtc = DateTimeOffset.UtcNow + }; + photo.Id = await store.Entries.UpsertAsync(photo); + var mutations = new Explorer.Application.LocalIndexMutations(store); + await mutations.SetUserCategoryByPathAsync(@"C:\media\shot.jpg", FileCategory.Installer, "Manual"); + var loaded = await store.Entries.GetAsync(photo.Id); + Assert.Equal(FileCategory.Installer, loaded!.Category); + Assert.Equal(CategorySources.User, loaded.CategorySource); + } + [Fact] public async Task Folder_aggregates_and_tombstones() {