Persist file categories on the index and classify archives and unknown types during idle maintenance.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-28 11:37:29 +02:00
parent b33a78dbbe
commit 222b5d9969
42 changed files with 1719 additions and 80 deletions

View File

@@ -457,14 +457,14 @@ Goal: Understand what files and folders represent rather than relying only on ex
## Detection signals
- [x] File extension
- [ ] MIME/content signature
- [x] MIME/content signature (light idle magic-byte; never hydrates)
- [x] Folder structure
- [x] Git metadata
- [x] Media metadata
- [x] Known application structures (`node_modules`, `bin`, `obj`, `.vs`)
- [x] File age (old installers flagged in preview)
- [ ] File relationships
- [ ] Index metadata
- [x] Index metadata
---

View File

@@ -343,7 +343,22 @@ Classification **suggests** moves. Nothing moves until you Preview and Queue.
Build output names (`node_modules`, `bin`, `obj`, `.vs`, and similar) are never moved. Online-only cloud items are skipped. Old installers (older than one year) still propose a move and show a warning. Destinations may sit *inside* the source folder (Downloads → Downloads\Software). Destinations are remembered in preferences.
Never auto-reorganizes. No MIME/content sniffing (that would hydrate cloud files).
Organize uses the **indexed category** when one exists (including ZIP contents that are mostly photos, and **Classify as…** overrides). Extension heuristics fill in the rest.
Never auto-reorganizes. A light magic-byte peek for unknown extensions runs only during idle maintenance on local files — never on online-only cloud items.
---
## Classification
The index stores a category on each entry (Photos, Video, Documents, Archive, and so on) plus a short reason.
- Details shows a **Category** column (tooltip has the reason)
- Search accepts `category:photos` or the category dropdown
- Storage analysis has **By category** next to **By file type**
- Context menu **Classify as…** writes a user override that later scans do not replace
ZIP/7z files stay Archive until their contents are indexed; idle maintenance then promotes them when a category dominates.
---
@@ -416,7 +431,7 @@ Left open on purpose:
- Multi-PC search, sharing, encrypted vaults
- Robocopy as a second transfer engine
- Scheduled profiles and folder-watcher triggers
- MIME/EXIF classification
- Full EXIF / ffprobe classification
- Duplicate “backup copy” auto-tagging
---

View File

@@ -76,6 +76,16 @@ public sealed class AnalysisService
ct => _store.Analysis.UsageByExtensionAsync(sourceId, pathRelPrefix, take, ct),
cancellationToken);
public Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(
long? sourceId,
string? pathRelPrefix,
int take = AppConstants.AnalysisTopN,
CancellationToken cancellationToken = default)
=> CachedAsync(
$"categories:{sourceId}:{pathRelPrefix}:{take}",
ct => _store.Analysis.UsageByCategoryAsync(sourceId, pathRelPrefix, take, ct),
cancellationToken);
public Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
=> CachedAsync("sources", ct => _store.Analysis.UsageBySourceAsync(ct), cancellationToken);

View File

@@ -44,6 +44,17 @@
</MenuItem>
<MenuItem Header="Organize this folder…" Click="OnOrganizeFolder"
Visibility="{Binding ShowOrganizeFolder, Converter={StaticResource BoolVis}}"/>
<MenuItem Header="Classify as" Visibility="{Binding ShowClassifyAs, Converter={StaticResource BoolVis}}">
<MenuItem Header="Photos" Command="{Binding ClassifyAsCommand}" CommandParameter="Photos"/>
<MenuItem Header="Video" Command="{Binding ClassifyAsCommand}" CommandParameter="Video"/>
<MenuItem Header="Audio" Command="{Binding ClassifyAsCommand}" CommandParameter="Audio"/>
<MenuItem Header="Documents" Command="{Binding ClassifyAsCommand}" CommandParameter="Documents"/>
<MenuItem Header="Installer" Command="{Binding ClassifyAsCommand}" CommandParameter="Installer"/>
<MenuItem Header="Archive" Command="{Binding ClassifyAsCommand}" CommandParameter="Archive"/>
<MenuItem Header="Backup" Command="{Binding ClassifyAsCommand}" CommandParameter="Backup"/>
<MenuItem Header="Code repository" Command="{Binding ClassifyAsCommand}" CommandParameter="CodeRepository"/>
<MenuItem Header="Unknown" Command="{Binding ClassifyAsCommand}" CommandParameter="Unknown"/>
</MenuItem>
<Separator Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
<MenuItem Header="Extract here" Click="OnExtractHere"
Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
@@ -537,6 +548,13 @@
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
<GridViewColumn Header="Category" Width="110">
<GridViewColumn.CellTemplate>
<DataTemplate>
<TextBlock Text="{Binding CategoryLabel}" ToolTip="{Binding CategoryTooltip}"/>
</DataTemplate>
</GridViewColumn.CellTemplate>
</GridViewColumn>
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
@@ -661,6 +679,13 @@
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
<GridViewColumn Header="Category" Width="110">
<GridViewColumn.CellTemplate>
<DataTemplate>
<TextBlock Text="{Binding CategoryLabel}" ToolTip="{Binding CategoryTooltip}"/>
</DataTemplate>
</GridViewColumn.CellTemplate>
</GridViewColumn>
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
@@ -768,6 +793,11 @@
<TextBlock Grid.Row="1" Text="Extension" Foreground="{DynamicResource FgMuted}" VerticalAlignment="Center" Margin="0,0,8,0"/>
<StackPanel Grid.Row="1" Grid.Column="1" Orientation="Horizontal" Margin="0,0,16,0">
<TextBox Width="90" Text="{Binding Search.Extension, UpdateSourceTrigger=PropertyChanged}"/>
<ComboBox Width="140" Margin="8,0,0,0"
ItemsSource="{Binding Search.CategoryChoices}"
DisplayMemberPath="Label"
SelectedValuePath="Value"
SelectedValue="{Binding Search.Category}"/>
<CheckBox Content="Files" Margin="16,0,8,0" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
IsChecked="{Binding Search.FilesOnly}"/>
<CheckBox Content="Folders" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
@@ -795,6 +825,7 @@
<GridView>
<GridViewColumn Header="Name" Width="220" CellTemplate="{StaticResource NameWithOverlays}"/>
<GridViewColumn Header="Path" Width="420" DisplayMemberBinding="{Binding FullPath}"/>
<GridViewColumn Header="Category" Width="110" DisplayMemberBinding="{Binding CategoryLabel}"/>
<GridViewColumn Header="Size" Width="100" DisplayMemberBinding="{Binding SizeLabel}"/>
<GridViewColumn Header="Free space" Width="100" DisplayMemberBinding="{Binding FreeSpaceLabel}"/>
</GridView>

View File

@@ -36,6 +36,27 @@ public interface IHistoryMaintenance
Task<bool> TryCaptureAsync(CancellationToken cancellationToken = default);
}
public interface IIdleClassifyWork
{
bool IsPaused { get; }
string? CurrentPath { get; }
void Pause();
void Resume();
/// <summary>Returns true when any classification work was performed.</summary>
Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default);
}
public sealed class NullIdleClassifyWork : IIdleClassifyWork
{
public static NullIdleClassifyWork Instance { get; } = new();
public bool IsPaused => true;
public string? CurrentPath => null;
public void Pause() { }
public void Resume() { }
public Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
=> Task.FromResult(false);
}
public interface IForegroundWorkSignal
{
bool HasForegroundWork();

View File

@@ -395,7 +395,11 @@ public sealed class BrowseService
Cloud = entry.CloudAvailability is { } availability
? new CloudPresence(null, availability, entry.SizeBytes, entry.AllocatedSizeBytes, availability == CloudAvailability.OnlineOnly)
: null,
Hydration = ItemHydrationFlags.All
Hydration = ItemHydrationFlags.All,
IndexEntryId = entry.Id,
Category = entry.Category,
CategoryReason = entry.CategoryReason,
CategorySource = entry.CategorySource
};
private async IAsyncEnumerable<BrowseDelta> ListLiveProgressiveAsync(
@@ -609,7 +613,11 @@ public sealed class BrowseService
fileId: item.FileId ?? entry.FileId,
cloud: cloud,
indexedChildCount: entry.ChildFileCount + entry.ChildDirCount,
hydration: item.Hydration | ItemHydrationFlags.Index);
hydration: item.Hydration | ItemHydrationFlags.Index,
indexEntryId: entry.Id,
category: entry.Category,
categoryReason: entry.CategoryReason,
categorySource: entry.CategorySource);
return Annotate(hydrated, sizeFromIndex: item.IsDirectory && entry.AggregateSize > 0, preferences, probeAccess: false);
}
@@ -785,7 +793,11 @@ public sealed class BrowseService
Attributes = c.Attributes,
FileId = c.FileId,
ReparseTag = c.ReparseTag,
AllocatedSizeBytes = c.AllocatedSizeBytes
AllocatedSizeBytes = c.AllocatedSizeBytes,
IndexEntryId = c.Id,
Category = c.Category,
CategoryReason = c.CategoryReason,
CategorySource = c.CategorySource
}).ToList();
var hint = isArchiveFile && items.Count == 0

View File

@@ -0,0 +1,173 @@
using Explorer.Domain;
using Explorer.Domain.Abstractions;
using Microsoft.Extensions.Logging;
namespace Explorer.Application;
public sealed class EntryClassificationService : IIdleClassifyWork
{
private readonly IIndexStore _store;
private readonly IFileSystemEnumerator _enumerator;
private readonly IHydrationGuard _hydration;
private readonly ILogger<EntryClassificationService> _logger;
private readonly IHostActivitySink _activity;
private volatile bool _paused = true;
private volatile string? _currentPath;
public EntryClassificationService(
IIndexStore store,
IFileSystemEnumerator enumerator,
IHydrationGuard hydration,
ILogger<EntryClassificationService> logger,
IHostActivitySink? activity = null)
{
_store = store;
_enumerator = enumerator;
_hydration = hydration;
_logger = logger;
_activity = activity ?? NullHostActivitySink.Instance;
}
public bool IsPaused => _paused;
public string? CurrentPath => _currentPath;
public void Pause() => _paused = true;
public void Resume() => _paused = false;
public async Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
{
if (_paused)
{
return false;
}
var backfill = await _store.Entries.BackfillCheapCategoriesAsync(80, cancellationToken)
.ConfigureAwait(false);
if (backfill > 0)
{
_activity.Record("Classify", "Backfilled " + backfill + " extension categories");
return true;
}
var archives = await _store.Entries.GetArchiveIdsNeedingContentClassifyAsync(20, cancellationToken)
.ConfigureAwait(false);
if (archives.Count > 0)
{
foreach (var id in archives)
{
if (_paused || cancellationToken.IsCancellationRequested)
{
break;
}
await _store.Entries.RefreshCategoryFromChildrenAsync(id, cancellationToken)
.ConfigureAwait(false);
}
_activity.Record("Classify", "Updated " + archives.Count + " archive(s) from contents");
return true;
}
return await TrySignatureClassifyAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<bool> TrySignatureClassifyAsync(CancellationToken cancellationToken)
{
var unknowns = await _store.Entries.GetUnknownFilesForSignatureAsync(12, cancellationToken)
.ConfigureAwait(false);
if (unknowns.Count == 0)
{
return false;
}
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
var byId = sources.ToDictionary(s => s.Id);
var changed = 0;
foreach (var entry in unknowns)
{
if (_paused || cancellationToken.IsCancellationRequested)
{
break;
}
if (!byId.TryGetValue(entry.SourceId, out var source) || string.IsNullOrEmpty(source.LastRootPath))
{
continue;
}
var full = PathRules.Combine(source.LastRootPath, entry.PathRel);
_currentPath = full;
try
{
var item = _enumerator.GetItem(full);
if (item is null)
{
MarkSignatureSkip(entry, "File was not found.");
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
continue;
}
if (_hydration.WouldHydrateOnRead(item)
|| await _hydration.WouldHydrateOnReadAsync(full, cancellationToken).ConfigureAwait(false))
{
MarkSignatureSkip(entry, "Skipped online-only cloud file.");
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
continue;
}
await using var stream = new FileStream(
PathRules.ToExtended(full),
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite | FileShare.Delete,
bufferSize: 64,
FileOptions.SequentialScan | FileOptions.Asynchronous);
var buffer = new byte[16];
var read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)
.ConfigureAwait(false);
var hit = FileSignatureClassifier.TryClassify(buffer.AsSpan(0, read));
if (hit is null)
{
entry.CategorySource = CategorySources.Mime;
entry.CategoryConfidence = 10;
entry.CategoryReason = "No known file signature.";
entry.CategoryUtc = DateTimeOffset.UtcNow;
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
continue;
}
EntryCategoryAssigner.Apply(entry, hit, CategorySources.Mime, 70);
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
changed++;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogDebug(ex, "Signature classify failed for {Path}", full);
try
{
MarkSignatureSkip(entry, "Could not read file signature.");
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
}
catch (Exception markEx) when (markEx is not OperationCanceledException)
{
_logger.LogDebug(markEx, "Could not mark signature skip for {Path}", full);
}
}
}
_currentPath = null;
if (changed > 0)
{
_activity.Record("Classify", "Signature-classified " + changed + " file(s)");
}
return changed > 0 || unknowns.Count > 0;
}
private static void MarkSignatureSkip(IndexEntry entry, string reason)
=> EntryCategoryAssigner.Apply(
entry,
new FileClassification(entry.Category, reason),
CategorySources.Mime,
5);
}

View File

@@ -15,7 +15,8 @@ public sealed class ReorganizePlanner
Func<string, bool> isRepoRoot,
Func<FileSystemItem, bool>? wouldHydrate = null,
Func<string, bool>? pathExists = null,
DateTimeOffset? now = null)
DateTimeOffset? now = null,
IReadOnlyDictionary<string, IndexEntry>? indexedByName = null)
{
if (string.IsNullOrWhiteSpace(sourceRoot))
{
@@ -63,13 +64,16 @@ public sealed class ReorganizePlanner
var childCategories = item.IsDirectory && !FileClassifier.SkipDescent(item.Name)
? ChildCategories(item, enumerator, isRepoRoot)
: null;
var classification = FileClassifier.Classify(
var classification = PreferIndexed(
item,
indexedByName,
FileClassifier.Classify(
item.Name,
item.FullPath,
item.IsDirectory,
item.Attributes,
item.IsDirectory && isRepoRoot(item.FullPath),
childCategories);
childCategories));
if (FileClassifier.ShouldLeave(classification.Category))
{
@@ -143,6 +147,28 @@ public sealed class ReorganizePlanner
};
}
private static FileClassification PreferIndexed(
FileSystemItem item,
IReadOnlyDictionary<string, IndexEntry>? indexedByName,
FileClassification fallback)
{
if (indexedByName is null
|| !indexedByName.TryGetValue(item.Name, out var stored))
{
return fallback;
}
if (CategorySources.IsUser(stored.CategorySource)
|| stored.Category is not FileCategory.Unknown)
{
return new FileClassification(
stored.Category,
stored.CategoryReason ?? "Indexed category.");
}
return fallback;
}
private static IReadOnlyList<FileCategory> ChildCategories(
FileSystemItem folder,
IFileSystemEnumerator enumerator,

View File

@@ -57,4 +57,53 @@ public sealed class LocalIndexMutations : IIndexMutations
=> _store.Hashes.EnqueueSizeCollisionsAsync(sourceId, cancellationToken);
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
=> _store.Relations.UpsertAsync(relation, cancellationToken);
public async Task SetUserCategoryByPathAsync(
string fullPath,
FileCategory category,
string? reason = null,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(fullPath))
{
return;
}
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
Source? best = null;
var bestLen = -1;
var path = PathRules.FromExtended(fullPath).TrimEnd('\\');
foreach (var source in sources)
{
if (string.IsNullOrEmpty(source.LastRootPath))
{
continue;
}
var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\');
if (path.Equals(root, StringComparison.OrdinalIgnoreCase)
|| path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase))
{
if (root.Length > bestLen)
{
best = source;
bestLen = root.Length;
}
}
}
if (best?.LastRootPath is null)
{
return;
}
var rel = PathRules.MakeRelative(best.LastRootPath, fullPath) ?? "";
var entry = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false);
if (entry is null)
{
return;
}
await _store.Entries.SetUserCategoryAsync(entry.Id, category, reason, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -83,4 +83,6 @@ public interface IIndexMutations
Task MarkRenameBatchUndoneAsync(long id, CancellationToken cancellationToken = default);
Task EnqueueHashCollisionsAsync(long? sourceId, CancellationToken cancellationToken = default);
Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default);
Task SetUserCategoryByPathAsync(string fullPath, FileCategory category, string? reason = null, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
}

View File

@@ -59,6 +59,11 @@ public interface IEntryStore
Task DeleteExpiredTombstonesAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken = default);
Task RenameSubtreePathAsync(long sourceId, string oldPathRel, string newPathRel, CancellationToken cancellationToken = default);
Task<long> CountPresentAsync(long sourceId, CancellationToken cancellationToken = default);
Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default);
Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default);
Task<int> BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<long>> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<IndexEntry>> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default);
}
public interface IExcludeStore
@@ -109,6 +114,7 @@ public sealed class SearchRequest
public bool DirectChildrenOnly { get; init; }
public bool IncludeOffline { get; init; } = true;
public bool IncludeDeleted { get; init; }
public FileCategory? Category { get; init; }
public int Skip { get; init; }
public int Take { get; init; } = 500;
}
@@ -121,6 +127,7 @@ public interface IAnalysisStore
Task<IReadOnlyList<IndexEntry>> LargestDirectoriesAsync(long? sourceId, long? parentId, int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<IndexEntry>> LargestFilesAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<ExtensionUsage>> UsageByExtensionAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default);
Task<IReadOnlyList<IndexEntry>> ChildrenBySizeAsync(long parentId, int take, CancellationToken cancellationToken = default);
}
@@ -132,6 +139,13 @@ public sealed class ExtensionUsage
public long FileCount { get; init; }
}
public sealed class CategoryUsage
{
public required string Category { get; init; }
public long TotalSize { get; init; }
public long FileCount { get; init; }
}
public sealed class SourceUsage
{
public long SourceId { get; init; }

View File

@@ -68,6 +68,11 @@ public sealed class IndexEntry
public long ScanGeneration { get; set; }
public long? AllocatedSizeBytes { get; set; }
public CloudAvailability? CloudAvailability { get; set; }
public FileCategory Category { get; set; } = FileCategory.Unknown;
public string? CategoryReason { get; set; }
public int CategoryConfidence { get; set; }
public string CategorySource { get; set; } = CategorySources.None;
public DateTimeOffset? CategoryUtc { get; set; }
}
public sealed class ExcludeRule
@@ -150,6 +155,10 @@ public sealed class FileSystemItem
public bool AvailableToImport { get; init; }
public ItemHydrationFlags Hydration { get; init; } = ItemHydrationFlags.All;
public int IndexedChildCount { get; init; }
public long? IndexEntryId { get; init; }
public FileCategory Category { get; init; }
public string? CategoryReason { get; init; }
public string? CategorySource { get; init; }
public bool IsReparsePoint => (Attributes & AttributeFlags.ReparsePoint) != 0;
public FileSystemItem Overlay(
@@ -168,7 +177,11 @@ public sealed class FileSystemItem
long? capacityBytes = null,
bool? availableToImport = null,
ItemHydrationFlags? hydration = null,
int? indexedChildCount = null)
int? indexedChildCount = null,
long? indexEntryId = null,
FileCategory? category = null,
string? categoryReason = null,
string? categorySource = null)
=> new()
{
FullPath = FullPath,
@@ -189,7 +202,11 @@ public sealed class FileSystemItem
CapacityBytes = capacityBytes ?? CapacityBytes,
AvailableToImport = availableToImport ?? AvailableToImport,
Hydration = hydration ?? Hydration,
IndexedChildCount = indexedChildCount ?? IndexedChildCount
IndexedChildCount = indexedChildCount ?? IndexedChildCount,
IndexEntryId = indexEntryId ?? IndexEntryId,
Category = category ?? Category,
CategoryReason = categoryReason ?? CategoryReason,
CategorySource = categorySource ?? CategorySource
};
}

View File

@@ -0,0 +1,129 @@
namespace Explorer.Domain;
/// <summary>Where an <see cref="IndexEntry"/> category came from.</summary>
public static class CategorySources
{
public const string None = "none";
public const string Extension = "extension";
public const string Folder = "folder";
public const string Children = "children";
public const string ArchiveContents = "archive_contents";
public const string Mime = "mime";
public const string User = "user";
public static bool IsUser(string? source)
=> string.Equals(source, User, StringComparison.OrdinalIgnoreCase);
public static bool IsAutomatic(string? source)
=> !IsUser(source);
}
public static class EntryCategoryAssigner
{
public static void ApplyAutomatic(
IndexEntry entry,
string? rootPath = null,
bool isRepoRoot = false,
IReadOnlyList<FileCategory>? childCategories = null)
{
if (CategorySources.IsUser(entry.CategorySource))
{
return;
}
var full = string.IsNullOrWhiteSpace(rootPath)
? (string.IsNullOrEmpty(entry.PathRel) ? entry.Name : entry.PathRel)
: PathRules.Combine(rootPath, entry.PathRel);
var result = FileClassifier.Classify(
entry.Name,
full,
entry.IsDirectory,
entry.Attributes,
isRepoRoot,
childCategories);
Apply(entry, result, InferSource(entry, result, childCategories), Confidence(result, childCategories));
}
public static void ApplyUser(IndexEntry entry, FileCategory category, string? reason = null)
=> Apply(
entry,
new FileClassification(category, reason ?? "Set by user."),
CategorySources.User,
100);
public static void Apply(
IndexEntry entry,
FileClassification classification,
string source,
int confidence)
{
entry.Category = classification.Category;
entry.CategoryReason = classification.Reason;
entry.CategorySource = source;
entry.CategoryConfidence = Math.Clamp(confidence, 0, 100);
entry.CategoryUtc = DateTimeOffset.UtcNow;
}
public static FileCategory ParseCategory(string? value)
=> Enum.TryParse<FileCategory>(value, ignoreCase: true, out var cat) ? cat : FileCategory.Unknown;
public static string InferSource(
IndexEntry entry,
FileClassification classification,
IReadOnlyList<FileCategory>? childCategories)
{
if (childCategories is { Count: > 0 } && classification.Category != FileCategory.Unknown)
{
return entry.IsDirectory
? CategorySources.Children
: CategorySources.ArchiveContents;
}
if (entry.IsDirectory)
{
return classification.Category == FileCategory.Unknown
? CategorySources.None
: CategorySources.Folder;
}
if (ArchiveFormats.IsArchive(entry.Name))
{
return CategorySources.Extension;
}
return classification.Category == FileCategory.Unknown
? CategorySources.None
: CategorySources.Extension;
}
public static int Confidence(FileClassification classification, IReadOnlyList<FileCategory>? childCategories)
{
if (classification.Category == FileCategory.Unknown)
{
return 0;
}
if (childCategories is { Count: > 0 })
{
var known = childCategories.Count(c => c is not FileCategory.Unknown);
if (known == 0)
{
return 40;
}
var majority = childCategories
.Where(c => c == classification.Category)
.Count();
return Math.Clamp(40 + majority * 60 / known, 40, 95);
}
return classification.Category switch
{
FileCategory.SystemData or FileCategory.BuildOutput => 95,
FileCategory.Archive or FileCategory.Photos or FileCategory.Video
or FileCategory.Audio or FileCategory.Documents => 85,
FileCategory.Installer or FileCategory.Backup or FileCategory.CodeRepository => 80,
_ => 50
};
}
}

View File

@@ -87,6 +87,13 @@ public static class FileClassifier
{
if (ArchiveFormats.IsArchive(name))
{
if (TryMajority(childCategories, out var archiveMajority))
{
return new FileClassification(
archiveMajority,
"Archive contents are mostly " + OrganizeDestinations.Label(archiveMajority).ToLowerInvariant() + ".");
}
return new FileClassification(FileCategory.Archive, "Archive file.");
}
@@ -147,24 +154,62 @@ public static class FileClassifier
return new FileClassification(FileCategory.Backup, "Backup folder name.");
}
if (childCategories is { Count: > 0 })
if (TryMajority(childCategories, out var folderMajority))
{
var known = childCategories.Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData).ToList();
if (known.Count > 0)
{
var majority = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First();
if (majority.Count() * 10 >= known.Count * 7)
{
return new FileClassification(majority.Key, "Folder contents are mostly " + OrganizeDestinations.Label(majority.Key).ToLowerInvariant() + ".");
}
}
return new FileClassification(
folderMajority,
"Folder contents are mostly " + OrganizeDestinations.Label(folderMajority).ToLowerInvariant() + ".");
}
return new FileClassification(FileCategory.Unknown, "No matching signal.");
}
private static bool TryMajority(IReadOnlyList<FileCategory>? childCategories, out FileCategory majority)
{
majority = FileCategory.Unknown;
if (childCategories is not { Count: > 0 })
{
return false;
}
var known = childCategories
.Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData)
.ToList();
if (known.Count == 0)
{
return false;
}
var top = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First();
if (top.Count() * 10 < known.Count * 7)
{
return false;
}
majority = top.Key;
return true;
}
public static bool ShouldLeave(FileCategory category)
=> category is FileCategory.Unknown or FileCategory.SystemData or FileCategory.BuildOutput or FileCategory.Backup;
public static bool SkipDescent(string name) => BuildNames.Contains(name);
public static bool IsPhotoFile(string name)
{
var ext = NameNormalizer.Extension(name);
return ext is not null && Photos.Contains(ext);
}
public static bool IsVideoFile(string name)
{
var ext = NameNormalizer.Extension(name);
return ext is not null && Videos.Contains(ext);
}
public static bool IsAudioFile(string name)
{
var ext = NameNormalizer.Extension(name);
return ext is not null && Audio.Contains(ext);
}
}

View File

@@ -0,0 +1,84 @@
namespace Explorer.Domain;
/// <summary>Light magic-byte hints for files with no useful extension category.</summary>
public static class FileSignatureClassifier
{
public static FileClassification? TryClassify(ReadOnlySpan<byte> header)
{
if (header.Length < 4)
{
return null;
}
if (header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF)
{
return new FileClassification(FileCategory.Photos, "JPEG signature.");
}
if (header.Length >= 8
&& header[0] == 0x89 && header[1] == 0x50 && header[2] == 0x4E && header[3] == 0x47)
{
return new FileClassification(FileCategory.Photos, "PNG signature.");
}
if (header[0] == 0x47 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x38)
{
return new FileClassification(FileCategory.Photos, "GIF signature.");
}
if (header[0] == 0x25 && header[1] == 0x50 && header[2] == 0x44 && header[3] == 0x46)
{
return new FileClassification(FileCategory.Documents, "PDF signature.");
}
if (header[0] == 0x50 && header[1] == 0x4B && (header[2] == 0x03 || header[2] == 0x05 || header[2] == 0x07))
{
return new FileClassification(FileCategory.Archive, "ZIP-family signature.");
}
if (header.Length >= 12
&& header[0] == 0x52 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x46)
{
var form = System.Text.Encoding.ASCII.GetString(header.Slice(8, 4));
if (form is "WAVE" or "AVI ")
{
return new FileClassification(
form == "WAVE" ? FileCategory.Audio : FileCategory.Video,
"RIFF " + form.Trim() + " signature.");
}
}
if (header.Length >= 12
&& header[4] == 0x66 && header[5] == 0x74 && header[6] == 0x79 && header[7] == 0x70)
{
return new FileClassification(FileCategory.Video, "ISO BMFF (ftyp) signature.");
}
if (header[0] == 0x49 && header[1] == 0x44 && header[2] == 0x33)
{
return new FileClassification(FileCategory.Audio, "ID3 signature.");
}
if (header[0] == 0x7F && header[1] == 0x45 && header[2] == 0x4C && header[3] == 0x46)
{
return new FileClassification(FileCategory.Installer, "ELF binary signature.");
}
if (header[0] == 0x4D && header[1] == 0x5A)
{
return new FileClassification(FileCategory.Installer, "PE/MZ signature.");
}
if (header[0] == 0x37 && header[1] == 0x7A && header[2] == 0xBC && header[3] == 0xAF)
{
return new FileClassification(FileCategory.Archive, "7z signature.");
}
if (header[0] == 0x52 && header[1] == 0x61 && header[2] == 0x72 && header[3] == 0x21)
{
return new FileClassification(FileCategory.Archive, "RAR signature.");
}
return null;
}
}

View File

@@ -48,6 +48,51 @@ public sealed class OrganizeDestinations
_ => "Unknown"
};
public static bool TryParse(string? value, out FileCategory category)
{
category = FileCategory.Unknown;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}
var key = value.Trim().Replace(" ", "", StringComparison.Ordinal).Replace("_", "", StringComparison.Ordinal);
if (Enum.TryParse(key, ignoreCase: true, out category) && category != FileCategory.Unknown)
{
return true;
}
category = key.ToLowerInvariant() switch
{
"photo" or "photos" or "image" or "images" or "picture" or "pictures" => FileCategory.Photos,
"video" or "videos" or "movie" or "movies" => FileCategory.Video,
"audio" or "music" or "sound" => FileCategory.Audio,
"doc" or "docs" or "document" or "documents" => FileCategory.Documents,
"code" or "repo" or "repository" or "git" or "development" => FileCategory.CodeRepository,
"installer" or "installers" or "setup" => FileCategory.Installer,
"backup" or "backups" => FileCategory.Backup,
"archive" or "archives" or "zip" => FileCategory.Archive,
"system" or "systemdata" => FileCategory.SystemData,
"build" or "buildoutput" => FileCategory.BuildOutput,
"unknown" => FileCategory.Unknown,
_ => (FileCategory)(-1)
};
return (int)category >= 0;
}
public static readonly FileCategory[] UserChoices =
[
FileCategory.Photos,
FileCategory.Video,
FileCategory.Audio,
FileCategory.Documents,
FileCategory.Installer,
FileCategory.Archive,
FileCategory.Backup,
FileCategory.CodeRepository,
FileCategory.Unknown
];
private static string? EmptyToNull(string value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}

View File

@@ -13,6 +13,7 @@ public sealed class ReorganizeService
private readonly IHydrationGuard _hydration;
private readonly IGitStatusProvider _git;
private readonly UiPreferencesStore _preferences;
private readonly IIndexStore _store;
public ReorganizeService(
ReorganizePlanner planner,
@@ -21,7 +22,8 @@ public sealed class ReorganizeService
IFileSystemEnumerator enumerator,
IHydrationGuard hydration,
IGitStatusProvider git,
UiPreferencesStore preferences)
UiPreferencesStore preferences,
IIndexStore store)
{
_planner = planner;
_ops = ops;
@@ -30,6 +32,7 @@ public sealed class ReorganizeService
_hydration = hydration;
_git = git;
_preferences = preferences;
_store = store;
}
public OrganizeDestinations LoadDestinations()
@@ -63,14 +66,24 @@ public sealed class ReorganizeService
}
public OperationPlan Preview(string sourceRoot, OrganizeDestinations destinations)
=> _planner.Build(
=> PreviewAsync(sourceRoot, destinations).GetAwaiter().GetResult();
public async Task<OperationPlan> PreviewAsync(
string sourceRoot,
OrganizeDestinations destinations,
CancellationToken cancellationToken = default)
{
var indexed = await LoadIndexedChildrenAsync(sourceRoot, cancellationToken).ConfigureAwait(false);
return _planner.Build(
sourceRoot,
destinations,
_enumerator,
path => _volumes.IsPathReachable(path),
path => _git.IsRepoRoot(path),
item => _hydration.WouldHydrateOnRead(item),
RenameBatchService.PathExists);
RenameBatchService.PathExists,
indexedByName: indexed);
}
public async Task<OperationPlan> EnqueueAsync(
string sourceRoot,
@@ -78,7 +91,7 @@ public sealed class ReorganizeService
OperationPlan? plan = null,
CancellationToken cancellationToken = default)
{
plan ??= Preview(sourceRoot, destinations);
plan ??= await PreviewAsync(sourceRoot, destinations, cancellationToken).ConfigureAwait(false);
if (!plan.CanEnqueue)
{
return plan;
@@ -97,6 +110,62 @@ public sealed class ReorganizeService
return plan;
}
private async Task<IReadOnlyDictionary<string, IndexEntry>> LoadIndexedChildrenAsync(
string sourceRoot,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(sourceRoot))
{
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
}
try
{
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
Source? best = null;
var bestLen = -1;
foreach (var source in sources)
{
if (string.IsNullOrEmpty(source.LastRootPath))
{
continue;
}
var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\');
var path = PathRules.FromExtended(sourceRoot).TrimEnd('\\');
if (path.Equals(root, StringComparison.OrdinalIgnoreCase)
|| path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase))
{
if (root.Length > bestLen)
{
best = source;
bestLen = root.Length;
}
}
}
if (best?.LastRootPath is null)
{
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
}
var rel = PathRules.MakeRelative(best.LastRootPath, sourceRoot) ?? "";
var dir = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false);
if (dir is null)
{
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
}
var children = await _store.Entries.GetChildrenAsync(best.Id, dir.Id, EntryStatus.Present, cancellationToken)
.ConfigureAwait(false);
return children.ToDictionary(c => c.Name, StringComparer.OrdinalIgnoreCase);
}
catch (Exception)
{
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
}
}
private static string Known(Environment.SpecialFolder folder)
=> Environment.GetFolderPath(folder);

View File

@@ -579,6 +579,17 @@ public sealed class WorkbenchPipeClient : IWorkbenchHost, ICloudOverlay, IHostCo
=> _client.CallAsync("Mutations.EnqueueHashCollisions", cancellationToken, n: sourceId ?? 0);
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
=> _client.CallAsync("Mutations.UpsertRelation", cancellationToken, payload: Json(relation));
public Task SetUserCategoryByPathAsync(
string fullPath,
FileCategory category,
string? reason = null,
CancellationToken cancellationToken = default)
=> _client.CallAsync(
"Mutations.SetUserCategory",
cancellationToken,
s: fullPath,
dest: reason,
payload: category.ToString());
private static string Json<T>(T value) => JsonSerializer.Serialize(value, WorkbenchIpc.Json);
}
}

View File

@@ -14,6 +14,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
private readonly IForegroundWorkSignal _foreground;
private readonly IIdleIndexWork _indexing;
private readonly IIdleHashWork _hash;
private readonly IIdleClassifyWork _classify;
private readonly IHistoryMaintenance _history;
private readonly IIndexStore _store;
private readonly IVolumeService _volumes;
@@ -44,7 +45,8 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
IIndexStore store,
IVolumeService volumes,
ILogger<BackgroundMaintenanceCoordinator> logger,
IHostActivitySink? activity = null)
IHostActivitySink? activity = null,
IIdleClassifyWork? classify = null)
{
_idle = idle;
_power = power;
@@ -52,6 +54,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
_foreground = foreground;
_indexing = indexing;
_hash = hash;
_classify = classify ?? NullIdleClassifyWork.Instance;
_history = history;
_store = store;
_volumes = volumes;
@@ -71,6 +74,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_hash.Pause();
_classify.Pause();
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1));
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
{
@@ -115,10 +119,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
{
_idleScanQueued.Clear();
_idleVerified.Clear();
if (!_hash.IsPaused)
{
_hash.Pause();
}
PauseBackgroundWorkers();
if (_lastAllowed)
{
@@ -147,10 +148,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
if (_indexing.IsBusy || _indexing.HasIdleWork)
{
if (!_hash.IsPaused)
{
_hash.Pause();
}
PauseBackgroundWorkers();
// Keep the idle-maintenance caption only while idle work is still queued/running.
// Watcher/user jobs must not keep showing a stale "checking External" line.
@@ -165,10 +163,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
var verify = BackgroundMaintenancePlanner.NextLocalVerify(sources, _idleVerified);
if (verify is not null && verify.LastRootPath is not null && _volumes.IsPathReachable(verify.LastRootPath))
{
if (!_hash.IsPaused)
{
_hash.Pause();
}
PauseBackgroundWorkers();
_idleVerified.Add(verify.Id);
_indexing.EnqueueIdleVerify(verify.Id, "");
@@ -183,10 +178,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
var scan = BackgroundMaintenancePlanner.NextLocalScan(sources, DateTimeOffset.UtcNow, _idleScanQueued);
if (scan is not null && scan.LastRootPath is not null && _volumes.IsPathReachable(scan.LastRootPath))
{
if (!_hash.IsPaused)
{
_hash.Pause();
}
PauseBackgroundWorkers();
_idleScanQueued.Add(scan.Id);
_indexing.EnqueueIdleFullScan(scan.Id);
@@ -196,6 +188,16 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
return;
}
PauseHash();
_classify.Resume();
if (await _classify.ProcessPendingAsync(cancellationToken).ConfigureAwait(false))
{
_workMessage = "Idle maintenance · classifying";
Publish(decision, _workMessage);
return;
}
_classify.Pause();
_hash.Resume();
if (hashPending)
{
@@ -232,6 +234,23 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
Publish(decision, _workMessage);
}
private void PauseBackgroundWorkers()
{
PauseHash();
if (!_classify.IsPaused)
{
_classify.Pause();
}
}
private void PauseHash()
{
if (!_hash.IsPaused)
{
_hash.Pause();
}
}
private string PauseMessage(BackgroundWorkDecision decision)
{
if (decision.SkipReason == MaintenanceSkipReason.ForegroundOperations)

View File

@@ -100,8 +100,10 @@ public static class ExplorerHostServices
sp.GetRequiredService<IIndexMutations>()));
services.AddSingleton<DuplicateHashWorker>();
services.AddSingleton<HistoryRollupService>();
services.AddSingleton<EntryClassificationService>();
services.AddSingleton<IIdleIndexWork>(sp => sp.GetRequiredService<IndexingCoordinator>());
services.AddSingleton<IIdleHashWork>(sp => sp.GetRequiredService<DuplicateHashWorker>());
services.AddSingleton<IIdleClassifyWork>(sp => sp.GetRequiredService<EntryClassificationService>());
services.AddSingleton<IHistoryMaintenance>(sp => sp.GetRequiredService<HistoryRollupService>());
services.AddSingleton<BackgroundMaintenanceCoordinator>();
services.AddSingleton<IBackgroundMaintenance>(sp => sp.GetRequiredService<BackgroundMaintenanceCoordinator>());

View File

@@ -462,6 +462,16 @@ public sealed class WorkbenchPipeServer : BackgroundService
case "Mutations.UpsertRelation":
await Workbench.Mutations.UpsertRelationAsync(Read<FileRelation>(request.Payload)).ConfigureAwait(false);
return reply;
case "Mutations.SetUserCategory":
if (OrganizeDestinations.TryParse(request.Payload, out var userCategory)
|| Enum.TryParse(request.Payload, true, out userCategory))
{
await Workbench.Mutations.SetUserCategoryByPathAsync(
request.S ?? "",
userCategory,
request.Dest).ConfigureAwait(false);
}
return reply;
case "Cloud.Places":
reply.Payload = JsonSerializer.Serialize(_overlay!.GetPlaces(), WorkbenchIpc.Json);
return reply;

View File

@@ -190,6 +190,9 @@ public sealed class FolderReconciler
}
}, cancellationToken).ConfigureAwait(false);
await _store.Entries.RefreshCategoryFromChildrenAsync(parent.Id, cancellationToken)
.ConfigureAwait(false);
foreach (var path in archivesToTomb)
{
await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken)
@@ -202,6 +205,8 @@ public sealed class FolderReconciler
{
await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
.ConfigureAwait(false);
await _store.Entries.RefreshCategoryFromChildrenAsync(archive.Id, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -69,6 +69,7 @@ public sealed partial class AnalysisViewModel : ObservableObject
public const string PageFolders = "Biggest folders";
public const string PageFiles = "Biggest files";
public const string PageTypes = "By file type";
public const string PageCategories = "By category";
public const string PageSources = "By source";
private readonly AnalysisService _analysis;
@@ -93,7 +94,7 @@ public sealed partial class AnalysisViewModel : ObservableObject
Rows = [];
}
public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageSources];
public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageCategories, PageSources];
public ObservableCollection<StorageScopeItem> Scopes { get; }
public ObservableCollection<StorageNodeViewModel> TreeRoots { get; }
public ObservableCollection<StorageNodeViewModel> VisibleNodes { get; }
@@ -461,6 +462,25 @@ public sealed partial class AnalysisViewModel : ObservableObject
0,
(string?)null)).ToList();
}
else if (page == PageCategories)
{
var cats = await _analysis.UsageByCategoryAsync(sourceId, null, AppConstants.AnalysisTopN, cancellationToken)
.ConfigureAwait(false);
items = cats.Select(t => (
OrganizeDestinations.TryParse(t.Category, out var cat)
? OrganizeDestinations.Label(cat)
: string.IsNullOrEmpty(t.Category) ? "Unknown" : t.Category,
t.TotalSize,
(string?)null,
(string?)null,
(string?)null,
(long?)null,
sourceId,
false,
(int)t.FileCount,
0,
(string?)null)).ToList();
}
else
{
var usage = await _analysis.UsageBySourceAsync(cancellationToken).ConfigureAwait(false);

View File

@@ -85,6 +85,27 @@ public sealed partial class FolderItemViewModel : ObservableObject
};
public bool HasCloudStatus => !string.IsNullOrEmpty(CloudLabel);
public string CategoryLabel
=> Item.Category == FileCategory.Unknown
&& (string.IsNullOrEmpty(Item.CategorySource) || Item.CategorySource == CategorySources.None)
? ""
: OrganizeDestinations.Label(Item.Category);
public string CategoryTooltip
{
get
{
var label = CategoryLabel;
if (string.IsNullOrEmpty(label))
{
return "";
}
var reason = Item.CategoryReason;
return string.IsNullOrEmpty(reason) ? label : label + " · " + reason;
}
}
[ObservableProperty] private string _gitLabel = "";
public bool HasGitLabel => !string.IsNullOrEmpty(GitLabel);
@@ -122,18 +143,8 @@ public sealed partial class FolderItemViewModel : ObservableObject
internal static class MediaKinds
{
private static readonly HashSet<string> Images = new(StringComparer.OrdinalIgnoreCase)
{
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tif", ".tiff", ".ico", ".jfif"
};
private static readonly HashSet<string> Videos = new(StringComparer.OrdinalIgnoreCase)
{
".mp4", ".mkv", ".avi", ".mov", ".wmv", ".webm", ".m4v", ".mpg", ".mpeg"
};
public static bool IsImage(string name) => Images.Contains(Path.GetExtension(name));
public static bool IsVideo(string name) => Videos.Contains(Path.GetExtension(name));
public static bool IsImage(string name) => FileClassifier.IsPhotoFile(name);
public static bool IsVideo(string name) => FileClassifier.IsVideoFile(name);
}
file static class ItemExt

View File

@@ -34,6 +34,7 @@ public sealed partial class MainViewModel : ObservableObject
private readonly ConversionPlanner _conversionPlanner;
private readonly IFileSystemEnumerator _enumerator;
private readonly IMediaConversionProvider _conversion;
private readonly IIndexMutations _mutations;
private readonly IThumbnailService? _thumbnails;
private readonly IHostConnection? _host;
private readonly IBackgroundMaintenance? _maintenance;
@@ -65,6 +66,7 @@ public sealed partial class MainViewModel : ObservableObject
[ObservableProperty] private bool _showTags;
[ObservableProperty] private bool _showRunProfile;
[ObservableProperty] private bool _showOrganizeFolder;
[ObservableProperty] private bool _showClassifyAs;
[ObservableProperty] private bool _canUndoRenameBatch;
[ObservableProperty] private bool _showExtractArchive;
[ObservableProperty] private bool _showCompress;
@@ -141,6 +143,7 @@ public sealed partial class MainViewModel : ObservableObject
_conversionPlanner = conversionPlanner;
_enumerator = enumerator;
_conversion = conversion;
_mutations = mutations;
_thumbnails = thumbnails;
_host = hostConnection;
_maintenance = maintenance;
@@ -737,6 +740,36 @@ public sealed partial class MainViewModel : ObservableObject
public ReorganizeViewModel CreateReorganizeViewModel()
=> new(_reorganize, OrganizeSourcePath());
[RelayCommand]
public async Task ClassifyAsAsync(string? category)
{
if (!OrganizeDestinations.TryParse(category, out var parsed)
&& !Enum.TryParse(category, true, out parsed))
{
Footer = "Unknown category.";
return;
}
var items = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList();
if (items.Count == 0)
{
Footer = "Select a file or folder to classify.";
return;
}
var n = 0;
foreach (var item in items)
{
await _mutations.SetUserCategoryByPathAsync(item.FullPath, parsed, "Set by user.").ConfigureAwait(true);
n++;
}
await ActivePane.RefreshAsync().ConfigureAwait(true);
Footer = n == 1
? "Classified as " + OrganizeDestinations.Label(parsed) + "."
: "Classified " + n + " items as " + OrganizeDestinations.Label(parsed) + ".";
}
public GitChangesViewModel CreateGitChangesViewModel()
=> new(_gitCommands, _workspace, _hydration);
@@ -984,6 +1017,7 @@ public sealed partial class MainViewModel : ObservableObject
ShowRunProfile = ShowBatchRename;
ShowOrganizeFolder = OrganizeSourcePath() is not null;
var real = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList();
ShowClassifyAs = real.Count > 0;
ShowExtractArchive = real.Count > 0 && real.All(i => !i.IsDirectory && ArchiveFormats.IsArchive(i.Item.Name));
ShowCompress = real.Count > 0;
ShowConvert = real.Any(i => i.IsDirectory || ConversionFormats.IsConvertible(i.Item.Name));

View File

@@ -44,7 +44,7 @@ public sealed partial class ReorganizeViewModel : ObservableObject
{
Status = "Analyzing…";
CanQueue = false;
var plan = await Task.Run(() => _organize.Preview(SourcePath.Trim(), CurrentDestinations())).ConfigureAwait(true);
var plan = await _organize.PreviewAsync(SourcePath.Trim(), CurrentDestinations()).ConfigureAwait(true);
ApplyPlan(plan);
}

View File

@@ -9,6 +9,7 @@ using Explorer.Search;
namespace Explorer.Presentation.ViewModels;
public sealed record SearchScopeChoice(SearchScopeKind Kind, string Label);
public sealed record CategoryChoice(string Value, string Label);
public sealed partial class SearchViewModel : ObservableObject
{
@@ -20,6 +21,7 @@ public sealed partial class SearchViewModel : ObservableObject
[ObservableProperty] private string _text = "";
[ObservableProperty] private SearchScopeKind _scope = SearchScopeKind.AllKnown;
[ObservableProperty] private string? _extension;
[ObservableProperty] private string _category = "";
[ObservableProperty] private bool _foldersOnly;
[ObservableProperty] private bool _filesOnly;
[ObservableProperty] private string? _minSizeText;
@@ -46,6 +48,12 @@ public sealed partial class SearchViewModel : ObservableObject
new(SearchScopeKind.OfflineMedia, "Offline media")
];
public CategoryChoice[] CategoryChoices { get; } =
[
new("", "Any category"),
.. OrganizeDestinations.UserChoices.Select(c => new CategoryChoice(c.ToString(), OrganizeDestinations.Label(c)))
];
public void OpenWithoutSearch()
{
IsOpen = true;
@@ -103,6 +111,7 @@ public sealed partial class SearchViewModel : ObservableObject
Scope = scope,
Text = Text,
Extension = Extension,
Category = OrganizeDestinations.TryParse(Category, out var cat) ? cat : null,
IsDirectory = DirectoryFilter(FoldersOnly, FilesOnly),
MinSize = ParseSize(MinSizeText),
MaxSize = ParseSize(MaxSizeText),
@@ -139,7 +148,11 @@ public sealed partial class SearchViewModel : ObservableObject
FileId = entry.FileId,
ReparseTag = entry.ReparseTag,
FreeSpaceBytes = space.FreeBytes,
CapacityBytes = space.CapacityBytes
CapacityBytes = space.CapacityBytes,
IndexEntryId = entry.Id,
Category = entry.Category,
CategoryReason = entry.CategoryReason,
CategorySource = entry.CategorySource
}, entry.IsDirectory));
}

View File

@@ -9,4 +9,7 @@
<ProjectReference Include="..\Explorer.Application\Explorer.Application.csproj" />
<ProjectReference Include="..\Explorer.Domain\Explorer.Domain.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="Explorer.Search.Tests" />
</ItemGroup>
</Project>

View File

@@ -15,6 +15,7 @@ public sealed class SearchQuery
public DateTimeOffset? CreatedBefore { get; init; }
public DateTimeOffset? ModifiedAfter { get; init; }
public DateTimeOffset? ModifiedBefore { get; init; }
public FileCategory? Category { get; init; }
public IReadOnlyList<long>? SourceIds { get; init; }
public string? PathRelPrefix { get; init; }
public bool IncludeOffline { get; init; } = true;
@@ -33,9 +34,19 @@ public sealed class SearchService
var text = query.Text?.Trim();
string? extension = query.Extension;
string? name = text;
if (!string.IsNullOrEmpty(text) && text.StartsWith("*.", StringComparison.Ordinal) && !text.Contains(' ', StringComparison.Ordinal))
var category = query.Category;
if (!string.IsNullOrEmpty(text))
{
extension = text[2..];
(name, var hinted) = SplitCategoryHint(text);
if (hinted is not null)
{
category = hinted;
}
}
if (!string.IsNullOrEmpty(name) && name.StartsWith("*.", StringComparison.Ordinal) && !name.Contains(' ', StringComparison.Ordinal))
{
extension = name[2..];
name = null;
}
@@ -43,6 +54,7 @@ public sealed class SearchService
{
Name = name,
Extension = extension,
Category = category,
IsDirectory = query.IsDirectory,
MinSize = query.MinSize,
MaxSize = query.MaxSize,
@@ -62,4 +74,25 @@ public sealed class SearchService
return _store.Search.SearchAsync(request, cancellationToken);
}
internal static (string? Name, FileCategory? Category) SplitCategoryHint(string text)
{
FileCategory? category = null;
var name = System.Text.RegularExpressions.Regex.Replace(
text,
@"\b(?:category|cat):([A-Za-z]+)\b",
match =>
{
if (OrganizeDestinations.TryParse(match.Groups[1].Value, out var parsed))
{
category = parsed;
return "";
}
return match.Value;
},
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
name = name.Trim();
return (string.IsNullOrEmpty(name) ? null : name, category);
}
}

View File

@@ -91,6 +91,24 @@ internal sealed class AnalysisStore : IAnalysisStore
return rows.AsList();
}
public async Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default)
{
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
var sql = """
SELECT ifnull(category, 'Unknown') AS Category, SUM(size_bytes) AS TotalSize, COUNT(*) AS FileCount
FROM entries
WHERE is_dir=0 AND status=0
""";
if (sourceId is not null) sql += " AND source_id=@sourceId";
if (!string.IsNullOrEmpty(pathRelPrefix))
sql += " AND (path_rel=@pathRelPrefix OR path_rel LIKE @like ESCAPE '\\')";
sql += " GROUP BY category ORDER BY TotalSize DESC LIMIT @take";
var like = string.IsNullOrEmpty(pathRelPrefix) ? null : pathRelPrefix.Replace("\\", "\\\\") + "\\\\%";
var rows = await conn.QueryAsync<CategoryUsage>(new CommandDefinition(
sql, new { sourceId, pathRelPrefix, like, take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
return rows.AsList();
}
public async Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
{
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);

View File

@@ -88,14 +88,41 @@ internal sealed class EntryStore : IEntryStore
internal static async Task<long> UpsertCore(SqliteConnection conn, IndexEntry entry)
{
var existing = await SqliteExec.ScalarAsync<long?>(
var existingId = await SqliteExec.ScalarAsync<long?>(
conn,
"SELECT id FROM entries WHERE source_id=@SourceId AND ifnull(parent_id,-1)=ifnull(@ParentId,-1) AND name_norm=@NameNorm",
new { entry.SourceId, entry.ParentId, entry.NameNorm }).ConfigureAwait(false);
if (existing is > 0)
if (existingId is > 0)
{
entry.Id = existing.Value;
entry.Id = existingId.Value;
var existingSource = await SqliteExec.ScalarAsync<string>(
conn,
"SELECT ifnull(category_source, 'none') FROM entries WHERE id=@Id",
new { entry.Id }).ConfigureAwait(false);
if (CategorySources.IsUser(existingSource))
{
entry.Category = EntryCategoryAssigner.ParseCategory(
await SqliteExec.ScalarAsync<string>(conn, "SELECT category FROM entries WHERE id=@Id", new { entry.Id })
.ConfigureAwait(false));
entry.CategoryReason = await SqliteExec.ScalarAsync<string>(
conn, "SELECT category_reason FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false);
entry.CategoryConfidence = await SqliteExec.ScalarAsync<long>(
conn, "SELECT ifnull(category_confidence, 0) FROM entries WHERE id=@Id", new { entry.Id })
.ConfigureAwait(false) is { } conf
? (int)conf
: 0;
entry.CategorySource = CategorySources.User;
var utc = await SqliteExec.ScalarAsync<string>(
conn, "SELECT category_utc FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false);
entry.CategoryUtc = string.IsNullOrEmpty(utc) ? null : DateTimeOffset.Parse(utc);
}
else if (string.IsNullOrWhiteSpace(entry.CategorySource)
|| entry.CategorySource == CategorySources.None)
{
EntryCategoryAssigner.ApplyAutomatic(entry);
}
await SqliteExec.ExecuteAsync(conn, """
UPDATE entries SET
name=@Name, extension=@Extension, is_dir=@IsDir, size_bytes=@SizeBytes,
@@ -103,21 +130,32 @@ internal sealed class EntryStore : IEntryStore
last_indexed_utc=@LastIndexedUtc, attributes=@Attributes, file_id=@FileId,
parent_file_id=@ParentFileId, reparse_tag=@ReparseTag, status=@Status,
deleted_utc=@DeletedUtc, path_rel=@PathRel, scan_generation=@ScanGeneration,
allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability
allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability,
category=@Category, category_reason=@CategoryReason,
category_confidence=@CategoryConfidence, category_source=@CategorySource,
category_utc=@CategoryUtc
WHERE id=@Id
""", ToArgs(entry)).ConfigureAwait(false);
return entry.Id;
}
if (string.IsNullOrWhiteSpace(entry.CategorySource)
|| entry.CategorySource == CategorySources.None)
{
EntryCategoryAssigner.ApplyAutomatic(entry);
}
var id = await SqliteExec.InsertAsync(conn, """
INSERT INTO entries (source_id, parent_id, name, name_norm, extension, is_dir, size_bytes, aggregate_size,
child_file_count, child_dir_count, created_utc, modified_utc, last_seen_utc, last_indexed_utc,
attributes, file_id, parent_file_id, reparse_tag, status, deleted_utc, path_rel, content_hash, hash_state, scan_generation,
allocated_size, cloud_availability)
allocated_size, cloud_availability,
category, category_reason, category_confidence, category_source, category_utc)
VALUES (@SourceId, @ParentId, @Name, @NameNorm, @Extension, @IsDir, @SizeBytes, @AggregateSize,
@ChildFileCount, @ChildDirCount, @CreatedUtc, @ModifiedUtc, @LastSeenUtc, @LastIndexedUtc,
@Attributes, @FileId, @ParentFileId, @ReparseTag, @Status, @DeletedUtc, @PathRel, @ContentHash, @HashState, @ScanGeneration,
@AllocatedSizeBytes, @CloudAvailability)
@AllocatedSizeBytes, @CloudAvailability,
@Category, @CategoryReason, @CategoryConfidence, @CategorySource, @CategoryUtc)
""", ToArgs(entry)).ConfigureAwait(false);
entry.Id = id;
return id;
@@ -262,6 +300,121 @@ internal sealed class EntryStore : IEntryStore
new { sourceId }).ConfigureAwait(false);
}
public Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default)
=> _store.WriteAsync(async conn =>
{
var row = await conn.QuerySingleOrDefaultAsync<EntryRow>(
"SELECT * FROM entries WHERE id=@id", new { id = folderOrArchiveId }).ConfigureAwait(false);
if (row is null)
{
return;
}
var entry = row.ToModel();
if (CategorySources.IsUser(entry.CategorySource))
{
return;
}
var children = (await conn.QueryAsync<string>(
"SELECT ifnull(category, 'Unknown') FROM entries WHERE parent_id=@id AND status=0",
new { id = folderOrArchiveId }).ConfigureAwait(false)).ToList();
if (children.Count == 0)
{
return;
}
var cats = children.Select(EntryCategoryAssigner.ParseCategory).ToList();
EntryCategoryAssigner.ApplyAutomatic(entry, childCategories: cats);
await SqliteExec.ExecuteAsync(conn, """
UPDATE entries SET
category=@Category, category_reason=@CategoryReason,
category_confidence=@CategoryConfidence, category_source=@CategorySource,
category_utc=@CategoryUtc
WHERE id=@Id
""", ToArgs(entry)).ConfigureAwait(false);
}, cancellationToken);
public Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default)
=> _store.WriteAsync(async conn =>
{
var row = await conn.QuerySingleOrDefaultAsync<EntryRow>(
"SELECT * FROM entries WHERE id=@id", new { id = entryId }).ConfigureAwait(false);
if (row is null)
{
return;
}
var entry = row.ToModel();
EntryCategoryAssigner.ApplyUser(entry, category, reason);
await SqliteExec.ExecuteAsync(conn, """
UPDATE entries SET
category=@Category, category_reason=@CategoryReason,
category_confidence=@CategoryConfidence, category_source=@CategorySource,
category_utc=@CategoryUtc
WHERE id=@Id
""", ToArgs(entry)).ConfigureAwait(false);
}, cancellationToken);
public Task<int> BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default)
=> _store.WriteAsync(async conn =>
{
take = Math.Clamp(take, 1, 500);
var rows = (await conn.QueryAsync<EntryRow>(new CommandDefinition("""
SELECT * FROM entries
WHERE status=0 AND ifnull(category_source, 'none') IN ('none', '')
AND category_utc IS NULL
ORDER BY id
LIMIT @take
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false)).ToList();
foreach (var row in rows)
{
var entry = row.ToModel();
EntryCategoryAssigner.ApplyAutomatic(entry);
await SqliteExec.ExecuteAsync(conn, """
UPDATE entries SET
category=@Category, category_reason=@CategoryReason,
category_confidence=@CategoryConfidence, category_source=@CategorySource,
category_utc=@CategoryUtc
WHERE id=@Id
""", ToArgs(entry)).ConfigureAwait(false);
}
return rows.Count;
}, cancellationToken);
public async Task<IReadOnlyList<long>> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default)
{
take = Math.Clamp(take, 1, 100);
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
var ids = await conn.QueryAsync<long>(new CommandDefinition("""
SELECT a.id
FROM entries a
WHERE a.status=0 AND a.is_dir=0 AND a.category='Archive'
AND ifnull(a.category_source, 'none') IN ('extension', 'none')
AND EXISTS (
SELECT 1 FROM entries c WHERE c.parent_id=a.id AND c.status=0 LIMIT 1)
ORDER BY a.id
LIMIT @take
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
return ids.ToList();
}
public async Task<IReadOnlyList<IndexEntry>> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default)
{
take = Math.Clamp(take, 1, 50);
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
var rows = await conn.QueryAsync<EntryRow>(new CommandDefinition("""
SELECT * FROM entries
WHERE status=0 AND is_dir=0 AND category='Unknown'
AND ifnull(category_source, 'none') IN ('none', '')
AND ifnull(cloud_availability, 0) != 1
ORDER BY id
LIMIT @take
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
return rows.Select(r => r.ToModel()).ToList();
}
private static string EscapeLike(string? value)
{
if (string.IsNullOrEmpty(value))
@@ -302,7 +455,12 @@ internal sealed class EntryStore : IEntryStore
HashState = (int)e.HashState,
e.ScanGeneration,
e.AllocatedSizeBytes,
CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null
CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null,
Category = e.Category.ToString(),
e.CategoryReason,
e.CategoryConfidence,
e.CategorySource,
CategoryUtc = e.CategoryUtc?.ToString("O")
};
}
@@ -335,6 +493,11 @@ internal sealed class EntryRow
public long scan_generation { get; set; }
public long? allocated_size { get; set; }
public int? cloud_availability { get; set; }
public string? category { get; set; }
public string? category_reason { get; set; }
public int category_confidence { get; set; }
public string? category_source { get; set; }
public string? category_utc { get; set; }
public IndexEntry ToModel() => new()
{
@@ -364,7 +527,12 @@ internal sealed class EntryRow
HashState = (HashState)hash_state,
ScanGeneration = scan_generation,
AllocatedSizeBytes = allocated_size,
CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null
CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null,
Category = EntryCategoryAssigner.ParseCategory(category),
CategoryReason = category_reason,
CategoryConfidence = category_confidence,
CategorySource = string.IsNullOrWhiteSpace(category_source) ? CategorySources.None : category_source,
CategoryUtc = Parse(category_utc)
};
private static DateTimeOffset? Parse(string? v)

View File

@@ -63,11 +63,17 @@ internal static class SchemaScript
hash_state INTEGER NOT NULL DEFAULT 0,
scan_generation INTEGER NOT NULL DEFAULT 0,
allocated_size INTEGER,
cloud_availability INTEGER
cloud_availability INTEGER,
category TEXT NOT NULL DEFAULT 'Unknown',
category_reason TEXT,
category_confidence INTEGER NOT NULL DEFAULT 0,
category_source TEXT NOT NULL DEFAULT 'none',
category_utc TEXT
)
""",
"CREATE UNIQUE INDEX ix_entries_identity ON entries(source_id, ifnull(parent_id, -1), name_norm)",
"CREATE INDEX ix_entries_parent ON entries(source_id, parent_id, status)",
"CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0",
"CREATE INDEX ix_entries_ext_size ON entries(source_id, extension, size_bytes) WHERE is_dir = 0",
"CREATE INDEX ix_entries_size ON entries(source_id, size_bytes) WHERE is_dir = 0 AND status = 0",
"CREATE INDEX ix_entries_modified ON entries(source_id, modified_utc)",

View File

@@ -63,6 +63,12 @@ internal sealed class SearchStore : ISearchStore
args.Add("Ext", NameNormalizer.Normalize(request.Extension.TrimStart('.')));
}
if (request.Category is { } category)
{
sql.Append(" AND e.category = @Category");
args.Add("Category", category.ToString());
}
if (request.MinSize is not null)
{
sql.Append(" AND e.size_bytes >= @MinSize");

View File

@@ -606,6 +606,25 @@ public sealed class SqliteIndexStore : IIndexStore, IAsyncDisposable
SetUserVersion(conn, 10);
_logger.LogInformation("Migrated SQLite schema to v10 (hash queue state index)");
version = 10;
}
if (version < 11)
{
EnsureColumn(conn, "entries", "category", "TEXT NOT NULL DEFAULT 'Unknown'");
EnsureColumn(conn, "entries", "category_reason", "TEXT");
EnsureColumn(conn, "entries", "category_confidence", "INTEGER NOT NULL DEFAULT 0");
EnsureColumn(conn, "entries", "category_source", "TEXT NOT NULL DEFAULT 'none'");
EnsureColumn(conn, "entries", "category_utc", "TEXT");
using (var idx = conn.CreateCommand())
{
idx.CommandText =
"CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0";
idx.ExecuteNonQuery();
}
SetUserVersion(conn, 11);
_logger.LogInformation("Migrated SQLite schema to v11 (entry categories)");
}
}

View File

@@ -39,6 +39,8 @@ public class AnalysisTests
Assert.Equal("a.mkv", files[0].Name);
var types = await analysis.UsageByExtensionAsync(source.Id, null);
Assert.Contains(types, t => t.Extension == "mkv" && t.TotalSize == 200);
var cats = await analysis.UsageByCategoryAsync(source.Id, null);
Assert.Contains(cats, c => c.Category == "Video" && c.TotalSize == 200);
var drill = await analysis.DrilldownAsync(root.Id);
Assert.Contains(drill, e => e.Name == "Movies");
Assert.Equal("Movies", dirs[0].PathRel);

View File

@@ -0,0 +1,160 @@
using Explorer.Application;
using Explorer.Domain;
using Explorer.Domain.Abstractions;
using Explorer.Storage.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
namespace Explorer.Application.Tests;
public class EntryClassificationServiceTests
{
[Fact]
public async Task Archive_children_override_zip_as_photos()
{
await using var store = await OpenStore();
var source = await AddSource(store, @"C:\media");
var zip = await Upsert(store, new IndexEntry
{
SourceId = source.Id,
Name = "photos.zip",
NameNorm = "photos.zip",
Extension = "zip",
PathRel = "photos.zip",
LastSeenUtc = DateTimeOffset.UtcNow
});
await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg"));
await Upsert(store, Child(source.Id, zip.Id, "b.jpg", "jpg"));
await Upsert(store, Child(source.Id, zip.Id, "c.jpg", "jpg"));
var service = Create(store, new MemoryEnumerator());
service.Resume();
Assert.True(await service.ProcessPendingAsync());
var loaded = await store.Entries.GetAsync(zip.Id);
Assert.Equal(FileCategory.Photos, loaded!.Category);
Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource);
Assert.False(await service.ProcessPendingAsync());
}
[Fact]
public async Task Signature_classifies_jpeg_without_extension()
{
var root = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(root);
var file = Path.Combine(root, "pic.bin");
File.WriteAllBytes(file, [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01]);
await using var store = await OpenStore();
var source = await AddSource(store, root);
var entry = await Upsert(store, new IndexEntry
{
SourceId = source.Id,
Name = "pic.bin",
NameNorm = "pic.bin",
Extension = "bin",
PathRel = "pic.bin",
LastSeenUtc = DateTimeOffset.UtcNow
});
Assert.Equal(FileCategory.Unknown, entry.Category);
var enumerator = new MemoryEnumerator();
enumerator.Add(new FileSystemItem { FullPath = file, Name = "pic.bin" });
var service = Create(store, enumerator);
service.Resume();
Assert.True(await service.ProcessPendingAsync());
var loaded = await store.Entries.GetAsync(entry.Id);
Assert.Equal(FileCategory.Photos, loaded!.Category);
Assert.Equal(CategorySources.Mime, loaded.CategorySource);
}
[Fact]
public async Task Pause_skips_work()
{
await using var store = await OpenStore();
var source = await AddSource(store, @"C:\media");
var zip = await Upsert(store, new IndexEntry
{
SourceId = source.Id,
Name = "photos.zip",
NameNorm = "photos.zip",
Extension = "zip",
PathRel = "photos.zip",
LastSeenUtc = DateTimeOffset.UtcNow
});
await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg"));
var service = Create(store, new MemoryEnumerator());
Assert.True(service.IsPaused);
Assert.False(await service.ProcessPendingAsync());
var loaded = await store.Entries.GetAsync(zip.Id);
Assert.Equal(FileCategory.Archive, loaded!.Category);
}
private static EntryClassificationService Create(IIndexStore store, IFileSystemEnumerator enumerator)
=> new(
store,
enumerator,
new NeverHydrate(),
NullLogger<EntryClassificationService>.Instance);
private static async Task<SqliteIndexStore> OpenStore()
{
var db = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N"), "index.db");
var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
await store.OpenAsync();
return store;
}
private static async Task<Source> AddSource(IIndexStore store, string root)
{
var source = new Source
{
StableKey = Guid.NewGuid().ToString("N"),
Kind = SourceKind.NtfsLocal,
DisplayName = "Test",
LastRootPath = root,
Status = SourceStatus.Online
};
source.Id = await store.Sources.UpsertAsync(source);
return source;
}
private static async Task<IndexEntry> Upsert(IIndexStore store, IndexEntry entry)
{
entry.Id = await store.Entries.UpsertAsync(entry);
return (await store.Entries.GetAsync(entry.Id))!;
}
private static IndexEntry Child(long sourceId, long parentId, string name, string ext)
=> new()
{
SourceId = sourceId,
ParentId = parentId,
Name = name,
NameNorm = name,
Extension = ext,
PathRel = name,
LastSeenUtc = DateTimeOffset.UtcNow
};
private sealed class NeverHydrate : IHydrationGuard
{
public bool WouldHydrateOnRead(FileSystemItem item) => false;
public bool WouldHydrateOnRead(int attributes, CloudAvailability? availability) => false;
public Task<bool> WouldHydrateOnReadAsync(string path, CancellationToken cancellationToken = default)
=> Task.FromResult(false);
}
private sealed class MemoryEnumerator : IFileSystemEnumerator
{
private readonly Dictionary<string, FileSystemItem> _items = new(StringComparer.OrdinalIgnoreCase);
public void Add(FileSystemItem item) => _items[item.FullPath] = item;
public IEnumerable<FileSystemItem> EnumerateChildren(string directoryPath) => [];
public FileSystemItem? GetItem(string path)
=> _items.TryGetValue(path, out var item) ? item : null;
public IReadOnlyList<FileSystemItem> EnumerateChildrenSafe(string directoryPath, out string? error)
{
error = null;
return [];
}
}
}

View File

@@ -132,6 +132,40 @@ public class ReorganizePlannerTests
Assert.Null(fs.GetItem(@"C:\Pictures\photo.jpg"));
}
[Fact]
public void Prefers_indexed_archive_contents_over_extension()
{
var fs = Tree()
.Dir(@"C:\Downloads")
.File(@"C:\Downloads\photos.zip", 4, T0)
.Dir(@"C:\Pictures")
.Dir(@"C:\Archive");
var indexed = new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase)
{
["photos.zip"] = new IndexEntry
{
Name = "photos.zip",
NameNorm = "photos.zip",
PathRel = "photos.zip",
Category = FileCategory.Photos,
CategorySource = CategorySources.ArchiveContents,
CategoryReason = "Archive contents are mostly photos."
}
};
var plan = new ReorganizePlanner().Build(
@"C:\Downloads",
Map(),
fs,
_ => true,
_ => false,
now: T0,
pathExists: _ => false,
indexedByName: indexed);
Assert.Contains(plan.Operations, o => o.SourcePath.EndsWith("photos.zip") && o.DestinationPath == @"C:\Pictures\photos.zip");
Assert.DoesNotContain(plan.Operations, o => o.DestinationPath == @"C:\Archive\photos.zip");
}
private static OperationPlan Build(
IFileSystemEnumerator fs,
Action<OrganizeDestinations>? configure = null,

View File

@@ -484,6 +484,108 @@ public class FileClassifierTests
=> FileClassifier.Classify(name, @"C:\Downloads\" + name, isDirectory, isDirectory ? AttributeFlags.Directory : 0, isRepoRoot).Category;
}
public class EntryCategoryAssignerTests
{
[Fact]
public void ApplyAutomatic_sets_extension_category_on_files()
{
var entry = new IndexEntry
{
Name = "clip.mkv",
NameNorm = "clip.mkv",
PathRel = "clip.mkv",
Extension = "mkv"
};
EntryCategoryAssigner.ApplyAutomatic(entry);
Assert.Equal(FileCategory.Video, entry.Category);
Assert.Equal(CategorySources.Extension, entry.CategorySource);
Assert.True(entry.CategoryConfidence >= 80);
}
[Fact]
public void User_category_is_not_overwritten()
{
var entry = new IndexEntry
{
Name = "clip.mkv",
NameNorm = "clip.mkv",
PathRel = "clip.mkv",
Extension = "mkv"
};
EntryCategoryAssigner.ApplyUser(entry, FileCategory.Documents, "Manual");
EntryCategoryAssigner.ApplyAutomatic(entry);
Assert.Equal(FileCategory.Documents, entry.Category);
Assert.Equal(CategorySources.User, entry.CategorySource);
}
[Fact]
public void Children_majority_sets_folder_or_archive_source()
{
var folder = new IndexEntry
{
Name = "Vacation",
NameNorm = "vacation",
PathRel = "Vacation",
IsDirectory = true,
Attributes = AttributeFlags.Directory
};
EntryCategoryAssigner.ApplyAutomatic(
folder,
childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos, FileCategory.Documents]);
Assert.Equal(FileCategory.Photos, folder.Category);
Assert.Equal(CategorySources.Children, folder.CategorySource);
var zip = new IndexEntry
{
Name = "photos.zip",
NameNorm = "photos.zip",
PathRel = "photos.zip",
Extension = "zip"
};
EntryCategoryAssigner.ApplyAutomatic(
zip,
childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos]);
Assert.Equal(FileCategory.Photos, zip.Category);
Assert.Equal(CategorySources.ArchiveContents, zip.CategorySource);
}
}
public class OrganizeDestinationsTests
{
[Fact]
public void Parses_category_aliases()
{
Assert.True(OrganizeDestinations.TryParse("photos", out var photos));
Assert.Equal(FileCategory.Photos, photos);
Assert.True(OrganizeDestinations.TryParse("zip", out var zip));
Assert.Equal(FileCategory.Archive, zip);
Assert.True(OrganizeDestinations.TryParse("unknown", out var unknown));
Assert.Equal(FileCategory.Unknown, unknown);
Assert.False(OrganizeDestinations.TryParse("not-a-category", out _));
}
}
public class FileSignatureClassifierTests
{
[Fact]
public void Recognizes_common_headers()
{
Assert.Equal(FileCategory.Photos, Hit([0xFF, 0xD8, 0xFF, 0xE0]));
Assert.Equal(FileCategory.Photos, Hit([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]));
Assert.Equal(FileCategory.Documents, Hit([0x25, 0x50, 0x44, 0x46]));
Assert.Equal(FileCategory.Archive, Hit([0x50, 0x4B, 0x03, 0x04]));
Assert.Equal(FileCategory.Archive, Hit([0x37, 0x7A, 0xBC, 0xAF]));
Assert.Equal(FileCategory.Installer, Hit([0x4D, 0x5A, 0x90, 0x00]));
Assert.Equal(FileCategory.Audio, Hit("RIFF....WAVE"u8.ToArray()));
Assert.Equal(FileCategory.Video, Hit("RIFF....AVI "u8.ToArray()));
Assert.Null(FileSignatureClassifier.TryClassify([0x00, 0x01, 0x02, 0x03]));
Assert.Null(FileSignatureClassifier.TryClassify([0xFF, 0xD8]));
}
private static FileCategory Hit(byte[] header)
=> FileSignatureClassifier.TryClassify(header)!.Category;
}
public class GitChangeTests
{
[Fact]

View File

@@ -149,6 +149,44 @@ public class BackgroundMaintenanceCoordinatorTests
Assert.Empty(indexing.IdleScans);
}
[Fact]
public async Task Idle_classifies_before_hashing()
{
var hash = new FakeHash { Pending = true };
var indexing = new FakeIndexing();
var classify = new FakeClassify { Work = true };
var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify);
await coordinator.TickAsync(CancellationToken.None);
Assert.Equal(1, classify.Calls);
Assert.True(hash.IsPaused);
Assert.Contains("classifying", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task Idle_hashes_when_classify_is_idle()
{
var hash = new FakeHash { Pending = true };
var indexing = new FakeIndexing();
var classify = new FakeClassify();
var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify);
await coordinator.TickAsync(CancellationToken.None);
Assert.Equal(1, classify.Calls);
Assert.False(hash.IsPaused);
Assert.Contains("hashing", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public async Task Activity_pauses_classify()
{
var hash = new FakeHash();
var indexing = new FakeIndexing();
var classify = new FakeClassify { Paused = false };
var coordinator = Create(hash, indexing, idle: TimeSpan.Zero, classify: classify);
await coordinator.TickAsync(CancellationToken.None);
Assert.True(classify.IsPaused);
Assert.Equal(0, classify.Calls);
}
private static Source StaleLocal()
=> new()
{
@@ -168,8 +206,9 @@ public class BackgroundMaintenanceCoordinatorTests
bool enabled = true,
bool ac = true,
bool foreground = false,
IReadOnlyList<Source>? sources = null)
=> Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources);
IReadOnlyList<Source>? sources = null,
FakeClassify? classify = null)
=> Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources, classify);
private static BackgroundMaintenanceCoordinator Create(
FakeHash hash,
@@ -178,7 +217,8 @@ public class BackgroundMaintenanceCoordinatorTests
bool enabled = true,
bool ac = true,
bool foreground = false,
IReadOnlyList<Source>? sources = null)
IReadOnlyList<Source>? sources = null,
FakeClassify? classify = null)
{
var dir = Path.Combine(Path.GetTempPath(), "ew-maint", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
@@ -199,7 +239,8 @@ public class BackgroundMaintenanceCoordinatorTests
new FakeHistory(),
new FakeStore(sources ?? []),
new FakeVolumes(),
NullLogger<BackgroundMaintenanceCoordinator>.Instance);
NullLogger<BackgroundMaintenanceCoordinator>.Instance,
classify: classify);
}
private sealed class FakeIdle(TimeSpan idle) : IUserIdleMonitor
@@ -233,14 +274,31 @@ public class BackgroundMaintenanceCoordinatorTests
private sealed class FakeHash : IIdleHashWork
{
public bool Pending { get; set; }
public bool Paused { get; set; } = true;
public bool IsPaused => Paused;
public string? CurrentPath => null;
public void Pause() => Paused = true;
public void Resume() => Paused = false;
public void BeginUserRequested() { }
public Task<bool> HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(false);
public Task<long> CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(0L);
public Task<bool> HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending);
public Task<long> CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending ? 1L : 0L);
}
private sealed class FakeClassify : IIdleClassifyWork
{
public bool Work { get; set; }
public int Calls { get; private set; }
public bool Paused { get; set; } = true;
public bool IsPaused => Paused;
public string? CurrentPath => null;
public void Pause() => Paused = true;
public void Resume() => Paused = false;
public Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
{
Calls++;
return Task.FromResult(Work);
}
}
private sealed class FakeHistory : IHistoryMaintenance

View File

@@ -36,6 +36,7 @@ public class CoreRegistrationTests
Assert.NotNull(sp.GetService<IBackgroundMaintenance>());
Assert.NotNull(sp.GetService<IUserIdleMonitor>());
Assert.NotNull(sp.GetService<IPowerSourceMonitor>());
Assert.NotNull(sp.GetService<IIdleClassifyWork>());
Assert.IsType<StorageProviderRegistry>(sp.GetService<ICloudOverlay>());
}
finally

View File

@@ -44,4 +44,37 @@ public class SearchServiceTests
Assert.Contains(both, e => e.Name == "Folder");
Assert.Contains(both, e => e.Name == "notes.txt");
}
[Fact]
public async Task Category_hint_and_filter()
{
var db = Path.Combine(Path.GetTempPath(), "ew-search", Guid.NewGuid().ToString("N"), "index.db");
await using var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
await store.OpenAsync();
var source = new Source { StableKey = "k", DisplayName = "d", Kind = SourceKind.NtfsLocal, LastRootPath = @"C:\d", Status = SourceStatus.Online };
source.Id = await store.Sources.UpsertAsync(source);
var root = new IndexEntry { SourceId = source.Id, Name = "d", NameNorm = "d", IsDirectory = true, PathRel = "", LastSeenUtc = DateTimeOffset.UtcNow };
root.Id = await store.Entries.UpsertAsync(root);
await store.Entries.UpsertAsync(new IndexEntry
{
SourceId = source.Id, ParentId = root.Id, Name = "clip.mkv", NameNorm = "clip.mkv", Extension = "mkv",
PathRel = "clip.mkv", LastSeenUtc = DateTimeOffset.UtcNow
});
await store.Entries.UpsertAsync(new IndexEntry
{
SourceId = source.Id, ParentId = root.Id, Name = "notes.pdf", NameNorm = "notes.pdf", Extension = "pdf",
PathRel = "notes.pdf", LastSeenUtc = DateTimeOffset.UtcNow
});
var search = new SearchService(store);
var hinted = await search.SearchAsync(new SearchQuery { Text = "category:video", SourceIds = [source.Id] });
Assert.Single(hinted);
Assert.Equal("clip.mkv", hinted[0].Name);
var filtered = await search.SearchAsync(new SearchQuery { Category = FileCategory.Documents, SourceIds = [source.Id] });
Assert.Single(filtered);
Assert.Equal("notes.pdf", filtered[0].Name);
var split = SearchService.SplitCategoryHint("vacation category:photos");
Assert.Equal("vacation", split.Name);
Assert.Equal(FileCategory.Photos, split.Category);
}
}

View File

@@ -114,6 +114,135 @@ public class EntryAndSearchTests
Assert.Contains(byDate, e => e.Name == "Movie.mkv");
}
[Fact]
public async Task Upsert_assigns_and_preserves_categories()
{
await using var store = await Stores.Open();
var source = await store.AddSourceAsync(@"C:\media");
var root = new IndexEntry
{
SourceId = source.Id,
Name = "media",
NameNorm = "media",
IsDirectory = true,
PathRel = "",
LastSeenUtc = DateTimeOffset.UtcNow
};
root.Id = await store.Entries.UpsertAsync(root);
var photo = new IndexEntry
{
SourceId = source.Id,
ParentId = root.Id,
Name = "shot.jpg",
NameNorm = "shot.jpg",
Extension = "jpg",
PathRel = "shot.jpg",
LastSeenUtc = DateTimeOffset.UtcNow
};
photo.Id = await store.Entries.UpsertAsync(photo);
var loaded = await store.Entries.GetAsync(photo.Id);
Assert.Equal(FileCategory.Photos, loaded!.Category);
Assert.Equal(CategorySources.Extension, loaded.CategorySource);
await store.Entries.SetUserCategoryAsync(photo.Id, FileCategory.Documents, "Manual");
photo.SizeBytes = 100;
await store.Entries.UpsertAsync(photo);
loaded = await store.Entries.GetAsync(photo.Id);
Assert.Equal(FileCategory.Documents, loaded!.Category);
Assert.Equal(CategorySources.User, loaded.CategorySource);
var other = new IndexEntry
{
SourceId = source.Id,
ParentId = root.Id,
Name = "clip.mkv",
NameNorm = "clip.mkv",
Extension = "mkv",
PathRel = "clip.mkv",
LastSeenUtc = DateTimeOffset.UtcNow
};
other.Id = await store.Entries.UpsertAsync(other);
await store.Entries.RefreshCategoryFromChildrenAsync(root.Id);
var folder = await store.Entries.GetAsync(root.Id);
Assert.Equal(FileCategory.Unknown, folder!.Category);
}
[Fact]
public async Task Archive_contents_and_signature_queues()
{
await using var store = await Stores.Open();
var source = await store.AddSourceAsync(@"C:\media");
var zip = new IndexEntry
{
SourceId = source.Id,
Name = "photos.zip",
NameNorm = "photos.zip",
Extension = "zip",
PathRel = "photos.zip",
LastSeenUtc = DateTimeOffset.UtcNow
};
zip.Id = await store.Entries.UpsertAsync(zip);
Assert.Equal(FileCategory.Archive, (await store.Entries.GetAsync(zip.Id))!.Category);
for (var i = 0; i < 3; i++)
{
await store.Entries.UpsertAsync(new IndexEntry
{
SourceId = source.Id,
ParentId = zip.Id,
Name = $"shot{i}.jpg",
NameNorm = $"shot{i}.jpg",
Extension = "jpg",
PathRel = $"photos.zip/shot{i}.jpg",
LastSeenUtc = DateTimeOffset.UtcNow
});
}
var needing = await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10);
Assert.Contains(zip.Id, needing);
await store.Entries.RefreshCategoryFromChildrenAsync(zip.Id);
var loaded = await store.Entries.GetAsync(zip.Id);
Assert.Equal(FileCategory.Photos, loaded!.Category);
Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource);
Assert.Empty(await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10));
var mystery = new IndexEntry
{
SourceId = source.Id,
Name = "mystery.bin",
NameNorm = "mystery.bin",
Extension = "bin",
PathRel = "mystery.bin",
LastSeenUtc = DateTimeOffset.UtcNow
};
mystery.Id = await store.Entries.UpsertAsync(mystery);
var unknowns = await store.Entries.GetUnknownFilesForSignatureAsync(10);
Assert.Contains(unknowns, e => e.Id == mystery.Id);
Assert.Equal(0, await store.Entries.BackfillCheapCategoriesAsync(50));
}
[Fact]
public async Task User_category_by_path_is_preserved()
{
await using var store = await Stores.Open();
var source = await store.AddSourceAsync(@"C:\media");
var photo = new IndexEntry
{
SourceId = source.Id,
Name = "shot.jpg",
NameNorm = "shot.jpg",
Extension = "jpg",
PathRel = "shot.jpg",
LastSeenUtc = DateTimeOffset.UtcNow
};
photo.Id = await store.Entries.UpsertAsync(photo);
var mutations = new Explorer.Application.LocalIndexMutations(store);
await mutations.SetUserCategoryByPathAsync(@"C:\media\shot.jpg", FileCategory.Installer, "Manual");
var loaded = await store.Entries.GetAsync(photo.Id);
Assert.Equal(FileCategory.Installer, loaded!.Category);
Assert.Equal(CategorySources.User, loaded.CategorySource);
}
[Fact]
public async Task Folder_aggregates_and_tombstones()
{