Persist file categories on the index and classify archives and unknown types during idle maintenance.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -457,14 +457,14 @@ Goal: Understand what files and folders represent rather than relying only on ex
|
|||||||
## Detection signals
|
## Detection signals
|
||||||
|
|
||||||
- [x] File extension
|
- [x] File extension
|
||||||
- [ ] MIME/content signature
|
- [x] MIME/content signature (light idle magic-byte; never hydrates)
|
||||||
- [x] Folder structure
|
- [x] Folder structure
|
||||||
- [x] Git metadata
|
- [x] Git metadata
|
||||||
- [x] Media metadata
|
- [x] Media metadata
|
||||||
- [x] Known application structures (`node_modules`, `bin`, `obj`, `.vs`)
|
- [x] Known application structures (`node_modules`, `bin`, `obj`, `.vs`)
|
||||||
- [x] File age (old installers flagged in preview)
|
- [x] File age (old installers flagged in preview)
|
||||||
- [ ] File relationships
|
- [ ] File relationships
|
||||||
- [ ] Index metadata
|
- [x] Index metadata
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -343,7 +343,22 @@ Classification **suggests** moves. Nothing moves until you Preview and Queue.
|
|||||||
|
|
||||||
Build output names (`node_modules`, `bin`, `obj`, `.vs`, and similar) are never moved. Online-only cloud items are skipped. Old installers (older than one year) still propose a move and show a warning. Destinations may sit *inside* the source folder (Downloads → Downloads\Software). Destinations are remembered in preferences.
|
Build output names (`node_modules`, `bin`, `obj`, `.vs`, and similar) are never moved. Online-only cloud items are skipped. Old installers (older than one year) still propose a move and show a warning. Destinations may sit *inside* the source folder (Downloads → Downloads\Software). Destinations are remembered in preferences.
|
||||||
|
|
||||||
Never auto-reorganizes. No MIME/content sniffing (that would hydrate cloud files).
|
Organize uses the **indexed category** when one exists (including ZIP contents that are mostly photos, and **Classify as…** overrides). Extension heuristics fill in the rest.
|
||||||
|
|
||||||
|
Never auto-reorganizes. A light magic-byte peek for unknown extensions runs only during idle maintenance on local files — never on online-only cloud items.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Classification
|
||||||
|
|
||||||
|
The index stores a category on each entry (Photos, Video, Documents, Archive, and so on) plus a short reason.
|
||||||
|
|
||||||
|
- Details shows a **Category** column (tooltip has the reason)
|
||||||
|
- Search accepts `category:photos` or the category dropdown
|
||||||
|
- Storage analysis has **By category** next to **By file type**
|
||||||
|
- Context menu **Classify as…** writes a user override that later scans do not replace
|
||||||
|
|
||||||
|
ZIP/7z files stay Archive until their contents are indexed; idle maintenance then promotes them when a category dominates.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -416,7 +431,7 @@ Left open on purpose:
|
|||||||
- Multi-PC search, sharing, encrypted vaults
|
- Multi-PC search, sharing, encrypted vaults
|
||||||
- Robocopy as a second transfer engine
|
- Robocopy as a second transfer engine
|
||||||
- Scheduled profiles and folder-watcher triggers
|
- Scheduled profiles and folder-watcher triggers
|
||||||
- MIME/EXIF classification
|
- Full EXIF / ffprobe classification
|
||||||
- Duplicate “backup copy” auto-tagging
|
- Duplicate “backup copy” auto-tagging
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -76,6 +76,16 @@ public sealed class AnalysisService
|
|||||||
ct => _store.Analysis.UsageByExtensionAsync(sourceId, pathRelPrefix, take, ct),
|
ct => _store.Analysis.UsageByExtensionAsync(sourceId, pathRelPrefix, take, ct),
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
|
public Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(
|
||||||
|
long? sourceId,
|
||||||
|
string? pathRelPrefix,
|
||||||
|
int take = AppConstants.AnalysisTopN,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
=> CachedAsync(
|
||||||
|
$"categories:{sourceId}:{pathRelPrefix}:{take}",
|
||||||
|
ct => _store.Analysis.UsageByCategoryAsync(sourceId, pathRelPrefix, take, ct),
|
||||||
|
cancellationToken);
|
||||||
|
|
||||||
public Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
|
public Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
|
||||||
=> CachedAsync("sources", ct => _store.Analysis.UsageBySourceAsync(ct), cancellationToken);
|
=> CachedAsync("sources", ct => _store.Analysis.UsageBySourceAsync(ct), cancellationToken);
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,17 @@
|
|||||||
</MenuItem>
|
</MenuItem>
|
||||||
<MenuItem Header="Organize this folder…" Click="OnOrganizeFolder"
|
<MenuItem Header="Organize this folder…" Click="OnOrganizeFolder"
|
||||||
Visibility="{Binding ShowOrganizeFolder, Converter={StaticResource BoolVis}}"/>
|
Visibility="{Binding ShowOrganizeFolder, Converter={StaticResource BoolVis}}"/>
|
||||||
|
<MenuItem Header="Classify as" Visibility="{Binding ShowClassifyAs, Converter={StaticResource BoolVis}}">
|
||||||
|
<MenuItem Header="Photos" Command="{Binding ClassifyAsCommand}" CommandParameter="Photos"/>
|
||||||
|
<MenuItem Header="Video" Command="{Binding ClassifyAsCommand}" CommandParameter="Video"/>
|
||||||
|
<MenuItem Header="Audio" Command="{Binding ClassifyAsCommand}" CommandParameter="Audio"/>
|
||||||
|
<MenuItem Header="Documents" Command="{Binding ClassifyAsCommand}" CommandParameter="Documents"/>
|
||||||
|
<MenuItem Header="Installer" Command="{Binding ClassifyAsCommand}" CommandParameter="Installer"/>
|
||||||
|
<MenuItem Header="Archive" Command="{Binding ClassifyAsCommand}" CommandParameter="Archive"/>
|
||||||
|
<MenuItem Header="Backup" Command="{Binding ClassifyAsCommand}" CommandParameter="Backup"/>
|
||||||
|
<MenuItem Header="Code repository" Command="{Binding ClassifyAsCommand}" CommandParameter="CodeRepository"/>
|
||||||
|
<MenuItem Header="Unknown" Command="{Binding ClassifyAsCommand}" CommandParameter="Unknown"/>
|
||||||
|
</MenuItem>
|
||||||
<Separator Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
|
<Separator Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
|
||||||
<MenuItem Header="Extract here" Click="OnExtractHere"
|
<MenuItem Header="Extract here" Click="OnExtractHere"
|
||||||
Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
|
Visibility="{Binding ShowExtractArchive, Converter={StaticResource BoolVis}}"/>
|
||||||
@@ -537,6 +548,13 @@
|
|||||||
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
|
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
|
||||||
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
|
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
|
||||||
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
|
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
|
||||||
|
<GridViewColumn Header="Category" Width="110">
|
||||||
|
<GridViewColumn.CellTemplate>
|
||||||
|
<DataTemplate>
|
||||||
|
<TextBlock Text="{Binding CategoryLabel}" ToolTip="{Binding CategoryTooltip}"/>
|
||||||
|
</DataTemplate>
|
||||||
|
</GridViewColumn.CellTemplate>
|
||||||
|
</GridViewColumn>
|
||||||
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
|
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
|
||||||
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
|
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
|
||||||
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
|
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
|
||||||
@@ -661,6 +679,13 @@
|
|||||||
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
|
<GridViewColumn Header="Date modified" Width="148" DisplayMemberBinding="{Binding ModifiedLabel}"/>
|
||||||
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
|
<GridViewColumn Header="Date created" Width="148" DisplayMemberBinding="{Binding CreatedLabel}"/>
|
||||||
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
|
<GridViewColumn Header="Type" Width="100" DisplayMemberBinding="{Binding TypeLabel}"/>
|
||||||
|
<GridViewColumn Header="Category" Width="110">
|
||||||
|
<GridViewColumn.CellTemplate>
|
||||||
|
<DataTemplate>
|
||||||
|
<TextBlock Text="{Binding CategoryLabel}" ToolTip="{Binding CategoryTooltip}"/>
|
||||||
|
</DataTemplate>
|
||||||
|
</GridViewColumn.CellTemplate>
|
||||||
|
</GridViewColumn>
|
||||||
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
|
<GridViewColumn Header="Size" Width="110" DisplayMemberBinding="{Binding SizeLabel}"/>
|
||||||
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
|
<GridViewColumn Header="Git" Width="110" DisplayMemberBinding="{Binding GitLabel}"/>
|
||||||
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
|
<GridViewColumn Header="Cloud" Width="100" DisplayMemberBinding="{Binding CloudLabel}"/>
|
||||||
@@ -768,6 +793,11 @@
|
|||||||
<TextBlock Grid.Row="1" Text="Extension" Foreground="{DynamicResource FgMuted}" VerticalAlignment="Center" Margin="0,0,8,0"/>
|
<TextBlock Grid.Row="1" Text="Extension" Foreground="{DynamicResource FgMuted}" VerticalAlignment="Center" Margin="0,0,8,0"/>
|
||||||
<StackPanel Grid.Row="1" Grid.Column="1" Orientation="Horizontal" Margin="0,0,16,0">
|
<StackPanel Grid.Row="1" Grid.Column="1" Orientation="Horizontal" Margin="0,0,16,0">
|
||||||
<TextBox Width="90" Text="{Binding Search.Extension, UpdateSourceTrigger=PropertyChanged}"/>
|
<TextBox Width="90" Text="{Binding Search.Extension, UpdateSourceTrigger=PropertyChanged}"/>
|
||||||
|
<ComboBox Width="140" Margin="8,0,0,0"
|
||||||
|
ItemsSource="{Binding Search.CategoryChoices}"
|
||||||
|
DisplayMemberPath="Label"
|
||||||
|
SelectedValuePath="Value"
|
||||||
|
SelectedValue="{Binding Search.Category}"/>
|
||||||
<CheckBox Content="Files" Margin="16,0,8,0" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
|
<CheckBox Content="Files" Margin="16,0,8,0" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
|
||||||
IsChecked="{Binding Search.FilesOnly}"/>
|
IsChecked="{Binding Search.FilesOnly}"/>
|
||||||
<CheckBox Content="Folders" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
|
<CheckBox Content="Folders" VerticalAlignment="Center" Foreground="{DynamicResource Fg}"
|
||||||
@@ -795,6 +825,7 @@
|
|||||||
<GridView>
|
<GridView>
|
||||||
<GridViewColumn Header="Name" Width="220" CellTemplate="{StaticResource NameWithOverlays}"/>
|
<GridViewColumn Header="Name" Width="220" CellTemplate="{StaticResource NameWithOverlays}"/>
|
||||||
<GridViewColumn Header="Path" Width="420" DisplayMemberBinding="{Binding FullPath}"/>
|
<GridViewColumn Header="Path" Width="420" DisplayMemberBinding="{Binding FullPath}"/>
|
||||||
|
<GridViewColumn Header="Category" Width="110" DisplayMemberBinding="{Binding CategoryLabel}"/>
|
||||||
<GridViewColumn Header="Size" Width="100" DisplayMemberBinding="{Binding SizeLabel}"/>
|
<GridViewColumn Header="Size" Width="100" DisplayMemberBinding="{Binding SizeLabel}"/>
|
||||||
<GridViewColumn Header="Free space" Width="100" DisplayMemberBinding="{Binding FreeSpaceLabel}"/>
|
<GridViewColumn Header="Free space" Width="100" DisplayMemberBinding="{Binding FreeSpaceLabel}"/>
|
||||||
</GridView>
|
</GridView>
|
||||||
|
|||||||
@@ -36,6 +36,27 @@ public interface IHistoryMaintenance
|
|||||||
Task<bool> TryCaptureAsync(CancellationToken cancellationToken = default);
|
Task<bool> TryCaptureAsync(CancellationToken cancellationToken = default);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public interface IIdleClassifyWork
|
||||||
|
{
|
||||||
|
bool IsPaused { get; }
|
||||||
|
string? CurrentPath { get; }
|
||||||
|
void Pause();
|
||||||
|
void Resume();
|
||||||
|
/// <summary>Returns true when any classification work was performed.</summary>
|
||||||
|
Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class NullIdleClassifyWork : IIdleClassifyWork
|
||||||
|
{
|
||||||
|
public static NullIdleClassifyWork Instance { get; } = new();
|
||||||
|
public bool IsPaused => true;
|
||||||
|
public string? CurrentPath => null;
|
||||||
|
public void Pause() { }
|
||||||
|
public void Resume() { }
|
||||||
|
public Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
|
||||||
|
=> Task.FromResult(false);
|
||||||
|
}
|
||||||
|
|
||||||
public interface IForegroundWorkSignal
|
public interface IForegroundWorkSignal
|
||||||
{
|
{
|
||||||
bool HasForegroundWork();
|
bool HasForegroundWork();
|
||||||
|
|||||||
@@ -395,7 +395,11 @@ public sealed class BrowseService
|
|||||||
Cloud = entry.CloudAvailability is { } availability
|
Cloud = entry.CloudAvailability is { } availability
|
||||||
? new CloudPresence(null, availability, entry.SizeBytes, entry.AllocatedSizeBytes, availability == CloudAvailability.OnlineOnly)
|
? new CloudPresence(null, availability, entry.SizeBytes, entry.AllocatedSizeBytes, availability == CloudAvailability.OnlineOnly)
|
||||||
: null,
|
: null,
|
||||||
Hydration = ItemHydrationFlags.All
|
Hydration = ItemHydrationFlags.All,
|
||||||
|
IndexEntryId = entry.Id,
|
||||||
|
Category = entry.Category,
|
||||||
|
CategoryReason = entry.CategoryReason,
|
||||||
|
CategorySource = entry.CategorySource
|
||||||
};
|
};
|
||||||
|
|
||||||
private async IAsyncEnumerable<BrowseDelta> ListLiveProgressiveAsync(
|
private async IAsyncEnumerable<BrowseDelta> ListLiveProgressiveAsync(
|
||||||
@@ -609,7 +613,11 @@ public sealed class BrowseService
|
|||||||
fileId: item.FileId ?? entry.FileId,
|
fileId: item.FileId ?? entry.FileId,
|
||||||
cloud: cloud,
|
cloud: cloud,
|
||||||
indexedChildCount: entry.ChildFileCount + entry.ChildDirCount,
|
indexedChildCount: entry.ChildFileCount + entry.ChildDirCount,
|
||||||
hydration: item.Hydration | ItemHydrationFlags.Index);
|
hydration: item.Hydration | ItemHydrationFlags.Index,
|
||||||
|
indexEntryId: entry.Id,
|
||||||
|
category: entry.Category,
|
||||||
|
categoryReason: entry.CategoryReason,
|
||||||
|
categorySource: entry.CategorySource);
|
||||||
return Annotate(hydrated, sizeFromIndex: item.IsDirectory && entry.AggregateSize > 0, preferences, probeAccess: false);
|
return Annotate(hydrated, sizeFromIndex: item.IsDirectory && entry.AggregateSize > 0, preferences, probeAccess: false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -785,7 +793,11 @@ public sealed class BrowseService
|
|||||||
Attributes = c.Attributes,
|
Attributes = c.Attributes,
|
||||||
FileId = c.FileId,
|
FileId = c.FileId,
|
||||||
ReparseTag = c.ReparseTag,
|
ReparseTag = c.ReparseTag,
|
||||||
AllocatedSizeBytes = c.AllocatedSizeBytes
|
AllocatedSizeBytes = c.AllocatedSizeBytes,
|
||||||
|
IndexEntryId = c.Id,
|
||||||
|
Category = c.Category,
|
||||||
|
CategoryReason = c.CategoryReason,
|
||||||
|
CategorySource = c.CategorySource
|
||||||
}).ToList();
|
}).ToList();
|
||||||
|
|
||||||
var hint = isArchiveFile && items.Count == 0
|
var hint = isArchiveFile && items.Count == 0
|
||||||
|
|||||||
173
src/Explorer.Application/EntryClassificationService.cs
Normal file
173
src/Explorer.Application/EntryClassificationService.cs
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
using Explorer.Domain;
|
||||||
|
using Explorer.Domain.Abstractions;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace Explorer.Application;
|
||||||
|
|
||||||
|
public sealed class EntryClassificationService : IIdleClassifyWork
|
||||||
|
{
|
||||||
|
private readonly IIndexStore _store;
|
||||||
|
private readonly IFileSystemEnumerator _enumerator;
|
||||||
|
private readonly IHydrationGuard _hydration;
|
||||||
|
private readonly ILogger<EntryClassificationService> _logger;
|
||||||
|
private readonly IHostActivitySink _activity;
|
||||||
|
private volatile bool _paused = true;
|
||||||
|
private volatile string? _currentPath;
|
||||||
|
|
||||||
|
public EntryClassificationService(
|
||||||
|
IIndexStore store,
|
||||||
|
IFileSystemEnumerator enumerator,
|
||||||
|
IHydrationGuard hydration,
|
||||||
|
ILogger<EntryClassificationService> logger,
|
||||||
|
IHostActivitySink? activity = null)
|
||||||
|
{
|
||||||
|
_store = store;
|
||||||
|
_enumerator = enumerator;
|
||||||
|
_hydration = hydration;
|
||||||
|
_logger = logger;
|
||||||
|
_activity = activity ?? NullHostActivitySink.Instance;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool IsPaused => _paused;
|
||||||
|
public string? CurrentPath => _currentPath;
|
||||||
|
|
||||||
|
public void Pause() => _paused = true;
|
||||||
|
public void Resume() => _paused = false;
|
||||||
|
|
||||||
|
public async Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (_paused)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var backfill = await _store.Entries.BackfillCheapCategoriesAsync(80, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
if (backfill > 0)
|
||||||
|
{
|
||||||
|
_activity.Record("Classify", "Backfilled " + backfill + " extension categories");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
var archives = await _store.Entries.GetArchiveIdsNeedingContentClassifyAsync(20, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
if (archives.Count > 0)
|
||||||
|
{
|
||||||
|
foreach (var id in archives)
|
||||||
|
{
|
||||||
|
if (_paused || cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
await _store.Entries.RefreshCategoryFromChildrenAsync(id, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
_activity.Record("Classify", "Updated " + archives.Count + " archive(s) from contents");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await TrySignatureClassifyAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<bool> TrySignatureClassifyAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var unknowns = await _store.Entries.GetUnknownFilesForSignatureAsync(12, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
if (unknowns.Count == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var byId = sources.ToDictionary(s => s.Id);
|
||||||
|
var changed = 0;
|
||||||
|
foreach (var entry in unknowns)
|
||||||
|
{
|
||||||
|
if (_paused || cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!byId.TryGetValue(entry.SourceId, out var source) || string.IsNullOrEmpty(source.LastRootPath))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var full = PathRules.Combine(source.LastRootPath, entry.PathRel);
|
||||||
|
_currentPath = full;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var item = _enumerator.GetItem(full);
|
||||||
|
if (item is null)
|
||||||
|
{
|
||||||
|
MarkSignatureSkip(entry, "File was not found.");
|
||||||
|
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_hydration.WouldHydrateOnRead(item)
|
||||||
|
|| await _hydration.WouldHydrateOnReadAsync(full, cancellationToken).ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
MarkSignatureSkip(entry, "Skipped online-only cloud file.");
|
||||||
|
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var stream = new FileStream(
|
||||||
|
PathRules.ToExtended(full),
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.ReadWrite | FileShare.Delete,
|
||||||
|
bufferSize: 64,
|
||||||
|
FileOptions.SequentialScan | FileOptions.Asynchronous);
|
||||||
|
var buffer = new byte[16];
|
||||||
|
var read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
var hit = FileSignatureClassifier.TryClassify(buffer.AsSpan(0, read));
|
||||||
|
if (hit is null)
|
||||||
|
{
|
||||||
|
entry.CategorySource = CategorySources.Mime;
|
||||||
|
entry.CategoryConfidence = 10;
|
||||||
|
entry.CategoryReason = "No known file signature.";
|
||||||
|
entry.CategoryUtc = DateTimeOffset.UtcNow;
|
||||||
|
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
EntryCategoryAssigner.Apply(entry, hit, CategorySources.Mime, 70);
|
||||||
|
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||||
|
changed++;
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
_logger.LogDebug(ex, "Signature classify failed for {Path}", full);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
MarkSignatureSkip(entry, "Could not read file signature.");
|
||||||
|
await _store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception markEx) when (markEx is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
_logger.LogDebug(markEx, "Could not mark signature skip for {Path}", full);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_currentPath = null;
|
||||||
|
if (changed > 0)
|
||||||
|
{
|
||||||
|
_activity.Record("Classify", "Signature-classified " + changed + " file(s)");
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed > 0 || unknowns.Count > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void MarkSignatureSkip(IndexEntry entry, string reason)
|
||||||
|
=> EntryCategoryAssigner.Apply(
|
||||||
|
entry,
|
||||||
|
new FileClassification(entry.Category, reason),
|
||||||
|
CategorySources.Mime,
|
||||||
|
5);
|
||||||
|
}
|
||||||
@@ -15,7 +15,8 @@ public sealed class ReorganizePlanner
|
|||||||
Func<string, bool> isRepoRoot,
|
Func<string, bool> isRepoRoot,
|
||||||
Func<FileSystemItem, bool>? wouldHydrate = null,
|
Func<FileSystemItem, bool>? wouldHydrate = null,
|
||||||
Func<string, bool>? pathExists = null,
|
Func<string, bool>? pathExists = null,
|
||||||
DateTimeOffset? now = null)
|
DateTimeOffset? now = null,
|
||||||
|
IReadOnlyDictionary<string, IndexEntry>? indexedByName = null)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(sourceRoot))
|
if (string.IsNullOrWhiteSpace(sourceRoot))
|
||||||
{
|
{
|
||||||
@@ -63,13 +64,16 @@ public sealed class ReorganizePlanner
|
|||||||
var childCategories = item.IsDirectory && !FileClassifier.SkipDescent(item.Name)
|
var childCategories = item.IsDirectory && !FileClassifier.SkipDescent(item.Name)
|
||||||
? ChildCategories(item, enumerator, isRepoRoot)
|
? ChildCategories(item, enumerator, isRepoRoot)
|
||||||
: null;
|
: null;
|
||||||
var classification = FileClassifier.Classify(
|
var classification = PreferIndexed(
|
||||||
|
item,
|
||||||
|
indexedByName,
|
||||||
|
FileClassifier.Classify(
|
||||||
item.Name,
|
item.Name,
|
||||||
item.FullPath,
|
item.FullPath,
|
||||||
item.IsDirectory,
|
item.IsDirectory,
|
||||||
item.Attributes,
|
item.Attributes,
|
||||||
item.IsDirectory && isRepoRoot(item.FullPath),
|
item.IsDirectory && isRepoRoot(item.FullPath),
|
||||||
childCategories);
|
childCategories));
|
||||||
|
|
||||||
if (FileClassifier.ShouldLeave(classification.Category))
|
if (FileClassifier.ShouldLeave(classification.Category))
|
||||||
{
|
{
|
||||||
@@ -143,6 +147,28 @@ public sealed class ReorganizePlanner
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static FileClassification PreferIndexed(
|
||||||
|
FileSystemItem item,
|
||||||
|
IReadOnlyDictionary<string, IndexEntry>? indexedByName,
|
||||||
|
FileClassification fallback)
|
||||||
|
{
|
||||||
|
if (indexedByName is null
|
||||||
|
|| !indexedByName.TryGetValue(item.Name, out var stored))
|
||||||
|
{
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (CategorySources.IsUser(stored.CategorySource)
|
||||||
|
|| stored.Category is not FileCategory.Unknown)
|
||||||
|
{
|
||||||
|
return new FileClassification(
|
||||||
|
stored.Category,
|
||||||
|
stored.CategoryReason ?? "Indexed category.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
private static IReadOnlyList<FileCategory> ChildCategories(
|
private static IReadOnlyList<FileCategory> ChildCategories(
|
||||||
FileSystemItem folder,
|
FileSystemItem folder,
|
||||||
IFileSystemEnumerator enumerator,
|
IFileSystemEnumerator enumerator,
|
||||||
|
|||||||
@@ -57,4 +57,53 @@ public sealed class LocalIndexMutations : IIndexMutations
|
|||||||
=> _store.Hashes.EnqueueSizeCollisionsAsync(sourceId, cancellationToken);
|
=> _store.Hashes.EnqueueSizeCollisionsAsync(sourceId, cancellationToken);
|
||||||
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
|
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
|
||||||
=> _store.Relations.UpsertAsync(relation, cancellationToken);
|
=> _store.Relations.UpsertAsync(relation, cancellationToken);
|
||||||
|
|
||||||
|
public async Task SetUserCategoryByPathAsync(
|
||||||
|
string fullPath,
|
||||||
|
FileCategory category,
|
||||||
|
string? reason = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(fullPath))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
Source? best = null;
|
||||||
|
var bestLen = -1;
|
||||||
|
var path = PathRules.FromExtended(fullPath).TrimEnd('\\');
|
||||||
|
foreach (var source in sources)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(source.LastRootPath))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\');
|
||||||
|
if (path.Equals(root, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
if (root.Length > bestLen)
|
||||||
|
{
|
||||||
|
best = source;
|
||||||
|
bestLen = root.Length;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (best?.LastRootPath is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var rel = PathRules.MakeRelative(best.LastRootPath, fullPath) ?? "";
|
||||||
|
var entry = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false);
|
||||||
|
if (entry is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await _store.Entries.SetUserCategoryAsync(entry.Id, category, reason, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,4 +83,6 @@ public interface IIndexMutations
|
|||||||
Task MarkRenameBatchUndoneAsync(long id, CancellationToken cancellationToken = default);
|
Task MarkRenameBatchUndoneAsync(long id, CancellationToken cancellationToken = default);
|
||||||
Task EnqueueHashCollisionsAsync(long? sourceId, CancellationToken cancellationToken = default);
|
Task EnqueueHashCollisionsAsync(long? sourceId, CancellationToken cancellationToken = default);
|
||||||
Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default);
|
Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default);
|
||||||
|
Task SetUserCategoryByPathAsync(string fullPath, FileCategory category, string? reason = null, CancellationToken cancellationToken = default)
|
||||||
|
=> Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,11 @@ public interface IEntryStore
|
|||||||
Task DeleteExpiredTombstonesAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken = default);
|
Task DeleteExpiredTombstonesAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken = default);
|
||||||
Task RenameSubtreePathAsync(long sourceId, string oldPathRel, string newPathRel, CancellationToken cancellationToken = default);
|
Task RenameSubtreePathAsync(long sourceId, string oldPathRel, string newPathRel, CancellationToken cancellationToken = default);
|
||||||
Task<long> CountPresentAsync(long sourceId, CancellationToken cancellationToken = default);
|
Task<long> CountPresentAsync(long sourceId, CancellationToken cancellationToken = default);
|
||||||
|
Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default);
|
||||||
|
Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default);
|
||||||
|
Task<int> BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default);
|
||||||
|
Task<IReadOnlyList<long>> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default);
|
||||||
|
Task<IReadOnlyList<IndexEntry>> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default);
|
||||||
}
|
}
|
||||||
|
|
||||||
public interface IExcludeStore
|
public interface IExcludeStore
|
||||||
@@ -109,6 +114,7 @@ public sealed class SearchRequest
|
|||||||
public bool DirectChildrenOnly { get; init; }
|
public bool DirectChildrenOnly { get; init; }
|
||||||
public bool IncludeOffline { get; init; } = true;
|
public bool IncludeOffline { get; init; } = true;
|
||||||
public bool IncludeDeleted { get; init; }
|
public bool IncludeDeleted { get; init; }
|
||||||
|
public FileCategory? Category { get; init; }
|
||||||
public int Skip { get; init; }
|
public int Skip { get; init; }
|
||||||
public int Take { get; init; } = 500;
|
public int Take { get; init; } = 500;
|
||||||
}
|
}
|
||||||
@@ -121,6 +127,7 @@ public interface IAnalysisStore
|
|||||||
Task<IReadOnlyList<IndexEntry>> LargestDirectoriesAsync(long? sourceId, long? parentId, int take, CancellationToken cancellationToken = default);
|
Task<IReadOnlyList<IndexEntry>> LargestDirectoriesAsync(long? sourceId, long? parentId, int take, CancellationToken cancellationToken = default);
|
||||||
Task<IReadOnlyList<IndexEntry>> LargestFilesAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
|
Task<IReadOnlyList<IndexEntry>> LargestFilesAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
|
||||||
Task<IReadOnlyList<ExtensionUsage>> UsageByExtensionAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
|
Task<IReadOnlyList<ExtensionUsage>> UsageByExtensionAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
|
||||||
|
Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default);
|
||||||
Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default);
|
Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default);
|
||||||
Task<IReadOnlyList<IndexEntry>> ChildrenBySizeAsync(long parentId, int take, CancellationToken cancellationToken = default);
|
Task<IReadOnlyList<IndexEntry>> ChildrenBySizeAsync(long parentId, int take, CancellationToken cancellationToken = default);
|
||||||
}
|
}
|
||||||
@@ -132,6 +139,13 @@ public sealed class ExtensionUsage
|
|||||||
public long FileCount { get; init; }
|
public long FileCount { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class CategoryUsage
|
||||||
|
{
|
||||||
|
public required string Category { get; init; }
|
||||||
|
public long TotalSize { get; init; }
|
||||||
|
public long FileCount { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class SourceUsage
|
public sealed class SourceUsage
|
||||||
{
|
{
|
||||||
public long SourceId { get; init; }
|
public long SourceId { get; init; }
|
||||||
|
|||||||
@@ -68,6 +68,11 @@ public sealed class IndexEntry
|
|||||||
public long ScanGeneration { get; set; }
|
public long ScanGeneration { get; set; }
|
||||||
public long? AllocatedSizeBytes { get; set; }
|
public long? AllocatedSizeBytes { get; set; }
|
||||||
public CloudAvailability? CloudAvailability { get; set; }
|
public CloudAvailability? CloudAvailability { get; set; }
|
||||||
|
public FileCategory Category { get; set; } = FileCategory.Unknown;
|
||||||
|
public string? CategoryReason { get; set; }
|
||||||
|
public int CategoryConfidence { get; set; }
|
||||||
|
public string CategorySource { get; set; } = CategorySources.None;
|
||||||
|
public DateTimeOffset? CategoryUtc { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class ExcludeRule
|
public sealed class ExcludeRule
|
||||||
@@ -150,6 +155,10 @@ public sealed class FileSystemItem
|
|||||||
public bool AvailableToImport { get; init; }
|
public bool AvailableToImport { get; init; }
|
||||||
public ItemHydrationFlags Hydration { get; init; } = ItemHydrationFlags.All;
|
public ItemHydrationFlags Hydration { get; init; } = ItemHydrationFlags.All;
|
||||||
public int IndexedChildCount { get; init; }
|
public int IndexedChildCount { get; init; }
|
||||||
|
public long? IndexEntryId { get; init; }
|
||||||
|
public FileCategory Category { get; init; }
|
||||||
|
public string? CategoryReason { get; init; }
|
||||||
|
public string? CategorySource { get; init; }
|
||||||
public bool IsReparsePoint => (Attributes & AttributeFlags.ReparsePoint) != 0;
|
public bool IsReparsePoint => (Attributes & AttributeFlags.ReparsePoint) != 0;
|
||||||
|
|
||||||
public FileSystemItem Overlay(
|
public FileSystemItem Overlay(
|
||||||
@@ -168,7 +177,11 @@ public sealed class FileSystemItem
|
|||||||
long? capacityBytes = null,
|
long? capacityBytes = null,
|
||||||
bool? availableToImport = null,
|
bool? availableToImport = null,
|
||||||
ItemHydrationFlags? hydration = null,
|
ItemHydrationFlags? hydration = null,
|
||||||
int? indexedChildCount = null)
|
int? indexedChildCount = null,
|
||||||
|
long? indexEntryId = null,
|
||||||
|
FileCategory? category = null,
|
||||||
|
string? categoryReason = null,
|
||||||
|
string? categorySource = null)
|
||||||
=> new()
|
=> new()
|
||||||
{
|
{
|
||||||
FullPath = FullPath,
|
FullPath = FullPath,
|
||||||
@@ -189,7 +202,11 @@ public sealed class FileSystemItem
|
|||||||
CapacityBytes = capacityBytes ?? CapacityBytes,
|
CapacityBytes = capacityBytes ?? CapacityBytes,
|
||||||
AvailableToImport = availableToImport ?? AvailableToImport,
|
AvailableToImport = availableToImport ?? AvailableToImport,
|
||||||
Hydration = hydration ?? Hydration,
|
Hydration = hydration ?? Hydration,
|
||||||
IndexedChildCount = indexedChildCount ?? IndexedChildCount
|
IndexedChildCount = indexedChildCount ?? IndexedChildCount,
|
||||||
|
IndexEntryId = indexEntryId ?? IndexEntryId,
|
||||||
|
Category = category ?? Category,
|
||||||
|
CategoryReason = categoryReason ?? CategoryReason,
|
||||||
|
CategorySource = categorySource ?? CategorySource
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
129
src/Explorer.Domain/EntryCategoryAssigner.cs
Normal file
129
src/Explorer.Domain/EntryCategoryAssigner.cs
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
namespace Explorer.Domain;
|
||||||
|
|
||||||
|
/// <summary>Where an <see cref="IndexEntry"/> category came from.</summary>
|
||||||
|
public static class CategorySources
|
||||||
|
{
|
||||||
|
public const string None = "none";
|
||||||
|
public const string Extension = "extension";
|
||||||
|
public const string Folder = "folder";
|
||||||
|
public const string Children = "children";
|
||||||
|
public const string ArchiveContents = "archive_contents";
|
||||||
|
public const string Mime = "mime";
|
||||||
|
public const string User = "user";
|
||||||
|
|
||||||
|
public static bool IsUser(string? source)
|
||||||
|
=> string.Equals(source, User, StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
public static bool IsAutomatic(string? source)
|
||||||
|
=> !IsUser(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class EntryCategoryAssigner
|
||||||
|
{
|
||||||
|
public static void ApplyAutomatic(
|
||||||
|
IndexEntry entry,
|
||||||
|
string? rootPath = null,
|
||||||
|
bool isRepoRoot = false,
|
||||||
|
IReadOnlyList<FileCategory>? childCategories = null)
|
||||||
|
{
|
||||||
|
if (CategorySources.IsUser(entry.CategorySource))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var full = string.IsNullOrWhiteSpace(rootPath)
|
||||||
|
? (string.IsNullOrEmpty(entry.PathRel) ? entry.Name : entry.PathRel)
|
||||||
|
: PathRules.Combine(rootPath, entry.PathRel);
|
||||||
|
var result = FileClassifier.Classify(
|
||||||
|
entry.Name,
|
||||||
|
full,
|
||||||
|
entry.IsDirectory,
|
||||||
|
entry.Attributes,
|
||||||
|
isRepoRoot,
|
||||||
|
childCategories);
|
||||||
|
Apply(entry, result, InferSource(entry, result, childCategories), Confidence(result, childCategories));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void ApplyUser(IndexEntry entry, FileCategory category, string? reason = null)
|
||||||
|
=> Apply(
|
||||||
|
entry,
|
||||||
|
new FileClassification(category, reason ?? "Set by user."),
|
||||||
|
CategorySources.User,
|
||||||
|
100);
|
||||||
|
|
||||||
|
public static void Apply(
|
||||||
|
IndexEntry entry,
|
||||||
|
FileClassification classification,
|
||||||
|
string source,
|
||||||
|
int confidence)
|
||||||
|
{
|
||||||
|
entry.Category = classification.Category;
|
||||||
|
entry.CategoryReason = classification.Reason;
|
||||||
|
entry.CategorySource = source;
|
||||||
|
entry.CategoryConfidence = Math.Clamp(confidence, 0, 100);
|
||||||
|
entry.CategoryUtc = DateTimeOffset.UtcNow;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static FileCategory ParseCategory(string? value)
|
||||||
|
=> Enum.TryParse<FileCategory>(value, ignoreCase: true, out var cat) ? cat : FileCategory.Unknown;
|
||||||
|
|
||||||
|
public static string InferSource(
|
||||||
|
IndexEntry entry,
|
||||||
|
FileClassification classification,
|
||||||
|
IReadOnlyList<FileCategory>? childCategories)
|
||||||
|
{
|
||||||
|
if (childCategories is { Count: > 0 } && classification.Category != FileCategory.Unknown)
|
||||||
|
{
|
||||||
|
return entry.IsDirectory
|
||||||
|
? CategorySources.Children
|
||||||
|
: CategorySources.ArchiveContents;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.IsDirectory)
|
||||||
|
{
|
||||||
|
return classification.Category == FileCategory.Unknown
|
||||||
|
? CategorySources.None
|
||||||
|
: CategorySources.Folder;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ArchiveFormats.IsArchive(entry.Name))
|
||||||
|
{
|
||||||
|
return CategorySources.Extension;
|
||||||
|
}
|
||||||
|
|
||||||
|
return classification.Category == FileCategory.Unknown
|
||||||
|
? CategorySources.None
|
||||||
|
: CategorySources.Extension;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int Confidence(FileClassification classification, IReadOnlyList<FileCategory>? childCategories)
|
||||||
|
{
|
||||||
|
if (classification.Category == FileCategory.Unknown)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (childCategories is { Count: > 0 })
|
||||||
|
{
|
||||||
|
var known = childCategories.Count(c => c is not FileCategory.Unknown);
|
||||||
|
if (known == 0)
|
||||||
|
{
|
||||||
|
return 40;
|
||||||
|
}
|
||||||
|
|
||||||
|
var majority = childCategories
|
||||||
|
.Where(c => c == classification.Category)
|
||||||
|
.Count();
|
||||||
|
return Math.Clamp(40 + majority * 60 / known, 40, 95);
|
||||||
|
}
|
||||||
|
|
||||||
|
return classification.Category switch
|
||||||
|
{
|
||||||
|
FileCategory.SystemData or FileCategory.BuildOutput => 95,
|
||||||
|
FileCategory.Archive or FileCategory.Photos or FileCategory.Video
|
||||||
|
or FileCategory.Audio or FileCategory.Documents => 85,
|
||||||
|
FileCategory.Installer or FileCategory.Backup or FileCategory.CodeRepository => 80,
|
||||||
|
_ => 50
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -87,6 +87,13 @@ public static class FileClassifier
|
|||||||
{
|
{
|
||||||
if (ArchiveFormats.IsArchive(name))
|
if (ArchiveFormats.IsArchive(name))
|
||||||
{
|
{
|
||||||
|
if (TryMajority(childCategories, out var archiveMajority))
|
||||||
|
{
|
||||||
|
return new FileClassification(
|
||||||
|
archiveMajority,
|
||||||
|
"Archive contents are mostly " + OrganizeDestinations.Label(archiveMajority).ToLowerInvariant() + ".");
|
||||||
|
}
|
||||||
|
|
||||||
return new FileClassification(FileCategory.Archive, "Archive file.");
|
return new FileClassification(FileCategory.Archive, "Archive file.");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,24 +154,62 @@ public static class FileClassifier
|
|||||||
return new FileClassification(FileCategory.Backup, "Backup folder name.");
|
return new FileClassification(FileCategory.Backup, "Backup folder name.");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (childCategories is { Count: > 0 })
|
if (TryMajority(childCategories, out var folderMajority))
|
||||||
{
|
{
|
||||||
var known = childCategories.Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData).ToList();
|
return new FileClassification(
|
||||||
if (known.Count > 0)
|
folderMajority,
|
||||||
{
|
"Folder contents are mostly " + OrganizeDestinations.Label(folderMajority).ToLowerInvariant() + ".");
|
||||||
var majority = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First();
|
|
||||||
if (majority.Count() * 10 >= known.Count * 7)
|
|
||||||
{
|
|
||||||
return new FileClassification(majority.Key, "Folder contents are mostly " + OrganizeDestinations.Label(majority.Key).ToLowerInvariant() + ".");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return new FileClassification(FileCategory.Unknown, "No matching signal.");
|
return new FileClassification(FileCategory.Unknown, "No matching signal.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool TryMajority(IReadOnlyList<FileCategory>? childCategories, out FileCategory majority)
|
||||||
|
{
|
||||||
|
majority = FileCategory.Unknown;
|
||||||
|
if (childCategories is not { Count: > 0 })
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var known = childCategories
|
||||||
|
.Where(c => c is not FileCategory.Unknown and not FileCategory.BuildOutput and not FileCategory.SystemData)
|
||||||
|
.ToList();
|
||||||
|
if (known.Count == 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var top = known.GroupBy(c => c).OrderByDescending(g => g.Count()).First();
|
||||||
|
if (top.Count() * 10 < known.Count * 7)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
majority = top.Key;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
public static bool ShouldLeave(FileCategory category)
|
public static bool ShouldLeave(FileCategory category)
|
||||||
=> category is FileCategory.Unknown or FileCategory.SystemData or FileCategory.BuildOutput or FileCategory.Backup;
|
=> category is FileCategory.Unknown or FileCategory.SystemData or FileCategory.BuildOutput or FileCategory.Backup;
|
||||||
|
|
||||||
public static bool SkipDescent(string name) => BuildNames.Contains(name);
|
public static bool SkipDescent(string name) => BuildNames.Contains(name);
|
||||||
|
|
||||||
|
public static bool IsPhotoFile(string name)
|
||||||
|
{
|
||||||
|
var ext = NameNormalizer.Extension(name);
|
||||||
|
return ext is not null && Photos.Contains(ext);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool IsVideoFile(string name)
|
||||||
|
{
|
||||||
|
var ext = NameNormalizer.Extension(name);
|
||||||
|
return ext is not null && Videos.Contains(ext);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool IsAudioFile(string name)
|
||||||
|
{
|
||||||
|
var ext = NameNormalizer.Extension(name);
|
||||||
|
return ext is not null && Audio.Contains(ext);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
84
src/Explorer.Domain/FileSignatureClassifier.cs
Normal file
84
src/Explorer.Domain/FileSignatureClassifier.cs
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
namespace Explorer.Domain;
|
||||||
|
|
||||||
|
/// <summary>Light magic-byte hints for files with no useful extension category.</summary>
|
||||||
|
public static class FileSignatureClassifier
|
||||||
|
{
|
||||||
|
public static FileClassification? TryClassify(ReadOnlySpan<byte> header)
|
||||||
|
{
|
||||||
|
if (header.Length < 4)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Photos, "JPEG signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header.Length >= 8
|
||||||
|
&& header[0] == 0x89 && header[1] == 0x50 && header[2] == 0x4E && header[3] == 0x47)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Photos, "PNG signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x47 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x38)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Photos, "GIF signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x25 && header[1] == 0x50 && header[2] == 0x44 && header[3] == 0x46)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Documents, "PDF signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x50 && header[1] == 0x4B && (header[2] == 0x03 || header[2] == 0x05 || header[2] == 0x07))
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Archive, "ZIP-family signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header.Length >= 12
|
||||||
|
&& header[0] == 0x52 && header[1] == 0x49 && header[2] == 0x46 && header[3] == 0x46)
|
||||||
|
{
|
||||||
|
var form = System.Text.Encoding.ASCII.GetString(header.Slice(8, 4));
|
||||||
|
if (form is "WAVE" or "AVI ")
|
||||||
|
{
|
||||||
|
return new FileClassification(
|
||||||
|
form == "WAVE" ? FileCategory.Audio : FileCategory.Video,
|
||||||
|
"RIFF " + form.Trim() + " signature.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header.Length >= 12
|
||||||
|
&& header[4] == 0x66 && header[5] == 0x74 && header[6] == 0x79 && header[7] == 0x70)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Video, "ISO BMFF (ftyp) signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x49 && header[1] == 0x44 && header[2] == 0x33)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Audio, "ID3 signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x7F && header[1] == 0x45 && header[2] == 0x4C && header[3] == 0x46)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Installer, "ELF binary signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x4D && header[1] == 0x5A)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Installer, "PE/MZ signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x37 && header[1] == 0x7A && header[2] == 0xBC && header[3] == 0xAF)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Archive, "7z signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header[0] == 0x52 && header[1] == 0x61 && header[2] == 0x72 && header[3] == 0x21)
|
||||||
|
{
|
||||||
|
return new FileClassification(FileCategory.Archive, "RAR signature.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -48,6 +48,51 @@ public sealed class OrganizeDestinations
|
|||||||
_ => "Unknown"
|
_ => "Unknown"
|
||||||
};
|
};
|
||||||
|
|
||||||
|
public static bool TryParse(string? value, out FileCategory category)
|
||||||
|
{
|
||||||
|
category = FileCategory.Unknown;
|
||||||
|
if (string.IsNullOrWhiteSpace(value))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var key = value.Trim().Replace(" ", "", StringComparison.Ordinal).Replace("_", "", StringComparison.Ordinal);
|
||||||
|
if (Enum.TryParse(key, ignoreCase: true, out category) && category != FileCategory.Unknown)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
category = key.ToLowerInvariant() switch
|
||||||
|
{
|
||||||
|
"photo" or "photos" or "image" or "images" or "picture" or "pictures" => FileCategory.Photos,
|
||||||
|
"video" or "videos" or "movie" or "movies" => FileCategory.Video,
|
||||||
|
"audio" or "music" or "sound" => FileCategory.Audio,
|
||||||
|
"doc" or "docs" or "document" or "documents" => FileCategory.Documents,
|
||||||
|
"code" or "repo" or "repository" or "git" or "development" => FileCategory.CodeRepository,
|
||||||
|
"installer" or "installers" or "setup" => FileCategory.Installer,
|
||||||
|
"backup" or "backups" => FileCategory.Backup,
|
||||||
|
"archive" or "archives" or "zip" => FileCategory.Archive,
|
||||||
|
"system" or "systemdata" => FileCategory.SystemData,
|
||||||
|
"build" or "buildoutput" => FileCategory.BuildOutput,
|
||||||
|
"unknown" => FileCategory.Unknown,
|
||||||
|
_ => (FileCategory)(-1)
|
||||||
|
};
|
||||||
|
return (int)category >= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static readonly FileCategory[] UserChoices =
|
||||||
|
[
|
||||||
|
FileCategory.Photos,
|
||||||
|
FileCategory.Video,
|
||||||
|
FileCategory.Audio,
|
||||||
|
FileCategory.Documents,
|
||||||
|
FileCategory.Installer,
|
||||||
|
FileCategory.Archive,
|
||||||
|
FileCategory.Backup,
|
||||||
|
FileCategory.CodeRepository,
|
||||||
|
FileCategory.Unknown
|
||||||
|
];
|
||||||
|
|
||||||
private static string? EmptyToNull(string value)
|
private static string? EmptyToNull(string value)
|
||||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ public sealed class ReorganizeService
|
|||||||
private readonly IHydrationGuard _hydration;
|
private readonly IHydrationGuard _hydration;
|
||||||
private readonly IGitStatusProvider _git;
|
private readonly IGitStatusProvider _git;
|
||||||
private readonly UiPreferencesStore _preferences;
|
private readonly UiPreferencesStore _preferences;
|
||||||
|
private readonly IIndexStore _store;
|
||||||
|
|
||||||
public ReorganizeService(
|
public ReorganizeService(
|
||||||
ReorganizePlanner planner,
|
ReorganizePlanner planner,
|
||||||
@@ -21,7 +22,8 @@ public sealed class ReorganizeService
|
|||||||
IFileSystemEnumerator enumerator,
|
IFileSystemEnumerator enumerator,
|
||||||
IHydrationGuard hydration,
|
IHydrationGuard hydration,
|
||||||
IGitStatusProvider git,
|
IGitStatusProvider git,
|
||||||
UiPreferencesStore preferences)
|
UiPreferencesStore preferences,
|
||||||
|
IIndexStore store)
|
||||||
{
|
{
|
||||||
_planner = planner;
|
_planner = planner;
|
||||||
_ops = ops;
|
_ops = ops;
|
||||||
@@ -30,6 +32,7 @@ public sealed class ReorganizeService
|
|||||||
_hydration = hydration;
|
_hydration = hydration;
|
||||||
_git = git;
|
_git = git;
|
||||||
_preferences = preferences;
|
_preferences = preferences;
|
||||||
|
_store = store;
|
||||||
}
|
}
|
||||||
|
|
||||||
public OrganizeDestinations LoadDestinations()
|
public OrganizeDestinations LoadDestinations()
|
||||||
@@ -63,14 +66,24 @@ public sealed class ReorganizeService
|
|||||||
}
|
}
|
||||||
|
|
||||||
public OperationPlan Preview(string sourceRoot, OrganizeDestinations destinations)
|
public OperationPlan Preview(string sourceRoot, OrganizeDestinations destinations)
|
||||||
=> _planner.Build(
|
=> PreviewAsync(sourceRoot, destinations).GetAwaiter().GetResult();
|
||||||
|
|
||||||
|
public async Task<OperationPlan> PreviewAsync(
|
||||||
|
string sourceRoot,
|
||||||
|
OrganizeDestinations destinations,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var indexed = await LoadIndexedChildrenAsync(sourceRoot, cancellationToken).ConfigureAwait(false);
|
||||||
|
return _planner.Build(
|
||||||
sourceRoot,
|
sourceRoot,
|
||||||
destinations,
|
destinations,
|
||||||
_enumerator,
|
_enumerator,
|
||||||
path => _volumes.IsPathReachable(path),
|
path => _volumes.IsPathReachable(path),
|
||||||
path => _git.IsRepoRoot(path),
|
path => _git.IsRepoRoot(path),
|
||||||
item => _hydration.WouldHydrateOnRead(item),
|
item => _hydration.WouldHydrateOnRead(item),
|
||||||
RenameBatchService.PathExists);
|
RenameBatchService.PathExists,
|
||||||
|
indexedByName: indexed);
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<OperationPlan> EnqueueAsync(
|
public async Task<OperationPlan> EnqueueAsync(
|
||||||
string sourceRoot,
|
string sourceRoot,
|
||||||
@@ -78,7 +91,7 @@ public sealed class ReorganizeService
|
|||||||
OperationPlan? plan = null,
|
OperationPlan? plan = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
plan ??= Preview(sourceRoot, destinations);
|
plan ??= await PreviewAsync(sourceRoot, destinations, cancellationToken).ConfigureAwait(false);
|
||||||
if (!plan.CanEnqueue)
|
if (!plan.CanEnqueue)
|
||||||
{
|
{
|
||||||
return plan;
|
return plan;
|
||||||
@@ -97,6 +110,62 @@ public sealed class ReorganizeService
|
|||||||
return plan;
|
return plan;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyDictionary<string, IndexEntry>> LoadIndexedChildrenAsync(
|
||||||
|
string sourceRoot,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(sourceRoot))
|
||||||
|
{
|
||||||
|
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
Source? best = null;
|
||||||
|
var bestLen = -1;
|
||||||
|
foreach (var source in sources)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(source.LastRootPath))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\');
|
||||||
|
var path = PathRules.FromExtended(sourceRoot).TrimEnd('\\');
|
||||||
|
if (path.Equals(root, StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| path.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
if (root.Length > bestLen)
|
||||||
|
{
|
||||||
|
best = source;
|
||||||
|
bestLen = root.Length;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (best?.LastRootPath is null)
|
||||||
|
{
|
||||||
|
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
var rel = PathRules.MakeRelative(best.LastRootPath, sourceRoot) ?? "";
|
||||||
|
var dir = await _store.Entries.GetByPathAsync(best.Id, rel, cancellationToken).ConfigureAwait(false);
|
||||||
|
if (dir is null)
|
||||||
|
{
|
||||||
|
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
var children = await _store.Entries.GetChildrenAsync(best.Id, dir.Id, EntryStatus.Present, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
return children.ToDictionary(c => c.Name, StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static string Known(Environment.SpecialFolder folder)
|
private static string Known(Environment.SpecialFolder folder)
|
||||||
=> Environment.GetFolderPath(folder);
|
=> Environment.GetFolderPath(folder);
|
||||||
|
|
||||||
|
|||||||
@@ -579,6 +579,17 @@ public sealed class WorkbenchPipeClient : IWorkbenchHost, ICloudOverlay, IHostCo
|
|||||||
=> _client.CallAsync("Mutations.EnqueueHashCollisions", cancellationToken, n: sourceId ?? 0);
|
=> _client.CallAsync("Mutations.EnqueueHashCollisions", cancellationToken, n: sourceId ?? 0);
|
||||||
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
|
public Task UpsertRelationAsync(FileRelation relation, CancellationToken cancellationToken = default)
|
||||||
=> _client.CallAsync("Mutations.UpsertRelation", cancellationToken, payload: Json(relation));
|
=> _client.CallAsync("Mutations.UpsertRelation", cancellationToken, payload: Json(relation));
|
||||||
|
public Task SetUserCategoryByPathAsync(
|
||||||
|
string fullPath,
|
||||||
|
FileCategory category,
|
||||||
|
string? reason = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
=> _client.CallAsync(
|
||||||
|
"Mutations.SetUserCategory",
|
||||||
|
cancellationToken,
|
||||||
|
s: fullPath,
|
||||||
|
dest: reason,
|
||||||
|
payload: category.ToString());
|
||||||
private static string Json<T>(T value) => JsonSerializer.Serialize(value, WorkbenchIpc.Json);
|
private static string Json<T>(T value) => JsonSerializer.Serialize(value, WorkbenchIpc.Json);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
private readonly IForegroundWorkSignal _foreground;
|
private readonly IForegroundWorkSignal _foreground;
|
||||||
private readonly IIdleIndexWork _indexing;
|
private readonly IIdleIndexWork _indexing;
|
||||||
private readonly IIdleHashWork _hash;
|
private readonly IIdleHashWork _hash;
|
||||||
|
private readonly IIdleClassifyWork _classify;
|
||||||
private readonly IHistoryMaintenance _history;
|
private readonly IHistoryMaintenance _history;
|
||||||
private readonly IIndexStore _store;
|
private readonly IIndexStore _store;
|
||||||
private readonly IVolumeService _volumes;
|
private readonly IVolumeService _volumes;
|
||||||
@@ -44,7 +45,8 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
IIndexStore store,
|
IIndexStore store,
|
||||||
IVolumeService volumes,
|
IVolumeService volumes,
|
||||||
ILogger<BackgroundMaintenanceCoordinator> logger,
|
ILogger<BackgroundMaintenanceCoordinator> logger,
|
||||||
IHostActivitySink? activity = null)
|
IHostActivitySink? activity = null,
|
||||||
|
IIdleClassifyWork? classify = null)
|
||||||
{
|
{
|
||||||
_idle = idle;
|
_idle = idle;
|
||||||
_power = power;
|
_power = power;
|
||||||
@@ -52,6 +54,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
_foreground = foreground;
|
_foreground = foreground;
|
||||||
_indexing = indexing;
|
_indexing = indexing;
|
||||||
_hash = hash;
|
_hash = hash;
|
||||||
|
_classify = classify ?? NullIdleClassifyWork.Instance;
|
||||||
_history = history;
|
_history = history;
|
||||||
_store = store;
|
_store = store;
|
||||||
_volumes = volumes;
|
_volumes = volumes;
|
||||||
@@ -71,6 +74,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
_hash.Pause();
|
_hash.Pause();
|
||||||
|
_classify.Pause();
|
||||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1));
|
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1));
|
||||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||||
{
|
{
|
||||||
@@ -115,10 +119,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
{
|
{
|
||||||
_idleScanQueued.Clear();
|
_idleScanQueued.Clear();
|
||||||
_idleVerified.Clear();
|
_idleVerified.Clear();
|
||||||
if (!_hash.IsPaused)
|
PauseBackgroundWorkers();
|
||||||
{
|
|
||||||
_hash.Pause();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_lastAllowed)
|
if (_lastAllowed)
|
||||||
{
|
{
|
||||||
@@ -147,10 +148,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
|
|
||||||
if (_indexing.IsBusy || _indexing.HasIdleWork)
|
if (_indexing.IsBusy || _indexing.HasIdleWork)
|
||||||
{
|
{
|
||||||
if (!_hash.IsPaused)
|
PauseBackgroundWorkers();
|
||||||
{
|
|
||||||
_hash.Pause();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep the idle-maintenance caption only while idle work is still queued/running.
|
// Keep the idle-maintenance caption only while idle work is still queued/running.
|
||||||
// Watcher/user jobs must not keep showing a stale "checking External" line.
|
// Watcher/user jobs must not keep showing a stale "checking External" line.
|
||||||
@@ -165,10 +163,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
var verify = BackgroundMaintenancePlanner.NextLocalVerify(sources, _idleVerified);
|
var verify = BackgroundMaintenancePlanner.NextLocalVerify(sources, _idleVerified);
|
||||||
if (verify is not null && verify.LastRootPath is not null && _volumes.IsPathReachable(verify.LastRootPath))
|
if (verify is not null && verify.LastRootPath is not null && _volumes.IsPathReachable(verify.LastRootPath))
|
||||||
{
|
{
|
||||||
if (!_hash.IsPaused)
|
PauseBackgroundWorkers();
|
||||||
{
|
|
||||||
_hash.Pause();
|
|
||||||
}
|
|
||||||
|
|
||||||
_idleVerified.Add(verify.Id);
|
_idleVerified.Add(verify.Id);
|
||||||
_indexing.EnqueueIdleVerify(verify.Id, "");
|
_indexing.EnqueueIdleVerify(verify.Id, "");
|
||||||
@@ -183,10 +178,7 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
var scan = BackgroundMaintenancePlanner.NextLocalScan(sources, DateTimeOffset.UtcNow, _idleScanQueued);
|
var scan = BackgroundMaintenancePlanner.NextLocalScan(sources, DateTimeOffset.UtcNow, _idleScanQueued);
|
||||||
if (scan is not null && scan.LastRootPath is not null && _volumes.IsPathReachable(scan.LastRootPath))
|
if (scan is not null && scan.LastRootPath is not null && _volumes.IsPathReachable(scan.LastRootPath))
|
||||||
{
|
{
|
||||||
if (!_hash.IsPaused)
|
PauseBackgroundWorkers();
|
||||||
{
|
|
||||||
_hash.Pause();
|
|
||||||
}
|
|
||||||
|
|
||||||
_idleScanQueued.Add(scan.Id);
|
_idleScanQueued.Add(scan.Id);
|
||||||
_indexing.EnqueueIdleFullScan(scan.Id);
|
_indexing.EnqueueIdleFullScan(scan.Id);
|
||||||
@@ -196,6 +188,16 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
PauseHash();
|
||||||
|
_classify.Resume();
|
||||||
|
if (await _classify.ProcessPendingAsync(cancellationToken).ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
_workMessage = "Idle maintenance · classifying";
|
||||||
|
Publish(decision, _workMessage);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_classify.Pause();
|
||||||
_hash.Resume();
|
_hash.Resume();
|
||||||
if (hashPending)
|
if (hashPending)
|
||||||
{
|
{
|
||||||
@@ -232,6 +234,23 @@ public sealed class BackgroundMaintenanceCoordinator : BackgroundService, IBackg
|
|||||||
Publish(decision, _workMessage);
|
Publish(decision, _workMessage);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void PauseBackgroundWorkers()
|
||||||
|
{
|
||||||
|
PauseHash();
|
||||||
|
if (!_classify.IsPaused)
|
||||||
|
{
|
||||||
|
_classify.Pause();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PauseHash()
|
||||||
|
{
|
||||||
|
if (!_hash.IsPaused)
|
||||||
|
{
|
||||||
|
_hash.Pause();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private string PauseMessage(BackgroundWorkDecision decision)
|
private string PauseMessage(BackgroundWorkDecision decision)
|
||||||
{
|
{
|
||||||
if (decision.SkipReason == MaintenanceSkipReason.ForegroundOperations)
|
if (decision.SkipReason == MaintenanceSkipReason.ForegroundOperations)
|
||||||
|
|||||||
@@ -100,8 +100,10 @@ public static class ExplorerHostServices
|
|||||||
sp.GetRequiredService<IIndexMutations>()));
|
sp.GetRequiredService<IIndexMutations>()));
|
||||||
services.AddSingleton<DuplicateHashWorker>();
|
services.AddSingleton<DuplicateHashWorker>();
|
||||||
services.AddSingleton<HistoryRollupService>();
|
services.AddSingleton<HistoryRollupService>();
|
||||||
|
services.AddSingleton<EntryClassificationService>();
|
||||||
services.AddSingleton<IIdleIndexWork>(sp => sp.GetRequiredService<IndexingCoordinator>());
|
services.AddSingleton<IIdleIndexWork>(sp => sp.GetRequiredService<IndexingCoordinator>());
|
||||||
services.AddSingleton<IIdleHashWork>(sp => sp.GetRequiredService<DuplicateHashWorker>());
|
services.AddSingleton<IIdleHashWork>(sp => sp.GetRequiredService<DuplicateHashWorker>());
|
||||||
|
services.AddSingleton<IIdleClassifyWork>(sp => sp.GetRequiredService<EntryClassificationService>());
|
||||||
services.AddSingleton<IHistoryMaintenance>(sp => sp.GetRequiredService<HistoryRollupService>());
|
services.AddSingleton<IHistoryMaintenance>(sp => sp.GetRequiredService<HistoryRollupService>());
|
||||||
services.AddSingleton<BackgroundMaintenanceCoordinator>();
|
services.AddSingleton<BackgroundMaintenanceCoordinator>();
|
||||||
services.AddSingleton<IBackgroundMaintenance>(sp => sp.GetRequiredService<BackgroundMaintenanceCoordinator>());
|
services.AddSingleton<IBackgroundMaintenance>(sp => sp.GetRequiredService<BackgroundMaintenanceCoordinator>());
|
||||||
|
|||||||
@@ -462,6 +462,16 @@ public sealed class WorkbenchPipeServer : BackgroundService
|
|||||||
case "Mutations.UpsertRelation":
|
case "Mutations.UpsertRelation":
|
||||||
await Workbench.Mutations.UpsertRelationAsync(Read<FileRelation>(request.Payload)).ConfigureAwait(false);
|
await Workbench.Mutations.UpsertRelationAsync(Read<FileRelation>(request.Payload)).ConfigureAwait(false);
|
||||||
return reply;
|
return reply;
|
||||||
|
case "Mutations.SetUserCategory":
|
||||||
|
if (OrganizeDestinations.TryParse(request.Payload, out var userCategory)
|
||||||
|
|| Enum.TryParse(request.Payload, true, out userCategory))
|
||||||
|
{
|
||||||
|
await Workbench.Mutations.SetUserCategoryByPathAsync(
|
||||||
|
request.S ?? "",
|
||||||
|
userCategory,
|
||||||
|
request.Dest).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
return reply;
|
||||||
case "Cloud.Places":
|
case "Cloud.Places":
|
||||||
reply.Payload = JsonSerializer.Serialize(_overlay!.GetPlaces(), WorkbenchIpc.Json);
|
reply.Payload = JsonSerializer.Serialize(_overlay!.GetPlaces(), WorkbenchIpc.Json);
|
||||||
return reply;
|
return reply;
|
||||||
|
|||||||
@@ -190,6 +190,9 @@ public sealed class FolderReconciler
|
|||||||
}
|
}
|
||||||
}, cancellationToken).ConfigureAwait(false);
|
}, cancellationToken).ConfigureAwait(false);
|
||||||
|
|
||||||
|
await _store.Entries.RefreshCategoryFromChildrenAsync(parent.Id, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
foreach (var path in archivesToTomb)
|
foreach (var path in archivesToTomb)
|
||||||
{
|
{
|
||||||
await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken)
|
await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken)
|
||||||
@@ -202,6 +205,8 @@ public sealed class FolderReconciler
|
|||||||
{
|
{
|
||||||
await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
|
await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
|
||||||
.ConfigureAwait(false);
|
.ConfigureAwait(false);
|
||||||
|
await _store.Entries.RefreshCategoryFromChildrenAsync(archive.Id, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ public sealed partial class AnalysisViewModel : ObservableObject
|
|||||||
public const string PageFolders = "Biggest folders";
|
public const string PageFolders = "Biggest folders";
|
||||||
public const string PageFiles = "Biggest files";
|
public const string PageFiles = "Biggest files";
|
||||||
public const string PageTypes = "By file type";
|
public const string PageTypes = "By file type";
|
||||||
|
public const string PageCategories = "By category";
|
||||||
public const string PageSources = "By source";
|
public const string PageSources = "By source";
|
||||||
|
|
||||||
private readonly AnalysisService _analysis;
|
private readonly AnalysisService _analysis;
|
||||||
@@ -93,7 +94,7 @@ public sealed partial class AnalysisViewModel : ObservableObject
|
|||||||
Rows = [];
|
Rows = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageSources];
|
public string[] Pages { get; } = [PageTree, PageFolders, PageFiles, PageTypes, PageCategories, PageSources];
|
||||||
public ObservableCollection<StorageScopeItem> Scopes { get; }
|
public ObservableCollection<StorageScopeItem> Scopes { get; }
|
||||||
public ObservableCollection<StorageNodeViewModel> TreeRoots { get; }
|
public ObservableCollection<StorageNodeViewModel> TreeRoots { get; }
|
||||||
public ObservableCollection<StorageNodeViewModel> VisibleNodes { get; }
|
public ObservableCollection<StorageNodeViewModel> VisibleNodes { get; }
|
||||||
@@ -461,6 +462,25 @@ public sealed partial class AnalysisViewModel : ObservableObject
|
|||||||
0,
|
0,
|
||||||
(string?)null)).ToList();
|
(string?)null)).ToList();
|
||||||
}
|
}
|
||||||
|
else if (page == PageCategories)
|
||||||
|
{
|
||||||
|
var cats = await _analysis.UsageByCategoryAsync(sourceId, null, AppConstants.AnalysisTopN, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
items = cats.Select(t => (
|
||||||
|
OrganizeDestinations.TryParse(t.Category, out var cat)
|
||||||
|
? OrganizeDestinations.Label(cat)
|
||||||
|
: string.IsNullOrEmpty(t.Category) ? "Unknown" : t.Category,
|
||||||
|
t.TotalSize,
|
||||||
|
(string?)null,
|
||||||
|
(string?)null,
|
||||||
|
(string?)null,
|
||||||
|
(long?)null,
|
||||||
|
sourceId,
|
||||||
|
false,
|
||||||
|
(int)t.FileCount,
|
||||||
|
0,
|
||||||
|
(string?)null)).ToList();
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var usage = await _analysis.UsageBySourceAsync(cancellationToken).ConfigureAwait(false);
|
var usage = await _analysis.UsageBySourceAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
|||||||
@@ -85,6 +85,27 @@ public sealed partial class FolderItemViewModel : ObservableObject
|
|||||||
};
|
};
|
||||||
public bool HasCloudStatus => !string.IsNullOrEmpty(CloudLabel);
|
public bool HasCloudStatus => !string.IsNullOrEmpty(CloudLabel);
|
||||||
|
|
||||||
|
public string CategoryLabel
|
||||||
|
=> Item.Category == FileCategory.Unknown
|
||||||
|
&& (string.IsNullOrEmpty(Item.CategorySource) || Item.CategorySource == CategorySources.None)
|
||||||
|
? ""
|
||||||
|
: OrganizeDestinations.Label(Item.Category);
|
||||||
|
|
||||||
|
public string CategoryTooltip
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
var label = CategoryLabel;
|
||||||
|
if (string.IsNullOrEmpty(label))
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
var reason = Item.CategoryReason;
|
||||||
|
return string.IsNullOrEmpty(reason) ? label : label + " · " + reason;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
[ObservableProperty] private string _gitLabel = "";
|
[ObservableProperty] private string _gitLabel = "";
|
||||||
public bool HasGitLabel => !string.IsNullOrEmpty(GitLabel);
|
public bool HasGitLabel => !string.IsNullOrEmpty(GitLabel);
|
||||||
|
|
||||||
@@ -122,18 +143,8 @@ public sealed partial class FolderItemViewModel : ObservableObject
|
|||||||
|
|
||||||
internal static class MediaKinds
|
internal static class MediaKinds
|
||||||
{
|
{
|
||||||
private static readonly HashSet<string> Images = new(StringComparer.OrdinalIgnoreCase)
|
public static bool IsImage(string name) => FileClassifier.IsPhotoFile(name);
|
||||||
{
|
public static bool IsVideo(string name) => FileClassifier.IsVideoFile(name);
|
||||||
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tif", ".tiff", ".ico", ".jfif"
|
|
||||||
};
|
|
||||||
|
|
||||||
private static readonly HashSet<string> Videos = new(StringComparer.OrdinalIgnoreCase)
|
|
||||||
{
|
|
||||||
".mp4", ".mkv", ".avi", ".mov", ".wmv", ".webm", ".m4v", ".mpg", ".mpeg"
|
|
||||||
};
|
|
||||||
|
|
||||||
public static bool IsImage(string name) => Images.Contains(Path.GetExtension(name));
|
|
||||||
public static bool IsVideo(string name) => Videos.Contains(Path.GetExtension(name));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file static class ItemExt
|
file static class ItemExt
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ public sealed partial class MainViewModel : ObservableObject
|
|||||||
private readonly ConversionPlanner _conversionPlanner;
|
private readonly ConversionPlanner _conversionPlanner;
|
||||||
private readonly IFileSystemEnumerator _enumerator;
|
private readonly IFileSystemEnumerator _enumerator;
|
||||||
private readonly IMediaConversionProvider _conversion;
|
private readonly IMediaConversionProvider _conversion;
|
||||||
|
private readonly IIndexMutations _mutations;
|
||||||
private readonly IThumbnailService? _thumbnails;
|
private readonly IThumbnailService? _thumbnails;
|
||||||
private readonly IHostConnection? _host;
|
private readonly IHostConnection? _host;
|
||||||
private readonly IBackgroundMaintenance? _maintenance;
|
private readonly IBackgroundMaintenance? _maintenance;
|
||||||
@@ -65,6 +66,7 @@ public sealed partial class MainViewModel : ObservableObject
|
|||||||
[ObservableProperty] private bool _showTags;
|
[ObservableProperty] private bool _showTags;
|
||||||
[ObservableProperty] private bool _showRunProfile;
|
[ObservableProperty] private bool _showRunProfile;
|
||||||
[ObservableProperty] private bool _showOrganizeFolder;
|
[ObservableProperty] private bool _showOrganizeFolder;
|
||||||
|
[ObservableProperty] private bool _showClassifyAs;
|
||||||
[ObservableProperty] private bool _canUndoRenameBatch;
|
[ObservableProperty] private bool _canUndoRenameBatch;
|
||||||
[ObservableProperty] private bool _showExtractArchive;
|
[ObservableProperty] private bool _showExtractArchive;
|
||||||
[ObservableProperty] private bool _showCompress;
|
[ObservableProperty] private bool _showCompress;
|
||||||
@@ -141,6 +143,7 @@ public sealed partial class MainViewModel : ObservableObject
|
|||||||
_conversionPlanner = conversionPlanner;
|
_conversionPlanner = conversionPlanner;
|
||||||
_enumerator = enumerator;
|
_enumerator = enumerator;
|
||||||
_conversion = conversion;
|
_conversion = conversion;
|
||||||
|
_mutations = mutations;
|
||||||
_thumbnails = thumbnails;
|
_thumbnails = thumbnails;
|
||||||
_host = hostConnection;
|
_host = hostConnection;
|
||||||
_maintenance = maintenance;
|
_maintenance = maintenance;
|
||||||
@@ -737,6 +740,36 @@ public sealed partial class MainViewModel : ObservableObject
|
|||||||
public ReorganizeViewModel CreateReorganizeViewModel()
|
public ReorganizeViewModel CreateReorganizeViewModel()
|
||||||
=> new(_reorganize, OrganizeSourcePath());
|
=> new(_reorganize, OrganizeSourcePath());
|
||||||
|
|
||||||
|
[RelayCommand]
|
||||||
|
public async Task ClassifyAsAsync(string? category)
|
||||||
|
{
|
||||||
|
if (!OrganizeDestinations.TryParse(category, out var parsed)
|
||||||
|
&& !Enum.TryParse(category, true, out parsed))
|
||||||
|
{
|
||||||
|
Footer = "Unknown category.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var items = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList();
|
||||||
|
if (items.Count == 0)
|
||||||
|
{
|
||||||
|
Footer = "Select a file or folder to classify.";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var n = 0;
|
||||||
|
foreach (var item in items)
|
||||||
|
{
|
||||||
|
await _mutations.SetUserCategoryByPathAsync(item.FullPath, parsed, "Set by user.").ConfigureAwait(true);
|
||||||
|
n++;
|
||||||
|
}
|
||||||
|
|
||||||
|
await ActivePane.RefreshAsync().ConfigureAwait(true);
|
||||||
|
Footer = n == 1
|
||||||
|
? "Classified as " + OrganizeDestinations.Label(parsed) + "."
|
||||||
|
: "Classified " + n + " items as " + OrganizeDestinations.Label(parsed) + ".";
|
||||||
|
}
|
||||||
|
|
||||||
public GitChangesViewModel CreateGitChangesViewModel()
|
public GitChangesViewModel CreateGitChangesViewModel()
|
||||||
=> new(_gitCommands, _workspace, _hydration);
|
=> new(_gitCommands, _workspace, _hydration);
|
||||||
|
|
||||||
@@ -984,6 +1017,7 @@ public sealed partial class MainViewModel : ObservableObject
|
|||||||
ShowRunProfile = ShowBatchRename;
|
ShowRunProfile = ShowBatchRename;
|
||||||
ShowOrganizeFolder = OrganizeSourcePath() is not null;
|
ShowOrganizeFolder = OrganizeSourcePath() is not null;
|
||||||
var real = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList();
|
var real = ActivePane.SelectedItems.Where(IsRealFileSystemItem).ToList();
|
||||||
|
ShowClassifyAs = real.Count > 0;
|
||||||
ShowExtractArchive = real.Count > 0 && real.All(i => !i.IsDirectory && ArchiveFormats.IsArchive(i.Item.Name));
|
ShowExtractArchive = real.Count > 0 && real.All(i => !i.IsDirectory && ArchiveFormats.IsArchive(i.Item.Name));
|
||||||
ShowCompress = real.Count > 0;
|
ShowCompress = real.Count > 0;
|
||||||
ShowConvert = real.Any(i => i.IsDirectory || ConversionFormats.IsConvertible(i.Item.Name));
|
ShowConvert = real.Any(i => i.IsDirectory || ConversionFormats.IsConvertible(i.Item.Name));
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ public sealed partial class ReorganizeViewModel : ObservableObject
|
|||||||
{
|
{
|
||||||
Status = "Analyzing…";
|
Status = "Analyzing…";
|
||||||
CanQueue = false;
|
CanQueue = false;
|
||||||
var plan = await Task.Run(() => _organize.Preview(SourcePath.Trim(), CurrentDestinations())).ConfigureAwait(true);
|
var plan = await _organize.PreviewAsync(SourcePath.Trim(), CurrentDestinations()).ConfigureAwait(true);
|
||||||
ApplyPlan(plan);
|
ApplyPlan(plan);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ using Explorer.Search;
|
|||||||
namespace Explorer.Presentation.ViewModels;
|
namespace Explorer.Presentation.ViewModels;
|
||||||
|
|
||||||
public sealed record SearchScopeChoice(SearchScopeKind Kind, string Label);
|
public sealed record SearchScopeChoice(SearchScopeKind Kind, string Label);
|
||||||
|
public sealed record CategoryChoice(string Value, string Label);
|
||||||
|
|
||||||
public sealed partial class SearchViewModel : ObservableObject
|
public sealed partial class SearchViewModel : ObservableObject
|
||||||
{
|
{
|
||||||
@@ -20,6 +21,7 @@ public sealed partial class SearchViewModel : ObservableObject
|
|||||||
[ObservableProperty] private string _text = "";
|
[ObservableProperty] private string _text = "";
|
||||||
[ObservableProperty] private SearchScopeKind _scope = SearchScopeKind.AllKnown;
|
[ObservableProperty] private SearchScopeKind _scope = SearchScopeKind.AllKnown;
|
||||||
[ObservableProperty] private string? _extension;
|
[ObservableProperty] private string? _extension;
|
||||||
|
[ObservableProperty] private string _category = "";
|
||||||
[ObservableProperty] private bool _foldersOnly;
|
[ObservableProperty] private bool _foldersOnly;
|
||||||
[ObservableProperty] private bool _filesOnly;
|
[ObservableProperty] private bool _filesOnly;
|
||||||
[ObservableProperty] private string? _minSizeText;
|
[ObservableProperty] private string? _minSizeText;
|
||||||
@@ -46,6 +48,12 @@ public sealed partial class SearchViewModel : ObservableObject
|
|||||||
new(SearchScopeKind.OfflineMedia, "Offline media")
|
new(SearchScopeKind.OfflineMedia, "Offline media")
|
||||||
];
|
];
|
||||||
|
|
||||||
|
public CategoryChoice[] CategoryChoices { get; } =
|
||||||
|
[
|
||||||
|
new("", "Any category"),
|
||||||
|
.. OrganizeDestinations.UserChoices.Select(c => new CategoryChoice(c.ToString(), OrganizeDestinations.Label(c)))
|
||||||
|
];
|
||||||
|
|
||||||
public void OpenWithoutSearch()
|
public void OpenWithoutSearch()
|
||||||
{
|
{
|
||||||
IsOpen = true;
|
IsOpen = true;
|
||||||
@@ -103,6 +111,7 @@ public sealed partial class SearchViewModel : ObservableObject
|
|||||||
Scope = scope,
|
Scope = scope,
|
||||||
Text = Text,
|
Text = Text,
|
||||||
Extension = Extension,
|
Extension = Extension,
|
||||||
|
Category = OrganizeDestinations.TryParse(Category, out var cat) ? cat : null,
|
||||||
IsDirectory = DirectoryFilter(FoldersOnly, FilesOnly),
|
IsDirectory = DirectoryFilter(FoldersOnly, FilesOnly),
|
||||||
MinSize = ParseSize(MinSizeText),
|
MinSize = ParseSize(MinSizeText),
|
||||||
MaxSize = ParseSize(MaxSizeText),
|
MaxSize = ParseSize(MaxSizeText),
|
||||||
@@ -139,7 +148,11 @@ public sealed partial class SearchViewModel : ObservableObject
|
|||||||
FileId = entry.FileId,
|
FileId = entry.FileId,
|
||||||
ReparseTag = entry.ReparseTag,
|
ReparseTag = entry.ReparseTag,
|
||||||
FreeSpaceBytes = space.FreeBytes,
|
FreeSpaceBytes = space.FreeBytes,
|
||||||
CapacityBytes = space.CapacityBytes
|
CapacityBytes = space.CapacityBytes,
|
||||||
|
IndexEntryId = entry.Id,
|
||||||
|
Category = entry.Category,
|
||||||
|
CategoryReason = entry.CategoryReason,
|
||||||
|
CategorySource = entry.CategorySource
|
||||||
}, entry.IsDirectory));
|
}, entry.IsDirectory));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,4 +9,7 @@
|
|||||||
<ProjectReference Include="..\Explorer.Application\Explorer.Application.csproj" />
|
<ProjectReference Include="..\Explorer.Application\Explorer.Application.csproj" />
|
||||||
<ProjectReference Include="..\Explorer.Domain\Explorer.Domain.csproj" />
|
<ProjectReference Include="..\Explorer.Domain\Explorer.Domain.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<InternalsVisibleTo Include="Explorer.Search.Tests" />
|
||||||
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ public sealed class SearchQuery
|
|||||||
public DateTimeOffset? CreatedBefore { get; init; }
|
public DateTimeOffset? CreatedBefore { get; init; }
|
||||||
public DateTimeOffset? ModifiedAfter { get; init; }
|
public DateTimeOffset? ModifiedAfter { get; init; }
|
||||||
public DateTimeOffset? ModifiedBefore { get; init; }
|
public DateTimeOffset? ModifiedBefore { get; init; }
|
||||||
|
public FileCategory? Category { get; init; }
|
||||||
public IReadOnlyList<long>? SourceIds { get; init; }
|
public IReadOnlyList<long>? SourceIds { get; init; }
|
||||||
public string? PathRelPrefix { get; init; }
|
public string? PathRelPrefix { get; init; }
|
||||||
public bool IncludeOffline { get; init; } = true;
|
public bool IncludeOffline { get; init; } = true;
|
||||||
@@ -33,9 +34,19 @@ public sealed class SearchService
|
|||||||
var text = query.Text?.Trim();
|
var text = query.Text?.Trim();
|
||||||
string? extension = query.Extension;
|
string? extension = query.Extension;
|
||||||
string? name = text;
|
string? name = text;
|
||||||
if (!string.IsNullOrEmpty(text) && text.StartsWith("*.", StringComparison.Ordinal) && !text.Contains(' ', StringComparison.Ordinal))
|
var category = query.Category;
|
||||||
|
if (!string.IsNullOrEmpty(text))
|
||||||
{
|
{
|
||||||
extension = text[2..];
|
(name, var hinted) = SplitCategoryHint(text);
|
||||||
|
if (hinted is not null)
|
||||||
|
{
|
||||||
|
category = hinted;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(name) && name.StartsWith("*.", StringComparison.Ordinal) && !name.Contains(' ', StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
extension = name[2..];
|
||||||
name = null;
|
name = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,6 +54,7 @@ public sealed class SearchService
|
|||||||
{
|
{
|
||||||
Name = name,
|
Name = name,
|
||||||
Extension = extension,
|
Extension = extension,
|
||||||
|
Category = category,
|
||||||
IsDirectory = query.IsDirectory,
|
IsDirectory = query.IsDirectory,
|
||||||
MinSize = query.MinSize,
|
MinSize = query.MinSize,
|
||||||
MaxSize = query.MaxSize,
|
MaxSize = query.MaxSize,
|
||||||
@@ -62,4 +74,25 @@ public sealed class SearchService
|
|||||||
|
|
||||||
return _store.Search.SearchAsync(request, cancellationToken);
|
return _store.Search.SearchAsync(request, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal static (string? Name, FileCategory? Category) SplitCategoryHint(string text)
|
||||||
|
{
|
||||||
|
FileCategory? category = null;
|
||||||
|
var name = System.Text.RegularExpressions.Regex.Replace(
|
||||||
|
text,
|
||||||
|
@"\b(?:category|cat):([A-Za-z]+)\b",
|
||||||
|
match =>
|
||||||
|
{
|
||||||
|
if (OrganizeDestinations.TryParse(match.Groups[1].Value, out var parsed))
|
||||||
|
{
|
||||||
|
category = parsed;
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
return match.Value;
|
||||||
|
},
|
||||||
|
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||||
|
name = name.Trim();
|
||||||
|
return (string.IsNullOrEmpty(name) ? null : name, category);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,6 +91,24 @@ internal sealed class AnalysisStore : IAnalysisStore
|
|||||||
return rows.AsList();
|
return rows.AsList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<CategoryUsage>> UsageByCategoryAsync(long? sourceId, string? pathRelPrefix, int take, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var sql = """
|
||||||
|
SELECT ifnull(category, 'Unknown') AS Category, SUM(size_bytes) AS TotalSize, COUNT(*) AS FileCount
|
||||||
|
FROM entries
|
||||||
|
WHERE is_dir=0 AND status=0
|
||||||
|
""";
|
||||||
|
if (sourceId is not null) sql += " AND source_id=@sourceId";
|
||||||
|
if (!string.IsNullOrEmpty(pathRelPrefix))
|
||||||
|
sql += " AND (path_rel=@pathRelPrefix OR path_rel LIKE @like ESCAPE '\\')";
|
||||||
|
sql += " GROUP BY category ORDER BY TotalSize DESC LIMIT @take";
|
||||||
|
var like = string.IsNullOrEmpty(pathRelPrefix) ? null : pathRelPrefix.Replace("\\", "\\\\") + "\\\\%";
|
||||||
|
var rows = await conn.QueryAsync<CategoryUsage>(new CommandDefinition(
|
||||||
|
sql, new { sourceId, pathRelPrefix, like, take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
|
||||||
|
return rows.AsList();
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
|
public async Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
|||||||
@@ -88,14 +88,41 @@ internal sealed class EntryStore : IEntryStore
|
|||||||
|
|
||||||
internal static async Task<long> UpsertCore(SqliteConnection conn, IndexEntry entry)
|
internal static async Task<long> UpsertCore(SqliteConnection conn, IndexEntry entry)
|
||||||
{
|
{
|
||||||
var existing = await SqliteExec.ScalarAsync<long?>(
|
var existingId = await SqliteExec.ScalarAsync<long?>(
|
||||||
conn,
|
conn,
|
||||||
"SELECT id FROM entries WHERE source_id=@SourceId AND ifnull(parent_id,-1)=ifnull(@ParentId,-1) AND name_norm=@NameNorm",
|
"SELECT id FROM entries WHERE source_id=@SourceId AND ifnull(parent_id,-1)=ifnull(@ParentId,-1) AND name_norm=@NameNorm",
|
||||||
new { entry.SourceId, entry.ParentId, entry.NameNorm }).ConfigureAwait(false);
|
new { entry.SourceId, entry.ParentId, entry.NameNorm }).ConfigureAwait(false);
|
||||||
|
|
||||||
if (existing is > 0)
|
if (existingId is > 0)
|
||||||
{
|
{
|
||||||
entry.Id = existing.Value;
|
entry.Id = existingId.Value;
|
||||||
|
var existingSource = await SqliteExec.ScalarAsync<string>(
|
||||||
|
conn,
|
||||||
|
"SELECT ifnull(category_source, 'none') FROM entries WHERE id=@Id",
|
||||||
|
new { entry.Id }).ConfigureAwait(false);
|
||||||
|
if (CategorySources.IsUser(existingSource))
|
||||||
|
{
|
||||||
|
entry.Category = EntryCategoryAssigner.ParseCategory(
|
||||||
|
await SqliteExec.ScalarAsync<string>(conn, "SELECT category FROM entries WHERE id=@Id", new { entry.Id })
|
||||||
|
.ConfigureAwait(false));
|
||||||
|
entry.CategoryReason = await SqliteExec.ScalarAsync<string>(
|
||||||
|
conn, "SELECT category_reason FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false);
|
||||||
|
entry.CategoryConfidence = await SqliteExec.ScalarAsync<long>(
|
||||||
|
conn, "SELECT ifnull(category_confidence, 0) FROM entries WHERE id=@Id", new { entry.Id })
|
||||||
|
.ConfigureAwait(false) is { } conf
|
||||||
|
? (int)conf
|
||||||
|
: 0;
|
||||||
|
entry.CategorySource = CategorySources.User;
|
||||||
|
var utc = await SqliteExec.ScalarAsync<string>(
|
||||||
|
conn, "SELECT category_utc FROM entries WHERE id=@Id", new { entry.Id }).ConfigureAwait(false);
|
||||||
|
entry.CategoryUtc = string.IsNullOrEmpty(utc) ? null : DateTimeOffset.Parse(utc);
|
||||||
|
}
|
||||||
|
else if (string.IsNullOrWhiteSpace(entry.CategorySource)
|
||||||
|
|| entry.CategorySource == CategorySources.None)
|
||||||
|
{
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry);
|
||||||
|
}
|
||||||
|
|
||||||
await SqliteExec.ExecuteAsync(conn, """
|
await SqliteExec.ExecuteAsync(conn, """
|
||||||
UPDATE entries SET
|
UPDATE entries SET
|
||||||
name=@Name, extension=@Extension, is_dir=@IsDir, size_bytes=@SizeBytes,
|
name=@Name, extension=@Extension, is_dir=@IsDir, size_bytes=@SizeBytes,
|
||||||
@@ -103,21 +130,32 @@ internal sealed class EntryStore : IEntryStore
|
|||||||
last_indexed_utc=@LastIndexedUtc, attributes=@Attributes, file_id=@FileId,
|
last_indexed_utc=@LastIndexedUtc, attributes=@Attributes, file_id=@FileId,
|
||||||
parent_file_id=@ParentFileId, reparse_tag=@ReparseTag, status=@Status,
|
parent_file_id=@ParentFileId, reparse_tag=@ReparseTag, status=@Status,
|
||||||
deleted_utc=@DeletedUtc, path_rel=@PathRel, scan_generation=@ScanGeneration,
|
deleted_utc=@DeletedUtc, path_rel=@PathRel, scan_generation=@ScanGeneration,
|
||||||
allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability
|
allocated_size=@AllocatedSizeBytes, cloud_availability=@CloudAvailability,
|
||||||
|
category=@Category, category_reason=@CategoryReason,
|
||||||
|
category_confidence=@CategoryConfidence, category_source=@CategorySource,
|
||||||
|
category_utc=@CategoryUtc
|
||||||
WHERE id=@Id
|
WHERE id=@Id
|
||||||
""", ToArgs(entry)).ConfigureAwait(false);
|
""", ToArgs(entry)).ConfigureAwait(false);
|
||||||
return entry.Id;
|
return entry.Id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(entry.CategorySource)
|
||||||
|
|| entry.CategorySource == CategorySources.None)
|
||||||
|
{
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry);
|
||||||
|
}
|
||||||
|
|
||||||
var id = await SqliteExec.InsertAsync(conn, """
|
var id = await SqliteExec.InsertAsync(conn, """
|
||||||
INSERT INTO entries (source_id, parent_id, name, name_norm, extension, is_dir, size_bytes, aggregate_size,
|
INSERT INTO entries (source_id, parent_id, name, name_norm, extension, is_dir, size_bytes, aggregate_size,
|
||||||
child_file_count, child_dir_count, created_utc, modified_utc, last_seen_utc, last_indexed_utc,
|
child_file_count, child_dir_count, created_utc, modified_utc, last_seen_utc, last_indexed_utc,
|
||||||
attributes, file_id, parent_file_id, reparse_tag, status, deleted_utc, path_rel, content_hash, hash_state, scan_generation,
|
attributes, file_id, parent_file_id, reparse_tag, status, deleted_utc, path_rel, content_hash, hash_state, scan_generation,
|
||||||
allocated_size, cloud_availability)
|
allocated_size, cloud_availability,
|
||||||
|
category, category_reason, category_confidence, category_source, category_utc)
|
||||||
VALUES (@SourceId, @ParentId, @Name, @NameNorm, @Extension, @IsDir, @SizeBytes, @AggregateSize,
|
VALUES (@SourceId, @ParentId, @Name, @NameNorm, @Extension, @IsDir, @SizeBytes, @AggregateSize,
|
||||||
@ChildFileCount, @ChildDirCount, @CreatedUtc, @ModifiedUtc, @LastSeenUtc, @LastIndexedUtc,
|
@ChildFileCount, @ChildDirCount, @CreatedUtc, @ModifiedUtc, @LastSeenUtc, @LastIndexedUtc,
|
||||||
@Attributes, @FileId, @ParentFileId, @ReparseTag, @Status, @DeletedUtc, @PathRel, @ContentHash, @HashState, @ScanGeneration,
|
@Attributes, @FileId, @ParentFileId, @ReparseTag, @Status, @DeletedUtc, @PathRel, @ContentHash, @HashState, @ScanGeneration,
|
||||||
@AllocatedSizeBytes, @CloudAvailability)
|
@AllocatedSizeBytes, @CloudAvailability,
|
||||||
|
@Category, @CategoryReason, @CategoryConfidence, @CategorySource, @CategoryUtc)
|
||||||
""", ToArgs(entry)).ConfigureAwait(false);
|
""", ToArgs(entry)).ConfigureAwait(false);
|
||||||
entry.Id = id;
|
entry.Id = id;
|
||||||
return id;
|
return id;
|
||||||
@@ -262,6 +300,121 @@ internal sealed class EntryStore : IEntryStore
|
|||||||
new { sourceId }).ConfigureAwait(false);
|
new { sourceId }).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Task RefreshCategoryFromChildrenAsync(long folderOrArchiveId, CancellationToken cancellationToken = default)
|
||||||
|
=> _store.WriteAsync(async conn =>
|
||||||
|
{
|
||||||
|
var row = await conn.QuerySingleOrDefaultAsync<EntryRow>(
|
||||||
|
"SELECT * FROM entries WHERE id=@id", new { id = folderOrArchiveId }).ConfigureAwait(false);
|
||||||
|
if (row is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var entry = row.ToModel();
|
||||||
|
if (CategorySources.IsUser(entry.CategorySource))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var children = (await conn.QueryAsync<string>(
|
||||||
|
"SELECT ifnull(category, 'Unknown') FROM entries WHERE parent_id=@id AND status=0",
|
||||||
|
new { id = folderOrArchiveId }).ConfigureAwait(false)).ToList();
|
||||||
|
if (children.Count == 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var cats = children.Select(EntryCategoryAssigner.ParseCategory).ToList();
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry, childCategories: cats);
|
||||||
|
await SqliteExec.ExecuteAsync(conn, """
|
||||||
|
UPDATE entries SET
|
||||||
|
category=@Category, category_reason=@CategoryReason,
|
||||||
|
category_confidence=@CategoryConfidence, category_source=@CategorySource,
|
||||||
|
category_utc=@CategoryUtc
|
||||||
|
WHERE id=@Id
|
||||||
|
""", ToArgs(entry)).ConfigureAwait(false);
|
||||||
|
}, cancellationToken);
|
||||||
|
|
||||||
|
public Task SetUserCategoryAsync(long entryId, FileCategory category, string? reason = null, CancellationToken cancellationToken = default)
|
||||||
|
=> _store.WriteAsync(async conn =>
|
||||||
|
{
|
||||||
|
var row = await conn.QuerySingleOrDefaultAsync<EntryRow>(
|
||||||
|
"SELECT * FROM entries WHERE id=@id", new { id = entryId }).ConfigureAwait(false);
|
||||||
|
if (row is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var entry = row.ToModel();
|
||||||
|
EntryCategoryAssigner.ApplyUser(entry, category, reason);
|
||||||
|
await SqliteExec.ExecuteAsync(conn, """
|
||||||
|
UPDATE entries SET
|
||||||
|
category=@Category, category_reason=@CategoryReason,
|
||||||
|
category_confidence=@CategoryConfidence, category_source=@CategorySource,
|
||||||
|
category_utc=@CategoryUtc
|
||||||
|
WHERE id=@Id
|
||||||
|
""", ToArgs(entry)).ConfigureAwait(false);
|
||||||
|
}, cancellationToken);
|
||||||
|
|
||||||
|
public Task<int> BackfillCheapCategoriesAsync(int take, CancellationToken cancellationToken = default)
|
||||||
|
=> _store.WriteAsync(async conn =>
|
||||||
|
{
|
||||||
|
take = Math.Clamp(take, 1, 500);
|
||||||
|
var rows = (await conn.QueryAsync<EntryRow>(new CommandDefinition("""
|
||||||
|
SELECT * FROM entries
|
||||||
|
WHERE status=0 AND ifnull(category_source, 'none') IN ('none', '')
|
||||||
|
AND category_utc IS NULL
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT @take
|
||||||
|
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false)).ToList();
|
||||||
|
foreach (var row in rows)
|
||||||
|
{
|
||||||
|
var entry = row.ToModel();
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry);
|
||||||
|
await SqliteExec.ExecuteAsync(conn, """
|
||||||
|
UPDATE entries SET
|
||||||
|
category=@Category, category_reason=@CategoryReason,
|
||||||
|
category_confidence=@CategoryConfidence, category_source=@CategorySource,
|
||||||
|
category_utc=@CategoryUtc
|
||||||
|
WHERE id=@Id
|
||||||
|
""", ToArgs(entry)).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return rows.Count;
|
||||||
|
}, cancellationToken);
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<long>> GetArchiveIdsNeedingContentClassifyAsync(int take, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
take = Math.Clamp(take, 1, 100);
|
||||||
|
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var ids = await conn.QueryAsync<long>(new CommandDefinition("""
|
||||||
|
SELECT a.id
|
||||||
|
FROM entries a
|
||||||
|
WHERE a.status=0 AND a.is_dir=0 AND a.category='Archive'
|
||||||
|
AND ifnull(a.category_source, 'none') IN ('extension', 'none')
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM entries c WHERE c.parent_id=a.id AND c.status=0 LIMIT 1)
|
||||||
|
ORDER BY a.id
|
||||||
|
LIMIT @take
|
||||||
|
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
|
||||||
|
return ids.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<IndexEntry>> GetUnknownFilesForSignatureAsync(int take, CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
take = Math.Clamp(take, 1, 50);
|
||||||
|
await using var conn = await _store.OpenReadAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
var rows = await conn.QueryAsync<EntryRow>(new CommandDefinition("""
|
||||||
|
SELECT * FROM entries
|
||||||
|
WHERE status=0 AND is_dir=0 AND category='Unknown'
|
||||||
|
AND ifnull(category_source, 'none') IN ('none', '')
|
||||||
|
AND ifnull(cloud_availability, 0) != 1
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT @take
|
||||||
|
""", new { take }, cancellationToken: cancellationToken)).ConfigureAwait(false);
|
||||||
|
return rows.Select(r => r.ToModel()).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
private static string EscapeLike(string? value)
|
private static string EscapeLike(string? value)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrEmpty(value))
|
if (string.IsNullOrEmpty(value))
|
||||||
@@ -302,7 +455,12 @@ internal sealed class EntryStore : IEntryStore
|
|||||||
HashState = (int)e.HashState,
|
HashState = (int)e.HashState,
|
||||||
e.ScanGeneration,
|
e.ScanGeneration,
|
||||||
e.AllocatedSizeBytes,
|
e.AllocatedSizeBytes,
|
||||||
CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null
|
CloudAvailability = e.CloudAvailability is { } a ? (int?)a : null,
|
||||||
|
Category = e.Category.ToString(),
|
||||||
|
e.CategoryReason,
|
||||||
|
e.CategoryConfidence,
|
||||||
|
e.CategorySource,
|
||||||
|
CategoryUtc = e.CategoryUtc?.ToString("O")
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -335,6 +493,11 @@ internal sealed class EntryRow
|
|||||||
public long scan_generation { get; set; }
|
public long scan_generation { get; set; }
|
||||||
public long? allocated_size { get; set; }
|
public long? allocated_size { get; set; }
|
||||||
public int? cloud_availability { get; set; }
|
public int? cloud_availability { get; set; }
|
||||||
|
public string? category { get; set; }
|
||||||
|
public string? category_reason { get; set; }
|
||||||
|
public int category_confidence { get; set; }
|
||||||
|
public string? category_source { get; set; }
|
||||||
|
public string? category_utc { get; set; }
|
||||||
|
|
||||||
public IndexEntry ToModel() => new()
|
public IndexEntry ToModel() => new()
|
||||||
{
|
{
|
||||||
@@ -364,7 +527,12 @@ internal sealed class EntryRow
|
|||||||
HashState = (HashState)hash_state,
|
HashState = (HashState)hash_state,
|
||||||
ScanGeneration = scan_generation,
|
ScanGeneration = scan_generation,
|
||||||
AllocatedSizeBytes = allocated_size,
|
AllocatedSizeBytes = allocated_size,
|
||||||
CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null
|
CloudAvailability = cloud_availability is { } a ? (CloudAvailability)a : null,
|
||||||
|
Category = EntryCategoryAssigner.ParseCategory(category),
|
||||||
|
CategoryReason = category_reason,
|
||||||
|
CategoryConfidence = category_confidence,
|
||||||
|
CategorySource = string.IsNullOrWhiteSpace(category_source) ? CategorySources.None : category_source,
|
||||||
|
CategoryUtc = Parse(category_utc)
|
||||||
};
|
};
|
||||||
|
|
||||||
private static DateTimeOffset? Parse(string? v)
|
private static DateTimeOffset? Parse(string? v)
|
||||||
|
|||||||
@@ -63,11 +63,17 @@ internal static class SchemaScript
|
|||||||
hash_state INTEGER NOT NULL DEFAULT 0,
|
hash_state INTEGER NOT NULL DEFAULT 0,
|
||||||
scan_generation INTEGER NOT NULL DEFAULT 0,
|
scan_generation INTEGER NOT NULL DEFAULT 0,
|
||||||
allocated_size INTEGER,
|
allocated_size INTEGER,
|
||||||
cloud_availability INTEGER
|
cloud_availability INTEGER,
|
||||||
|
category TEXT NOT NULL DEFAULT 'Unknown',
|
||||||
|
category_reason TEXT,
|
||||||
|
category_confidence INTEGER NOT NULL DEFAULT 0,
|
||||||
|
category_source TEXT NOT NULL DEFAULT 'none',
|
||||||
|
category_utc TEXT
|
||||||
)
|
)
|
||||||
""",
|
""",
|
||||||
"CREATE UNIQUE INDEX ix_entries_identity ON entries(source_id, ifnull(parent_id, -1), name_norm)",
|
"CREATE UNIQUE INDEX ix_entries_identity ON entries(source_id, ifnull(parent_id, -1), name_norm)",
|
||||||
"CREATE INDEX ix_entries_parent ON entries(source_id, parent_id, status)",
|
"CREATE INDEX ix_entries_parent ON entries(source_id, parent_id, status)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0",
|
||||||
"CREATE INDEX ix_entries_ext_size ON entries(source_id, extension, size_bytes) WHERE is_dir = 0",
|
"CREATE INDEX ix_entries_ext_size ON entries(source_id, extension, size_bytes) WHERE is_dir = 0",
|
||||||
"CREATE INDEX ix_entries_size ON entries(source_id, size_bytes) WHERE is_dir = 0 AND status = 0",
|
"CREATE INDEX ix_entries_size ON entries(source_id, size_bytes) WHERE is_dir = 0 AND status = 0",
|
||||||
"CREATE INDEX ix_entries_modified ON entries(source_id, modified_utc)",
|
"CREATE INDEX ix_entries_modified ON entries(source_id, modified_utc)",
|
||||||
|
|||||||
@@ -63,6 +63,12 @@ internal sealed class SearchStore : ISearchStore
|
|||||||
args.Add("Ext", NameNormalizer.Normalize(request.Extension.TrimStart('.')));
|
args.Add("Ext", NameNormalizer.Normalize(request.Extension.TrimStart('.')));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (request.Category is { } category)
|
||||||
|
{
|
||||||
|
sql.Append(" AND e.category = @Category");
|
||||||
|
args.Add("Category", category.ToString());
|
||||||
|
}
|
||||||
|
|
||||||
if (request.MinSize is not null)
|
if (request.MinSize is not null)
|
||||||
{
|
{
|
||||||
sql.Append(" AND e.size_bytes >= @MinSize");
|
sql.Append(" AND e.size_bytes >= @MinSize");
|
||||||
|
|||||||
@@ -606,6 +606,25 @@ public sealed class SqliteIndexStore : IIndexStore, IAsyncDisposable
|
|||||||
|
|
||||||
SetUserVersion(conn, 10);
|
SetUserVersion(conn, 10);
|
||||||
_logger.LogInformation("Migrated SQLite schema to v10 (hash queue state index)");
|
_logger.LogInformation("Migrated SQLite schema to v10 (hash queue state index)");
|
||||||
|
version = 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (version < 11)
|
||||||
|
{
|
||||||
|
EnsureColumn(conn, "entries", "category", "TEXT NOT NULL DEFAULT 'Unknown'");
|
||||||
|
EnsureColumn(conn, "entries", "category_reason", "TEXT");
|
||||||
|
EnsureColumn(conn, "entries", "category_confidence", "INTEGER NOT NULL DEFAULT 0");
|
||||||
|
EnsureColumn(conn, "entries", "category_source", "TEXT NOT NULL DEFAULT 'none'");
|
||||||
|
EnsureColumn(conn, "entries", "category_utc", "TEXT");
|
||||||
|
using (var idx = conn.CreateCommand())
|
||||||
|
{
|
||||||
|
idx.CommandText =
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_entries_category ON entries(source_id, category) WHERE status = 0";
|
||||||
|
idx.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
|
||||||
|
SetUserVersion(conn, 11);
|
||||||
|
_logger.LogInformation("Migrated SQLite schema to v11 (entry categories)");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ public class AnalysisTests
|
|||||||
Assert.Equal("a.mkv", files[0].Name);
|
Assert.Equal("a.mkv", files[0].Name);
|
||||||
var types = await analysis.UsageByExtensionAsync(source.Id, null);
|
var types = await analysis.UsageByExtensionAsync(source.Id, null);
|
||||||
Assert.Contains(types, t => t.Extension == "mkv" && t.TotalSize == 200);
|
Assert.Contains(types, t => t.Extension == "mkv" && t.TotalSize == 200);
|
||||||
|
var cats = await analysis.UsageByCategoryAsync(source.Id, null);
|
||||||
|
Assert.Contains(cats, c => c.Category == "Video" && c.TotalSize == 200);
|
||||||
var drill = await analysis.DrilldownAsync(root.Id);
|
var drill = await analysis.DrilldownAsync(root.Id);
|
||||||
Assert.Contains(drill, e => e.Name == "Movies");
|
Assert.Contains(drill, e => e.Name == "Movies");
|
||||||
Assert.Equal("Movies", dirs[0].PathRel);
|
Assert.Equal("Movies", dirs[0].PathRel);
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
using Explorer.Application;
|
||||||
|
using Explorer.Domain;
|
||||||
|
using Explorer.Domain.Abstractions;
|
||||||
|
using Explorer.Storage.Sqlite;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace Explorer.Application.Tests;
|
||||||
|
|
||||||
|
public class EntryClassificationServiceTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task Archive_children_override_zip_as_photos()
|
||||||
|
{
|
||||||
|
await using var store = await OpenStore();
|
||||||
|
var source = await AddSource(store, @"C:\media");
|
||||||
|
var zip = await Upsert(store, new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "photos.zip",
|
||||||
|
NameNorm = "photos.zip",
|
||||||
|
Extension = "zip",
|
||||||
|
PathRel = "photos.zip",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg"));
|
||||||
|
await Upsert(store, Child(source.Id, zip.Id, "b.jpg", "jpg"));
|
||||||
|
await Upsert(store, Child(source.Id, zip.Id, "c.jpg", "jpg"));
|
||||||
|
|
||||||
|
var service = Create(store, new MemoryEnumerator());
|
||||||
|
service.Resume();
|
||||||
|
Assert.True(await service.ProcessPendingAsync());
|
||||||
|
var loaded = await store.Entries.GetAsync(zip.Id);
|
||||||
|
Assert.Equal(FileCategory.Photos, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource);
|
||||||
|
Assert.False(await service.ProcessPendingAsync());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Signature_classifies_jpeg_without_extension()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(root);
|
||||||
|
var file = Path.Combine(root, "pic.bin");
|
||||||
|
File.WriteAllBytes(file, [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01]);
|
||||||
|
await using var store = await OpenStore();
|
||||||
|
var source = await AddSource(store, root);
|
||||||
|
var entry = await Upsert(store, new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "pic.bin",
|
||||||
|
NameNorm = "pic.bin",
|
||||||
|
Extension = "bin",
|
||||||
|
PathRel = "pic.bin",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
Assert.Equal(FileCategory.Unknown, entry.Category);
|
||||||
|
|
||||||
|
var enumerator = new MemoryEnumerator();
|
||||||
|
enumerator.Add(new FileSystemItem { FullPath = file, Name = "pic.bin" });
|
||||||
|
var service = Create(store, enumerator);
|
||||||
|
service.Resume();
|
||||||
|
Assert.True(await service.ProcessPendingAsync());
|
||||||
|
var loaded = await store.Entries.GetAsync(entry.Id);
|
||||||
|
Assert.Equal(FileCategory.Photos, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.Mime, loaded.CategorySource);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Pause_skips_work()
|
||||||
|
{
|
||||||
|
await using var store = await OpenStore();
|
||||||
|
var source = await AddSource(store, @"C:\media");
|
||||||
|
var zip = await Upsert(store, new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "photos.zip",
|
||||||
|
NameNorm = "photos.zip",
|
||||||
|
Extension = "zip",
|
||||||
|
PathRel = "photos.zip",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
await Upsert(store, Child(source.Id, zip.Id, "a.jpg", "jpg"));
|
||||||
|
var service = Create(store, new MemoryEnumerator());
|
||||||
|
Assert.True(service.IsPaused);
|
||||||
|
Assert.False(await service.ProcessPendingAsync());
|
||||||
|
var loaded = await store.Entries.GetAsync(zip.Id);
|
||||||
|
Assert.Equal(FileCategory.Archive, loaded!.Category);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static EntryClassificationService Create(IIndexStore store, IFileSystemEnumerator enumerator)
|
||||||
|
=> new(
|
||||||
|
store,
|
||||||
|
enumerator,
|
||||||
|
new NeverHydrate(),
|
||||||
|
NullLogger<EntryClassificationService>.Instance);
|
||||||
|
|
||||||
|
private static async Task<SqliteIndexStore> OpenStore()
|
||||||
|
{
|
||||||
|
var db = Path.Combine(Path.GetTempPath(), "ew-classify", Guid.NewGuid().ToString("N"), "index.db");
|
||||||
|
var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
|
||||||
|
await store.OpenAsync();
|
||||||
|
return store;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<Source> AddSource(IIndexStore store, string root)
|
||||||
|
{
|
||||||
|
var source = new Source
|
||||||
|
{
|
||||||
|
StableKey = Guid.NewGuid().ToString("N"),
|
||||||
|
Kind = SourceKind.NtfsLocal,
|
||||||
|
DisplayName = "Test",
|
||||||
|
LastRootPath = root,
|
||||||
|
Status = SourceStatus.Online
|
||||||
|
};
|
||||||
|
source.Id = await store.Sources.UpsertAsync(source);
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<IndexEntry> Upsert(IIndexStore store, IndexEntry entry)
|
||||||
|
{
|
||||||
|
entry.Id = await store.Entries.UpsertAsync(entry);
|
||||||
|
return (await store.Entries.GetAsync(entry.Id))!;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IndexEntry Child(long sourceId, long parentId, string name, string ext)
|
||||||
|
=> new()
|
||||||
|
{
|
||||||
|
SourceId = sourceId,
|
||||||
|
ParentId = parentId,
|
||||||
|
Name = name,
|
||||||
|
NameNorm = name,
|
||||||
|
Extension = ext,
|
||||||
|
PathRel = name,
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
private sealed class NeverHydrate : IHydrationGuard
|
||||||
|
{
|
||||||
|
public bool WouldHydrateOnRead(FileSystemItem item) => false;
|
||||||
|
public bool WouldHydrateOnRead(int attributes, CloudAvailability? availability) => false;
|
||||||
|
public Task<bool> WouldHydrateOnReadAsync(string path, CancellationToken cancellationToken = default)
|
||||||
|
=> Task.FromResult(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class MemoryEnumerator : IFileSystemEnumerator
|
||||||
|
{
|
||||||
|
private readonly Dictionary<string, FileSystemItem> _items = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
public void Add(FileSystemItem item) => _items[item.FullPath] = item;
|
||||||
|
|
||||||
|
public IEnumerable<FileSystemItem> EnumerateChildren(string directoryPath) => [];
|
||||||
|
public FileSystemItem? GetItem(string path)
|
||||||
|
=> _items.TryGetValue(path, out var item) ? item : null;
|
||||||
|
public IReadOnlyList<FileSystemItem> EnumerateChildrenSafe(string directoryPath, out string? error)
|
||||||
|
{
|
||||||
|
error = null;
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -132,6 +132,40 @@ public class ReorganizePlannerTests
|
|||||||
Assert.Null(fs.GetItem(@"C:\Pictures\photo.jpg"));
|
Assert.Null(fs.GetItem(@"C:\Pictures\photo.jpg"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Prefers_indexed_archive_contents_over_extension()
|
||||||
|
{
|
||||||
|
var fs = Tree()
|
||||||
|
.Dir(@"C:\Downloads")
|
||||||
|
.File(@"C:\Downloads\photos.zip", 4, T0)
|
||||||
|
.Dir(@"C:\Pictures")
|
||||||
|
.Dir(@"C:\Archive");
|
||||||
|
|
||||||
|
var indexed = new Dictionary<string, IndexEntry>(StringComparer.OrdinalIgnoreCase)
|
||||||
|
{
|
||||||
|
["photos.zip"] = new IndexEntry
|
||||||
|
{
|
||||||
|
Name = "photos.zip",
|
||||||
|
NameNorm = "photos.zip",
|
||||||
|
PathRel = "photos.zip",
|
||||||
|
Category = FileCategory.Photos,
|
||||||
|
CategorySource = CategorySources.ArchiveContents,
|
||||||
|
CategoryReason = "Archive contents are mostly photos."
|
||||||
|
}
|
||||||
|
};
|
||||||
|
var plan = new ReorganizePlanner().Build(
|
||||||
|
@"C:\Downloads",
|
||||||
|
Map(),
|
||||||
|
fs,
|
||||||
|
_ => true,
|
||||||
|
_ => false,
|
||||||
|
now: T0,
|
||||||
|
pathExists: _ => false,
|
||||||
|
indexedByName: indexed);
|
||||||
|
Assert.Contains(plan.Operations, o => o.SourcePath.EndsWith("photos.zip") && o.DestinationPath == @"C:\Pictures\photos.zip");
|
||||||
|
Assert.DoesNotContain(plan.Operations, o => o.DestinationPath == @"C:\Archive\photos.zip");
|
||||||
|
}
|
||||||
|
|
||||||
private static OperationPlan Build(
|
private static OperationPlan Build(
|
||||||
IFileSystemEnumerator fs,
|
IFileSystemEnumerator fs,
|
||||||
Action<OrganizeDestinations>? configure = null,
|
Action<OrganizeDestinations>? configure = null,
|
||||||
|
|||||||
@@ -484,6 +484,108 @@ public class FileClassifierTests
|
|||||||
=> FileClassifier.Classify(name, @"C:\Downloads\" + name, isDirectory, isDirectory ? AttributeFlags.Directory : 0, isRepoRoot).Category;
|
=> FileClassifier.Classify(name, @"C:\Downloads\" + name, isDirectory, isDirectory ? AttributeFlags.Directory : 0, isRepoRoot).Category;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class EntryCategoryAssignerTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void ApplyAutomatic_sets_extension_category_on_files()
|
||||||
|
{
|
||||||
|
var entry = new IndexEntry
|
||||||
|
{
|
||||||
|
Name = "clip.mkv",
|
||||||
|
NameNorm = "clip.mkv",
|
||||||
|
PathRel = "clip.mkv",
|
||||||
|
Extension = "mkv"
|
||||||
|
};
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry);
|
||||||
|
Assert.Equal(FileCategory.Video, entry.Category);
|
||||||
|
Assert.Equal(CategorySources.Extension, entry.CategorySource);
|
||||||
|
Assert.True(entry.CategoryConfidence >= 80);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void User_category_is_not_overwritten()
|
||||||
|
{
|
||||||
|
var entry = new IndexEntry
|
||||||
|
{
|
||||||
|
Name = "clip.mkv",
|
||||||
|
NameNorm = "clip.mkv",
|
||||||
|
PathRel = "clip.mkv",
|
||||||
|
Extension = "mkv"
|
||||||
|
};
|
||||||
|
EntryCategoryAssigner.ApplyUser(entry, FileCategory.Documents, "Manual");
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(entry);
|
||||||
|
Assert.Equal(FileCategory.Documents, entry.Category);
|
||||||
|
Assert.Equal(CategorySources.User, entry.CategorySource);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void Children_majority_sets_folder_or_archive_source()
|
||||||
|
{
|
||||||
|
var folder = new IndexEntry
|
||||||
|
{
|
||||||
|
Name = "Vacation",
|
||||||
|
NameNorm = "vacation",
|
||||||
|
PathRel = "Vacation",
|
||||||
|
IsDirectory = true,
|
||||||
|
Attributes = AttributeFlags.Directory
|
||||||
|
};
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(
|
||||||
|
folder,
|
||||||
|
childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos, FileCategory.Documents]);
|
||||||
|
Assert.Equal(FileCategory.Photos, folder.Category);
|
||||||
|
Assert.Equal(CategorySources.Children, folder.CategorySource);
|
||||||
|
|
||||||
|
var zip = new IndexEntry
|
||||||
|
{
|
||||||
|
Name = "photos.zip",
|
||||||
|
NameNorm = "photos.zip",
|
||||||
|
PathRel = "photos.zip",
|
||||||
|
Extension = "zip"
|
||||||
|
};
|
||||||
|
EntryCategoryAssigner.ApplyAutomatic(
|
||||||
|
zip,
|
||||||
|
childCategories: [FileCategory.Photos, FileCategory.Photos, FileCategory.Photos]);
|
||||||
|
Assert.Equal(FileCategory.Photos, zip.Category);
|
||||||
|
Assert.Equal(CategorySources.ArchiveContents, zip.CategorySource);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class OrganizeDestinationsTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void Parses_category_aliases()
|
||||||
|
{
|
||||||
|
Assert.True(OrganizeDestinations.TryParse("photos", out var photos));
|
||||||
|
Assert.Equal(FileCategory.Photos, photos);
|
||||||
|
Assert.True(OrganizeDestinations.TryParse("zip", out var zip));
|
||||||
|
Assert.Equal(FileCategory.Archive, zip);
|
||||||
|
Assert.True(OrganizeDestinations.TryParse("unknown", out var unknown));
|
||||||
|
Assert.Equal(FileCategory.Unknown, unknown);
|
||||||
|
Assert.False(OrganizeDestinations.TryParse("not-a-category", out _));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class FileSignatureClassifierTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void Recognizes_common_headers()
|
||||||
|
{
|
||||||
|
Assert.Equal(FileCategory.Photos, Hit([0xFF, 0xD8, 0xFF, 0xE0]));
|
||||||
|
Assert.Equal(FileCategory.Photos, Hit([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]));
|
||||||
|
Assert.Equal(FileCategory.Documents, Hit([0x25, 0x50, 0x44, 0x46]));
|
||||||
|
Assert.Equal(FileCategory.Archive, Hit([0x50, 0x4B, 0x03, 0x04]));
|
||||||
|
Assert.Equal(FileCategory.Archive, Hit([0x37, 0x7A, 0xBC, 0xAF]));
|
||||||
|
Assert.Equal(FileCategory.Installer, Hit([0x4D, 0x5A, 0x90, 0x00]));
|
||||||
|
Assert.Equal(FileCategory.Audio, Hit("RIFF....WAVE"u8.ToArray()));
|
||||||
|
Assert.Equal(FileCategory.Video, Hit("RIFF....AVI "u8.ToArray()));
|
||||||
|
Assert.Null(FileSignatureClassifier.TryClassify([0x00, 0x01, 0x02, 0x03]));
|
||||||
|
Assert.Null(FileSignatureClassifier.TryClassify([0xFF, 0xD8]));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileCategory Hit(byte[] header)
|
||||||
|
=> FileSignatureClassifier.TryClassify(header)!.Category;
|
||||||
|
}
|
||||||
|
|
||||||
public class GitChangeTests
|
public class GitChangeTests
|
||||||
{
|
{
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
@@ -149,6 +149,44 @@ public class BackgroundMaintenanceCoordinatorTests
|
|||||||
Assert.Empty(indexing.IdleScans);
|
Assert.Empty(indexing.IdleScans);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Idle_classifies_before_hashing()
|
||||||
|
{
|
||||||
|
var hash = new FakeHash { Pending = true };
|
||||||
|
var indexing = new FakeIndexing();
|
||||||
|
var classify = new FakeClassify { Work = true };
|
||||||
|
var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify);
|
||||||
|
await coordinator.TickAsync(CancellationToken.None);
|
||||||
|
Assert.Equal(1, classify.Calls);
|
||||||
|
Assert.True(hash.IsPaused);
|
||||||
|
Assert.Contains("classifying", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Idle_hashes_when_classify_is_idle()
|
||||||
|
{
|
||||||
|
var hash = new FakeHash { Pending = true };
|
||||||
|
var indexing = new FakeIndexing();
|
||||||
|
var classify = new FakeClassify();
|
||||||
|
var coordinator = Create(hash, indexing, idle: TimeSpan.FromMinutes(20), classify: classify);
|
||||||
|
await coordinator.TickAsync(CancellationToken.None);
|
||||||
|
Assert.Equal(1, classify.Calls);
|
||||||
|
Assert.False(hash.IsPaused);
|
||||||
|
Assert.Contains("hashing", coordinator.Snapshot.Message, StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Activity_pauses_classify()
|
||||||
|
{
|
||||||
|
var hash = new FakeHash();
|
||||||
|
var indexing = new FakeIndexing();
|
||||||
|
var classify = new FakeClassify { Paused = false };
|
||||||
|
var coordinator = Create(hash, indexing, idle: TimeSpan.Zero, classify: classify);
|
||||||
|
await coordinator.TickAsync(CancellationToken.None);
|
||||||
|
Assert.True(classify.IsPaused);
|
||||||
|
Assert.Equal(0, classify.Calls);
|
||||||
|
}
|
||||||
|
|
||||||
private static Source StaleLocal()
|
private static Source StaleLocal()
|
||||||
=> new()
|
=> new()
|
||||||
{
|
{
|
||||||
@@ -168,8 +206,9 @@ public class BackgroundMaintenanceCoordinatorTests
|
|||||||
bool enabled = true,
|
bool enabled = true,
|
||||||
bool ac = true,
|
bool ac = true,
|
||||||
bool foreground = false,
|
bool foreground = false,
|
||||||
IReadOnlyList<Source>? sources = null)
|
IReadOnlyList<Source>? sources = null,
|
||||||
=> Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources);
|
FakeClassify? classify = null)
|
||||||
|
=> Create(hash, indexing, new FakeIdle(idle), enabled, ac, foreground, sources, classify);
|
||||||
|
|
||||||
private static BackgroundMaintenanceCoordinator Create(
|
private static BackgroundMaintenanceCoordinator Create(
|
||||||
FakeHash hash,
|
FakeHash hash,
|
||||||
@@ -178,7 +217,8 @@ public class BackgroundMaintenanceCoordinatorTests
|
|||||||
bool enabled = true,
|
bool enabled = true,
|
||||||
bool ac = true,
|
bool ac = true,
|
||||||
bool foreground = false,
|
bool foreground = false,
|
||||||
IReadOnlyList<Source>? sources = null)
|
IReadOnlyList<Source>? sources = null,
|
||||||
|
FakeClassify? classify = null)
|
||||||
{
|
{
|
||||||
var dir = Path.Combine(Path.GetTempPath(), "ew-maint", Guid.NewGuid().ToString("N"));
|
var dir = Path.Combine(Path.GetTempPath(), "ew-maint", Guid.NewGuid().ToString("N"));
|
||||||
Directory.CreateDirectory(dir);
|
Directory.CreateDirectory(dir);
|
||||||
@@ -199,7 +239,8 @@ public class BackgroundMaintenanceCoordinatorTests
|
|||||||
new FakeHistory(),
|
new FakeHistory(),
|
||||||
new FakeStore(sources ?? []),
|
new FakeStore(sources ?? []),
|
||||||
new FakeVolumes(),
|
new FakeVolumes(),
|
||||||
NullLogger<BackgroundMaintenanceCoordinator>.Instance);
|
NullLogger<BackgroundMaintenanceCoordinator>.Instance,
|
||||||
|
classify: classify);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class FakeIdle(TimeSpan idle) : IUserIdleMonitor
|
private sealed class FakeIdle(TimeSpan idle) : IUserIdleMonitor
|
||||||
@@ -233,14 +274,31 @@ public class BackgroundMaintenanceCoordinatorTests
|
|||||||
|
|
||||||
private sealed class FakeHash : IIdleHashWork
|
private sealed class FakeHash : IIdleHashWork
|
||||||
{
|
{
|
||||||
|
public bool Pending { get; set; }
|
||||||
public bool Paused { get; set; } = true;
|
public bool Paused { get; set; } = true;
|
||||||
public bool IsPaused => Paused;
|
public bool IsPaused => Paused;
|
||||||
public string? CurrentPath => null;
|
public string? CurrentPath => null;
|
||||||
public void Pause() => Paused = true;
|
public void Pause() => Paused = true;
|
||||||
public void Resume() => Paused = false;
|
public void Resume() => Paused = false;
|
||||||
public void BeginUserRequested() { }
|
public void BeginUserRequested() { }
|
||||||
public Task<bool> HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(false);
|
public Task<bool> HasPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending);
|
||||||
public Task<long> CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(0L);
|
public Task<long> CountPendingAsync(CancellationToken cancellationToken = default) => Task.FromResult(Pending ? 1L : 0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class FakeClassify : IIdleClassifyWork
|
||||||
|
{
|
||||||
|
public bool Work { get; set; }
|
||||||
|
public int Calls { get; private set; }
|
||||||
|
public bool Paused { get; set; } = true;
|
||||||
|
public bool IsPaused => Paused;
|
||||||
|
public string? CurrentPath => null;
|
||||||
|
public void Pause() => Paused = true;
|
||||||
|
public void Resume() => Paused = false;
|
||||||
|
public Task<bool> ProcessPendingAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Calls++;
|
||||||
|
return Task.FromResult(Work);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class FakeHistory : IHistoryMaintenance
|
private sealed class FakeHistory : IHistoryMaintenance
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ public class CoreRegistrationTests
|
|||||||
Assert.NotNull(sp.GetService<IBackgroundMaintenance>());
|
Assert.NotNull(sp.GetService<IBackgroundMaintenance>());
|
||||||
Assert.NotNull(sp.GetService<IUserIdleMonitor>());
|
Assert.NotNull(sp.GetService<IUserIdleMonitor>());
|
||||||
Assert.NotNull(sp.GetService<IPowerSourceMonitor>());
|
Assert.NotNull(sp.GetService<IPowerSourceMonitor>());
|
||||||
|
Assert.NotNull(sp.GetService<IIdleClassifyWork>());
|
||||||
Assert.IsType<StorageProviderRegistry>(sp.GetService<ICloudOverlay>());
|
Assert.IsType<StorageProviderRegistry>(sp.GetService<ICloudOverlay>());
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
|
|||||||
@@ -44,4 +44,37 @@ public class SearchServiceTests
|
|||||||
Assert.Contains(both, e => e.Name == "Folder");
|
Assert.Contains(both, e => e.Name == "Folder");
|
||||||
Assert.Contains(both, e => e.Name == "notes.txt");
|
Assert.Contains(both, e => e.Name == "notes.txt");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Category_hint_and_filter()
|
||||||
|
{
|
||||||
|
var db = Path.Combine(Path.GetTempPath(), "ew-search", Guid.NewGuid().ToString("N"), "index.db");
|
||||||
|
await using var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
|
||||||
|
await store.OpenAsync();
|
||||||
|
var source = new Source { StableKey = "k", DisplayName = "d", Kind = SourceKind.NtfsLocal, LastRootPath = @"C:\d", Status = SourceStatus.Online };
|
||||||
|
source.Id = await store.Sources.UpsertAsync(source);
|
||||||
|
var root = new IndexEntry { SourceId = source.Id, Name = "d", NameNorm = "d", IsDirectory = true, PathRel = "", LastSeenUtc = DateTimeOffset.UtcNow };
|
||||||
|
root.Id = await store.Entries.UpsertAsync(root);
|
||||||
|
await store.Entries.UpsertAsync(new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id, ParentId = root.Id, Name = "clip.mkv", NameNorm = "clip.mkv", Extension = "mkv",
|
||||||
|
PathRel = "clip.mkv", LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
await store.Entries.UpsertAsync(new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id, ParentId = root.Id, Name = "notes.pdf", NameNorm = "notes.pdf", Extension = "pdf",
|
||||||
|
PathRel = "notes.pdf", LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
|
||||||
|
var search = new SearchService(store);
|
||||||
|
var hinted = await search.SearchAsync(new SearchQuery { Text = "category:video", SourceIds = [source.Id] });
|
||||||
|
Assert.Single(hinted);
|
||||||
|
Assert.Equal("clip.mkv", hinted[0].Name);
|
||||||
|
var filtered = await search.SearchAsync(new SearchQuery { Category = FileCategory.Documents, SourceIds = [source.Id] });
|
||||||
|
Assert.Single(filtered);
|
||||||
|
Assert.Equal("notes.pdf", filtered[0].Name);
|
||||||
|
var split = SearchService.SplitCategoryHint("vacation category:photos");
|
||||||
|
Assert.Equal("vacation", split.Name);
|
||||||
|
Assert.Equal(FileCategory.Photos, split.Category);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,6 +114,135 @@ public class EntryAndSearchTests
|
|||||||
Assert.Contains(byDate, e => e.Name == "Movie.mkv");
|
Assert.Contains(byDate, e => e.Name == "Movie.mkv");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Upsert_assigns_and_preserves_categories()
|
||||||
|
{
|
||||||
|
await using var store = await Stores.Open();
|
||||||
|
var source = await store.AddSourceAsync(@"C:\media");
|
||||||
|
var root = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "media",
|
||||||
|
NameNorm = "media",
|
||||||
|
IsDirectory = true,
|
||||||
|
PathRel = "",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
root.Id = await store.Entries.UpsertAsync(root);
|
||||||
|
var photo = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
ParentId = root.Id,
|
||||||
|
Name = "shot.jpg",
|
||||||
|
NameNorm = "shot.jpg",
|
||||||
|
Extension = "jpg",
|
||||||
|
PathRel = "shot.jpg",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
photo.Id = await store.Entries.UpsertAsync(photo);
|
||||||
|
var loaded = await store.Entries.GetAsync(photo.Id);
|
||||||
|
Assert.Equal(FileCategory.Photos, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.Extension, loaded.CategorySource);
|
||||||
|
|
||||||
|
await store.Entries.SetUserCategoryAsync(photo.Id, FileCategory.Documents, "Manual");
|
||||||
|
photo.SizeBytes = 100;
|
||||||
|
await store.Entries.UpsertAsync(photo);
|
||||||
|
loaded = await store.Entries.GetAsync(photo.Id);
|
||||||
|
Assert.Equal(FileCategory.Documents, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.User, loaded.CategorySource);
|
||||||
|
|
||||||
|
var other = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
ParentId = root.Id,
|
||||||
|
Name = "clip.mkv",
|
||||||
|
NameNorm = "clip.mkv",
|
||||||
|
Extension = "mkv",
|
||||||
|
PathRel = "clip.mkv",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
other.Id = await store.Entries.UpsertAsync(other);
|
||||||
|
await store.Entries.RefreshCategoryFromChildrenAsync(root.Id);
|
||||||
|
var folder = await store.Entries.GetAsync(root.Id);
|
||||||
|
Assert.Equal(FileCategory.Unknown, folder!.Category);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Archive_contents_and_signature_queues()
|
||||||
|
{
|
||||||
|
await using var store = await Stores.Open();
|
||||||
|
var source = await store.AddSourceAsync(@"C:\media");
|
||||||
|
var zip = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "photos.zip",
|
||||||
|
NameNorm = "photos.zip",
|
||||||
|
Extension = "zip",
|
||||||
|
PathRel = "photos.zip",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
zip.Id = await store.Entries.UpsertAsync(zip);
|
||||||
|
Assert.Equal(FileCategory.Archive, (await store.Entries.GetAsync(zip.Id))!.Category);
|
||||||
|
|
||||||
|
for (var i = 0; i < 3; i++)
|
||||||
|
{
|
||||||
|
await store.Entries.UpsertAsync(new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
ParentId = zip.Id,
|
||||||
|
Name = $"shot{i}.jpg",
|
||||||
|
NameNorm = $"shot{i}.jpg",
|
||||||
|
Extension = "jpg",
|
||||||
|
PathRel = $"photos.zip/shot{i}.jpg",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
var needing = await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10);
|
||||||
|
Assert.Contains(zip.Id, needing);
|
||||||
|
await store.Entries.RefreshCategoryFromChildrenAsync(zip.Id);
|
||||||
|
var loaded = await store.Entries.GetAsync(zip.Id);
|
||||||
|
Assert.Equal(FileCategory.Photos, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.ArchiveContents, loaded.CategorySource);
|
||||||
|
Assert.Empty(await store.Entries.GetArchiveIdsNeedingContentClassifyAsync(10));
|
||||||
|
|
||||||
|
var mystery = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "mystery.bin",
|
||||||
|
NameNorm = "mystery.bin",
|
||||||
|
Extension = "bin",
|
||||||
|
PathRel = "mystery.bin",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
mystery.Id = await store.Entries.UpsertAsync(mystery);
|
||||||
|
var unknowns = await store.Entries.GetUnknownFilesForSignatureAsync(10);
|
||||||
|
Assert.Contains(unknowns, e => e.Id == mystery.Id);
|
||||||
|
Assert.Equal(0, await store.Entries.BackfillCheapCategoriesAsync(50));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task User_category_by_path_is_preserved()
|
||||||
|
{
|
||||||
|
await using var store = await Stores.Open();
|
||||||
|
var source = await store.AddSourceAsync(@"C:\media");
|
||||||
|
var photo = new IndexEntry
|
||||||
|
{
|
||||||
|
SourceId = source.Id,
|
||||||
|
Name = "shot.jpg",
|
||||||
|
NameNorm = "shot.jpg",
|
||||||
|
Extension = "jpg",
|
||||||
|
PathRel = "shot.jpg",
|
||||||
|
LastSeenUtc = DateTimeOffset.UtcNow
|
||||||
|
};
|
||||||
|
photo.Id = await store.Entries.UpsertAsync(photo);
|
||||||
|
var mutations = new Explorer.Application.LocalIndexMutations(store);
|
||||||
|
await mutations.SetUserCategoryByPathAsync(@"C:\media\shot.jpg", FileCategory.Installer, "Manual");
|
||||||
|
var loaded = await store.Entries.GetAsync(photo.Id);
|
||||||
|
Assert.Equal(FileCategory.Installer, loaded!.Category);
|
||||||
|
Assert.Equal(CategorySources.User, loaded.CategorySource);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task Folder_aggregates_and_tombstones()
|
public async Task Folder_aggregates_and_tombstones()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user