Add Explorer Workbench with hierarchical, off-UI Storage analysis.

Storage queries run in the background with cancellation and covering indexes so switching views no longer freezes the UI.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/Explorer.Analysis/AnalysisResultCache.cs

@@ -0,0 +1,45 @@
+namespace Explorer.Analysis;
+
+internal sealed class AnalysisResultCache
+{
+    private const int MaxEntries = 32;
+    private static readonly TimeSpan Ttl = TimeSpan.FromSeconds(20);
+
+    private readonly object _gate = new();
+    private readonly Dictionary<string, Entry> _items = new(StringComparer.Ordinal);
+
+    public bool TryGet<T>(string key, long stamp, out T value)
+    {
+        lock (_gate)
+        {
+            if (_items.TryGetValue(key, out var entry)
+                && entry.Stamp == stamp
+                && entry.Utc + Ttl > DateTime.UtcNow
+                && entry.Value is T typed)
+            {
+                value = typed;
+                return true;
+            }
+        }
+
+        value = default!;
+        return false;
+    }
+
+    public void Set<T>(string key, long stamp, T value)
+    {
+        lock (_gate)
+        {
+            _items[key] = new Entry(stamp, value!, DateTime.UtcNow);
+            if (_items.Count <= MaxEntries)
+            {
+                return;
+            }
+
+            var oldest = _items.OrderBy(p => p.Value.Utc).First().Key;
+            _items.Remove(oldest);
+        }
+    }
+
+    private readonly record struct Entry(long Stamp, object Value, DateTime Utc);
+}

src/Explorer.Analysis/AnalysisService.cs

@@ -0,0 +1,117 @@
+using Explorer.Domain;
+using Explorer.Domain.Abstractions;
+
+namespace Explorer.Analysis;
+
+public sealed class AnalysisService
+{
+    private readonly IIndexStore _store;
+    private readonly AnalysisResultCache _cache = new();
+    private readonly SemaphoreSlim _ready = new(1, 1);
+    private bool _readyDone;
+
+    public AnalysisService(IIndexStore store) => _store = store;
+
+    public Task EnsureReadyAsync(CancellationToken cancellationToken = default)
+        => RunOffUiAsync(async ct =>
+        {
+            if (_readyDone)
+            {
+                return 0;
+            }
+
+            await _ready.WaitAsync(ct).ConfigureAwait(false);
+            try
+            {
+                if (!_readyDone)
+                {
+                    await _store.Analysis.EnsureReadyAsync(CancellationToken.None).ConfigureAwait(false);
+                    _readyDone = true;
+                }
+            }
+            finally
+            {
+                _ready.Release();
+            }
+
+            return 0;
+        }, cancellationToken);
+
+    public Task<long> GetIndexStampAsync(CancellationToken cancellationToken = default)
+        => RunOffUiAsync(ct => _store.Analysis.GetIndexStampAsync(ct), cancellationToken);
+
+    public Task<IReadOnlyList<IndexEntry>> GetDirectoryRootsAsync(CancellationToken cancellationToken = default)
+        => CachedAsync("roots", ct => _store.Analysis.GetDirectoryRootsAsync(ct), cancellationToken);
+
+    public Task<IReadOnlyList<IndexEntry>> LargestDirectoriesAsync(
+        long? sourceId,
+        long? parentId,
+        int take = AppConstants.AnalysisTopN,
+        CancellationToken cancellationToken = default)
+        => CachedAsync(
+            $"dirs:{sourceId}:{parentId}:{take}",
+            ct => _store.Analysis.LargestDirectoriesAsync(sourceId, parentId, take, ct),
+            cancellationToken);
+
+    public Task<IReadOnlyList<IndexEntry>> LargestFilesAsync(
+        long? sourceId,
+        string? pathRelPrefix,
+        int take = AppConstants.AnalysisTopN,
+        CancellationToken cancellationToken = default)
+        => CachedAsync(
+            $"files:{sourceId}:{pathRelPrefix}:{take}",
+            ct => _store.Analysis.LargestFilesAsync(sourceId, pathRelPrefix, take, ct),
+            cancellationToken);
+
+    public Task<IReadOnlyList<ExtensionUsage>> UsageByExtensionAsync(
+        long? sourceId,
+        string? pathRelPrefix,
+        int take = AppConstants.AnalysisTopN,
+        CancellationToken cancellationToken = default)
+        => CachedAsync(
+            $"types:{sourceId}:{pathRelPrefix}:{take}",
+            ct => _store.Analysis.UsageByExtensionAsync(sourceId, pathRelPrefix, take, ct),
+            cancellationToken);
+
+    public Task<IReadOnlyList<SourceUsage>> UsageBySourceAsync(CancellationToken cancellationToken = default)
+        => CachedAsync("sources", ct => _store.Analysis.UsageBySourceAsync(ct), cancellationToken);
+
+    public Task<IReadOnlyList<IndexEntry>> DrilldownAsync(
+        long parentId,
+        int take = AppConstants.AnalysisTopN,
+        CancellationToken cancellationToken = default)
+        => CachedAsync(
+            $"children:{parentId}:{take}",
+            ct => _store.Analysis.ChildrenBySizeAsync(parentId, take, ct),
+            cancellationToken);
+
+    public Task<IReadOnlyList<Source>> GetKnownSourcesAsync(CancellationToken cancellationToken = default)
+        => RunOffUiAsync(ct => _store.Sources.GetAllAsync(ct), cancellationToken);
+
+    private async Task<T> CachedAsync<T>(string key, Func<CancellationToken, Task<T>> query, CancellationToken cancellationToken)
+    {
+        await EnsureReadyAsync(cancellationToken).ConfigureAwait(false);
+        var stamp = await GetIndexStampAsync(cancellationToken).ConfigureAwait(false);
+        if (_cache.TryGet<T>(key, stamp, out var hit))
+        {
+            return hit;
+        }
+
+        var value = await RunOffUiAsync(query, cancellationToken).ConfigureAwait(false);
+        cancellationToken.ThrowIfCancellationRequested();
+        _cache.Set(key, stamp, value);
+        return value;
+    }
+
+    public static async Task<T> RunOffUiAsync<T>(Func<CancellationToken, Task<T>> work, CancellationToken cancellationToken)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+        if (SynchronizationContext.Current is null)
+        {
+            return await work(cancellationToken).ConfigureAwait(false);
+        }
+
+        return await Task.Run(async () => await work(cancellationToken).ConfigureAwait(false), cancellationToken)
+            .ConfigureAwait(false);
+    }
+}

src/Explorer.Analysis/DuplicateAndHistory.cs

@@ -0,0 +1,158 @@
+using System.Security.Cryptography;
+using Explorer.Application;
+using Explorer.Domain;
+using Explorer.Domain.Abstractions;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Explorer.Analysis;
+
+public sealed class DuplicateHashWorker : BackgroundService
+{
+    private readonly IIndexStore _store;
+    private readonly IHydrationGuard _hydration;
+    private readonly ILogger<DuplicateHashWorker> _logger;
+    private volatile bool _paused;
+
+    public DuplicateHashWorker(IIndexStore store, IHydrationGuard hydration, ILogger<DuplicateHashWorker> logger)
+    {
+        _store = store;
+        _hydration = hydration;
+        _logger = logger;
+    }
+
+    public void Pause() => _paused = true;
+    public void Resume() => _paused = false;
+
+    public async Task ProcessPendingAsync(CancellationToken cancellationToken)
+    {
+        var batch = await _store.Hashes.DequeueAsync(8, cancellationToken).ConfigureAwait(false);
+        foreach (var item in batch)
+        {
+            if (_paused || cancellationToken.IsCancellationRequested)
+            {
+                break;
+            }
+
+            if (item.RootPath is null)
+            {
+                continue;
+            }
+
+            var path = PathRules.Combine(item.RootPath, item.PathRel);
+            try
+            {
+                if (_hydration.WouldHydrateOnRead(item.Attributes, item.CloudAvailability)
+                    || await _hydration.WouldHydrateOnReadAsync(path, cancellationToken).ConfigureAwait(false))
+                {
+                    await _store.Hashes.MarkSkippedAsync(item.EntryId, cancellationToken).ConfigureAwait(false);
+                    continue;
+                }
+
+                if (item.State == "Pending")
+                {
+                    var hash = await HashAsync(path, AppConstants.PartialHashBytes, cancellationToken).ConfigureAwait(false);
+                    if (hash is not null)
+                    {
+                        await _store.Hashes.CompletePartialAsync(item.EntryId, hash, cancellationToken).ConfigureAwait(false);
+                    }
+                }
+                else if (item.State == "PartialDone")
+                {
+                    if (!await _store.Hashes.HasPartialCollisionAsync(item.EntryId, item.SizeBytes, cancellationToken)
+                            .ConfigureAwait(false))
+                    {
+                        await _store.Hashes.MarkUniquePartialAsync(item.EntryId, cancellationToken).ConfigureAwait(false);
+                        continue;
+                    }
+
+                    var hash = await HashAsync(path, null, cancellationToken).ConfigureAwait(false);
+                    if (hash is not null)
+                    {
+                        await _store.Hashes.CompleteFullAsync(item.EntryId, hash, cancellationToken).ConfigureAwait(false);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Hash failed for {Path}", path);
+                await _store.Hashes.MarkErrorAsync(item.EntryId, cancellationToken).ConfigureAwait(false);
+            }
+        }
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromSeconds(5));
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
+        {
+            if (_paused)
+            {
+                continue;
+            }
+
+            try
+            {
+                await ProcessPendingAsync(stoppingToken).ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Hash worker loop error");
+            }
+        }
+    }
+
+    private static async Task<byte[]?> HashAsync(string path, int? limit, CancellationToken cancellationToken)
+    {
+        var ext = PathRules.ToExtended(path);
+        await using var stream = new FileStream(ext, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan);
+        if (limit is int n)
+        {
+            var buffer = new byte[n];
+            var read = await stream.ReadAsync(buffer.AsMemory(0, n), cancellationToken).ConfigureAwait(false);
+            return SHA256.HashData(buffer.AsSpan(0, read));
+        }
+
+        return await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
+    }
+}
+
+public sealed class HistoryRollupService : BackgroundService
+{
+    private readonly IIndexStore _store;
+    private DateTime _last = DateTime.MinValue;
+
+    public HistoryRollupService(IIndexStore store) => _store = store;
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        using var timer = new PeriodicTimer(TimeSpan.FromHours(6));
+        await CaptureAsync(stoppingToken).ConfigureAwait(false);
+        while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
+        {
+            await CaptureAsync(stoppingToken).ConfigureAwait(false);
+        }
+    }
+
+    private async Task CaptureAsync(CancellationToken cancellationToken)
+    {
+        if ((DateTime.UtcNow - _last).TotalHours < 20)
+        {
+            return;
+        }
+
+        var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
+        var utc = DateTimeOffset.UtcNow;
+        foreach (var source in sources.Where(s => s.IsIndexed))
+        {
+            await _store.History.CaptureSourceSnapshotAsync(source.Id, utc, cancellationToken).ConfigureAwait(false);
+            await _store.History.CaptureDirectorySnapshotsAsync(
+                source.Id, utc, AppConstants.DirectoryHistoryThresholdBytes, AppConstants.DirectoryHistoryTopN, cancellationToken)
+                .ConfigureAwait(false);
+        }
+
+        var days = AppConstants.DefaultTombstoneRetentionDays;
+        await _store.Entries.DeleteExpiredTombstonesAsync(utc.AddDays(-days), cancellationToken).ConfigureAwait(false);
+        _last = DateTime.UtcNow;
+    }
+}

src/Explorer.Analysis/Explorer.Analysis.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <RootNamespace>Explorer.Analysis</RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Explorer.Application\Explorer.Application.csproj" />
+    <ProjectReference Include="..\Explorer.Domain\Explorer.Domain.csproj" />
+  </ItemGroup>
+</Project>