From b33a78dbbea170d926d7a73195a398e794252ae4 Mon Sep 17 00:00:00 2001 From: netquick Date: Fri, 28 Aug 2026 02:03:52 +0200 Subject: [PATCH] Add host activity and DB browser, and keep dialogs, drag-drop, and idle maintenance responsive. Co-authored-by: Cursor --- docs/Documentation.md | 8 +- src/Explorer.Analysis/AnalysisService.cs | 161 ++++++++- src/Explorer.Analysis/DuplicateAndHistory.cs | 75 +++- src/Explorer.App/DatabaseWindow.xaml | 78 ++++ src/Explorer.App/DatabaseWindow.xaml.cs | 159 +++++++++ src/Explorer.App/DocumentationWindow.xaml | 4 +- src/Explorer.App/DocumentationWindow.xaml.cs | 40 ++- src/Explorer.App/GitChangesWindow.xaml | 2 +- src/Explorer.App/GitChangesWindow.xaml.cs | 3 + src/Explorer.App/GitDiffWindow.xaml | 2 +- src/Explorer.App/GitDiffWindow.xaml.cs | 3 + .../HostActivityMonitorWindow.xaml | 130 +++++++ .../HostActivityMonitorWindow.xaml.cs | 139 ++++++++ src/Explorer.App/MainWindow.xaml | 132 ++++--- src/Explorer.App/MainWindow.xaml.cs | 125 ++++++- src/Explorer.App/ModelessWindowClose.cs | 23 ++ .../BackgroundMaintenance.cs | 2 + src/Explorer.Application/HostActivityLog.cs | 87 +++++ .../ISqliteDatabaseSession.cs | 39 ++ .../IndexedPathPresence.cs | 116 ++++++ src/Explorer.Contracts/HostActivity.cs | 52 +++ .../Abstractions/IIndexStore.cs | 5 + src/Explorer.Domain/DragDropPolicy.cs | 42 +++ .../ExplorerHostClientServices.cs | 10 + .../Ipc/WorkbenchPipeClient.cs | 14 +- .../BackgroundMaintenanceCoordinator.cs | 32 +- src/Explorer.Hosting/ExplorerHostServices.cs | 1 + .../Ipc/WorkbenchPipeServer.cs | 39 ++ src/Explorer.Indexing/FolderReconciler.cs | 18 +- src/Explorer.Indexing/IndexingCoordinator.cs | 105 +++++- .../ViewModels/DatabaseViewerViewModel.cs | 327 +++++++++++++++++ .../ViewModels/DuplicateViewModel.cs | 308 ++++++++++++++-- .../HostActivityMonitorViewModel.cs | 324 +++++++++++++++++ .../ViewModels/MainViewModel.cs | 84 ++++- .../AnalysisHistoryHashStores.cs | 115 ++++-- src/Explorer.Storage.Sqlite/SchemaScript.cs | 1 + .../SqliteDatabaseSession.cs | 336 ++++++++++++++++++ .../SqliteIndexStore.cs | 14 + .../Explorer.Analysis.Tests/AnalysisTests.cs | 36 +- .../HostActivityLogTests.cs | 31 ++ .../IndexedPathPresenceTests.cs | 85 +++++ tests/Explorer.Domain.Tests/DomainTests.cs | 19 + .../BackgroundMaintenanceCoordinatorTests.cs | 24 ++ tests/Explorer.Indexing.Tests/ScannerTests.cs | 24 ++ .../SqliteDatabaseSessionTests.cs | 51 +++ 45 files changed, 3254 insertions(+), 171 deletions(-) create mode 100644 src/Explorer.App/DatabaseWindow.xaml create mode 100644 src/Explorer.App/DatabaseWindow.xaml.cs create mode 100644 src/Explorer.App/HostActivityMonitorWindow.xaml create mode 100644 src/Explorer.App/HostActivityMonitorWindow.xaml.cs create mode 100644 src/Explorer.App/ModelessWindowClose.cs create mode 100644 src/Explorer.Application/HostActivityLog.cs create mode 100644 src/Explorer.Application/ISqliteDatabaseSession.cs create mode 100644 src/Explorer.Application/IndexedPathPresence.cs create mode 100644 src/Explorer.Contracts/HostActivity.cs create mode 100644 src/Explorer.Presentation/ViewModels/DatabaseViewerViewModel.cs create mode 100644 src/Explorer.Presentation/ViewModels/HostActivityMonitorViewModel.cs create mode 100644 src/Explorer.Storage.Sqlite/SqliteDatabaseSession.cs create mode 100644 tests/Explorer.Application.Tests/HostActivityLogTests.cs create mode 100644 tests/Explorer.Application.Tests/IndexedPathPresenceTests.cs create mode 100644 tests/Explorer.Storage.Tests/SqliteDatabaseSessionTests.cs diff --git a/docs/Documentation.md b/docs/Documentation.md index b85df02..ac88452 100644 --- a/docs/Documentation.md +++ b/docs/Documentation.md @@ -182,9 +182,15 @@ even when that NAS is currently offline — if the archive was indexed earlier. --- +## Development tools + +**Tools → Development → Host activity…** opens a live monitor of the background host: maintenance, indexing jobs, hashing, transfers, and a rolling activity log. It refreshes about every 1.5 seconds while open and also reacts to push events; closing it stops the polling. + +**Tools → Development → Database…** opens a general SQLite viewer. It starts on the Workbench index (`%LocalAppData%\ExplorerWorkbench\index.db`) in read-only mode while the host holds the write lock. Use **Open file…** for any other `.db`, and choose write mode when the file is not locked. You can browse tables, run SQL, and — when writable — edit cells, insert rows, and delete rows. + ## Duplicates -**Tools → Storage → Duplicates**. Groups are hashed in the background (size → partial hash → full hash only when needed). Workbench distinguishes: +**Tools → Storage → Duplicates**. Groups come from the **index** (hashed in the background: size → partial hash → full hash only when needed), not from a live walk of the disk. The list fills from the index first (largest groups at the top); missing copies are dropped afterwards without blocking the window. A finished full scan of the drive also marks missing trees deleted; cancelling a scan does not. Unmarked groups have no class label. After you mark a group, Workbench shows: | Class | Meaning | | --- | --- | diff --git a/src/Explorer.Analysis/AnalysisService.cs b/src/Explorer.Analysis/AnalysisService.cs index 7ccee98..234eaa9 100644 --- a/src/Explorer.Analysis/AnalysisService.cs +++ b/src/Explorer.Analysis/AnalysisService.cs @@ -1,3 +1,4 @@ +using Explorer.Application; using Explorer.Domain; using Explorer.Domain.Abstractions; @@ -5,6 +6,8 @@ namespace Explorer.Analysis; public sealed class AnalysisService { + private const int DuplicateHashChunk = 24; + private readonly IIndexStore _store; private readonly AnalysisResultCache _cache = new(); private readonly SemaphoreSlim _ready = new(1, 1); @@ -93,15 +96,152 @@ public sealed class AnalysisService CancellationToken cancellationToken = default) => RunOffUiAsync(async ct => { - var raw = await _store.Hashes.GetDuplicateGroupsAsync(null, null, Math.Max(take * 8, 400), ct) - .ConfigureAwait(false); - var ids = raw.SelectMany(g => g.Entries.Select(e => e.Id)).Distinct().ToList(); - var relations = await _store.Relations.GetAmongAsync(ids, ct).ConfigureAwait(false); - return (IReadOnlyList)raw - .Select(g => DuplicateClassifier.ClassifyGroup(g, DuplicateClassifier.RelationsFor(g.Entries, relations))) - .ToList(); + var list = new List(); + await StreamCoreAsync(take, list.Add, verifyPresence: true, ct).ConfigureAwait(false); + return (IReadOnlyList)list; }, cancellationToken); + public Task> StreamClassifiedDuplicatesAsync( + int take, + IProgress progress, + CancellationToken cancellationToken = default) + => StreamClassifiedDuplicatesAsync(take, progress, verifyPresence: false, cancellationToken); + + public Task> StreamClassifiedDuplicatesAsync( + int take, + IProgress progress, + bool verifyPresence, + CancellationToken cancellationToken = default) + => RunOffUiAsync( + ct => StreamCoreAsync(take, progress.Report, verifyPresence, ct), + cancellationToken); + + private async Task> StreamCoreAsync( + int take, + Action emit, + bool verifyPresence, + CancellationToken cancellationToken) + { + var limit = Math.Max(take * 8, 400); + var hashes = await _store.Hashes.GetDuplicateHashesAsync(null, null, limit, cancellationToken) + .ConfigureAwait(false); + var sources = (await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false)) + .ToDictionary(s => s.Id); + var missing = new List(); + for (var offset = 0; offset < hashes.Count; offset += DuplicateHashChunk) + { + cancellationToken.ThrowIfCancellationRequested(); + var chunk = hashes.Skip(offset).Take(DuplicateHashChunk).ToList(); + var groups = await _store.Hashes.GetDuplicateGroupsByHashesAsync(chunk, cancellationToken) + .ConfigureAwait(false); + var kept = new List(); + foreach (var group in groups) + { + var present = verifyPresence + ? KeepPresentCopies(group, sources, missing) + : group.Entries.ToList(); + if (present.Count < 2) + { + continue; + } + + kept.Add(new DuplicateGroup + { + SizeBytes = group.SizeBytes, + Hash = group.Hash, + Entries = present, + SameFileId = DuplicateClassifier.IsHardlinkOnly(present) + }); + } + + var ids = kept.SelectMany(g => g.Entries.Select(e => e.Id)).Distinct().ToList(); + var relations = await _store.Relations.GetAmongAsync(ids, cancellationToken).ConfigureAwait(false); + foreach (var group in kept) + { + emit(DuplicateClassifier.ClassifyGroup( + group, + DuplicateClassifier.RelationsFor(group.Entries, relations))); + } + } + + if (_store.CanWrite && missing.Count > 0) + { + await TombstoneMissingAsync(missing, sources, cancellationToken).ConfigureAwait(false); + } + + return missing + .Select(m => (m.SourceId, IndexedPathPresence.ReconcilePath(m.PathRel, m.Prefix))) + .Distinct() + .ToList(); + } + + private static List KeepPresentCopies( + DuplicateGroup group, + IReadOnlyDictionary sources, + List missing) + { + var kept = new List(group.Entries.Count); + foreach (var entry in group.Entries) + { + if (!sources.TryGetValue(entry.SourceId, out var source) + || string.IsNullOrWhiteSpace(source.LastRootPath) + || !IndexedPathPresence.RootReachable(source.LastRootPath)) + { + kept.Add(entry); + continue; + } + + if (IndexedPathPresence.FileExists(source.LastRootPath, entry.PathRel)) + { + kept.Add(entry); + continue; + } + + var prefix = IndexedPathPresence.HighestMissingPrefix(source.LastRootPath, entry.PathRel) + ?? entry.PathRel; + missing.Add(new MissingCopy(entry.SourceId, entry.Id, entry.PathRel, prefix, source.LastRootPath)); + } + + return kept; + } + + private async Task TombstoneMissingAsync( + IReadOnlyList missing, + IReadOnlyDictionary sources, + CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var prefixes = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var item in missing) + { + if (!item.Prefix.Equals(item.PathRel, StringComparison.OrdinalIgnoreCase) + && sources.TryGetValue(item.SourceId, out var source) + && !string.IsNullOrWhiteSpace(source.LastRootPath) + && !IndexedPathPresence.DirectoryExists(PathRules.Combine(source.LastRootPath, item.Prefix))) + { + var key = item.SourceId + "|" + item.Prefix; + if (!prefixes.Add(key)) + { + continue; + } + + var folder = await _store.Entries.GetByPathAsync(item.SourceId, item.Prefix, cancellationToken) + .ConfigureAwait(false); + if (folder is not null) + { + await _store.Entries.TombstoneAsync(folder.Id, now, cancellationToken).ConfigureAwait(false); + } + + await _store.Entries.TombstoneByPathPrefixAsync(item.SourceId, item.Prefix, now, cancellationToken) + .ConfigureAwait(false); + } + else + { + await _store.Entries.TombstoneAsync(item.EntryId, now, cancellationToken).ConfigureAwait(false); + } + } + } + public Task MarkDuplicateGroupAsync( IReadOnlyList entries, FileRelationKind kind, @@ -156,4 +296,11 @@ public sealed class AnalysisService return await Task.Run(async () => await work(cancellationToken).ConfigureAwait(false), cancellationToken) .ConfigureAwait(false); } + + private readonly record struct MissingCopy( + long SourceId, + long EntryId, + string PathRel, + string Prefix, + string Root); } diff --git a/src/Explorer.Analysis/DuplicateAndHistory.cs b/src/Explorer.Analysis/DuplicateAndHistory.cs index 0af3118..34ee245 100644 --- a/src/Explorer.Analysis/DuplicateAndHistory.cs +++ b/src/Explorer.Analysis/DuplicateAndHistory.cs @@ -11,18 +11,30 @@ public sealed class DuplicateHashWorker : BackgroundService, IIdleHashWork { private readonly IIndexStore _store; private readonly IHydrationGuard _hydration; + private readonly IHostActivitySink _activity; private readonly ILogger _logger; + private readonly object _pendingGate = new(); + private long _pendingCount; + private DateTimeOffset _pendingCountUtc = DateTimeOffset.MinValue; + private int _pendingRefreshBusy; private volatile bool _paused = true; private volatile bool _userRequested; + private volatile string? _currentPath; - public DuplicateHashWorker(IIndexStore store, IHydrationGuard hydration, ILogger logger) + public DuplicateHashWorker( + IIndexStore store, + IHydrationGuard hydration, + ILogger logger, + IHostActivitySink? activity = null) { _store = store; _hydration = hydration; _logger = logger; + _activity = activity ?? NullHostActivitySink.Instance; } public bool IsPaused => _paused && !_userRequested; + public string? CurrentPath => _currentPath; public void Pause() => _paused = true; public void Resume() => _paused = false; public void BeginUserRequested() => _userRequested = true; @@ -30,6 +42,61 @@ public sealed class DuplicateHashWorker : BackgroundService, IIdleHashWork public async Task HasPendingAsync(CancellationToken cancellationToken = default) => await _store.Hashes.HasPendingAsync(cancellationToken).ConfigureAwait(false); + public Task CountPendingAsync(CancellationToken cancellationToken = default) + { + long cached; + var never = false; + lock (_pendingGate) + { + cached = _pendingCount; + never = _pendingCountUtc == DateTimeOffset.MinValue; + if (!never && DateTimeOffset.UtcNow - _pendingCountUtc < TimeSpan.FromSeconds(15)) + { + return Task.FromResult(cached); + } + } + + if (never) + { + return RefreshPendingCountAsync(cancellationToken); + } + + if (Interlocked.CompareExchange(ref _pendingRefreshBusy, 1, 0) == 0) + { + _ = RefreshPendingCountInBackgroundAsync(); + } + + return Task.FromResult(cached); + } + + private async Task RefreshPendingCountInBackgroundAsync() + { + try + { + await RefreshPendingCountAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Background hash-pending count failed"); + } + finally + { + Interlocked.Exchange(ref _pendingRefreshBusy, 0); + } + } + + private async Task RefreshPendingCountAsync(CancellationToken cancellationToken) + { + var count = await _store.Hashes.CountPendingAsync(cancellationToken).ConfigureAwait(false); + lock (_pendingGate) + { + _pendingCount = count; + _pendingCountUtc = DateTimeOffset.UtcNow; + } + + return count; + } + public async Task ProcessPendingAsync(CancellationToken cancellationToken) { var batch = await _store.Hashes.DequeueAsync(8, cancellationToken).ConfigureAwait(false); @@ -46,6 +113,8 @@ public sealed class DuplicateHashWorker : BackgroundService, IIdleHashWork } var path = PathRules.Combine(item.RootPath, item.PathRel); + _currentPath = path; + _activity.Record("Hash", item.State + " · " + path); try { if (!File.Exists(path)) @@ -90,6 +159,10 @@ public sealed class DuplicateHashWorker : BackgroundService, IIdleHashWork _logger.LogDebug(ex, "Hash failed for {Path}", path); await _store.Hashes.MarkErrorAsync(item.EntryId, cancellationToken).ConfigureAwait(false); } + finally + { + _currentPath = null; + } } } diff --git a/src/Explorer.App/DatabaseWindow.xaml b/src/Explorer.App/DatabaseWindow.xaml new file mode 100644 index 0000000..a349f84 --- /dev/null +++ b/src/Explorer.App/DatabaseWindow.xaml @@ -0,0 +1,78 @@ + + + +