Add Explorer Workbench with hierarchical, off-UI Storage analysis.
Storage queries run in the background with cancellation and covering indexes so switching views no longer freezes the UI. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
18
tests/Explorer.Indexing.Tests/Explorer.Indexing.Tests.csproj
Normal file
18
tests/Explorer.Indexing.Tests/Explorer.Indexing.Tests.csproj
Normal file
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Explorer.Domain\Explorer.Domain.csproj" />
|
||||
<ProjectReference Include="..\..\src\Explorer.Indexing\Explorer.Indexing.csproj" />
|
||||
<ProjectReference Include="..\..\src\Explorer.Application\Explorer.Application.csproj" />
|
||||
<ProjectReference Include="..\..\src\Explorer.Storage.Sqlite\Explorer.Storage.Sqlite.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
221
tests/Explorer.Indexing.Tests/ScannerTests.cs
Normal file
221
tests/Explorer.Indexing.Tests/ScannerTests.cs
Normal file
@@ -0,0 +1,221 @@
|
||||
using Explorer.Application;
|
||||
using Explorer.Domain;
|
||||
using Explorer.Domain.Abstractions;
|
||||
using Explorer.Indexing;
|
||||
using Explorer.Storage.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace Explorer.Indexing.Tests;
|
||||
|
||||
public sealed class IoEnumerator : IFileSystemEnumerator
|
||||
{
|
||||
public IEnumerable<FileSystemItem> EnumerateChildren(string directoryPath)
|
||||
=> EnumerateChildrenSafe(directoryPath, out _);
|
||||
|
||||
public IReadOnlyList<FileSystemItem> EnumerateChildrenSafe(string directoryPath, out string? error)
|
||||
{
|
||||
error = null;
|
||||
try
|
||||
{
|
||||
return Directory.EnumerateFileSystemEntries(directoryPath)
|
||||
.Select(p => GetItem(p)!)
|
||||
.Where(i => i is not null)
|
||||
.ToList()!;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
error = "Access denied";
|
||||
return [];
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex.Message;
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public FileSystemItem? GetItem(string path)
|
||||
{
|
||||
if (Directory.Exists(path))
|
||||
{
|
||||
var d = new DirectoryInfo(path);
|
||||
return new FileSystemItem
|
||||
{
|
||||
FullPath = d.FullName,
|
||||
Name = d.Name,
|
||||
IsDirectory = true,
|
||||
Attributes = (int)d.Attributes,
|
||||
CreatedUtc = d.CreationTimeUtc,
|
||||
ModifiedUtc = d.LastWriteTimeUtc
|
||||
};
|
||||
}
|
||||
|
||||
if (File.Exists(path))
|
||||
{
|
||||
var f = new FileInfo(path);
|
||||
return new FileSystemItem
|
||||
{
|
||||
FullPath = f.FullName,
|
||||
Name = f.Name,
|
||||
IsDirectory = false,
|
||||
SizeBytes = f.Length,
|
||||
Attributes = (int)f.Attributes,
|
||||
CreatedUtc = f.CreationTimeUtc,
|
||||
ModifiedUtc = f.LastWriteTimeUtc
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public class ScannerTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Full_scan_indexes_tree_and_folder_sizes()
|
||||
{
|
||||
var root = CreateTree();
|
||||
try
|
||||
{
|
||||
await using var store = await OpenStore();
|
||||
var source = await AddSource(store, root);
|
||||
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
||||
var job = await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
||||
Assert.Equal(ScanJobStatus.Done, job.Status);
|
||||
Assert.True(job.FilesSeen >= 2);
|
||||
var movies = await store.Entries.GetByPathAsync(source.Id, "Movies");
|
||||
Assert.NotNull(movies);
|
||||
Assert.True(movies!.AggregateSize >= 50);
|
||||
var hit = await store.Search.SearchAsync(new SearchRequest { Name = "a.txt", SourceIds = [source.Id] });
|
||||
Assert.Contains(hit, e => e.Name == "a.txt");
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Repeated_scan_upserts_without_duplicating()
|
||||
{
|
||||
var root = CreateTree();
|
||||
try
|
||||
{
|
||||
await using var store = await OpenStore();
|
||||
var source = await AddSource(store, root);
|
||||
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
||||
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
||||
source = (await store.Sources.GetAsync(source.Id))!;
|
||||
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
||||
var count = await store.Entries.CountPresentAsync(source.Id);
|
||||
var second = await store.Entries.CountPresentAsync(source.Id);
|
||||
Assert.Equal(count, second);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancelled_scan_keeps_committed_batches()
|
||||
{
|
||||
var root = CreateTree();
|
||||
try
|
||||
{
|
||||
await using var store = await OpenStore();
|
||||
var source = await AddSource(store, root);
|
||||
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
||||
using var cts = new CancellationTokenSource();
|
||||
cts.Cancel();
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
|
||||
scanner.ScanAsync(source, ScanKind.Full, null, null, cts.Token));
|
||||
var loaded = await store.Sources.GetAsync(source.Id);
|
||||
Assert.Equal(SourceStatus.Stale, loaded!.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Excludes_skip_matching_names()
|
||||
{
|
||||
var root = CreateTree();
|
||||
Directory.CreateDirectory(Path.Combine(root, "node_modules"));
|
||||
await File.WriteAllTextAsync(Path.Combine(root, "node_modules", "x.js"), "x");
|
||||
try
|
||||
{
|
||||
await using var store = await OpenStore();
|
||||
await store.Excludes.EnsureDefaultsAsync(DefaultExcludes.Create());
|
||||
var source = await AddSource(store, root);
|
||||
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
||||
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
||||
var hit = await store.Entries.GetByPathAsync(source.Id, "node_modules");
|
||||
Assert.Null(hit);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Folder_reconcile_tombs_removed_file()
|
||||
{
|
||||
var root = CreateTree();
|
||||
try
|
||||
{
|
||||
await using var store = await OpenStore();
|
||||
var source = await AddSource(store, root);
|
||||
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
||||
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
||||
File.Delete(Path.Combine(root, "a.txt"));
|
||||
var reconciler = new FolderReconciler(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance));
|
||||
await reconciler.ReconcileAsync(source, "", CancellationToken.None);
|
||||
var entry = await store.Entries.GetByPathAsync(source.Id, "a.txt");
|
||||
Assert.Equal(EntryStatus.Deleted, entry!.Status);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(root);
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateTree()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), "ew-scan", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.Combine(root, "Movies"));
|
||||
File.WriteAllText(Path.Combine(root, "a.txt"), new string('a', 100));
|
||||
File.WriteAllText(Path.Combine(root, "Movies", "b.txt"), new string('b', 50));
|
||||
return root;
|
||||
}
|
||||
|
||||
private static async Task<SqliteIndexStore> OpenStore()
|
||||
{
|
||||
var db = Path.Combine(Path.GetTempPath(), "ew-scan", Guid.NewGuid().ToString("N"), "index.db");
|
||||
var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
|
||||
await store.OpenAsync();
|
||||
return store;
|
||||
}
|
||||
|
||||
private static async Task<Source> AddSource(IIndexStore store, string root)
|
||||
{
|
||||
var s = new Source
|
||||
{
|
||||
StableKey = Guid.NewGuid().ToString("N"),
|
||||
Kind = SourceKind.NtfsLocal,
|
||||
DisplayName = "t",
|
||||
LastRootPath = root,
|
||||
Status = SourceStatus.Online
|
||||
};
|
||||
s.Id = await store.Sources.UpsertAsync(s);
|
||||
return s;
|
||||
}
|
||||
|
||||
private static void TryDelete(string root)
|
||||
{
|
||||
try { Directory.Delete(root, true); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user