430 lines
18 KiB
C#
430 lines
18 KiB
C#
using Explorer.Application;
|
|
using Explorer.Domain;
|
|
using Explorer.Domain.Abstractions;
|
|
using Explorer.FileOperations;
|
|
using Explorer.Indexing;
|
|
using Explorer.Storage.Sqlite;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
|
|
namespace Explorer.Indexing.Tests;
|
|
|
|
public sealed class IoEnumerator : IFileSystemEnumerator
|
|
{
|
|
public IEnumerable<FileSystemItem> EnumerateChildren(string directoryPath)
|
|
=> EnumerateChildrenSafe(directoryPath, out _);
|
|
|
|
public IReadOnlyList<FileSystemItem> EnumerateChildrenSafe(string directoryPath, out string? error)
|
|
{
|
|
error = null;
|
|
try
|
|
{
|
|
return Directory.EnumerateFileSystemEntries(directoryPath)
|
|
.Select(p => GetItem(p)!)
|
|
.Where(i => i is not null)
|
|
.ToList()!;
|
|
}
|
|
catch (UnauthorizedAccessException)
|
|
{
|
|
error = "Access denied";
|
|
return [];
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
error = ex.Message;
|
|
return [];
|
|
}
|
|
}
|
|
|
|
public FileSystemItem? GetItem(string path)
|
|
{
|
|
if (Directory.Exists(path))
|
|
{
|
|
var d = new DirectoryInfo(path);
|
|
return new FileSystemItem
|
|
{
|
|
FullPath = d.FullName,
|
|
Name = d.Name,
|
|
IsDirectory = true,
|
|
Attributes = (int)d.Attributes,
|
|
CreatedUtc = d.CreationTimeUtc,
|
|
ModifiedUtc = d.LastWriteTimeUtc
|
|
};
|
|
}
|
|
|
|
if (File.Exists(path))
|
|
{
|
|
var f = new FileInfo(path);
|
|
return new FileSystemItem
|
|
{
|
|
FullPath = f.FullName,
|
|
Name = f.Name,
|
|
IsDirectory = false,
|
|
SizeBytes = f.Length,
|
|
Attributes = (int)f.Attributes,
|
|
CreatedUtc = f.CreationTimeUtc,
|
|
ModifiedUtc = f.LastWriteTimeUtc
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public class ScannerTests
|
|
{
|
|
[Fact]
|
|
public async Task Full_scan_indexes_tree_and_folder_sizes()
|
|
{
|
|
var root = CreateTree();
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
var job = await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
Assert.Equal(ScanJobStatus.Done, job.Status);
|
|
Assert.True(job.FilesSeen >= 2);
|
|
var movies = await store.Entries.GetByPathAsync(source.Id, "Movies");
|
|
Assert.NotNull(movies);
|
|
Assert.True(movies!.AggregateSize >= 50);
|
|
var hit = await store.Search.SearchAsync(new SearchRequest { Name = "a.txt", SourceIds = [source.Id] });
|
|
Assert.Contains(hit, e => e.Name == "a.txt");
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Repeated_scan_upserts_without_duplicating()
|
|
{
|
|
var root = CreateTree();
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
source = (await store.Sources.GetAsync(source.Id))!;
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
var count = await store.Entries.CountPresentAsync(source.Id);
|
|
var second = await store.Entries.CountPresentAsync(source.Id);
|
|
Assert.Equal(count, second);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Cancelled_scan_keeps_committed_batches()
|
|
{
|
|
var root = CreateTree();
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
using var cts = new CancellationTokenSource();
|
|
cts.Cancel();
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
|
|
scanner.ScanAsync(source, ScanKind.Full, null, null, cts.Token));
|
|
var loaded = await store.Sources.GetAsync(source.Id);
|
|
Assert.Equal(SourceStatus.Stale, loaded!.Status);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Excludes_skip_matching_names()
|
|
{
|
|
var root = CreateTree();
|
|
Directory.CreateDirectory(Path.Combine(root, "node_modules"));
|
|
await File.WriteAllTextAsync(Path.Combine(root, "node_modules", "x.js"), "x");
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
await store.Excludes.EnsureDefaultsAsync(DefaultExcludes.Create());
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
var hit = await store.Entries.GetByPathAsync(source.Id, "node_modules");
|
|
Assert.Null(hit);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Full_scan_indexes_archive_members_without_double_counting_folder_size()
|
|
{
|
|
var root = CreateTree();
|
|
var zipPath = Path.Combine(root, "pack.zip");
|
|
CreateZip(zipPath, ("docs/inner.txt", new string('x', 40)));
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var indexer = CreateIndexer(store, enabled: true);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance, indexer);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
|
|
var inner = await store.Entries.GetByPathAsync(source.Id, @"pack.zip\docs\inner.txt");
|
|
Assert.NotNull(inner);
|
|
Assert.Equal(40, inner.SizeBytes);
|
|
Assert.False(inner.IsDirectory);
|
|
|
|
var zip = await store.Entries.GetByPathAsync(source.Id, "pack.zip");
|
|
Assert.NotNull(zip);
|
|
Assert.False(zip.IsDirectory);
|
|
Assert.Equal(new FileInfo(zipPath).Length, zip.SizeBytes);
|
|
|
|
var scanRoot = await store.Entries.GetRootAsync(source.Id);
|
|
Assert.Equal(100 + 50 + zip.SizeBytes, scanRoot!.AggregateSize);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Full_scan_skips_archive_contents_when_setting_is_off()
|
|
{
|
|
var root = CreateTree();
|
|
CreateZip(Path.Combine(root, "pack.zip"), ("inner.txt", "hello"));
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
Assert.Null(await store.Entries.GetByPathAsync(source.Id, @"pack.zip\inner.txt"));
|
|
Assert.NotNull(await store.Entries.GetByPathAsync(source.Id, "pack.zip"));
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Folder_reconcile_tombs_archive_members_when_zip_is_removed()
|
|
{
|
|
var root = CreateTree();
|
|
var zipPath = Path.Combine(root, "pack.zip");
|
|
CreateZip(zipPath, ("inner.txt", "hello"));
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var indexer = CreateIndexer(store, enabled: true);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance, indexer);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
File.Delete(zipPath);
|
|
var reconciler = new FolderReconciler(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), indexer);
|
|
await reconciler.ReconcileAsync(source, "", CancellationToken.None);
|
|
var zip = await store.Entries.GetByPathAsync(source.Id, "pack.zip");
|
|
var inner = await store.Entries.GetByPathAsync(source.Id, @"pack.zip\inner.txt");
|
|
Assert.Equal(EntryStatus.Deleted, zip!.Status);
|
|
Assert.Equal(EntryStatus.Deleted, inner!.Status);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Archive_catalog_lists_zip_entry_sizes_without_extracting()
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "ew-scan", Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(dir);
|
|
var zipPath = Path.Combine(dir, "pack.zip");
|
|
CreateZip(zipPath, ("docs/inner.txt", new string('z', 25)));
|
|
try
|
|
{
|
|
var members = new ArchiveCatalog().TryList(zipPath);
|
|
var inner = Assert.Single(members, m => m.RelativePath.Equals(@"docs\inner.txt", StringComparison.OrdinalIgnoreCase));
|
|
Assert.False(inner.IsDirectory);
|
|
Assert.Equal(25, inner.SizeBytes);
|
|
Assert.DoesNotContain(Directory.GetFiles(dir, "*", SearchOption.AllDirectories), p => p.EndsWith("inner.txt", StringComparison.OrdinalIgnoreCase) && !p.EndsWith(".zip", StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(dir);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Verify_reconcile_updates_child_folder_size_after_uninstall()
|
|
{
|
|
var root = CreateTree();
|
|
var unity = Path.Combine(root, "Unity");
|
|
var editor = Path.Combine(unity, "Editor");
|
|
Directory.CreateDirectory(editor);
|
|
await File.WriteAllTextAsync(Path.Combine(editor, "big.bin"), new string('x', 10_000));
|
|
await File.WriteAllTextAsync(Path.Combine(unity, "leftover.txt"), "ok");
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
var before = await store.Entries.GetByPathAsync(source.Id, "Unity");
|
|
Assert.True(before!.AggregateSize >= 10_000);
|
|
|
|
Directory.Delete(editor, recursive: true);
|
|
var reconciler = new FolderReconciler(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance));
|
|
await reconciler.ReconcileAsync(source, "", CancellationToken.None);
|
|
var shallow = await store.Entries.GetByPathAsync(source.Id, "Unity");
|
|
Assert.True(shallow!.AggregateSize >= 10_000);
|
|
|
|
await reconciler.ReconcileAsync(source, "", CancellationToken.None, verifyChildren: true);
|
|
var verified = await store.Entries.GetByPathAsync(source.Id, "Unity");
|
|
Assert.True(verified!.AggregateSize < 1_000);
|
|
var editorEntry = await store.Entries.GetByPathAsync(source.Id, @"Unity\Editor");
|
|
Assert.Equal(EntryStatus.Deleted, editorEntry!.Status);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Verify_from_root_finds_nested_emptied_folder()
|
|
{
|
|
var root = CreateTree();
|
|
var programFiles = Path.Combine(root, "Program Files");
|
|
var unity = Path.Combine(programFiles, "Unity");
|
|
var editor = Path.Combine(unity, "Editor");
|
|
Directory.CreateDirectory(editor);
|
|
await File.WriteAllTextAsync(Path.Combine(editor, "big.bin"), new string('x', 10_000));
|
|
await File.WriteAllTextAsync(Path.Combine(unity, "leftover.txt"), "ok");
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
Directory.Delete(editor, recursive: true);
|
|
var reconciler = new FolderReconciler(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance));
|
|
await reconciler.ReconcileAsync(source, "", CancellationToken.None, verifyChildren: true);
|
|
var verified = await store.Entries.GetByPathAsync(source.Id, @"Program Files\Unity");
|
|
Assert.True(verified!.AggregateSize < 1_000);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Folder_reconcile_tombs_removed_file()
|
|
{
|
|
var root = CreateTree();
|
|
try
|
|
{
|
|
await using var store = await OpenStore();
|
|
var source = await AddSource(store, root);
|
|
var scanner = new FilesystemScanner(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance), NullLogger<FilesystemScanner>.Instance);
|
|
await scanner.ScanAsync(source, ScanKind.Full, null, null, CancellationToken.None);
|
|
File.Delete(Path.Combine(root, "a.txt"));
|
|
var reconciler = new FolderReconciler(store, new IoEnumerator(), new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance));
|
|
await reconciler.ReconcileAsync(source, "", CancellationToken.None);
|
|
var entry = await store.Entries.GetByPathAsync(source.Id, "a.txt");
|
|
Assert.Equal(EntryStatus.Deleted, entry!.Status);
|
|
}
|
|
finally
|
|
{
|
|
TryDelete(root);
|
|
}
|
|
}
|
|
|
|
private static string CreateTree()
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), "ew-scan", Guid.NewGuid().ToString("N"));
|
|
Directory.CreateDirectory(Path.Combine(root, "Movies"));
|
|
File.WriteAllText(Path.Combine(root, "a.txt"), new string('a', 100));
|
|
File.WriteAllText(Path.Combine(root, "Movies", "b.txt"), new string('b', 50));
|
|
return root;
|
|
}
|
|
|
|
private static async Task<SqliteIndexStore> OpenStore()
|
|
{
|
|
var db = Path.Combine(Path.GetTempPath(), "ew-scan", Guid.NewGuid().ToString("N"), "index.db");
|
|
var store = new SqliteIndexStore(db, NullLogger<SqliteIndexStore>.Instance);
|
|
await store.OpenAsync();
|
|
return store;
|
|
}
|
|
|
|
private static async Task<Source> AddSource(IIndexStore store, string root)
|
|
{
|
|
var s = new Source
|
|
{
|
|
StableKey = Guid.NewGuid().ToString("N"),
|
|
Kind = SourceKind.NtfsLocal,
|
|
DisplayName = "t",
|
|
LastRootPath = root,
|
|
Status = SourceStatus.Online
|
|
};
|
|
s.Id = await store.Sources.UpsertAsync(s);
|
|
return s;
|
|
}
|
|
|
|
private static void TryDelete(string root)
|
|
{
|
|
try { Directory.Delete(root, true); } catch { /* ignore */ }
|
|
}
|
|
|
|
private static void CreateZip(string zipPath, params (string Name, string Content)[] files)
|
|
{
|
|
using var zip = System.IO.Compression.ZipFile.Open(zipPath, System.IO.Compression.ZipArchiveMode.Create);
|
|
foreach (var (name, content) in files)
|
|
{
|
|
var entry = zip.CreateEntry(name);
|
|
using var stream = entry.Open();
|
|
using var writer = new StreamWriter(stream);
|
|
writer.Write(content);
|
|
}
|
|
}
|
|
|
|
private static ArchiveContentsIndexer CreateIndexer(IIndexStore store, bool enabled)
|
|
{
|
|
var dir = Path.Combine(Path.GetTempPath(), "ew-scan-prefs", Guid.NewGuid().ToString("N"));
|
|
var prefs = new UiPreferencesStore(new ScanPrefsEnv(dir));
|
|
prefs.Save(new UiPreferences("Dark", false, false, enabled));
|
|
return new ArchiveContentsIndexer(
|
|
store,
|
|
new IoEnumerator(),
|
|
new HydrationGuard(new StorageProviderRegistry([], NullLogger<StorageProviderRegistry>.Instance)),
|
|
new ArchiveCatalog(),
|
|
prefs,
|
|
NullLogger<ArchiveContentsIndexer>.Instance);
|
|
}
|
|
}
|
|
|
|
file sealed class ScanPrefsEnv : IAppEnvironment
|
|
{
|
|
public ScanPrefsEnv(string dir)
|
|
{
|
|
DataDirectory = dir;
|
|
Directory.CreateDirectory(dir);
|
|
DatabasePath = Path.Combine(dir, "index.db");
|
|
LogDirectory = Path.Combine(dir, "logs");
|
|
Directory.CreateDirectory(LogDirectory);
|
|
}
|
|
|
|
public string DataDirectory { get; }
|
|
public string DatabasePath { get; }
|
|
public string LogDirectory { get; }
|
|
}
|