quick/Explorer-Workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity Open in Code-Server

Add settings, cloud places, and optional archive-content indexing.

Browse Source 
Keep official clients in charge of sync while Explorer can group locations, persist UI prefs, and list zip/rar/7z members without extracting them.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
netquick
2026-08-23 12:16:09 +02:00
parent e9aba73552
commit 09a8cfafa3
57 changed files with 3130 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/Explorer.Indexing/ArchiveCatalog.cs Normal file
View File

@@ -0,0 +1,96 @@
using System.IO.Compression;
using Explorer.Application;
using Explorer.Domain;
using SharpCompress.Archives;
using SharpCompress.Readers;
namespace Explorer.Indexing;
public sealed class ArchiveCatalog : IArchiveCatalog
{
public IReadOnlyList<ArchiveMember> TryList(string archivePath, CancellationToken cancellationToken = default)
{
try
{
if (!File.Exists(archivePath))
{
return [];
}
var name = Path.GetFileName(archivePath);
if (ArchiveFormats.IsZipFamily(name))
{
return ListZip(archivePath, cancellationToken);
}
return ListGeneric(archivePath, cancellationToken);
}
catch
{
return [];
}
}
private static IReadOnlyList<ArchiveMember> ListZip(string archivePath, CancellationToken cancellationToken)
{
using var stream = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
using var zip = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: false);
return Collect(zip.Entries.Select(e => (
e.FullName,
e.FullName.EndsWith('/') || e.FullName.EndsWith('\\'),
e.Length)), cancellationToken);
}
private static IReadOnlyList<ArchiveMember> ListGeneric(string archivePath, CancellationToken cancellationToken)
{
try
{
using var archive = ArchiveFactory.OpenArchive(archivePath);
return Collect(archive.Entries.Select(e => (
e.Key ?? "",
e.IsDirectory,
e.Size)), cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
using var reader = ReaderFactory.OpenReader(archivePath);
var raw = new List<(string Path, bool IsDirectory, long Size)>();
while (reader.MoveToNextEntry())
{
cancellationToken.ThrowIfCancellationRequested();
raw.Add((reader.Entry.Key ?? "", reader.Entry.IsDirectory, reader.Entry.Size));
}
return Collect(raw, cancellationToken);
}
}
private static IReadOnlyList<ArchiveMember> Collect(
IEnumerable<(string Path, bool IsDirectory, long Size)> entries,
CancellationToken cancellationToken)
{
var seen = new Dictionary<string, ArchiveMember>(StringComparer.OrdinalIgnoreCase);
foreach (var (path, isDirectory, size) in entries)
{
cancellationToken.ThrowIfCancellationRequested();
if (seen.Count >= AppConstants.MaxArchiveEntries)
{
break;
}
if (!ArchiveFormats.TryNormalizeEntryPath(path, out var relative))
{
continue;
}
var dir = isDirectory || path.EndsWith('/') || path.EndsWith('\\');
seen[relative] = new ArchiveMember(relative, dir, dir ? 0 : Math.Max(0, size));
}
return seen.Values.ToList();
}
}

111
src/Explorer.Indexing/ArchiveContentsIndexer.cs Normal file
View File

@@ -0,0 +1,111 @@
using Explorer.Application;
using Explorer.Domain;
using Explorer.Domain.Abstractions;
using Microsoft.Extensions.Logging;
namespace Explorer.Indexing;
public sealed class ArchiveContentsIndexer
{
private readonly IIndexStore _store;
private readonly IFileSystemEnumerator _enumerator;
private readonly IHydrationGuard _hydration;
private readonly IArchiveCatalog _catalog;
private readonly UiPreferencesStore _preferences;
private readonly ILogger<ArchiveContentsIndexer> _logger;
public ArchiveContentsIndexer(
IIndexStore store,
IFileSystemEnumerator enumerator,
IHydrationGuard hydration,
IArchiveCatalog catalog,
UiPreferencesStore preferences,
ILogger<ArchiveContentsIndexer> logger)
{
_store = store;
_enumerator = enumerator;
_hydration = hydration;
_catalog = catalog;
_preferences = preferences;
_logger = logger;
}
public bool IsEnabled => _preferences.Load().IndexArchiveContents;
public Task TombstoneInnerAsync(long sourceId, string archivePathRel, DateTimeOffset utc, CancellationToken cancellationToken)
=> _store.Entries.TombstoneByPathPrefixAsync(sourceId, archivePathRel, utc, cancellationToken);
public async Task ExpandIfNeededAsync(
Source source,
IndexEntry archive,
DateTimeOffset now,
long generation,
CancellationToken cancellationToken)
{
if (!IsEnabled
|| archive.IsDirectory
|| archive.Id <= 0
|| string.IsNullOrEmpty(source.LastRootPath)
|| !ArchiveFormats.IsArchive(archive.Name))
{
return;
}
var full = PathRules.Combine(source.LastRootPath, archive.PathRel);
var item = _enumerator.GetItem(full);
if (item is null)
{
return;
}
if (_hydration.WouldHydrateOnRead(item)
|| await _hydration.WouldHydrateOnReadAsync(full, cancellationToken).ConfigureAwait(false))
{
return;
}
IReadOnlyList<ArchiveMember> members;
try
{
members = _catalog.TryList(full, cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Archive listing failed for {Path}", full);
return;
}
var nodes = ArchiveTreeBuilder.Build(archive, members);
await _store.RunWriteAsync(async s =>
{
await s.Entries.TombstoneByPathPrefixAsync(source.Id, archive.PathRel, now, cancellationToken)
.ConfigureAwait(false);
var ids = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase)
{
[archive.PathRel] = archive.Id
};
foreach (var node in nodes)
{
if (!ids.TryGetValue(node.ParentPathRel, out var parentId))
{
continue;
}
var entry = node.ToEntry(source, parentId, now, generation);
entry.Id = await s.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
ids[node.PathRel] = entry.Id;
if (node.IsDirectory)
{
await s.Entries.UpdateAggregatesAsync(
entry.Id, node.AggregateSize, node.ChildFiles, node.ChildDirs, cancellationToken)
.ConfigureAwait(false);
}
}
}, cancellationToken).ConfigureAwait(false);
}
}

139
src/Explorer.Indexing/ArchiveTreeBuilder.cs Normal file
View File

@@ -0,0 +1,139 @@
using Explorer.Application;
using Explorer.Domain;
namespace Explorer.Indexing;
internal sealed record ArchiveNode(
string PathRel,
string ParentPathRel,
string Name,
bool IsDirectory,
long SizeBytes,
long AggregateSize,
int ChildFiles,
int ChildDirs);
internal static class ArchiveTreeBuilder
{
public static IReadOnlyList<ArchiveNode> Build(
IndexEntry archive,
IReadOnlyList<ArchiveMember> members)
{
var dirs = new Dictionary<string, DirAcc>(StringComparer.OrdinalIgnoreCase);
var files = new List<(string PathRel, string Name, long Size)>();
foreach (var member in members)
{
var parts = member.RelativePath.Split('\\', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 0)
{
continue;
}
var dirCount = member.IsDirectory ? parts.Length : parts.Length - 1;
var prefix = "";
for (var i = 0; i < dirCount; i++)
{
prefix = prefix.Length == 0 ? parts[i] : prefix + "\\" + parts[i];
dirs.TryAdd(prefix, new DirAcc(parts[i]));
}
if (!member.IsDirectory)
{
files.Add((member.RelativePath, parts[^1], member.SizeBytes));
}
}
foreach (var file in files)
{
var parent = PathRules.RelativeParent(file.PathRel);
while (parent.Length > 0)
{
if (dirs.TryGetValue(parent, out var acc))
{
acc.Size += file.Size;
acc.Files++;
}
parent = PathRules.RelativeParent(parent);
}
}
foreach (var dirRel in dirs.Keys)
{
var parent = PathRules.RelativeParent(dirRel);
while (parent.Length > 0)
{
if (dirs.TryGetValue(parent, out var acc))
{
acc.Dirs++;
}
parent = PathRules.RelativeParent(parent);
}
}
var nodes = new List<ArchiveNode>(dirs.Count + files.Count);
foreach (var (rel, acc) in dirs)
{
nodes.Add(ToNode(archive.PathRel, rel, acc.Name, true, 0, acc.Size, acc.Files, acc.Dirs));
}
foreach (var file in files)
{
nodes.Add(ToNode(archive.PathRel, file.PathRel, file.Name, false, file.Size, file.Size, 0, 0));
}
return nodes
.OrderBy(n => n.PathRel.Count(c => c == '\\'))
.ThenBy(n => n.PathRel, StringComparer.OrdinalIgnoreCase)
.ToList();
}
public static IndexEntry ToEntry(this ArchiveNode node, Source source, long parentId, DateTimeOffset now, long generation)
=> new()
{
SourceId = source.Id,
ParentId = parentId,
Name = node.Name,
NameNorm = NameNormalizer.Normalize(node.Name),
Extension = node.IsDirectory ? null : NameNormalizer.Extension(node.Name),
IsDirectory = node.IsDirectory,
SizeBytes = node.IsDirectory ? 0 : node.SizeBytes,
AggregateSize = node.AggregateSize,
ChildFileCount = node.ChildFiles,
ChildDirCount = node.ChildDirs,
LastSeenUtc = now,
LastIndexedUtc = now,
Status = EntryStatus.Present,
PathRel = node.PathRel,
ScanGeneration = generation,
HashState = HashState.Skipped
};
private static ArchiveNode ToNode(
string archivePathRel,
string innerRel,
string name,
bool isDir,
long size,
long aggregate,
int files,
int dirs)
{
var pathRel = string.IsNullOrEmpty(archivePathRel) ? innerRel : archivePathRel + "\\" + innerRel;
var parentInner = PathRules.RelativeParent(innerRel);
var parentPathRel = parentInner.Length == 0
? archivePathRel
: (string.IsNullOrEmpty(archivePathRel) ? parentInner : archivePathRel + "\\" + parentInner);
return new ArchiveNode(pathRel, parentPathRel, name, isDir, size, aggregate, files, dirs);
}
private sealed class DirAcc(string name)
{
public string Name { get; } = name;
public long Size { get; set; }
public int Files { get; set; }
public int Dirs { get; set; }
}
}

1
src/Explorer.Indexing/Explorer.Indexing.csproj
View File

@@ -5,6 +5,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
<PackageReference Include="SharpCompress" Version="0.50.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Explorer.Application\Explorer.Application.csproj" />

40
src/Explorer.Indexing/FilesystemScanner.cs
View File

@@ -11,17 +11,20 @@ public sealed class FilesystemScanner
private readonly IFileSystemEnumerator _enumerator;
private readonly StorageProviderRegistry _providers;
private readonly ILogger<FilesystemScanner> _logger;
private readonly ArchiveContentsIndexer? _archives;
public FilesystemScanner(
IIndexStore store,
IFileSystemEnumerator enumerator,
StorageProviderRegistry providers,
ILogger<FilesystemScanner> logger)
ILogger<FilesystemScanner> logger,
ArchiveContentsIndexer? archives = null)
{
_store = store;
_enumerator = enumerator;
_providers = providers;
_logger = logger;
_archives = archives;
}
public async Task<ScanJob> ScanAsync(
@@ -155,7 +158,8 @@ public sealed class FilesystemScanner
}
}
await FlushAsync(pending, childDirs, cancellationToken).ConfigureAwait(false);
await FlushAsync(pending, childDirs, source, now, generation, expandArchives: true, cancellationToken)
.ConfigureAwait(false);
frame.Expanded = true;
for (var i = childDirs.Count - 1; i >= 0; i--)
{
@@ -184,7 +188,8 @@ public sealed class FilesystemScanner
}
}
await FlushAsync(pending, [], cancellationToken).ConfigureAwait(false);
await FlushAsync(pending, [], source, now, generation, expandArchives: true, cancellationToken)
.ConfigureAwait(false);
await _store.Entries.MarkMissingAsDeletedAsync(source.Id, generation, DateTimeOffset.UtcNow, string.IsNullOrEmpty(startRel) ? null : startRel, cancellationToken)
.ConfigureAwait(false);
await _store.Sources.UpdateIndexedAsync(source.Id, DateTimeOffset.UtcNow, generation, cancellationToken)
@@ -198,7 +203,8 @@ public sealed class FilesystemScanner
}
catch (OperationCanceledException)
{
await FlushAsync(pending, [], CancellationToken.None).ConfigureAwait(false);
await FlushAsync(pending, [], source, now, generation, expandArchives: true, CancellationToken.None)
.ConfigureAwait(false);
job.Status = ScanJobStatus.Cancelled;
job.FinishedUtc = DateTimeOffset.UtcNow;
await _store.ScanJobs.UpdateAsync(job, CancellationToken.None).ConfigureAwait(false);
@@ -219,7 +225,14 @@ public sealed class FilesystemScanner
}
}
private async Task FlushAsync(List<IndexEntry> pending, List<Frame> dirs, CancellationToken cancellationToken)
private async Task FlushAsync(
List<IndexEntry> pending,
List<Frame> dirs,
Source source,
DateTimeOffset now,
long generation,
bool expandArchives,
CancellationToken cancellationToken)
{
if (pending.Count == 0)
{
@@ -244,7 +257,24 @@ public sealed class FilesystemScanner
}
}
var flushed = pending.ToList();
pending.Clear();
if (!expandArchives || _archives is null)
{
return;
}
foreach (var entry in flushed)
{
if (entry.IsDirectory || !ArchiveFormats.IsArchive(entry.Name))
{
continue;
}
await _archives.ExpandIfNeededAsync(source, entry, now, generation, cancellationToken)
.ConfigureAwait(false);
}
}
private static ScanProgress ToProgress(ScanJob job, string path) => new()

62
src/Explorer.Indexing/FolderReconciler.cs
View File

@@ -1,7 +1,6 @@
using Explorer.Application;
using Explorer.Domain;
using Explorer.Domain.Abstractions;
using Microsoft.Extensions.Logging;
namespace Explorer.Indexing;
@@ -10,12 +9,18 @@ public sealed class FolderReconciler
private readonly IIndexStore _store;
private readonly IFileSystemEnumerator _enumerator;
private readonly StorageProviderRegistry _providers;
private readonly ArchiveContentsIndexer? _archives;
public FolderReconciler(IIndexStore store, IFileSystemEnumerator enumerator, StorageProviderRegistry providers)
public FolderReconciler(
IIndexStore store,
IFileSystemEnumerator enumerator,
StorageProviderRegistry providers,
ArchiveContentsIndexer? archives = null)
{
_store = store;
_enumerator = enumerator;
_providers = providers;
_archives = archives;
}
public async Task ReconcileAsync(Source source, string pathRel, CancellationToken cancellationToken)
@@ -32,12 +37,38 @@ public sealed class FolderReconciler
return;
}
if (!parent.IsDirectory)
{
if (ArchiveFormats.IsArchive(parent.Name))
{
if (_archives is { IsEnabled: true } && File.Exists(full))
{
await _archives.ExpandIfNeededAsync(source, parent, DateTimeOffset.UtcNow, source.ScanGeneration, cancellationToken)
.ConfigureAwait(false);
}
else
{
await _store.Entries.TombstoneByPathPrefixAsync(source.Id, parent.PathRel, DateTimeOffset.UtcNow, cancellationToken)
.ConfigureAwait(false);
}
}
return;
}
if (!Directory.Exists(full))
{
return;
}
var live = await _providers.EnrichAsync(_enumerator.EnumerateChildrenSafe(full, out _), cancellationToken)
.ConfigureAwait(false);
var indexed = await _store.Entries.GetChildrenAsync(source.Id, parent.Id, EntryStatus.Present, cancellationToken)
.ConfigureAwait(false);
var now = DateTimeOffset.UtcNow;
var liveNames = new HashSet<string>(live.Select(i => NameNormalizer.Normalize(i.Name)), StringComparer.Ordinal);
var archivesToExpand = new List<IndexEntry>();
var archivesToTomb = new List<string>();
await _store.RunWriteAsync(async s =>
{
@@ -71,6 +102,7 @@ public sealed class FolderReconciler
var existing = indexed.FirstOrDefault(e => e.NameNorm == entry.NameNorm);
var oldSize = existing is { IsDirectory: false } ? existing.SizeBytes : 0;
var id = await s.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
entry.Id = id;
if (!item.IsDirectory)
{
var delta = item.SizeBytes - oldSize;
@@ -84,14 +116,17 @@ public sealed class FolderReconciler
await s.Entries.ApplySizeDeltaToAncestorsAsync(parent.Id, delta, 0, 0, cancellationToken)
.ConfigureAwait(false);
}
if (ArchiveFormats.IsArchive(item.Name))
{
archivesToExpand.Add(entry);
}
}
else if (existing is null)
{
await s.Entries.ApplySizeDeltaToAncestorsAsync(parent.Id, 0, 0, 1, cancellationToken)
.ConfigureAwait(false);
}
_ = id;
}
foreach (var old in indexed)
@@ -99,8 +134,27 @@ public sealed class FolderReconciler
if (!liveNames.Contains(old.NameNorm))
{
await s.Entries.TombstoneAsync(old.Id, now, cancellationToken).ConfigureAwait(false);
if (!old.IsDirectory && ArchiveFormats.IsArchive(old.Name))
{
archivesToTomb.Add(old.PathRel);
}
}
}
}, cancellationToken).ConfigureAwait(false);
foreach (var path in archivesToTomb)
{
await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken)
.ConfigureAwait(false);
}
if (_archives is { IsEnabled: true })
{
foreach (var archive in archivesToExpand)
{
await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
.ConfigureAwait(false);
}
}
}
}

49
src/Explorer.Indexing/UsnChangeApplier.cs
View File

@@ -9,12 +9,18 @@ public sealed class UsnChangeApplier
private readonly IIndexStore _store;
private readonly IFileSystemEnumerator _enumerator;
private readonly ILogger<UsnChangeApplier> _logger;
private readonly ArchiveContentsIndexer? _archives;
public UsnChangeApplier(IIndexStore store, IFileSystemEnumerator enumerator, ILogger<UsnChangeApplier> logger)
public UsnChangeApplier(
IIndexStore store,
IFileSystemEnumerator enumerator,
ILogger<UsnChangeApplier> logger,
ArchiveContentsIndexer? archives = null)
{
_store = store;
_enumerator = enumerator;
_logger = logger;
_archives = archives;
}
public async Task<UsnReadStatus> ApplyAsync(Source source, IUsnJournal journal, CancellationToken cancellationToken)
@@ -64,6 +70,7 @@ public sealed class UsnChangeApplier
var coalesced = Coalesce(records);
var now = DateTimeOffset.UtcNow;
var expand = new List<IndexEntry>();
await _store.RunWriteAsync(async s =>
{
foreach (var rec in coalesced)
@@ -71,7 +78,11 @@ public sealed class UsnChangeApplier
cancellationToken.ThrowIfCancellationRequested();
try
{
await ApplyRecord(s, source, rec, now, cancellationToken).ConfigureAwait(false);
var updated = await ApplyRecord(s, source, rec, now, cancellationToken).ConfigureAwait(false);
if (updated is not null)
{
expand.Add(updated);
}
}
catch (Exception ex)
{
@@ -82,10 +93,19 @@ public sealed class UsnChangeApplier
await s.Sources.UpdateUsnAsync(source.Id, next.JournalId, next.NextUsn, cancellationToken).ConfigureAwait(false);
}, cancellationToken).ConfigureAwait(false);
if (_archives is { IsEnabled: true })
{
foreach (var archive in expand)
{
await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
.ConfigureAwait(false);
}
}
return UsnReadStatus.Ok;
}
private async Task ApplyRecord(IIndexStore store, Source source, UsnRecord rec, DateTimeOffset now, CancellationToken cancellationToken)
private async Task<IndexEntry?> ApplyRecord(IIndexStore store, Source source, UsnRecord rec, DateTimeOffset now, CancellationToken cancellationToken)
{
var existing = rec.FileReferenceNumber != 0
? await store.Entries.GetByFileIdAsync(source.Id, rec.FileReferenceNumber, cancellationToken).ConfigureAwait(false)
@@ -96,9 +116,14 @@ public sealed class UsnChangeApplier
if (existing is not null)
{
await store.Entries.TombstoneAsync(existing.Id, now, cancellationToken).ConfigureAwait(false);
if (!existing.IsDirectory && ArchiveFormats.IsArchive(existing.Name))
{
await store.Entries.TombstoneByPathPrefixAsync(source.Id, existing.PathRel, now, cancellationToken)
.ConfigureAwait(false);
}
}
return;
return null;
}
var parent = rec.ParentFileReferenceNumber != 0
@@ -115,9 +140,14 @@ public sealed class UsnChangeApplier
if (existing is not null)
{
await store.Entries.TombstoneAsync(existing.Id, now, cancellationToken).ConfigureAwait(false);
if (!existing.IsDirectory && ArchiveFormats.IsArchive(existing.Name))
{
await store.Entries.TombstoneByPathPrefixAsync(source.Id, existing.PathRel, now, cancellationToken)
.ConfigureAwait(false);
}
}
return;
return null;
}
var oldSize = existing is { IsDirectory: false } ? existing.SizeBytes : 0;
@@ -150,7 +180,7 @@ public sealed class UsnChangeApplier
.ConfigureAwait(false);
}
await store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
entry.Id = await store.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
if (!live.IsDirectory)
{
var delta = live.SizeBytes - oldSize;
@@ -164,7 +194,14 @@ public sealed class UsnChangeApplier
await store.Entries.ApplySizeDeltaToAncestorsAsync(parent?.Id, delta, 0, 0, cancellationToken)
.ConfigureAwait(false);
}
if (ArchiveFormats.IsArchive(live.Name))
{
return entry;
}
}
return null;
}
private static List<UsnRecord> Coalesce(IReadOnlyList<UsnRecord> records)