Explorer-Workbench/src/Explorer.Indexing/FolderReconciler.cs

using Explorer.Application;
using Explorer.Domain;
using Explorer.Domain.Abstractions;

namespace Explorer.Indexing;

public sealed class FolderReconciler
{
    private readonly IIndexStore _store;
    private readonly IFileSystemEnumerator _enumerator;
    private readonly StorageProviderRegistry _providers;
    private readonly ArchiveContentsIndexer? _archives;
    private readonly UiPreferencesStore? _preferences;

    public FolderReconciler(
        IIndexStore store,
        IFileSystemEnumerator enumerator,
        StorageProviderRegistry providers,
        ArchiveContentsIndexer? archives = null,
        UiPreferencesStore? preferences = null)
    {
        _store = store;
        _enumerator = enumerator;
        _providers = providers;
        _archives = archives;
        _preferences = preferences;
    }

    public Task<bool> ReconcileAsync(Source source, string pathRel, CancellationToken cancellationToken)
        => ReconcileAsync(source, pathRel, cancellationToken, verifyChildren: false);

    public Task<bool> ReconcileAsync(
        Source source,
        string pathRel,
        CancellationToken cancellationToken,
        bool verifyChildren)
        => ReconcileCoreAsync(source, pathRel, verifyChildren, remaining: 48, cancellationToken);

    private async Task<bool> ReconcileCoreAsync(
        Source source,
        string pathRel,
        bool verifyChildren,
        int remaining,
        CancellationToken cancellationToken)
    {
        if (string.IsNullOrEmpty(source.LastRootPath))
        {
            return false;
        }

        var full = PathRules.Combine(source.LastRootPath, pathRel);
        var parent = await _store.Entries.GetByPathAsync(source.Id, pathRel, cancellationToken).ConfigureAwait(false);
        if (parent is null)
        {
            return false;
        }

        if (!parent.IsDirectory)
        {
            if (ArchiveFormats.IsArchive(parent.Name))
            {
                if (_archives is { IsEnabled: true } && File.Exists(full))
                {
                    await _archives.ExpandIfNeededAsync(source, parent, DateTimeOffset.UtcNow, source.ScanGeneration, cancellationToken)
                        .ConfigureAwait(false);
                    return false;
                }

                await _store.Entries.TombstoneByPathPrefixAsync(source.Id, parent.PathRel, DateTimeOffset.UtcNow, cancellationToken)
                    .ConfigureAwait(false);
                return true;
            }

            return false;
        }

        if (LocationClassifier.IsRecycleBinName(parent.Name))
        {
            return false;
        }

        if (!IndexedPathPresence.DirectoryExists(full))
        {
            if (string.IsNullOrEmpty(pathRel))
            {
                return false;
            }

            var gone = DateTimeOffset.UtcNow;
            await _store.Entries.TombstoneByPathPrefixAsync(source.Id, parent.PathRel, gone, cancellationToken)
                .ConfigureAwait(false);
            await _store.Entries.TombstoneAsync(parent.Id, gone, cancellationToken).ConfigureAwait(false);
            return true;
        }

        var live = await _providers.EnrichAsync(_enumerator.EnumerateChildrenSafe(full, out _), cancellationToken)
            .ConfigureAwait(false);
        var indexed = await _store.Entries.GetChildrenAsync(source.Id, parent.Id, EntryStatus.Present, cancellationToken)
            .ConfigureAwait(false);
        var now = DateTimeOffset.UtcNow;
        var liveNames = new HashSet<string>(live.Select(i => NameNormalizer.Normalize(i.Name)), StringComparer.Ordinal);
        var archivesToExpand = new List<IndexEntry>();
        var archivesToTomb = new List<string>();
        var prefs = _preferences?.Load() ?? UiPreferences.Default;
        var changed = false;

        await _store.RunWriteAsync(async s =>
        {
            foreach (var item in live)
            {
                var location = LocationClassifier.Classify(item.FullPath, item.Name, item.Attributes, item.IsDirectory);
                if (!LocationVisibility.ShouldShow(location, prefs) || location.IsRecycleBin)
                {
                    continue;
                }

                var rel = PathRules.MakeRelative(source.LastRootPath, item.FullPath);
                var entry = new IndexEntry
                {
                    SourceId = source.Id,
                    ParentId = parent.Id,
                    Name = item.Name,
                    NameNorm = NameNormalizer.Normalize(item.Name),
                    Extension = item.IsDirectory ? null : NameNormalizer.Extension(item.Name),
                    IsDirectory = item.IsDirectory,
                    SizeBytes = item.IsDirectory ? 0 : item.SizeBytes,
                    AggregateSize = item.IsDirectory ? 0 : item.SizeBytes,
                    CreatedUtc = item.CreatedUtc,
                    ModifiedUtc = item.ModifiedUtc,
                    LastSeenUtc = now,
                    LastIndexedUtc = now,
                    Attributes = item.Attributes,
                    FileId = item.FileId,
                    ReparseTag = item.ReparseTag,
                    Status = EntryStatus.Present,
                    PathRel = rel,
                    ScanGeneration = source.ScanGeneration,
                    AllocatedSizeBytes = item.AllocatedSizeBytes ?? item.Cloud?.AllocatedSizeBytes,
                    CloudAvailability = item.Cloud?.Availability
                };

                var existing = indexed.FirstOrDefault(e => e.NameNorm == entry.NameNorm);
                var oldSize = existing is { IsDirectory: false } ? existing.SizeBytes : 0;
                var id = await s.Entries.UpsertAsync(entry, cancellationToken).ConfigureAwait(false);
                entry.Id = id;
                if (!item.IsDirectory)
                {
                    var delta = item.SizeBytes - oldSize;
                    if (existing is null)
                    {
                        changed = true;
                        await s.Entries.ApplySizeDeltaToAncestorsAsync(parent.Id, item.SizeBytes, 1, 0, cancellationToken)
                            .ConfigureAwait(false);
                    }
                    else if (delta != 0)
                    {
                        changed = true;
                        await s.Entries.ApplySizeDeltaToAncestorsAsync(parent.Id, delta, 0, 0, cancellationToken)
                            .ConfigureAwait(false);
                    }

                    if (ArchiveFormats.IsArchive(item.Name))
                    {
                        archivesToExpand.Add(entry);
                    }
                }
                else if (existing is null)
                {
                    changed = true;
                    await s.Entries.ApplySizeDeltaToAncestorsAsync(parent.Id, 0, 0, 1, cancellationToken)
                        .ConfigureAwait(false);
                }
            }

            foreach (var old in indexed)
            {
                if (!liveNames.Contains(old.NameNorm))
                {
                    changed = true;
                    await s.Entries.TombstoneAsync(old.Id, now, cancellationToken).ConfigureAwait(false);
                    if (old.IsDirectory)
                    {
                        archivesToTomb.Add(old.PathRel);
                    }
                    else if (ArchiveFormats.IsArchive(old.Name))
                    {
                        archivesToTomb.Add(old.PathRel);
                    }
                }
            }
        }, cancellationToken).ConfigureAwait(false);

        await _store.Entries.RefreshCategoryFromChildrenAsync(parent.Id, cancellationToken)
            .ConfigureAwait(false);

        foreach (var path in archivesToTomb)
        {
            await _store.Entries.TombstoneByPathPrefixAsync(source.Id, path, now, cancellationToken)
                .ConfigureAwait(false);
        }

        if (_archives is { IsEnabled: true })
        {
            foreach (var archive in archivesToExpand)
            {
                await _archives.ExpandIfNeededAsync(source, archive, now, source.ScanGeneration, cancellationToken)
                    .ConfigureAwait(false);
                await _store.Entries.RefreshCategoryFromChildrenAsync(archive.Id, cancellationToken)
                    .ConfigureAwait(false);
            }
        }

        if (!verifyChildren || remaining <= 0)
        {
            return changed;
        }

        changed |= await WalkSizedChildrenAsync(source, indexed, liveNames, remaining, cancellationToken)
            .ConfigureAwait(false);
        return changed;
    }

    private async Task<bool> WalkSizedChildrenAsync(
        Source source,
        IReadOnlyList<IndexEntry> indexed,
        HashSet<string> liveNames,
        int remaining,
        CancellationToken cancellationToken)
    {
        var changed = false;
        foreach (var child in indexed
                     .Where(e => e.IsDirectory
                                 && liveNames.Contains(e.NameNorm)
                                 && (e.AggregateSize > 0 || e.ChildFileCount > 0 || e.ChildDirCount > 0))
                     .OrderByDescending(e => e.AggregateSize))
        {
            if (remaining <= 0)
            {
                break;
            }

            remaining--;
            var full = PathRules.Combine(source.LastRootPath!, child.PathRel);
            var liveKids = _enumerator.EnumerateChildrenSafe(full, out var error);
            if (error is not null)
            {
                continue;
            }

            var indexedKids = await _store.Entries.GetChildrenAsync(source.Id, child.Id, EntryStatus.Present, cancellationToken)
                .ConfigureAwait(false);
            var childLiveNames = new HashSet<string>(liveKids.Select(i => NameNormalizer.Normalize(i.Name)), StringComparer.Ordinal);
            var matches = liveKids.Count == indexedKids.Count && indexedKids.All(e => childLiveNames.Contains(e.NameNorm));
            if (!matches)
            {
                changed |= await ReconcileCoreAsync(source, child.PathRel, verifyChildren: true, remaining, cancellationToken)
                    .ConfigureAwait(false);
                continue;
            }

            changed |= await WalkSizedChildrenAsync(source, indexedKids, childLiveNames, remaining, cancellationToken)
                .ConfigureAwait(false);
        }

        return changed;
    }
}