Files
Explorer-Workbench/src/Explorer.Application/SourceManager.cs

690 lines
24 KiB
C#

using System.Diagnostics;
using Explorer.Contracts;
using Explorer.Domain;
using Explorer.Domain.Abstractions;
using Microsoft.Extensions.Logging;
namespace Explorer.Application;
public sealed class SourceManager
{
private readonly IIndexStore _store;
private readonly IVolumeService _volumes;
private readonly IAppEnvironment _env;
private readonly IClock _clock;
private readonly ILogger<SourceManager> _logger;
private readonly ISourceHost? _remote;
private readonly object _refreshLock = new();
private Task<IReadOnlyList<Source>>? _refreshInFlight;
private long _refreshCacheTimestamp;
private static readonly TimeSpan RefreshCacheTtl = TimeSpan.FromSeconds(2);
public event EventHandler<Source>? PresenceChanged;
public SourceManager(
IIndexStore store,
IVolumeService volumes,
IAppEnvironment env,
IClock clock,
ILogger<SourceManager> logger,
ISourceHost? remote = null)
{
_store = store;
_volumes = volumes;
_env = env;
_clock = clock;
_logger = logger;
_remote = remote;
}
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
await _store.OpenAsync(cancellationToken).ConfigureAwait(false);
if (!_store.CanWrite)
{
if (_remote is not null)
{
await _remote.RefreshAsync(cancellationToken).ConfigureAwait(false);
}
InvalidateRefreshCache();
return;
}
await _store.ScanJobs.InterruptRunningAsync(cancellationToken).ConfigureAwait(false);
await _store.Excludes.EnsureDefaultsAsync(DefaultExcludes.Create(), cancellationToken).ConfigureAwait(false);
await RefreshOnlineStateAsync(cancellationToken).ConfigureAwait(false);
}
public Task<IReadOnlyList<Source>> RefreshOnlineStateAsync(CancellationToken cancellationToken = default)
=> RefreshOnlineStateAsync(forceRefresh: false, cancellationToken);
public Task<IReadOnlyList<Source>> RefreshOnlineStateAsync(bool forceRefresh, CancellationToken cancellationToken = default)
{
lock (_refreshLock)
{
if (!forceRefresh && _refreshInFlight is { IsCompleted: false })
{
return _refreshInFlight;
}
if (!forceRefresh
&& BoundedWait.IsFresh(_refreshCacheTimestamp, RefreshCacheTtl))
{
return _store.Sources.GetAllAsync(cancellationToken);
}
if (forceRefresh && _refreshInFlight is { IsCompleted: false })
{
return _refreshInFlight;
}
_refreshInFlight = RefreshOnlineStateCoreAsync(cancellationToken);
return _refreshInFlight;
}
}
private async Task<IReadOnlyList<Source>> RefreshOnlineStateCoreAsync(CancellationToken cancellationToken)
{
if (!_store.CanWrite)
{
if (_remote is not null)
{
await _remote.RefreshAsync(cancellationToken).ConfigureAwait(false);
}
var snapshot = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
lock (_refreshLock)
{
_refreshCacheTimestamp = Stopwatch.GetTimestamp();
}
return snapshot;
}
var started = Stopwatch.GetTimestamp();
var known = (await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false)).ToList();
var online = _volumes.EnumerateOnlineVolumes();
var seenIds = new HashSet<long>();
foreach (var fp in online)
{
var match = VolumeIdentityMatcher.Match(fp, known);
Source source;
if (match.Source is not null && !match.Ambiguous)
{
source = match.Source;
var wasOffline = source.Status == SourceStatus.Offline;
source.LastRootPath = fp.RootPath;
source.DisplayName = fp.DisplayName ?? source.DisplayName;
source.Label = fp.Label ?? source.Label;
source.Filesystem = fp.Filesystem ?? source.Filesystem;
source.CapacityBytes = fp.CapacityBytes ?? source.CapacityBytes;
source.VolumeGuid = fp.VolumeGuid ?? source.VolumeGuid;
source.VolumeSerial = fp.VolumeSerial ?? source.VolumeSerial;
source.Kind = fp.Kind;
source.LastSeenUtc = _clock.UtcNow;
source.Status = await ResolveReachableStatusAsync(source, cancellationToken).ConfigureAwait(false);
source.LastError = null;
await TryIndexWrite(
() => _store.Sources.UpsertAsync(source, cancellationToken),
"source upsert",
source.Id).ConfigureAwait(false);
if (source.IsIndexed && wasOffline)
{
await TryIndexWrite(
() => _store.Entries.MarkSourceOnlinePresentAsync(source.Id, cancellationToken),
"mark online",
source.Id).ConfigureAwait(false);
}
}
else if (fp.Kind.IsNetwork())
{
continue;
}
else
{
source = new Source
{
StableKey = Guid.NewGuid().ToString("N"),
Kind = fp.Kind,
DisplayName = fp.DisplayName ?? fp.RootPath,
VolumeGuid = fp.VolumeGuid,
VolumeSerial = fp.VolumeSerial,
Filesystem = fp.Filesystem,
Label = fp.Label,
CapacityBytes = fp.CapacityBytes,
DeviceInstanceId = fp.DeviceInstanceId,
LastRootPath = fp.RootPath,
Status = SourceStatus.Online,
LastSeenUtc = _clock.UtcNow
};
source.Id = await _store.Sources.UpsertAsync(source, cancellationToken).ConfigureAwait(false);
known.Add(source);
}
seenIds.Add(source.Id);
}
foreach (var source in known)
{
if (seenIds.Contains(source.Id))
{
continue;
}
if (source.LastRootPath is not null
&& (source.Kind.IsNetwork() || PathRules.IsUnc(source.LastRootPath))
&& _volumes.IsPathReachable(source.LastRootPath))
{
await BringOnlineAsync(source, cancellationToken).ConfigureAwait(false);
continue;
}
if (source.Status != SourceStatus.Offline)
{
await TryIndexWrite(
() => _store.Sources.UpdateStatusAsync(source.Id, SourceStatus.Offline, null, cancellationToken),
"mark source offline",
source.Id).ConfigureAwait(false);
await TryIndexWrite(
() => _store.Entries.MarkSourceOfflineAsync(source.Id, cancellationToken),
"mark entries offline",
source.Id).ConfigureAwait(false);
source.Status = SourceStatus.Offline;
}
}
foreach (var unc in LoadRecents())
{
if (known.Any(s => s.LastRootPath is not null
&& PathRules.CanonicalUncRoot(s.LastRootPath)
.Equals(PathRules.CanonicalUncRoot(unc), StringComparison.OrdinalIgnoreCase)))
{
continue;
}
await AddUncAsync(unc, cancellationToken).ConfigureAwait(false);
}
var result = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
lock (_refreshLock)
{
_refreshCacheTimestamp = Stopwatch.GetTimestamp();
}
_logger.LogInformation(
"Refreshed {Count} sources in {Ms} ms",
result.Count,
(long)Stopwatch.GetElapsedTime(started).TotalMilliseconds);
return result;
}
private async Task TryIndexWrite(Func<Task> work, string what, long id)
{
try
{
await work().ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Skipped {What} for source {Id}; index may be busy", what, id);
}
}
public async Task<Source> AddUncAsync(string path, CancellationToken cancellationToken = default)
{
if (!_store.CanWrite)
{
if (_remote is null)
{
throw new InvalidOperationException("Cannot add a network location while the index is read-only.");
}
var added = await _remote.AddUncAsync(path, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return added;
}
var root = PathRules.CanonicalUncRoot(path);
var known = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
var fp = new VolumeFingerprint
{
Kind = SourceKind.Smb,
RootPath = root,
DisplayName = root,
Filesystem = "SMB"
};
var match = VolumeIdentityMatcher.Match(fp, known);
if (match.Source is not null)
{
var existing = match.Source;
existing.LastRootPath = root;
existing.LastSeenUtc = _clock.UtcNow;
existing.Status = _volumes.IsPathReachable(root) ? SourceStatus.Online : SourceStatus.Offline;
await _store.Sources.UpsertAsync(existing, cancellationToken).ConfigureAwait(false);
RememberUnc(root);
InvalidateRefreshCache();
return existing;
}
var source = new Source
{
StableKey = Guid.NewGuid().ToString("N"),
Kind = SourceKind.Smb,
DisplayName = root,
Filesystem = "SMB",
LastRootPath = root,
Status = _volumes.IsPathReachable(root) ? SourceStatus.Online : SourceStatus.Offline,
LastSeenUtc = _clock.UtcNow
};
source.Id = await _store.Sources.UpsertAsync(source, cancellationToken).ConfigureAwait(false);
RememberUnc(root);
InvalidateRefreshCache();
return source;
}
public async Task<Source?> MarkReachableAsync(string path, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(path) || LocationRoots.IsVirtual(path))
{
return null;
}
if (!_store.CanWrite)
{
if (_remote is null)
{
return await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
}
var current = await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (current is { Status: SourceStatus.Online or SourceStatus.Stale or SourceStatus.Scanning })
{
return current;
}
var updated = await _remote.MarkReachableAsync(path, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
var source = updated ?? await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (source is not null)
{
RaisePresence(source);
}
return source;
}
var found = await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (found is null)
{
return null;
}
await BringOnlineAsync(found, cancellationToken).ConfigureAwait(false);
return found;
}
public async Task ConfirmAccessedAsync(string path, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(path) || LocationRoots.IsVirtual(path))
{
return;
}
var source = await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (source is null || source.Status != SourceStatus.Offline)
{
return;
}
if (_volumes.IsPathReachable(path))
{
await MarkReachableAsync(path, cancellationToken).ConfigureAwait(false);
return;
}
for (var attempt = 0; attempt < 4; attempt++)
{
await Task.Delay(500, cancellationToken).ConfigureAwait(false);
if (_volumes.IsPathReachable(path))
{
await MarkReachableAsync(path, cancellationToken).ConfigureAwait(false);
return;
}
}
}
public bool IsPresentInWindows(Source source)
{
var online = _volumes.EnumerateOnlineVolumes();
foreach (var fp in online)
{
var match = VolumeIdentityMatcher.Match(fp, [source]);
if (match.Source is not null)
{
return true;
}
if (source.LastRootPath is not null && RootsEqual(fp.RootPath, source.LastRootPath))
{
return true;
}
}
return source.LastRootPath is not null && _volumes.IsPathReachable(source.LastRootPath);
}
public bool CanForget(Source source) => !IsPresentInWindows(source);
public async Task<bool> CanForgetPathAsync(string path, CancellationToken cancellationToken = default)
{
var source = await FindSourceRootAsync(path, cancellationToken).ConfigureAwait(false);
return source is not null && CanForget(source);
}
public async Task<bool> ForgetDisconnectedAsync(string path, CancellationToken cancellationToken = default)
{
if (!_store.CanWrite)
{
if (_remote is null)
{
return false;
}
var forgotten = await _remote.ForgetAsync(path, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return forgotten;
}
var source = await FindSourceRootAsync(path, cancellationToken).ConfigureAwait(false);
if (source is null || !CanForget(source))
{
return false;
}
await _store.RunWriteAsync(store => store.Sources.DeleteAsync(source.Id, cancellationToken), cancellationToken)
.ConfigureAwait(false);
if (source.LastRootPath is not null && PathRules.IsUnc(source.LastRootPath))
{
ForgetUnc(PathRules.CanonicalUncRoot(source.LastRootPath));
}
_logger.LogInformation("Forgot disconnected source {DisplayName} ({Path})", source.DisplayName, source.LastRootPath);
InvalidateRefreshCache();
return true;
}
public async Task<Source?> FindByPathAsync(string path, CancellationToken cancellationToken = default)
{
var sources = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
var normalized = PathRules.FromExtended(path);
Source? best = null;
var bestLen = -1;
foreach (var source in sources)
{
if (source.LastRootPath is null)
{
continue;
}
var root = PathRules.FromExtended(source.LastRootPath).TrimEnd('\\');
var candidate = normalized.TrimEnd('\\');
if (candidate.Equals(root, StringComparison.OrdinalIgnoreCase)
|| candidate.StartsWith(root + "\\", StringComparison.OrdinalIgnoreCase)
|| (root.Length == 2 && root[1] == ':' && candidate.StartsWith(root, StringComparison.OrdinalIgnoreCase)))
{
if (root.Length > bestLen)
{
best = source;
bestLen = root.Length;
}
}
}
return best;
}
public async Task<Source?> EnsureForPathAsync(string path, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(path) || LocationRoots.IsVirtual(path))
{
return null;
}
if (!_store.CanWrite)
{
if (_remote is null)
{
return await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
}
var ensured = await _remote.EnsureForPathAsync(path, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return ensured;
}
var existing = await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (existing is not null)
{
existing.LastSeenUtc = _clock.UtcNow;
existing.Status = _volumes.IsPathReachable(existing.LastRootPath ?? path)
? await ResolveReachableStatusAsync(existing, cancellationToken).ConfigureAwait(false)
: SourceStatus.Offline;
await _store.Sources.UpsertAsync(existing, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return existing;
}
var fp = _volumes.Probe(path);
if (fp is null)
{
return null;
}
var known = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
var match = VolumeIdentityMatcher.Match(fp, known);
if (match.Source is not null && !match.Ambiguous)
{
var source = match.Source;
source.LastRootPath = fp.RootPath;
source.DisplayName = fp.DisplayName ?? source.DisplayName;
source.Kind = fp.Kind;
source.Filesystem = fp.Filesystem ?? source.Filesystem;
source.Label = fp.Label ?? source.Label;
source.LastSeenUtc = _clock.UtcNow;
source.Status = _volumes.IsPathReachable(fp.RootPath) ? SourceStatus.Online : SourceStatus.Offline;
await _store.Sources.UpsertAsync(source, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return source;
}
if (fp.Kind == SourceKind.Smb && PathRules.IsUnc(fp.RootPath))
{
return await AddUncAsync(fp.RootPath, cancellationToken).ConfigureAwait(false);
}
var created = new Source
{
StableKey = Guid.NewGuid().ToString("N"),
Kind = fp.Kind,
DisplayName = fp.DisplayName ?? fp.RootPath,
VolumeGuid = fp.VolumeGuid,
VolumeSerial = fp.VolumeSerial,
Filesystem = fp.Filesystem,
Label = fp.Label,
CapacityBytes = fp.CapacityBytes,
LastRootPath = fp.RootPath,
Status = _volumes.IsPathReachable(fp.RootPath) ? SourceStatus.Online : SourceStatus.Offline,
LastSeenUtc = _clock.UtcNow
};
created.Id = await _store.Sources.UpsertAsync(created, cancellationToken).ConfigureAwait(false);
InvalidateRefreshCache();
return created;
}
public async Task<IReadOnlyList<VolumeFingerprint>> ListUntrackedOnlineVolumesAsync(CancellationToken cancellationToken = default)
{
var known = await _store.Sources.GetAllAsync(cancellationToken).ConfigureAwait(false);
var untracked = new List<VolumeFingerprint>();
foreach (var fp in _volumes.EnumerateOnlineVolumes())
{
var match = VolumeIdentityMatcher.Match(fp, known);
if (match.Source is not null && !match.Ambiguous)
{
continue;
}
if (known.Any(s => s.LastRootPath is not null && RootsEqual(s.LastRootPath, fp.RootPath)))
{
continue;
}
untracked.Add(fp);
}
return untracked;
}
private async Task<Source?> FindSourceRootAsync(string path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path) || LocationRoots.IsVirtual(path))
{
return null;
}
var source = await FindByPathAsync(path, cancellationToken).ConfigureAwait(false);
if (source?.LastRootPath is null)
{
return null;
}
return RootsEqual(source.LastRootPath, path) ? source : null;
}
private static bool RootsEqual(string a, string b)
{
var left = PathRules.EnsureDirectoryTrailingSlashIfRoot(PathRules.FromExtended(a).TrimEnd('\\'));
var right = PathRules.EnsureDirectoryTrailingSlashIfRoot(PathRules.FromExtended(b).TrimEnd('\\'));
return left.Equals(right, StringComparison.OrdinalIgnoreCase);
}
private void ForgetUnc(string root)
{
try
{
var file = Path.Combine(_env.DataDirectory, "recents.txt");
if (!File.Exists(file))
{
return;
}
var lines = LoadRecents()
.Where(l => !PathRules.CanonicalUncRoot(l).Equals(root, StringComparison.OrdinalIgnoreCase))
.ToList();
File.WriteAllLines(file, lines);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed updating recents");
}
}
private void InvalidateRefreshCache()
{
lock (_refreshLock)
{
_refreshCacheTimestamp = 0;
}
}
public Task<Source?> GetAsync(long id, CancellationToken cancellationToken = default)
=> _store.Sources.GetAsync(id, cancellationToken);
private async Task BringOnlineAsync(Source source, CancellationToken cancellationToken)
{
if (source.Status == SourceStatus.Scanning
&& await _store.ScanJobs.HasActiveAsync(source.Id, cancellationToken).ConfigureAwait(false))
{
return;
}
if (source.Status is SourceStatus.Online or SourceStatus.Stale)
{
return;
}
var wasOffline = source.Status == SourceStatus.Offline;
source.Status = SourceStatus.Online;
source.LastSeenUtc = _clock.UtcNow;
source.LastError = null;
await TryIndexWrite(
() => _store.Sources.UpsertAsync(source, cancellationToken),
"mark source online",
source.Id).ConfigureAwait(false);
if (source.IsIndexed && wasOffline)
{
await TryIndexWrite(
() => _store.Entries.MarkSourceOnlinePresentAsync(source.Id, cancellationToken),
"mark online",
source.Id).ConfigureAwait(false);
}
InvalidateRefreshCache();
RaisePresence(source);
}
private void RaisePresence(Source source)
=> PresenceChanged?.Invoke(this, source);
private async Task<SourceStatus> ResolveReachableStatusAsync(Source source, CancellationToken cancellationToken)
{
if (source.Status == SourceStatus.Scanning
&& await _store.ScanJobs.HasActiveAsync(source.Id, cancellationToken).ConfigureAwait(false))
{
return SourceStatus.Scanning;
}
return SourceStatus.Online;
}
private IReadOnlyList<string> LoadRecents()
{
var file = Path.Combine(_env.DataDirectory, "recents.txt");
if (!File.Exists(file))
{
return [];
}
try
{
return File.ReadAllLines(file)
.Where(l => !string.IsNullOrWhiteSpace(l))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed reading recents");
return [];
}
}
private void RememberUnc(string root)
{
try
{
var file = Path.Combine(_env.DataDirectory, "recents.txt");
var lines = LoadRecents().ToList();
lines.RemoveAll(l => l.Equals(root, StringComparison.OrdinalIgnoreCase));
lines.Insert(0, root);
File.WriteAllLines(file, lines.Take(30));
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed writing recents");
}
}
}