Compare commits

..

48 Commits

Author SHA1 Message Date
Flaminel 2b766300ff fixed processing files marked as skipped 2025-03-26 23:51:29 +02:00
Flaminel 2d3ff04172 fixed Deluge failing when WebUI is not connected 2025-03-26 22:15:37 +02:00
Flaminel 874351aed7 added dry run for category creation 2025-03-26 21:50:02 +02:00
Flaminel 1a89822bad fixed some logs; updated test files; fixed deluge not working after fresh start 2025-03-26 20:49:23 +02:00
Flaminel ac086fcd47 fixed missing download location 2025-03-26 02:05:05 +02:00
Flaminel 4b38a6fee1 Merge branch 'main' into add_cleanup_on_no_hardlinks 2025-03-26 00:28:59 +02:00
Flaminel 9f770473e5 Remove Transmission downloads cache (#105) 2025-03-26 00:26:10 +02:00
Flaminel 60e838cba7 fixed Deluge processing for changing category 2025-03-25 22:15:04 +02:00
Flaminel 7b95ec579c fixed category change notification 2025-03-25 22:14:50 +02:00
Flaminel ab8fbc4b6e fixed Deluge get labels 2025-03-25 22:13:57 +02:00
Flaminel f2130ad734 fixed label being init only 2025-03-25 22:13:45 +02:00
Flaminel c86e9c97b8 removed commented and unused code 2025-03-25 16:21:31 +02:00
Flaminel 7639b0787e added set torrent label for Deluge 2025-03-25 16:13:46 +02:00
Flaminel 5e362d4af8 added Deluge label creation 2025-03-25 16:11:53 +02:00
Flaminel 3b63d1b7e5 fixed some logs and comments 2025-03-25 10:55:37 +02:00
Flaminel 4a1e0f6896 renamed vars; added root dir; fixed root dir file counts; fixed qbit flow 2025-03-24 18:05:50 +02:00
Flaminel a83809eef7 Merge branch 'main' into add_cleanup_on_no_hardlinks 2025-03-24 16:34:11 +02:00
Flaminel d993cd30a7 qbit test 2025-03-24 15:51:00 +02:00
Flaminel 6bc59c8389 added implementation for Deluge and Transmission 2025-03-24 15:50:56 +02:00
Flaminel 5fe0f5750a Fix qBit queued items being processed (#102) 2025-03-21 23:06:31 +02:00
Flaminel b1d98c2b62 fixed category changing 2025-02-27 22:23:59 +02:00
Flaminel 46ac50c393 fixed hard links check 2025-02-27 22:01:59 +02:00
Flaminel b834a8bc01 fixed downloads to change category value 2025-02-27 21:52:46 +02:00
Flaminel 3c8ef3db91 fixed service type 2025-02-27 21:51:27 +02:00
Flaminel bc642d8f80 fixed typo 2025-02-27 21:44:54 +02:00
Flaminel a994bc4526 fixed missing methods 2025-02-27 21:41:07 +02:00
Flaminel 1243da3d22 streamlined downloads processing after category changed 2025-02-26 23:39:08 +02:00
Flaminel 6b33075a21 fixed merge conflicts 2025-02-26 22:29:24 +02:00
Flaminel c27ee326f7 updated test data 2025-02-26 22:20:23 +02:00
Flaminel d27562a889 explicit dispose and clean on some objects 2025-02-26 22:20:23 +02:00
Flaminel e8d287de84 fixed missing notification configuration 2025-02-26 22:20:23 +02:00
Flaminel c65c85a0c5 fixed categories not being filtered when fetching downloads; fixed inconsistent var naming 2025-02-26 22:20:22 +02:00
Flaminel bd81f2ffca fixed using root directory instead of root path 2025-02-26 22:20:22 +02:00
Flaminel 5bd2a9cbea added trace logs 2025-02-26 22:20:06 +02:00
Flaminel e006521dc9 refactored method names; fixed qbit category creation 2025-02-26 22:20:05 +02:00
Flaminel 1ad07b1f51 refactored names; fixed return values for hard links service 2025-02-26 22:19:09 +02:00
Flaminel 9b68792ea9 added handling for windows files 2025-02-26 22:17:31 +02:00
Flaminel 8c8d412ef1 refactored hardlink service 2025-02-26 22:17:31 +02:00
Flaminel 029f255351 added debug logs 2025-02-26 22:17:30 +02:00
Flaminel 19ac8cbd28 trying to account for cross-seed 2025-02-26 22:17:30 +02:00
Flaminel f91e85651f removed debug logs 2025-02-26 22:17:30 +02:00
Flaminel fbe6ebaa6b trying to fix Unix stat yet again 2025-02-26 22:17:30 +02:00
Flaminel 268ede8a9c trying to fix Unix stat again 2025-02-26 22:17:30 +02:00
Flaminel a63bae0bb9 added debug logs 2025-02-26 22:17:29 +02:00
Flaminel d454a094a0 fixed qbit file path 2025-02-26 22:17:29 +02:00
Flaminel 1650b0e5a4 trying to fix Unix stat #2 2025-02-26 22:17:29 +02:00
Flaminel 2d6f16692c trying to fix Unix stat 2025-02-26 22:17:29 +02:00
Flaminel 017e25fb06 added category change for downloads with no additional hardlinks 2025-02-26 22:17:28 +02:00
44 changed files with 1249 additions and 192 deletions
@@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration;
namespace Common.Configuration.DownloadCleaner;
public sealed record Category : IConfig
public sealed record CleanCategory : IConfig
{
public required string Name { get; init; }
@@ -8,8 +8,8 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
public const string SectionName = "DownloadCleaner";
public bool Enabled { get; init; }
public List<Category>? Categories { get; init; }
public List<CleanCategory>? Categories { get; init; }
[ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; init; }
@@ -17,6 +17,15 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
public string? IgnoredDownloadsPath { get; init; }
[ConfigurationKeyName("UNLINKED_TARGET_CATEGORY")]
public string UnlinkedTargetCategory { get; init; } = "cleanuperr-unlinked";
[ConfigurationKeyName("UNLINKED_IGNORED_ROOT_DIR")]
public string UnlinkedIgnoredRootDir { get; init; } = string.Empty;
[ConfigurationKeyName("UNLINKED_CATEGORIES")]
public List<string>? UnlinkedCategories { get; init; }
public void Validate()
{
if (!Enabled)
@@ -31,9 +40,34 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
if (Categories?.GroupBy(x => x.Name).Any(x => x.Count() > 1) is true)
{
throw new ValidationException("duplicated categories found");
throw new ValidationException("duplicated clean categories found");
}
Categories?.ForEach(x => x.Validate());
if (string.IsNullOrEmpty(UnlinkedTargetCategory))
{
return;
}
if (UnlinkedCategories?.Count is null or 0)
{
throw new ValidationException("no unlinked categories configured");
}
if (UnlinkedCategories.Contains(UnlinkedTargetCategory))
{
throw new ValidationException($"{SectionName.ToUpperInvariant()}__UNLINKED_TARGET_CATEGORY should not be present in {SectionName.ToUpperInvariant()}__UNLINKED_CATEGORIES");
}
if (UnlinkedCategories.Any(string.IsNullOrEmpty))
{
throw new ValidationException("empty unlinked category filter found");
}
if (!string.IsNullOrEmpty(UnlinkedIgnoredRootDir) && !Directory.Exists(UnlinkedIgnoredRootDir))
{
throw new ValidationException($"{UnlinkedIgnoredRootDir} root directory does not exist");
}
}
}
@@ -15,8 +15,16 @@ public abstract record NotificationConfig
[ConfigurationKeyName("ON_DOWNLOAD_CLEANED")]
public bool OnDownloadCleaned { get; init; }
[ConfigurationKeyName("ON_CATEGORY_CHANGED")]
public bool OnCategoryChanged { get; init; }
public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnQueueItemDeleted || OnDownloadCleaned;
public bool IsEnabled =>
OnImportFailedStrike ||
OnStalledStrike ||
OnQueueItemDeleted ||
OnDownloadCleaned ||
OnCategoryChanged;
public abstract bool IsValid();
}
+12
View File
@@ -0,0 +1,12 @@
namespace Common.Exceptions;
public class FatalException : Exception
{
public FatalException()
{
}
public FatalException(string message) : base(message)
{
}
}
+5 -1
View File
@@ -2,7 +2,11 @@
public enum DeleteReason
{
None,
Stalled,
ImportFailed,
AllFilesBlocked
DownloadingMetadata,
AllFilesSkipped,
AllFilesSkippedByQBit,
AllFilesBlocked,
}
+1
View File
@@ -3,5 +3,6 @@
public enum StrikeType
{
Stalled,
DownloadingMetadata,
ImportFailed
}
@@ -17,7 +17,7 @@ public sealed record TorrentStatus
[JsonProperty("total_done")]
public long TotalDone { get; init; }
public string? Label { get; init; }
public string? Label { get; set; }
[JsonProperty("seeding_time")]
public long SeedingTime { get; init; }
@@ -25,6 +25,9 @@ public sealed record TorrentStatus
public float Ratio { get; init; }
public required IReadOnlyList<Tracker> Trackers { get; init; }
[JsonProperty("download_location")]
public required string DownloadLocation { get; init; }
}
public sealed record Tracker
@@ -17,7 +17,9 @@ public static class MainDI
.AddLogging(builder => builder.ClearProviders().AddConsole())
.AddHttpClients(configuration)
.AddConfiguration(configuration)
.AddMemoryCache()
.AddMemoryCache(options => {
options.ExpirationScanFrequency = TimeSpan.FromMinutes(1);
})
.AddServices()
.AddQuartzServices(configuration)
.AddNotifications(configuration)
@@ -27,6 +29,7 @@ public static class MainDI
config.AddConsumer<NotificationConsumer<StalledStrikeNotification>>();
config.AddConsumer<NotificationConsumer<QueueItemDeletedNotification>>();
config.AddConsumer<NotificationConsumer<DownloadCleanedNotification>>();
config.AddConsumer<NotificationConsumer<CategoryChangedNotification>>();
config.UsingInMemory((context, cfg) =>
{
@@ -36,6 +39,7 @@ public static class MainDI
e.ConfigureConsumer<NotificationConsumer<StalledStrikeNotification>>(context);
e.ConfigureConsumer<NotificationConsumer<QueueItemDeletedNotification>>(context);
e.ConfigureConsumer<NotificationConsumer<DownloadCleanedNotification>>(context);
e.ConfigureConsumer<NotificationConsumer<CategoryChangedNotification>>(context);
e.ConcurrentMessageLimit = 1;
e.PrefetchCount = 1;
});
@@ -10,6 +10,7 @@ using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.DownloadClient.Deluge;
using Infrastructure.Verticals.DownloadClient.QBittorrent;
using Infrastructure.Verticals.DownloadClient.Transmission;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.QueueCleaner;
@@ -27,6 +28,9 @@ public static class ServicesDI
.AddTransient<ContentBlocker>()
.AddTransient<DownloadCleaner>()
.AddTransient<IFilenameEvaluator, FilenameEvaluator>()
.AddTransient<IHardLinkFileService, HardLinkFileService>()
.AddTransient<UnixHardLinkFileService>()
.AddTransient<WindowsHardLinkFileService>()
.AddTransient<DummyDownloadService>()
.AddTransient<QBitService>()
.AddTransient<DelugeService>()
+10 -4
View File
@@ -3,7 +3,7 @@
"HTTP_MAX_RETRIES": 0,
"HTTP_TIMEOUT": 10,
"Logging": {
"LogLevel": "Debug",
"LogLevel": "Verbose",
"Enhanced": true,
"File": {
"Enabled": false,
@@ -25,13 +25,13 @@
"Enabled": true,
"RunSequentially": true,
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads",
"IMPORT_FAILED_MAX_STRIKES": 5,
"IMPORT_FAILED_MAX_STRIKES": 3,
"IMPORT_FAILED_IGNORE_PRIVATE": true,
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [
"file is a sample"
],
"STALLED_MAX_STRIKES": 5,
"STALLED_MAX_STRIKES": 3,
"STALLED_RESET_STRIKES_ON_PROGRESS": true,
"STALLED_IGNORE_PRIVATE": true,
"STALLED_DELETE_PRIVATE": false
@@ -44,9 +44,15 @@
"Name": "tv-sonarr",
"MAX_RATIO": -1,
"MIN_SEED_TIME": 0,
"MAX_SEED_TIME": -1
"MAX_SEED_TIME": 240
}
],
"UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked",
"UNLINKED_IGNORED_ROOT_DIR": "",
"UNLINKED_CATEGORIES": [
"tv-sonarr",
"radarr"
],
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads"
},
"DOWNLOAD_CLIENT": "qbittorrent",
+3
View File
@@ -37,6 +37,9 @@
"Enabled": false,
"DELETE_PRIVATE": false,
"CATEGORIES": [],
"UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked",
"UNLINKED_IGNORED_ROOT_DIR": "",
"UNLINKED_CATEGORIES": [],
"IGNORED_DOWNLOADS_PATH": ""
},
"DOWNLOAD_CLIENT": "none",
@@ -1,9 +1,10 @@
using Common.Configuration.ContentBlocker;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -56,6 +57,7 @@ public class DownloadServiceFixture : IDisposable
var filenameEvaluator = Substitute.For<IFilenameEvaluator>();
var notifier = Substitute.For<INotificationPublisher>();
var dryRunInterceptor = Substitute.For<IDryRunInterceptor>();
var hardlinkFileService = Substitute.For<IHardLinkFileService>();
return new TestDownloadService(
Logger,
@@ -66,7 +68,8 @@ public class DownloadServiceFixture : IDisposable
filenameEvaluator,
Striker,
notifier,
dryRunInterceptor
dryRunInterceptor,
hardlinkFileService
);
}
@@ -105,13 +105,14 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
// Arrange
const string hash = "test-hash";
const string itemName = "test-item";
_fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, StrikeType.Stalled)
StrikeType strikeType = StrikeType.Stalled;
_fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, strikeType)
.Returns(true);
TestDownloadService sut = _fixture.CreateSut();
// Act
bool result = await sut.StrikeAndCheckLimit(hash, itemName);
bool result = await sut.StrikeAndCheckLimit(hash, itemName, strikeType);
// Assert
result.ShouldBeTrue();
@@ -132,7 +133,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue()
{
// Arrange
Category category = new()
CleanCategory category = new()
{
Name = "test",
MaxRatio = 1.0,
@@ -158,7 +159,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse()
{
// Arrange
Category category = new()
CleanCategory category = new()
{
Name = "test",
MaxRatio = 1.0,
@@ -184,7 +185,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenMaxSeedTimeReached_ShouldReturnTrue()
{
// Arrange
Category category = new()
CleanCategory category = new()
{
Name = "test",
MaxRatio = -1,
@@ -210,7 +211,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenNeitherConditionMet_ShouldReturnFalse()
{
// Arrange
Category category = new()
CleanCategory category = new()
{
Name = "test",
MaxRatio = 2.0,
@@ -1,11 +1,13 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -25,10 +27,11 @@ public class TestDownloadService : DownloadService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
}
@@ -39,12 +42,14 @@ public class TestDownloadService : DownloadService
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads) => Task.FromResult(new BlockFilesResult());
public override Task DeleteDownload(string hash) => Task.CompletedTask;
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => Task.FromResult<List<object>?>(null);
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
public override Task CreateCategoryAsync(string name) => Task.CompletedTask;
public override Task<List<object>?> GetSeedingDownloads() => Task.FromResult<List<object>?>(null);
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) => null;
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) => null;
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
public override Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
// Expose protected methods for testing
public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded);
public new Task<bool> StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName);
public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category);
public new Task<bool> StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType) => base.StrikeAndCheckLimit(hash, itemName, strikeType);
public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) => base.ShouldCleanDownload(ratio, seedingTime, category);
}
+2 -1
View File
@@ -13,11 +13,12 @@
<ItemGroup>
<PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
<PackageReference Include="FLM.Transmission" Version="1.0.2" />
<PackageReference Include="FLM.Transmission" Version="1.0.3" />
<PackageReference Include="Mapster" Version="7.4.0" />
<PackageReference Include="MassTransit" Version="8.3.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageReference Include="Quartz" Version="3.13.1" />
<PackageReference Include="Scrutor" Version="6.0.1" />
</ItemGroup>
@@ -114,7 +114,12 @@ public abstract class ArrClient : IArrClient
return false;
}
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient)
public virtual async Task DeleteQueueItemAsync(
ArrInstance arrInstance,
QueueRecord record,
bool removeFromClient,
DeleteReason deleteReason
)
{
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}";
@@ -130,8 +135,9 @@ public abstract class ArrClient : IArrClient
_logger.LogInformation(
removeFromClient
? "queue item deleted | {url} | {title}"
: "queue item removed from arr | {url} | {title}",
? "queue item deleted with reason {reason} | {url} | {title}"
: "queue item removed from arr with reason {reason} | {url} | {title}",
deleteReason.ToString(),
arrInstance.Url,
record.Title
);
@@ -11,7 +11,7 @@ public interface IArrClient
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason);
Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);
@@ -43,7 +43,7 @@ public sealed class BlocklistProvider
{
if (_initialized)
{
_logger.LogDebug("blocklists already loaded");
_logger.LogTrace("blocklists already loaded");
return;
}
@@ -142,7 +142,7 @@ public sealed class ContentBlocker : GenericHandler
removeFromClient = false;
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, DeleteReason.AllFilesBlocked);
await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked);
}
});
@@ -21,6 +21,8 @@ public sealed class DownloadCleaner : GenericHandler
private readonly IgnoredDownloadsProvider<DownloadCleanerConfig> _ignoredDownloadsProvider;
private readonly HashSet<string> _excludedHashes = [];
private static bool _hardLinkCategoryCreated;
public DownloadCleaner(
ILogger<DownloadCleaner> logger,
IOptions<DownloadCleanerConfig> config,
@@ -65,13 +67,20 @@ public sealed class DownloadCleaner : GenericHandler
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories);
if (downloads?.Count is null or 0)
List<object>? downloads = await _downloadService.GetSeedingDownloads();
List<object>? downloadsToChangeCategory = null;
if (!string.IsNullOrEmpty(_config.UnlinkedTargetCategory) && _config.UnlinkedCategories?.Count > 0)
{
_logger.LogDebug("no downloads found in the download client");
return;
if (!_hardLinkCategoryCreated)
{
_logger.LogDebug("creating category {cat}", _config.UnlinkedTargetCategory);
await _downloadService.CreateCategoryAsync(_config.UnlinkedTargetCategory);
_hardLinkCategoryCreated = true;
}
downloadsToChangeCategory = _downloadService.FilterDownloadsToChangeCategoryAsync(downloads, _config.UnlinkedCategories);
}
// wait for the downloads to appear in the arr queue
@@ -81,7 +90,16 @@ public sealed class DownloadCleaner : GenericHandler
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads);
_logger.LogTrace("looking for downloads to change category");
await _downloadService.ChangeCategoryForNoHardLinksAsync(downloadsToChangeCategory, _excludedHashes, ignoredDownloads);
List<object>? downloadsToClean = _downloadService.FilterDownloadsToBeCleanedAsync(downloads, _config.Categories);
// release unused objects
downloads = null;
_logger.LogTrace("looking for downloads to clean");
await _downloadService.CleanDownloadsAsync(downloadsToClean, _config.Categories, _excludedHashes, ignoredDownloads);
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
@@ -2,6 +2,7 @@
using System.Text.Json.Serialization;
using Common.Configuration;
using Common.Configuration.DownloadClient;
using Common.Exceptions;
using Domain.Models.Deluge.Exceptions;
using Domain.Models.Deluge.Request;
using Domain.Models.Deluge.Response;
@@ -27,7 +28,8 @@ public sealed class DelugeClient
"label",
"seeding_time",
"ratio",
"trackers"
"trackers",
"download_location"
];
public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory)
@@ -42,11 +44,42 @@ public sealed class DelugeClient
return await SendRequest<bool>("auth.login", _config.Password);
}
public async Task<bool> IsConnected()
{
return await SendRequest<bool>("web.connected");
}
public async Task<bool> Connect()
{
string? firstHost = await GetHost();
if (string.IsNullOrEmpty(firstHost))
{
return false;
}
var result = await SendRequest<List<string>?>("web.connect", firstHost);
return result?.Count > 0;
}
public async Task<bool> Logout()
{
return await SendRequest<bool>("auth.delete_session");
}
public async Task<string?> GetHost()
{
var hosts = await SendRequest<List<List<string>?>?>("web.get_hosts");
if (hosts?.Count > 1)
{
throw new FatalException("multiple Deluge hosts found - please connect to only one host");
}
return hosts?.FirstOrDefault()?.FirstOrDefault();
}
public async Task<List<DelugeTorrent>> ListTorrents(Dictionary<string, string>? filters = null)
{
filters ??= new Dictionary<string, string>();
@@ -147,7 +180,7 @@ public sealed class DelugeClient
return responseJson;
}
private DelugeRequest CreateRequest(string method, params object[] parameters)
private static DelugeRequest CreateRequest(string method, params object[] parameters)
{
if (String.IsNullOrWhiteSpace(method))
{
@@ -193,4 +226,19 @@ public sealed class DelugeClient
return webResponse.Result;
}
public async Task<IReadOnlyList<string>> GetLabels()
{
return await SendRequest<IReadOnlyList<string>>("label.get_labels");
}
public async Task CreateLabel(string label)
{
await SendRequest<DelugeResponse<object>>("label.add", label);
}
public async Task SetTorrentLabel(string hash, string newLabel)
{
await SendRequest<DelugeResponse<object>>("label.set_torrent", hash, newLabel);
}
}
@@ -5,12 +5,14 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Exceptions;
using Domain.Enums;
using Domain.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -34,10 +36,11 @@ public class DelugeService : DownloadService, IDelugeService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
config.Value.Validate();
@@ -47,6 +50,11 @@ public class DelugeService : DownloadService, IDelugeService
public override async Task LoginAsync()
{
await _client.LoginAsync();
if (!await _client.IsConnected() && !await _client.Connect())
{
throw new FatalException("Deluge WebUI is not connected to the daemon");
}
}
/// <inheritdoc/>
@@ -65,6 +73,8 @@ public class DelugeService : DownloadService, IDelugeService
return result;
}
result.IsPrivate = download.Private;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
@@ -79,6 +89,7 @@ public class DelugeService : DownloadService, IDelugeService
{
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
}
bool shouldRemove = contents?.Contents?.Count > 0;
@@ -92,17 +103,15 @@ public class DelugeService : DownloadService, IDelugeService
if (shouldRemove)
{
result.DeleteReason = DeleteReason.AllFilesBlocked;
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download);
result.IsPrivate = download.Private;
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download);
if (!shouldRemove && result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
return result;
}
@@ -123,9 +132,6 @@ public class DelugeService : DownloadService, IDelugeService
return result;
}
var ceva = await _client.GetTorrentExtended(hash);
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
@@ -208,26 +214,51 @@ public class DelugeService : DownloadService, IDelugeService
return result;
}
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override async Task<List<object>?> GetSeedingDownloads()
{
return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<TorrentStatus>()
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<TorrentStatus>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentStatus download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
@@ -235,19 +266,13 @@ public class DelugeService : DownloadService, IDelugeService
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
{
@@ -279,7 +304,107 @@ public class DelugeService : DownloadService, IDelugeService
await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyList<string> existingLabels = await _client.GetLabels();
if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateLabel, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentStatus download in downloads.Cast<TorrentStatus>())
{
if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
DelugeContents? contents = null;
try
{
contents = await _client.GetTorrentFiles(download.Hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name);
continue;
}
bool hasHardlinks = false;
ProcessFiles(contents?.Contents, (_, file) =>
{
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/']));
if (file.Priority <= 0)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
return;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
return;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
}
});
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _notifier.NotifyCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Label = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
@@ -288,6 +413,12 @@ public class DelugeService : DownloadService, IDelugeService
await _client.DeleteTorrents([hash]);
}
[DryRunSafeguard]
protected async Task CreateLabel(string name)
{
await _client.CreateLabel(name);
}
[DryRunSafeguard]
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
@@ -295,33 +426,39 @@ public class DelugeService : DownloadService, IDelugeService
await _client.ChangeFilesPriority(hash, sortedPriorities);
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentStatus status)
[DryRunSafeguard]
protected virtual async Task ChangeLabel(string hash, string newLabel)
{
await _client.SetTorrentLabel(hash, newLabel);
}
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentStatus status)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
return false;
return (false, default);
}
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return false;
return (false, default);
}
if (status.Eta > 0)
{
return false;
return (false, default);
}
ResetStrikesOnProgress(status.Hash!, status.TotalDone);
return await StrikeAndCheckLimit(status.Hash!, status.Name!);
return (await StrikeAndCheckLimit(status.Hash!, status.Name!, StrikeType.Stalled), DeleteReason.Stalled);
}
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
@@ -1,4 +1,4 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
@@ -10,6 +10,7 @@ using Infrastructure.Helpers;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -30,6 +31,7 @@ public abstract class DownloadService : IDownloadService
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected readonly INotificationPublisher _notifier;
protected readonly IDryRunInterceptor _dryRunInterceptor;
protected readonly IHardLinkFileService _hardLinkFileService;
protected DownloadService(
ILogger<DownloadService> logger,
@@ -40,7 +42,8 @@ public abstract class DownloadService : IDownloadService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
)
{
_logger = logger;
@@ -52,6 +55,7 @@ public abstract class DownloadService : IDownloadService
_striker = striker;
_notifier = notifier;
_dryRunInterceptor = dryRunInterceptor;
_hardLinkFileService = hardLinkFileService;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
@@ -72,12 +76,23 @@ public abstract class DownloadService : IDownloadService
public abstract Task DeleteDownload(string hash);
/// <inheritdoc/>
public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
public abstract Task<List<object>?> GetSeedingDownloads();
/// <inheritdoc/>
public abstract List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
/// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
public abstract List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
/// <inheritdoc/>
public abstract Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task CreateCategoryAsync(string name);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
@@ -100,13 +115,14 @@ public abstract class DownloadService : IDownloadService
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="itemName">The name or title of the item.</param>
/// <param name="strikeType"></param>
/// <returns>True if the limit has been reached; otherwise, false.</returns>
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName)
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType)
{
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, strikeType);
}
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category)
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category)
{
// check ratio
if (DownloadReachedRatio(ratio, seedingTime, category))
@@ -130,8 +146,28 @@ public abstract class DownloadService : IDownloadService
return new();
}
protected string? GetRootWithFirstDirectory(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category)
string? root = Path.GetPathRoot(path);
if (root is null)
{
return null;
}
string relativePath = path[root.Length..].TrimStart(Path.DirectorySeparatorChar);
string[] parts = relativePath.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries);
return parts.Length > 0 ? Path.Combine(root, parts[0]) : root;
}
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category)
{
if (category.MaxRatio < 0)
{
@@ -157,7 +193,7 @@ public abstract class DownloadService : IDownloadService
return true;
}
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category)
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category)
{
if (category.MaxSeedTime < 0)
{
@@ -1,10 +1,11 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -15,7 +16,21 @@ namespace Infrastructure.Verticals.DownloadClient;
public class DummyDownloadService : DownloadService
{
public DummyDownloadService(ILogger<DownloadService> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IOptions<DownloadCleanerConfig> downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier, dryRunInterceptor)
public DummyDownloadService(
ILogger<DownloadService> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig,
cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
}
@@ -39,13 +54,32 @@ public class DummyDownloadService : DownloadService
throw new NotImplementedException();
}
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override Task<List<object>?> GetSeedingDownloads()
{
throw new NotImplementedException();
}
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
throw new NotImplementedException();
}
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
throw new NotImplementedException();
}
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
public override Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
public override Task CreateCategoryAsync(string name)
{
throw new NotImplementedException();
}
@@ -34,24 +34,52 @@ public interface IDownloadService : IDisposable
);
/// <summary>
/// Fetches all downloads.
/// Fetches all seeding downloads.
/// </summary>
/// <returns>A list of downloads that are seeding.</returns>
Task<List<object>?> GetSeedingDownloads();
/// <summary>
/// Filters downloads that should be cleaned.
/// </summary>
/// <param name="downloads">The downloads to filter.</param>
/// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns>
Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
/// <summary>
/// Filters downloads that should have their category changed.
/// </summary>
/// <param name="downloads">The downloads to filter.</param>
/// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns>
List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
/// <summary>
/// Cleans the downloads.
/// </summary>
/// <param name="downloads"></param>
/// <param name="downloads">The downloads to clean.</param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Changes the category for downloads that have no hardlinks.
/// </summary>
/// <param name="downloads">The downloads to change.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Deletes a download item.
/// </summary>
public Task DeleteDownload(string hash);
/// <summary>
/// Creates a category.
/// </summary>
/// <param name="name">The category name.</param>
public Task CreateCategoryAsync(string name);
}
@@ -5,19 +5,20 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Exceptions;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using QBittorrent.Client;
using Category = Common.Configuration.DownloadCleaner.Category;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
@@ -37,10 +38,11 @@ public class QBitService : DownloadService, IQBitService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
_config = config.Value;
@@ -96,30 +98,26 @@ public class QBitService : DownloadService, IQBitService
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
return result;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
// if all files are marked as skip
if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
{
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
return result;
}
// remove if all files are unwanted
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
result.ShouldRemove = await IsItemStuckAndShouldRemove(download, result.IsPrivate);
if (result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download, result.IsPrivate);
return result;
}
@@ -230,20 +228,42 @@ public class QBitService : DownloadService, IQBitService
}
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) =>
public override async Task<List<object>?> GetSeedingDownloads() =>
(await _client.GetTorrentListAsync(new()
{
Filter = TorrentListFilter.Seeding
}))
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
@@ -251,16 +271,22 @@ public class QBitService : DownloadService, IQBitService
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
@@ -268,12 +294,6 @@ public class QBitService : DownloadService, IQBitService
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate)
{
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
@@ -319,12 +339,125 @@ public class QBitService : DownloadService, IQBitService
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyDictionary<string, Category>? existingCategories = await _client.GetCategoriesAsync();
if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateCategory, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
bool hasHardlinks = false;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
_logger.LogDebug("skip | file index is null for {name}", download.Name);
hasHardlinks = true;
break;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/']));
if (file.Priority is TorrentContentPriority.Skip)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
continue;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Category = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
await _client.DeleteAsync(hash, deleteDownloadedData: true);
}
[DryRunSafeguard]
protected async Task CreateCategory(string name)
{
await _client.AddCategoryAsync(name);
}
[DryRunSafeguard]
protected virtual async Task SkipFile(string hash, int fileIndex)
@@ -332,35 +465,46 @@ public class QBitService : DownloadService, IQBitService
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
[DryRunSafeguard]
protected virtual async Task ChangeCategory(string hash, string newCategory)
{
await _client.SetTorrentCategoryAsync([hash], newCategory);
}
public override void Dispose()
{
_client.Dispose();
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
return false;
return (false, default);
}
if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
and not TorrentState.ForcedFetchingMetadata)
{
// ignore other states
return false;
return (false, default);
}
ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return await StrikeAndCheckLimit(torrent.Hash, torrent.Name);
if (torrent.State is TorrentState.StalledDownload)
{
return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.Stalled), DeleteReason.Stalled);
}
return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
}
private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
@@ -5,12 +5,14 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Exceptions;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -26,7 +28,6 @@ public class TransmissionService : DownloadService, ITransmissionService
{
private readonly TransmissionConfig _config;
private readonly Client _client;
private TorrentInfo[]? _torrentsCache;
private static readonly string[] Fields =
[
@@ -56,10 +57,11 @@ public class TransmissionService : DownloadService, ITransmissionService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
_config = config.Value;
@@ -119,17 +121,15 @@ public class TransmissionService : DownloadService, ITransmissionService
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
return result;
}
// remove if all files are unwanted or download is stuck
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download);
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download);
if (!shouldRemove && result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
return result;
}
@@ -211,14 +211,38 @@ public class TransmissionService : DownloadService, ITransmissionService
return result;
}
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override async Task<List<object>?> GetSeedingDownloads()
{
return (await _client.TorrentGetAsync(Fields))
string[] fields = [
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER,
TorrentFields.DOWNLOAD_DIR,
TorrentFields.SECONDS_SEEDING,
TorrentFields.UPLOAD_RATIO
];
return (await _client.TorrentGetAsync(fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
return downloads
?
.Cast<TorrentInfo>()
.Where(x => categories
.Any(cat =>
{
@@ -235,10 +259,20 @@ public class TransmissionService : DownloadService, ITransmissionService
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
throw new NotImplementedException();
}
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.HashString))
@@ -252,7 +286,7 @@ public class TransmissionService : DownloadService, ITransmissionService
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x =>
{
if (download.DownloadDir is null)
@@ -306,6 +340,138 @@ public class TransmissionService : DownloadService, ITransmissionService
}
}
public override async Task CreateCategoryAsync(string name)
{
throw new NotImplementedException();
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
// TODO ignored downloads
throw new NotImplementedException();
// if (_downloadCleanerConfig.NoHardLinksIgnoreRootDir)
// {
// downloads
// .Cast<TorrentInfo>()
// .Select(x =>
// {
// if (x.DownloadDir == null)
// {
// return string.Empty;
// }
//
// string? firstDir = GetRootWithFirstDirectory(x.DownloadDir);
//
// if (string.IsNullOrEmpty(firstDir))
// {
// return string.Empty;
// }
//
// if (firstDir == Path.GetPathRoot(x.DownloadDir))
// {
// return string.Empty;
// }
//
// return firstDir;
// })
// .Where(x => !string.IsNullOrEmpty(x))
// .Distinct()
// .ToList()
// .ForEach(x =>
// {
// _logger.LogTrace("populating file counts from {dir}", x);
//
// if (!Directory.Exists(x))
// {
// throw new ValidationException($"directory \"{x}\" does not exist");
// }
//
// _hardLinkFileService.PopulateFileCounts(x);
// });
// }
//
// foreach (TorrentInfo download in downloads.Cast<TorrentInfo>())
// {
// if (string.IsNullOrEmpty(download.HashString) || download.DownloadDir == null)
// {
// _logger.LogDebug("skip | download hash or download directory is null for {name}", download.Name);
// continue;
// }
//
// if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
// {
// _logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
// continue;
// }
//
// ContextProvider.Set("downloadName", download.Name);
// ContextProvider.Set("hash", download.HashString);
//
// bool hasHardlinks = false;
//
// if (download.Files != null)
// {
// foreach (TransmissionTorrentFiles file in download.Files)
// {
// string filePath = Path.Combine(download.DownloadDir, file.Name);
//
// long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, _downloadCleanerConfig.NoHardLinksIgnoreRootDir);
//
// if (hardlinkCount < 0)
// {
// _logger.LogDebug("skip | could not get file properties | {name}", download.Name);
// hasHardlinks = true;
// break;
// }
//
// if (hardlinkCount > 0)
// {
// hasHardlinks = true;
// break;
// }
// }
// }
//
// if (hasHardlinks)
// {
// _logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
// continue;
// }
//
// // Get the current category (directory name)
// string currentCategory = Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir));
//
// // Create the new location path
// string newLocation = Path.Combine(
// Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(download.DownloadDir)) ?? string.Empty,
// _downloadCleanerConfig.NoHardLinksCategory
// );
//
// await _dryRunInterceptor.InterceptAsync(MoveDownload, download.Id, newLocation);
//
// _logger.LogInformation("category changed for {name}", download.Name);
//
// await _notifier.NotifyCategoryChanged(currentCategory, _downloadCleanerConfig.NoHardLinksCategory);
// }
}
[DryRunSafeguard]
protected virtual async Task MoveDownload(long downloadId, string newLocation)
{
await _client.TorrentSetAsync(new TorrentSettings
{
Ids = [downloadId],
Location = newLocation,
// Move = true
});
}
public override async Task DeleteDownload(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
@@ -338,60 +504,38 @@ public class TransmissionService : DownloadService, ITransmissionService
});
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent)
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
return false;
return (false, default);
}
if (torrent.Status is not 4)
{
// not in downloading state
return false;
return (false, default);
}
if (torrent.Eta > 0)
{
return false;
return (false, default);
}
ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
return await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
return (await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!, StrikeType.Stalled), DeleteReason.Stalled);
}
private async Task<TorrentInfo?> GetTorrentAsync(string hash)
{
TorrentInfo? torrent = _torrentsCache?
.FirstOrDefault(x => x.HashString.Equals(hash, StringComparison.InvariantCultureIgnoreCase));
if (_torrentsCache is null || torrent is null)
{
// refresh cache
_torrentsCache = (await _client.TorrentGetAsync(Fields))
?.Torrents;
}
if (_torrentsCache?.Length is null or 0)
{
_logger.LogDebug("could not list torrents | {url}", _config.Url);
}
torrent = _torrentsCache?.FirstOrDefault(x => x.HashString.Equals(hash, StringComparison.InvariantCultureIgnoreCase));
if (torrent is null)
{
_logger.LogDebug("could not find torrent | {hash} | {url}", hash, _config.Url);
}
return torrent;
}
private async Task<TorrentInfo?> GetTorrentAsync(string hash) =>
(await _client.TorrentGetAsync(Fields, hash))
?.Torrents
?.FirstOrDefault();
}
@@ -0,0 +1,51 @@
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.Files;
public class HardLinkFileService : IHardLinkFileService
{
private readonly ILogger<HardLinkFileService> _logger;
private readonly UnixHardLinkFileService _unixHardLinkFileService;
private readonly WindowsHardLinkFileService _windowsHardLinkFileService;
public HardLinkFileService(
ILogger<HardLinkFileService> logger,
UnixHardLinkFileService unixHardLinkFileService,
WindowsHardLinkFileService windowsHardLinkFileService
)
{
_logger = logger;
_unixHardLinkFileService = unixHardLinkFileService;
_windowsHardLinkFileService = windowsHardLinkFileService;
}
public void PopulateFileCounts(string directoryPath)
{
_logger.LogTrace("populating file counts from {dir}", directoryPath);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_windowsHardLinkFileService.PopulateFileCounts(directoryPath);
return;
}
_unixHardLinkFileService.PopulateFileCounts(directoryPath);
}
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
if (!File.Exists(filePath))
{
_logger.LogDebug("file {file} does not exist", filePath);
return -1;
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return _windowsHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
}
return _unixHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
}
}
@@ -0,0 +1,19 @@
namespace Infrastructure.Verticals.Files;
public interface IHardLinkFileService
{
/// <summary>
/// Populates the inode counts for Unix and the file index counts for Windows.
/// Needs to be called before <see cref="GetHardLinkCount"/> to populate the inode counts.
/// </summary>
/// <param name="directoryPath">The root directory where to search for hardlinks.</param>
void PopulateFileCounts(string directoryPath);
/// <summary>
/// Get the hardlink count of a file.
/// </summary>
/// <param name="filePath">File path.</param>
/// <param name="ignoreRootDir">Whether to ignore hardlinks found in the same root dir.</param>
/// <returns>-1 on error, 0 if there are no hardlinks and 1 otherwise.</returns>
long GetHardLinkCount(string filePath, bool ignoreRootDir);
}
@@ -0,0 +1,87 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Mono.Unix.Native;
namespace Infrastructure.Verticals.Files;
public class UnixHardLinkFileService : IHardLinkFileService, IDisposable
{
private readonly ILogger<UnixHardLinkFileService> _logger;
private readonly ConcurrentDictionary<ulong, int> _inodeCounts = new();
public UnixHardLinkFileService(ILogger<UnixHardLinkFileService> logger)
{
_logger = logger;
}
/// <inheritdoc/>
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
try
{
if (Syscall.stat(filePath, out Stat stat) != 0)
{
_logger.LogDebug("failed to stat file {file}", filePath);
return -1;
}
if (!ignoreRootDir)
{
_logger.LogDebug("stat file | hardlinks: {nlink} | {file}", stat.st_nlink, filePath);
return (long)stat.st_nlink == 1 ? 0 : 1;
}
// get the number of hardlinks in the same root directory
int linksInIgnoredDir = _inodeCounts.TryGetValue(stat.st_ino, out int count)
? count
: 1; // default to 1 if not found
_logger.LogDebug("stat file | hardlinks: {nlink} | ignored: {ignored} | {file}", stat.st_nlink, linksInIgnoredDir, filePath);
return (long)stat.st_nlink - linksInIgnoredDir;
}
catch (Exception exception)
{
_logger.LogError(exception, "failed to stat file {file}", filePath);
return -1;
}
}
/// <inheritdoc/>
public void PopulateFileCounts(string directoryPath)
{
try
{
// traverse all files in the ignored path and subdirectories
foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories))
{
AddInodeToCount(file);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to populate inode counts from {dir}", directoryPath);
throw;
}
}
private void AddInodeToCount(string path)
{
try
{
if (Syscall.stat(path, out Stat stat) == 0)
{
_inodeCounts.AddOrUpdate(stat.st_ino, 1, (_, count) => count + 1);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "could not stat {path} during inode counting", path);
throw;
}
}
public void Dispose()
{
_inodeCounts.Clear();
}
}
@@ -0,0 +1,113 @@
using System.Collections.Concurrent;
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
namespace Infrastructure.Verticals.Files;
public class WindowsHardLinkFileService : IHardLinkFileService, IDisposable
{
private readonly ILogger<WindowsHardLinkFileService> _logger;
private readonly ConcurrentDictionary<ulong, int> _fileIndexCounts = new();
public WindowsHardLinkFileService(ILogger<WindowsHardLinkFileService> logger)
{
_logger = logger;
}
/// <inheritdoc/>
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
try
{
using SafeFileHandle fileStream = File.OpenHandle(filePath);
if (!GetFileInformationByHandle(fileStream, out var file))
{
_logger.LogDebug("failed to get file handle {file}", filePath);
return -1;
}
if (!ignoreRootDir)
{
_logger.LogDebug("stat file | hardlinks: {nlink} | {file}", file.NumberOfLinks, filePath);
return file.NumberOfLinks == 1 ? 0 : 1;
}
// Get unique file ID (combination of high and low indices)
ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow;
// get the number of hardlinks in the same root directory
int linksInIgnoredDir = _fileIndexCounts.TryGetValue(fileIndex, out int count)
? count
: 1; // default to 1 if not found
_logger.LogDebug("stat file | hardlinks: {links} | ignored: {ignored} | {file}", file.NumberOfLinks, linksInIgnoredDir, filePath);
return file.NumberOfLinks - linksInIgnoredDir;
}
catch (Exception exception)
{
_logger.LogError(exception, "failed to stat file {file}", filePath);
return -1;
}
}
/// <inheritdoc/>
public void PopulateFileCounts(string directoryPath)
{
try
{
// traverse all files in the ignored path and subdirectories
foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories))
{
AddFileIndexToCount(file);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to populate file index counts from {dir}", directoryPath);
}
}
private void AddFileIndexToCount(string path)
{
try
{
using SafeFileHandle fileStream = File.OpenHandle(path);
if (GetFileInformationByHandle(fileStream, out var file))
{
ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow;
_fileIndexCounts.AddOrUpdate(fileIndex, 1, (_, count) => count + 1);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Couldn't stat {path} during file index counting", path);
}
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool GetFileInformationByHandle(
SafeFileHandle hFile,
out BY_HANDLE_FILE_INFORMATION lpFileInformation
);
private struct BY_HANDLE_FILE_INFORMATION
{
public uint FileAttributes;
public System.Runtime.InteropServices.ComTypes.FILETIME CreationTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastAccessTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastWriteTime;
public uint VolumeSerialNumber;
public uint FileSizeHigh;
public uint FileSizeLow;
public uint NumberOfLinks;
public uint FileIndexHigh;
public uint FileIndexLow;
}
public void Dispose()
{
_fileIndexCounts.Clear();
}
}
@@ -33,6 +33,9 @@ public sealed class NotificationConsumer<T> : IConsumer<T> where T : Notificatio
case DownloadCleanedNotification downloadCleanedNotification:
await _notificationService.Notify(downloadCleanedNotification);
break;
case CategoryChangedNotification categoryChangedNotification:
await _notificationService.Notify(categoryChangedNotification);
break;
default:
throw new NotImplementedException();
}
@@ -9,4 +9,6 @@ public interface INotificationFactory
List<INotificationProvider> OnQueueItemDeletedEnabled();
List<INotificationProvider> OnDownloadCleanedEnabled();
List<INotificationProvider> OnCategoryChangedEnabled();
}
@@ -16,4 +16,6 @@ public interface INotificationProvider
Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
Task OnDownloadCleaned(DownloadCleanedNotification notification);
Task OnCategoryChanged(CategoryChangedNotification notification);
}
@@ -9,4 +9,6 @@ public interface INotificationPublisher
Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason);
Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason);
Task NotifyCategoryChanged(string oldCategory, string newCategory);
}
@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record CategoryChangedNotification : Notification
{
}
@@ -41,6 +41,11 @@ public class NotifiarrProvider : NotificationProvider
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
public override async Task OnCategoryChanged(CategoryChangedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
private NotifiarrPayload BuildPayload(ArrNotification notification, string color)
{
@@ -105,4 +110,32 @@ public class NotifiarrProvider : NotificationProvider
return payload;
}
private NotifiarrPayload BuildPayload(CategoryChangedNotification notification)
{
NotifiarrPayload payload = new()
{
Discord = new()
{
Color = WarningColor,
Text = new()
{
Title = notification.Title,
Icon = Logo,
Description = notification.Description,
Fields = notification.Fields?.Adapt<List<Field>>() ?? []
},
Ids = new Ids
{
Channel = _config.ChannelId
},
Images = new()
{
Thumbnail = new Uri(Logo)
}
}
};
return payload;
}
}
@@ -1,5 +1,6 @@
using System.Text;
using Common.Helpers;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
@@ -7,12 +8,14 @@ namespace Infrastructure.Verticals.Notifications.Notifiarr;
public class NotifiarrProxy : INotifiarrProxy
{
private readonly ILogger<NotifiarrProxy> _logger;
private readonly HttpClient _httpClient;
private const string Url = "https://notifiarr.com/api/v1/notification/passthrough/";
public NotifiarrProxy(IHttpClientFactory httpClientFactory)
public NotifiarrProxy(ILogger<NotifiarrProxy> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
}
@@ -25,6 +28,8 @@ public class NotifiarrProxy : INotifiarrProxy
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
_logger.LogTrace("sending notification to Notifiarr: {content}", content);
using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, $"{Url}{config.ApiKey}");
request.Method = HttpMethod.Post;
request.Content = new StringContent(content, Encoding.UTF8, "application/json");
@@ -34,4 +34,9 @@ public class NotificationFactory : INotificationFactory
ActiveProviders()
.Where(n => n.Config.OnDownloadCleaned)
.ToList();
public List<INotificationProvider> OnCategoryChangedEnabled() =>
ActiveProviders()
.Where(n => n.Config.OnCategoryChanged)
.ToList();
}
@@ -22,4 +22,6 @@ public abstract class NotificationProvider : INotificationProvider
public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification);
public abstract Task OnCategoryChanged(CategoryChangedNotification notification);
}
@@ -48,10 +48,10 @@ public class NotificationPublisher : INotificationPublisher
switch (strikeType)
{
case StrikeType.Stalled:
await _dryRunInterceptor.InterceptAsync(Notify<StalledStrikeNotification>, notification.Adapt<StalledStrikeNotification>());
await NotifyInternal(notification.Adapt<StalledStrikeNotification>());
break;
case StrikeType.ImportFailed:
await _dryRunInterceptor.InterceptAsync(Notify<FailedImportStrikeNotification>, notification.Adapt<FailedImportStrikeNotification>());
await NotifyInternal(notification.Adapt<FailedImportStrikeNotification>());
break;
}
}
@@ -81,7 +81,7 @@ public class NotificationPublisher : INotificationPublisher
Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }]
};
await _dryRunInterceptor.InterceptAsync(Notify<QueueItemDeletedNotification>, notification);
await NotifyInternal(notification);
}
catch (Exception ex)
{
@@ -110,13 +110,36 @@ public class NotificationPublisher : INotificationPublisher
Level = NotificationLevel.Important
};
await _dryRunInterceptor.InterceptAsync(Notify<DownloadCleanedNotification>, notification);
await NotifyInternal(notification);
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to notify download cleaned");
}
}
public virtual async Task NotifyCategoryChanged(string oldCategory, string newCategory)
{
CategoryChangedNotification notification = new()
{
Title = "Category changed",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Old category", Text = oldCategory },
new() { Title = "New category", Text = newCategory }
],
Level = NotificationLevel.Important
};
await NotifyInternal(notification);
}
private Task NotifyInternal<T>(T message) where T: notnull
{
return _dryRunInterceptor.InterceptAsync(Notify<T>, message);
}
[DryRunSafeguard]
private Task Notify<T>(T message) where T: notnull
@@ -73,4 +73,19 @@ public class NotificationService
}
}
}
public async Task Notify(CategoryChangedNotification notification)
{
foreach (INotificationProvider provider in _notificationFactory.OnCategoryChangedEnabled())
{
try
{
await provider.OnCategoryChanged(notification);
}
catch (Exception exception)
{
_logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name);
}
}
}
}
@@ -132,7 +132,7 @@ public sealed class QueueCleaner : GenericHandler
}
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, deleteReason);
await _notifier.NotifyQueueItemDeleted(removeFromClient, deleteReason);
}
});
+10 -4
View File
@@ -178,7 +178,7 @@ services:
- TZ=Europe/Bucharest
- DRY_RUN=false
- LOGGING__LOGLEVEL=Debug
- LOGGING__LOGLEVEL=Verbose
- LOGGING__FILE__ENABLED=true
- LOGGING__FILE__PATH=/var/logs
- LOGGING__ENHANCED=true
@@ -212,11 +212,15 @@ services:
- DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr
- DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1
- DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0
- DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=0.01
- DOWNLOADCLEANER__CATEGORIES__1__NAME=radarr
- DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=99999
- DOWNLOADCLEANER__CATEGORIES__1__NAME=nohardlink
- DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1
- DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0
- DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=0.01
- DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=99999
- DOWNLOADCLEANER__UNLINKED_TARGET_CATEGORY=cleanuperr-unlinked
- DOWNLOADCLEANER__UNLINKED_IGNORED_ROOT_DIR=/downloads
- DOWNLOADCLEANER__UNLINKED_CATEGORIES__0=tv-sonarr
- DOWNLOADCLEANER__UNLINKED_CATEGORIES__1=radarr
- DOWNLOAD_CLIENT=qbittorrent
- QBITTORRENT__URL=http://qbittorrent:8080
@@ -255,11 +259,13 @@ services:
# - NOTIFIARR__ON_STALLED_STRIKE=true
# - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true
# - NOTIFIARR__ON_DOWNLOAD_CLEANED=true
# - NOTIFIARR__ON_CATEGORY_CHANGED=true
# - NOTIFIARR__API_KEY=notifiarr_secret
# - NOTIFIARR__CHANNEL_ID=discord_channel_id
volumes:
- ./data/cleanuperr/logs:/var/logs
- ./data/cleanuperr/ignored_downloads:/ignored
- ./data/qbittorrent/downloads:/downloads
restart: unless-stopped
depends_on:
- qbittorrent