From 017e25fb06b710682f8197a5d2c73fe172ff017e Mon Sep 17 00:00:00 2001 From: Flaminel Date: Wed, 19 Feb 2025 23:46:12 +0200 Subject: [PATCH] added category change for downloads with no additional hardlinks --- .../{Category.cs => CleanCategory.cs} | 2 +- .../DownloadCleaner/DownloadCleanerConfig.cs | 32 +++++- .../Notification/NotificationConfig.cs | 10 +- .../DependencyInjection/ServicesDI.cs | 2 + code/Executable/appsettings.Development.json | 5 + .../DownloadClient/DownloadServiceFixture.cs | 7 +- .../DownloadClient/DownloadServiceTests.cs | 8 +- .../DownloadClient/TestDownloadService.cs | 19 ++-- .../DownloadCleaner/DownloadCleaner.cs | 33 ++++++- .../DownloadClient/Deluge/DelugeService.cs | 24 +++-- .../DownloadClient/DownloadService.cs | 24 +++-- .../DownloadClient/DummyDownloadService.cs | 33 ++++++- .../DownloadClient/IDownloadService.cs | 15 ++- .../DownloadClient/QBittorrent/QBitService.cs | 76 ++++++++++++-- .../Transmission/TransmissionService.cs | 24 +++-- .../Verticals/Files/HardlinkFileService.cs | 99 +++++++++++++++++++ .../Verticals/Files/IHardlinkFileService.cs | 6 ++ .../Consumers/NotificationConsumer.cs | 3 + .../Notifications/INotificationFactory.cs | 2 + .../Notifications/INotificationProvider.cs | 2 + .../Models/CategoryChangedNotification.cs | 5 + .../Notifiarr/NotifiarrProvider.cs | 33 +++++++ .../Notifications/NotificationFactory.cs | 5 + .../Notifications/NotificationProvider.cs | 2 + .../Notifications/NotificationPublisher.cs | 19 ++++ .../Notifications/NotificationService.cs | 15 +++ code/test/docker-compose.yml | 9 +- 27 files changed, 458 insertions(+), 56 deletions(-) rename code/Common/Configuration/DownloadCleaner/{Category.cs => CleanCategory.cs} (96%) create mode 100644 code/Infrastructure/Verticals/Files/HardlinkFileService.cs create mode 100644 code/Infrastructure/Verticals/Files/IHardlinkFileService.cs create mode 100644 code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs diff --git a/code/Common/Configuration/DownloadCleaner/Category.cs b/code/Common/Configuration/DownloadCleaner/CleanCategory.cs similarity index 96% rename from code/Common/Configuration/DownloadCleaner/Category.cs rename to code/Common/Configuration/DownloadCleaner/CleanCategory.cs index 67d12c5..48574cf 100644 --- a/code/Common/Configuration/DownloadCleaner/Category.cs +++ b/code/Common/Configuration/DownloadCleaner/CleanCategory.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadCleaner; -public sealed record Category : IConfig +public sealed record CleanCategory : IConfig { public required string Name { get; init; } diff --git a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs index 7f08fdb..c9a0459 100644 --- a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs +++ b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs @@ -8,11 +8,17 @@ public sealed record DownloadCleanerConfig : IJobConfig public const string SectionName = "DownloadCleaner"; public bool Enabled { get; init; } - - public List? Categories { get; init; } + + public List? Categories { get; init; } [ConfigurationKeyName("DELETE_PRIVATE")] - public bool DeletePrivate { get; set; } + public bool DeletePrivate { get; init; } + + [ConfigurationKeyName("NO_HARDLINKS_CATEGORY")] + public string NoHardlinksCategory { get; init; } = ""; + + [ConfigurationKeyName("HARDLINK_CATEGORIES")] + public List? HardlinkCategories { get; init; } public void Validate() { @@ -32,5 +38,25 @@ public sealed record DownloadCleanerConfig : IJobConfig } Categories?.ForEach(x => x.Validate()); + + if (string.IsNullOrEmpty(NoHardlinksCategory)) + { + return; + } + + if (HardlinkCategories?.Count is null or 0) + { + throw new ValidationException("no categories configured"); + } + + if (HardlinkCategories.Contains(NoHardlinksCategory)) + { + throw new ValidationException("NO_HARDLINKS_CATEGORY is present in the list of filtered categories"); + } + + if (HardlinkCategories.Any(string.IsNullOrEmpty)) + { + throw new ValidationException("empty hardlink filter category found"); + } } } \ No newline at end of file diff --git a/code/Common/Configuration/Notification/NotificationConfig.cs b/code/Common/Configuration/Notification/NotificationConfig.cs index 18e7415..7c45b5f 100644 --- a/code/Common/Configuration/Notification/NotificationConfig.cs +++ b/code/Common/Configuration/Notification/NotificationConfig.cs @@ -15,8 +15,16 @@ public abstract record NotificationConfig [ConfigurationKeyName("ON_DOWNLOAD_CLEANED")] public bool OnDownloadCleaned { get; init; } + + [ConfigurationKeyName("ON_CATEGORY_CHANGED")] + public bool OnCategoryChanged { get; init; } - public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnQueueItemDeleted || OnDownloadCleaned; + public bool IsEnabled => + OnImportFailedStrike || + OnStalledStrike || + OnQueueItemDeleted || + OnDownloadCleaned || + OnCategoryChanged; public abstract bool IsValid(); } \ No newline at end of file diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index e1becad..b737dd8 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -6,6 +6,7 @@ using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient.Deluge; using Infrastructure.Verticals.DownloadClient.QBittorrent; using Infrastructure.Verticals.DownloadClient.Transmission; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.QueueCleaner; @@ -23,6 +24,7 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddTransient() + .AddTransient() .AddTransient() .AddTransient() .AddTransient() diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 27dc69b..e273531 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -44,6 +44,11 @@ "MIN_SEED_TIME": 0, "MAX_SEED_TIME": -1 } + ], + "NO_HARDLINKS_CATEGORY": "nohardlinks", + "HARDLINK_CATEGORIES": [ + "tv-sonarr", + "radarr" ] }, "DOWNLOAD_CLIENT": "qbittorrent", diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs index e1d3559..69be2eb 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs @@ -1,9 +1,10 @@ -using Common.Configuration.ContentBlocker; +using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -56,6 +57,7 @@ public class DownloadServiceFixture : IDisposable var filenameEvaluator = Substitute.For(); var notifier = Substitute.For(); var dryRunInterceptor = Substitute.For(); + var hardlinkFileService = Substitute.For(); return new TestDownloadService( Logger, @@ -66,7 +68,8 @@ public class DownloadServiceFixture : IDisposable filenameEvaluator, Striker, notifier, - dryRunInterceptor + dryRunInterceptor, + hardlinkFileService ); } diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs index 9fcba54..3a5fc35 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs @@ -132,7 +132,7 @@ public class DownloadServiceTests : IClassFixture public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 1.0, @@ -158,7 +158,7 @@ public class DownloadServiceTests : IClassFixture public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 1.0, @@ -184,7 +184,7 @@ public class DownloadServiceTests : IClassFixture public void WhenMaxSeedTimeReached_ShouldReturnTrue() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = -1, @@ -210,7 +210,7 @@ public class DownloadServiceTests : IClassFixture public void WhenNeitherConditionMet_ShouldReturnFalse() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 2.0, diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index abf0f76..d37282b 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; @@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -25,10 +26,11 @@ public class TestDownloadService : DownloadService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService ) { } @@ -38,12 +40,15 @@ public class TestDownloadService : DownloadService public override Task ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult()); public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, ConcurrentBag regexes) => Task.FromResult(new BlockFilesResult()); - public override Task DeleteDownload(string hash) => Task.CompletedTask; - public override Task?> GetAllDownloadsToBeCleaned(List categories) => Task.FromResult?>(null); - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) => Task.CompletedTask; + public override Task DeleteDownload(string hash) => Task.CompletedTask; + public override Task?> GetDownloadsToBeCleaned(List categories) => Task.FromResult?>(null); + public override Task?> GetDownloadsToChangeCategory(List categories) => Task.FromResult?>(null); + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) => Task.CompletedTask; + public override Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes) => Task.CompletedTask; + // Expose protected methods for testing public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); public new Task StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName); - public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category); + public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) => base.ShouldCleanDownload(ratio, seedingTime, category); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index 2732694..b922da4 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -60,11 +60,20 @@ public sealed class DownloadCleaner : GenericHandler await _downloadService.LoginAsync(); - List? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); + List? downloadsToBeCleaned = await _downloadService.GetDownloadsToBeCleaned(_config.Categories); + List? downloadsToChangeCategory = null; - if (downloads?.Count is null or 0) + if (!string.IsNullOrEmpty(_config.NoHardlinksCategory) && _config.HardlinkCategories?.Count > 0) { - _logger.LogDebug("no downloads found in the download client"); + downloadsToChangeCategory = await _downloadService.GetDownloadsToChangeCategory(_config.HardlinkCategories); + } + + bool hasDownloadsToClean = downloadsToBeCleaned?.Count > 0; + bool hasDownloadsToChange = downloadsToChangeCategory?.Count > 0; + + if (!hasDownloadsToClean && !hasDownloadsToChange) + { + _logger.LogDebug("no downloads to process"); return; } @@ -75,7 +84,23 @@ public sealed class DownloadCleaner : GenericHandler await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); - await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes); + if (hasDownloadsToChange) + { + await _downloadService.ChangeCategoryForNoHardlinksAsync(downloadsToChangeCategory, _excludedHashes); + } + else + { + _logger.LogDebug("no downloads found to change category"); + } + + if (hasDownloadsToClean) + { + await _downloadService.CleanDownloads(downloadsToBeCleaned, _config.Categories, _excludedHashes); + } + else + { + _logger.LogDebug("no downloads found to be cleaned"); + } } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 47cfc3f..ca75243 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -10,6 +10,7 @@ using Domain.Models.Deluge.Response; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -33,10 +34,11 @@ public class DelugeService : DownloadService, IDelugeService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService ) { config.Value.Validate(); @@ -194,7 +196,7 @@ public class DelugeService : DownloadService, IDelugeService return result; } - public override async Task?> GetAllDownloadsToBeCleaned(List categories) + public override async Task?> GetDownloadsToBeCleaned(List categories) { return (await _client.GetStatusForAllTorrents()) ?.Where(x => !string.IsNullOrEmpty(x.Hash)) @@ -204,8 +206,13 @@ public class DelugeService : DownloadService, IDelugeService .ToList(); } + public override Task?> GetDownloadsToChangeCategory(List categories) + { + throw new NotImplementedException(); + } + /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) { foreach (TorrentStatus download in downloads) { @@ -214,7 +221,7 @@ public class DelugeService : DownloadService, IDelugeService continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); if (category is null) @@ -258,7 +265,12 @@ public class DelugeService : DownloadService, IDelugeService await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason); } } - + + public override Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes) + { + throw new NotImplementedException(); + } + /// [DryRunSafeguard] public override async Task DeleteDownload(string hash) diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index 3f528d3..fe76c16 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; @@ -10,6 +10,7 @@ using Infrastructure.Helpers; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -30,6 +31,7 @@ public abstract class DownloadService : IDownloadService protected readonly MemoryCacheEntryOptions _cacheOptions; protected readonly INotificationPublisher _notifier; protected readonly IDryRunInterceptor _dryRunInterceptor; + protected readonly IHardlinkFileService _hardlinkFileService; protected DownloadService( ILogger logger, @@ -40,7 +42,8 @@ public abstract class DownloadService : IDownloadService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService ) { _logger = logger; @@ -52,6 +55,7 @@ public abstract class DownloadService : IDownloadService _striker = striker; _notifier = notifier; _dryRunInterceptor = dryRunInterceptor; + _hardlinkFileService = hardlinkFileService; _cacheOptions = new MemoryCacheEntryOptions() .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); } @@ -74,11 +78,17 @@ public abstract class DownloadService : IDownloadService public abstract Task DeleteDownload(string hash); /// - public abstract Task?> GetAllDownloadsToBeCleaned(List categories); + public abstract Task?> GetDownloadsToBeCleaned(List categories); /// - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + public abstract Task?> GetDownloadsToChangeCategory(List categories); + /// + public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + + /// + public abstract Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes); + protected void ResetStrikesOnProgress(string hash, long downloaded) { if (!_queueCleanerConfig.StalledResetStrikesOnProgress) @@ -107,7 +117,7 @@ public abstract class DownloadService : IDownloadService return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled); } - protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) + protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) { // check ratio if (DownloadReachedRatio(ratio, seedingTime, category)) @@ -132,7 +142,7 @@ public abstract class DownloadService : IDownloadService return new(); } - private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) + private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category) { if (category.MaxRatio < 0) { @@ -158,7 +168,7 @@ public abstract class DownloadService : IDownloadService return true; } - private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category) + private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category) { if (category.MaxSeedTime < 0) { diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 9126328..3fecdb2 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -1,10 +1,11 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -15,7 +16,21 @@ namespace Infrastructure.Verticals.DownloadClient; public class DummyDownloadService : DownloadService { - public DummyDownloadService(ILogger logger, IOptions queueCleanerConfig, IOptions contentBlockerConfig, IOptions downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier, dryRunInterceptor) + public DummyDownloadService( + ILogger logger, + IOptions queueCleanerConfig, + IOptions contentBlockerConfig, + IOptions downloadCleanerConfig, + IMemoryCache cache, + IFilenameEvaluator filenameEvaluator, + IStriker striker, + NotificationPublisher notifier, + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService + ) : base( + logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, + cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService + ) { } @@ -38,12 +53,22 @@ public class DummyDownloadService : DownloadService throw new NotImplementedException(); } - public override Task?> GetAllDownloadsToBeCleaned(List categories) + public override Task?> GetDownloadsToBeCleaned(List categories) { throw new NotImplementedException(); } - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override Task?> GetDownloadsToChangeCategory(List categories) + { + throw new NotImplementedException(); + } + + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + { + throw new NotImplementedException(); + } + + public override Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index c2d446f..e5c9d2f 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -36,16 +36,25 @@ public interface IDownloadService : IDisposable /// /// The categories by which to filter the downloads. /// A list of downloads for the provided categories. - Task?> GetAllDownloadsToBeCleaned(List categories); + Task?> GetDownloadsToBeCleaned(List categories); + + Task?> GetDownloadsToChangeCategory(List categories); /// /// Cleans the downloads. /// - /// + /// The downloads to clean. /// The categories that should be cleaned. /// The hashes that should not be cleaned. - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + /// + /// Changes the category for downloads that have no hardlinks. + /// + /// The downloads to change. + /// + Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes); + /// /// Deletes a download item. /// diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 0e14776..7090f1b 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -10,13 +10,13 @@ using Domain.Enums; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using QBittorrent.Client; -using Category = Common.Configuration.DownloadCleaner.Category; namespace Infrastructure.Verticals.DownloadClient.QBittorrent; @@ -36,10 +36,11 @@ public class QBitService : DownloadService, IQBitService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService ) { _config = config.Value; @@ -207,7 +208,7 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task?> GetAllDownloadsToBeCleaned(List categories) => + public override async Task?> GetDownloadsToBeCleaned(List categories) => (await _client.GetTorrentListAsync(new() { Filter = TorrentListFilter.Seeding @@ -217,8 +218,19 @@ public class QBitService : DownloadService, IQBitService .Cast() .ToList(); + public override async Task?> GetDownloadsToChangeCategory(List categories) + { + return (await _client.GetTorrentListAsync(new() + { + Filter = TorrentListFilter.Seeding + })) + ?.Where(x => !string.IsNullOrEmpty(x.Hash)) + .Cast() + .ToList(); + } + /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) { foreach (TorrentInfo download in downloads) { @@ -227,7 +239,7 @@ public class QBitService : DownloadService, IQBitService continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); if (category is null) @@ -286,6 +298,52 @@ public class QBitService : DownloadService, IQBitService } } + public override async Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes) + { + // TODO account for cross-seed + foreach (TorrentInfo download in downloads) + { + IReadOnlyList? files = await _client.GetTorrentContentsAsync(download.Hash); + + if (files is null) + { + _logger.LogDebug("failed to find files for {name}", download.Name); + return; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + bool hasHardlinks = false; + + foreach (TorrentContent file in files) + { + if (!file.Index.HasValue) + { + _logger.LogDebug("skip | file index is null for {name}", download.Name); + return; + } + + if (_hardlinkFileService.GetHardLinkCount(file.Name) > 1) + { + hasHardlinks = true; + }; + } + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + await ((QBitService)Proxy).ChangeCategory(download.Hash, _downloadCleanerConfig.NoHardlinksCategory); + await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.NoHardlinksCategory); + } + } + /// [DryRunSafeguard] public override async Task DeleteDownload(string hash) @@ -299,6 +357,12 @@ public class QBitService : DownloadService, IQBitService await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); } + [DryRunSafeguard] + protected virtual async Task ChangeCategory(string hash, string newCategory) + { + await _client.SetTorrentCategoryAsync([hash], newCategory); + } + public override void Dispose() { _client.Dispose(); diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 80abfa7..568a67b 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -10,6 +10,7 @@ using Domain.Enums; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -38,10 +39,11 @@ public class TransmissionService : DownloadService, ITransmissionService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardlinkFileService hardlinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService ) { _config = config.Value; @@ -181,7 +183,7 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task?> GetAllDownloadsToBeCleaned(List categories) + public override async Task?> GetDownloadsToBeCleaned(List categories) { string[] fields = [ TorrentFields.FILES, @@ -218,8 +220,13 @@ public class TransmissionService : DownloadService, ITransmissionService .ToList(); } + public override Task?> GetDownloadsToChangeCategory(List categories) + { + throw new NotImplementedException(); + } + /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) { foreach (TorrentInfo download in downloads) { @@ -228,7 +235,7 @@ public class TransmissionService : DownloadService, ITransmissionService continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => { if (download.DownloadDir is null) @@ -281,7 +288,12 @@ public class TransmissionService : DownloadService, ITransmissionService await _notifier.NotifyDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason); } } - + + public override Task ChangeCategoryForNoHardlinksAsync(List downloads, HashSet excludedHashes) + { + throw new NotImplementedException(); + } + public override async Task DeleteDownload(string hash) { TorrentInfo? torrent = await GetTorrentAsync(hash); diff --git a/code/Infrastructure/Verticals/Files/HardlinkFileService.cs b/code/Infrastructure/Verticals/Files/HardlinkFileService.cs new file mode 100644 index 0000000..5e3e4b3 --- /dev/null +++ b/code/Infrastructure/Verticals/Files/HardlinkFileService.cs @@ -0,0 +1,99 @@ +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Win32.SafeHandles; + +namespace Infrastructure.Verticals.Files; + +public class HardlinkFileService : IHardlinkFileService +{ + private readonly ILogger _logger; + + public HardlinkFileService(ILogger logger) + { + _logger = logger; + } + + public uint GetHardLinkCount(string filePath) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return GetWindowsHardLinkCount(filePath); + } + + return GetUnixHardLinkCount(filePath); + } + + private uint GetWindowsHardLinkCount(string filePath) + { + try + { + using SafeFileHandle fileStream = File.OpenHandle(filePath); + + if (GetFileInformationByHandle(fileStream, out var file)) + { + return file.NumberOfLinks; + } + } + catch (Exception exception) + { + // TODO log download name? + _logger.LogError(exception, "failed to stat Windows file {file}", filePath); + } + + return default; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetFileInformationByHandle( + SafeFileHandle hFile, + out BY_HANDLE_FILE_INFORMATION lpFileInformation + ); + + private struct BY_HANDLE_FILE_INFORMATION + { + public uint FileAttributes; + public System.Runtime.InteropServices.ComTypes.FILETIME CreationTime; + public System.Runtime.InteropServices.ComTypes.FILETIME LastAccessTime; + public System.Runtime.InteropServices.ComTypes.FILETIME LastWriteTime; + public uint VolumeSerialNumber; + public uint FileSizeHigh; + public uint FileSizeLow; + public uint NumberOfLinks; + public uint FileIndexHigh; + public uint FileIndexLow; + } + + private uint GetUnixHardLinkCount(string filePath) + { + try + { + if (stat64(filePath, out Stat stat) == 0) + { + return stat.st_nlink; + } + } + catch (Exception exception) + { + // TODO log download name? + _logger.LogError(exception, "failed to stat Unix file {file}", filePath); + } + + return 0; + } + + [DllImport("libc", SetLastError = true)] + private static extern int stat64(string path, out Stat stat); + + [StructLayout(LayoutKind.Sequential)] + private struct Stat + { + public uint st_dev; + public ulong st_ino; + public uint st_mode; + public uint st_nlink; // Hard link count + public uint st_uid; + public uint st_gid; + public uint st_rdev; + public long st_size; + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Files/IHardlinkFileService.cs b/code/Infrastructure/Verticals/Files/IHardlinkFileService.cs new file mode 100644 index 0000000..26bc80c --- /dev/null +++ b/code/Infrastructure/Verticals/Files/IHardlinkFileService.cs @@ -0,0 +1,6 @@ +namespace Infrastructure.Verticals.Files; + +public interface IHardlinkFileService +{ + uint GetHardLinkCount(string filePath); +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs b/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs index 297ce56..6abc990 100644 --- a/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs +++ b/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs @@ -33,6 +33,9 @@ public sealed class NotificationConsumer : IConsumer where T : Notificatio case DownloadCleanedNotification downloadCleanedNotification: await _notificationService.Notify(downloadCleanedNotification); break; + case CategoryChangedNotification categoryChangedNotification: + await _notificationService.Notify(categoryChangedNotification); + break; default: throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs b/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs index 5cf3d2e..ed8fa7c 100644 --- a/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs +++ b/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs @@ -9,4 +9,6 @@ public interface INotificationFactory List OnQueueItemDeletedEnabled(); List OnDownloadCleanedEnabled(); + + List OnCategoryChangedEnabled(); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs b/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs index ea69253..a2cd74e 100644 --- a/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs @@ -16,4 +16,6 @@ public interface INotificationProvider Task OnQueueItemDeleted(QueueItemDeletedNotification notification); Task OnDownloadCleaned(DownloadCleanedNotification notification); + + Task OnCategoryChanged(CategoryChangedNotification notification); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs b/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs new file mode 100644 index 0000000..4da1dc8 --- /dev/null +++ b/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs @@ -0,0 +1,5 @@ +namespace Infrastructure.Verticals.Notifications.Models; + +public sealed record CategoryChangedNotification : Notification +{ +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs index 6ad8c1b..16d1f7a 100644 --- a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs @@ -41,6 +41,11 @@ public class NotifiarrProvider : NotificationProvider { await _proxy.SendNotification(BuildPayload(notification), _config); } + + public override async Task OnCategoryChanged(CategoryChangedNotification notification) + { + await _proxy.SendNotification(BuildPayload(notification), _config); + } private NotifiarrPayload BuildPayload(ArrNotification notification, string color) { @@ -105,4 +110,32 @@ public class NotifiarrProvider : NotificationProvider return payload; } + + private NotifiarrPayload BuildPayload(CategoryChangedNotification notification) + { + NotifiarrPayload payload = new() + { + Discord = new() + { + Color = WarningColor, + Text = new() + { + Title = notification.Title, + Icon = Logo, + Description = notification.Description, + Fields = notification.Fields?.Adapt>() ?? [] + }, + Ids = new Ids + { + Channel = _config.ChannelId + }, + Images = new() + { + Thumbnail = new Uri(Logo) + } + } + }; + + return payload; + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs b/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs index 10c5ba0..a26db9a 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs @@ -34,4 +34,9 @@ public class NotificationFactory : INotificationFactory ActiveProviders() .Where(n => n.Config.OnDownloadCleaned) .ToList(); + + public List OnCategoryChangedEnabled() => + ActiveProviders() + .Where(n => n.Config.OnCategoryChanged) + .ToList(); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs b/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs index e1b6cc7..373ab54 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs @@ -22,4 +22,6 @@ public abstract class NotificationProvider : INotificationProvider public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification); public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification); + + public abstract Task OnCategoryChanged(CategoryChangedNotification notification); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs index 28b3011..fcdb4a8 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs @@ -106,6 +106,25 @@ public class NotificationPublisher : INotificationPublisher { return _messageBus.Publish(message); } + + [DryRunSafeguard] + public virtual async Task NotifyCategoryChanged(string oldCategory, string newCategory) + { + CategoryChangedNotification notification = new() + { + Title = "Category changed", + Description = ContextProvider.Get("downloadName"), + Fields = + [ + new() { Title = "Hash", Text = ContextProvider.Get("hash").ToLowerInvariant() }, + new() { Title = "Old category", Text = oldCategory }, + new() { Title = "New category", Text = newCategory } + ], + Level = NotificationLevel.Important + }; + + await _messageBus.Publish(notification); + } private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) => instanceType switch diff --git a/code/Infrastructure/Verticals/Notifications/NotificationService.cs b/code/Infrastructure/Verticals/Notifications/NotificationService.cs index fbe8fd6..0775b9e 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationService.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationService.cs @@ -73,4 +73,19 @@ public class NotificationService } } } + + public async Task Notify(CategoryChangedNotification notification) + { + foreach (INotificationProvider provider in _notificationFactory.OnCategoryChangedEnabled()) + { + try + { + await provider.OnCategoryChanged(notification); + } + catch (Exception exception) + { + _logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name); + } + } + } } \ No newline at end of file diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 010531d..cd6b446 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -209,11 +209,14 @@ services: - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0 - - DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=0.01 + - DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=-1 - DOWNLOADCLEANER__CATEGORIES__1__NAME=radarr - DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0 - - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=0.01 + - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=-1 + + - DOWNLOADCLEANER__HARDLINK_CATEGORIES__0=tv-sonarr + - DOWNLOADCLEANER__HARDLINK_CATEGORIES__1=radarr - DOWNLOAD_CLIENT=qbittorrent - QBITTORRENT__URL=http://qbittorrent:8080 @@ -252,10 +255,12 @@ services: # - NOTIFIARR__ON_STALLED_STRIKE=true # - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true # - NOTIFIARR__ON_DOWNLOAD_CLEANED=true + # - NOTIFIARR__ON_CATEGORY_CHANGED=true # - NOTIFIARR__API_KEY=notifiarr_secret # - NOTIFIARR__CHANNEL_ID=discord_channel_id volumes: - ./data/cleanuperr/logs:/var/logs + - ./data/qbittorrent/downloads:/downloads restart: unless-stopped depends_on: - qbittorrent