From ba02aa0e49b5b098738cc5260511ad406a1f4809 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Sun, 2 Mar 2025 22:48:21 +0200 Subject: [PATCH 01/16] Fix notifications failing when poster image is not set (#78) --- .../Notifications/NotificationPublisher.cs | 92 ++++++++++++------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs index 28b3011..3d7aed3 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs @@ -32,7 +32,7 @@ public class NotificationPublisher : INotificationPublisher QueueRecord record = ContextProvider.Get(nameof(QueueRecord)); InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); Uri instanceUrl = ContextProvider.Get(nameof(ArrInstance) + nameof(ArrInstance.Url)); - Uri imageUrl = GetImageFromContext(record, instanceType); + Uri? imageUrl = GetImageFromContext(record, instanceType); ArrNotification notification = new() { @@ -63,42 +63,59 @@ public class NotificationPublisher : INotificationPublisher public virtual async Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason) { - QueueRecord record = ContextProvider.Get(nameof(QueueRecord)); - InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); - Uri instanceUrl = ContextProvider.Get(nameof(ArrInstance) + nameof(ArrInstance.Url)); - Uri imageUrl = GetImageFromContext(record, instanceType); - - QueueItemDeletedNotification notification = new() + try { - InstanceType = instanceType, - InstanceUrl = instanceUrl, - Hash = record.DownloadId.ToLowerInvariant(), - Title = $"Deleting item from queue with reason: {reason}", - Description = record.Title, - Image = imageUrl, - Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }] - }; - - await _dryRunInterceptor.InterceptAsync(Notify, notification); + QueueRecord record = ContextProvider.Get(nameof(QueueRecord)); + InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); + Uri instanceUrl = ContextProvider.Get(nameof(ArrInstance) + nameof(ArrInstance.Url)); + Uri? imageUrl = GetImageFromContext(record, instanceType); + + QueueItemDeletedNotification notification = new() + { + InstanceType = instanceType, + InstanceUrl = instanceUrl, + Hash = record.DownloadId.ToLowerInvariant(), + Title = $"Deleting item from queue with reason: {reason}", + Description = record.Title, + Image = imageUrl, + Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }] + }; + + await _dryRunInterceptor.InterceptAsync(Notify, notification); + } + catch (Exception ex) + { + _logger.LogError(ex, "failed to notify queue item deleted"); + } } public virtual async Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason) { - DownloadCleanedNotification notification = new() + try { - Title = $"Cleaned item from download client with reason: {reason}", - Description = ContextProvider.Get("downloadName"), - Fields = - [ - new() { Title = "Hash", Text = ContextProvider.Get("hash").ToLowerInvariant() }, - new() { Title = "Category", Text = categoryName.ToLowerInvariant() }, - new() { Title = "Ratio", Text = $"{ratio.ToString(CultureInfo.InvariantCulture)}%" }, - new() { Title = "Seeding hours", Text = $"{Math.Round(seedingTime.TotalHours, 0).ToString(CultureInfo.InvariantCulture)}h" } - ], - Level = NotificationLevel.Important - }; + DownloadCleanedNotification notification = new() + { + Title = $"Cleaned item from download client with reason: {reason}", + Description = ContextProvider.Get("downloadName"), + Fields = + [ + new() { Title = "Hash", Text = ContextProvider.Get("hash").ToLowerInvariant() }, + new() { Title = "Category", Text = categoryName.ToLowerInvariant() }, + new() { Title = "Ratio", Text = $"{ratio.ToString(CultureInfo.InvariantCulture)}%" }, + new() + { + Title = "Seeding hours", Text = $"{Math.Round(seedingTime.TotalHours, 0).ToString(CultureInfo.InvariantCulture)}h" + } + ], + Level = NotificationLevel.Important + }; - await _dryRunInterceptor.InterceptAsync(Notify, notification); + await _dryRunInterceptor.InterceptAsync(Notify, notification); + } + catch (Exception ex) + { + _logger.LogError(ex, "failed to notify download cleaned"); + } } [DryRunSafeguard] @@ -107,12 +124,21 @@ public class NotificationPublisher : INotificationPublisher return _messageBus.Publish(message); } - private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) => - instanceType switch + private Uri? GetImageFromContext(QueueRecord record, InstanceType instanceType) + { + Uri? image = instanceType switch { InstanceType.Sonarr => record.Series!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, InstanceType.Radarr => record.Movie!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, InstanceType.Lidarr => record.Album!.Images.FirstOrDefault(x => x.CoverType == "cover")?.Url, _ => throw new ArgumentOutOfRangeException(nameof(instanceType)) - } ?? throw new Exception("failed to get image url from context"); + }; + + if (image is null) + { + _logger.LogWarning("no poster found for {title}", record.Title); + } + + return image; + } } \ No newline at end of file From 81990c6768d6bfdd5a2d0c596e17087e6c709e84 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Mon, 3 Mar 2025 22:37:22 +0200 Subject: [PATCH 02/16] fixed missing README link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e20b712..2c3ae20 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ I've seen a few discussions on this type of naming and I've decided that I didn' ## Using cleanuperr's blocklist (works with all supported download clients) 1. Set both `QUEUECLEANER__ENABLED` and `CONTENTBLOCKER__ENABLED` to `true` in your environment variables. -2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Arr variables](#Arr-variables) section. +2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Arr variables](variables.md#Arr-settings) section. 3. Once configured, cleanuperr will perform the following tasks: - Execute the **content blocker** job, as explained in the [How it works](#how-it-works) section. - Execute the **queue cleaner** job, as explained in the [How it works](#how-it-works) section. From 027c4a0f4d8832fbc3598b856fd1dde9b35cc589 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Sun, 9 Mar 2025 23:38:27 +0200 Subject: [PATCH 03/16] Add option to ignore specific downloads (#79) --- README.md | 4 + .../ContentBlocker/ContentBlockerConfig.cs | 5 +- .../DownloadCleaner/DownloadCleanerConfig.cs | 7 +- .../Configuration/IIgnoredDownloadsConfig.cs | 6 + .../QueueCleaner/QueueCleanerConfig.cs | 5 +- .../Models/Deluge/Response/TorrentStatus.cs | 7 ++ .../DependencyInjection/ServicesDI.cs | 11 +- code/Executable/appsettings.Development.json | 7 +- code/Executable/appsettings.json | 7 +- .../DownloadClient/TestDownloadService.cs | 9 +- .../Extensions/DelugeExtensions.cs | 29 +++++ .../Extensions/QBitExtensions.cs | 42 +++++++ .../Extensions/TransmissionExtensions.cs | 42 +++++++ code/Infrastructure/Helpers/CacheKeys.cs | 2 + .../Providers/IgnoredDownloadsProvider.cs | 82 +++++++++++++ .../ContentBlocker/ContentBlocker.cs | 18 ++- .../DownloadCleaner/DownloadCleaner.cs | 10 +- .../DownloadClient/Deluge/DelugeClient.cs | 18 ++- .../DownloadClient/Deluge/DelugeService.cs | 53 ++++++--- .../DownloadClient/DownloadService.cs | 13 +-- .../DownloadClient/DummyDownloadService.cs | 8 +- .../DownloadClient/IDownloadService.cs | 16 ++- .../DownloadClient/QBittorrent/QBitService.cs | 62 +++++++--- .../Transmission/TransmissionService.cs | 110 ++++++++++-------- .../Verticals/QueueCleaner/QueueCleaner.cs | 18 ++- code/test/data/cleanuperr/ignored_downloads | 1 + code/test/docker-compose.yml | 4 + variables.md | 77 +++++++++++- 28 files changed, 552 insertions(+), 121 deletions(-) create mode 100644 code/Common/Configuration/IIgnoredDownloadsConfig.cs create mode 100644 code/Infrastructure/Extensions/DelugeExtensions.cs create mode 100644 code/Infrastructure/Extensions/QBitExtensions.cs create mode 100644 code/Infrastructure/Extensions/TransmissionExtensions.cs create mode 100644 code/Infrastructure/Providers/IgnoredDownloadsProvider.cs create mode 100644 code/test/data/cleanuperr/ignored_downloads diff --git a/README.md b/README.md index 2c3ae20..aad2468 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ services: restart: unless-stopped volumes: - ./cleanuperr/logs:/var/logs + - ./cleanuperr/ignored.txt:/ignored.txt environment: - TZ=America/New_York - DRY_RUN=false @@ -153,6 +154,7 @@ services: - TRIGGERS__DOWNLOADCLEANER=0 0 * * * ? - QUEUECLEANER__ENABLED=true + - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false @@ -165,10 +167,12 @@ services: - QUEUECLEANER__STALLED_DELETE_PRIVATE=false - CONTENTBLOCKER__ENABLED=true + - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored.txt - CONTENTBLOCKER__IGNORE_PRIVATE=false - CONTENTBLOCKER__DELETE_PRIVATE=false - DOWNLOADCLEANER__ENABLED=true + - DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt - DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 diff --git a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs b/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs index a0fb4b4..4b071ef 100644 --- a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs +++ b/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs @@ -2,7 +2,7 @@ namespace Common.Configuration.ContentBlocker; -public sealed record ContentBlockerConfig : IJobConfig +public sealed record ContentBlockerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "ContentBlocker"; @@ -13,6 +13,9 @@ public sealed record ContentBlockerConfig : IJobConfig [ConfigurationKeyName("DELETE_PRIVATE")] public bool DeletePrivate { get; init; } + + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } public void Validate() { diff --git a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs index 7f08fdb..b5658fa 100644 --- a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs +++ b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadCleaner; -public sealed record DownloadCleanerConfig : IJobConfig +public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "DownloadCleaner"; @@ -12,7 +12,10 @@ public sealed record DownloadCleanerConfig : IJobConfig public List? Categories { get; init; } [ConfigurationKeyName("DELETE_PRIVATE")] - public bool DeletePrivate { get; set; } + public bool DeletePrivate { get; init; } + + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } public void Validate() { diff --git a/code/Common/Configuration/IIgnoredDownloadsConfig.cs b/code/Common/Configuration/IIgnoredDownloadsConfig.cs new file mode 100644 index 0000000..f08e445 --- /dev/null +++ b/code/Common/Configuration/IIgnoredDownloadsConfig.cs @@ -0,0 +1,6 @@ +namespace Common.Configuration; + +public interface IIgnoredDownloadsConfig +{ + string? IgnoredDownloadsPath { get; } +} \ No newline at end of file diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 85c2992..f0d7049 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -2,7 +2,7 @@ namespace Common.Configuration.QueueCleaner; -public sealed record QueueCleanerConfig : IJobConfig +public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "QueueCleaner"; @@ -10,6 +10,9 @@ public sealed record QueueCleanerConfig : IJobConfig public required bool RunSequentially { get; init; } + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } + [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")] public ushort ImportFailedMaxStrikes { get; init; } diff --git a/code/Domain/Models/Deluge/Response/TorrentStatus.cs b/code/Domain/Models/Deluge/Response/TorrentStatus.cs index 5ad65a5..b33dd8f 100644 --- a/code/Domain/Models/Deluge/Response/TorrentStatus.cs +++ b/code/Domain/Models/Deluge/Response/TorrentStatus.cs @@ -23,4 +23,11 @@ public sealed record TorrentStatus public long SeedingTime { get; init; } public float Ratio { get; init; } + + public required IReadOnlyList Trackers { get; init; } +} + +public sealed record Tracker +{ + public required Uri Url { get; init; } } \ No newline at end of file diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index e1becad..395d164 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -1,4 +1,8 @@ -using Infrastructure.Interceptors; +using Common.Configuration.ContentBlocker; +using Common.Configuration.DownloadCleaner; +using Common.Configuration.QueueCleaner; +using Infrastructure.Interceptors; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadCleaner; @@ -30,5 +34,8 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton>() + .AddSingleton>() + .AddSingleton>(); } \ No newline at end of file diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 27dc69b..26a69d1 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -18,11 +18,13 @@ "ContentBlocker": { "Enabled": true, "IGNORE_PRIVATE": true, - "DELETE_PRIVATE": false + "DELETE_PRIVATE": false, + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads" }, "QueueCleaner": { "Enabled": true, "RunSequentially": true, + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads", "IMPORT_FAILED_MAX_STRIKES": 5, "IMPORT_FAILED_IGNORE_PRIVATE": true, "IMPORT_FAILED_DELETE_PRIVATE": false, @@ -44,7 +46,8 @@ "MIN_SEED_TIME": 0, "MAX_SEED_TIME": -1 } - ] + ], + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads" }, "DOWNLOAD_CLIENT": "qbittorrent", "qBittorrent": { diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index cf834a4..3b901e8 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -17,11 +17,13 @@ }, "ContentBlocker": { "Enabled": false, - "IGNORE_PRIVATE": false + "IGNORE_PRIVATE": false, + "IGNORED_DOWNLOADS_PATH": "" }, "QueueCleaner": { "Enabled": true, "RunSequentially": true, + "IGNORED_DOWNLOADS_PATH": "", "IMPORT_FAILED_MAX_STRIKES": 0, "IMPORT_FAILED_IGNORE_PRIVATE": false, "IMPORT_FAILED_DELETE_PRIVATE": false, @@ -34,7 +36,8 @@ "DownloadCleaner": { "Enabled": false, "DELETE_PRIVATE": false, - "CATEGORIES": [] + "CATEGORIES": [], + "IGNORED_DOWNLOADS_PATH": "" }, "DOWNLOAD_CLIENT": "none", "qBittorrent": { diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index abf0f76..60029be 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -35,12 +35,13 @@ public class TestDownloadService : DownloadService public override void Dispose() { } public override Task LoginAsync() => Task.CompletedTask; - public override Task ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult()); - public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, - ConcurrentBag patterns, ConcurrentBag regexes) => Task.FromResult(new BlockFilesResult()); + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) => Task.FromResult(new StalledResult()); + public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, + ConcurrentBag patterns, ConcurrentBag regexes, IReadOnlyList ignoredDownloads) => Task.FromResult(new BlockFilesResult()); public override Task DeleteDownload(string hash) => Task.CompletedTask; public override Task?> GetAllDownloadsToBeCleaned(List categories) => Task.FromResult?>(null); - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) => Task.CompletedTask; + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) => Task.CompletedTask; // Expose protected methods for testing public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); diff --git a/code/Infrastructure/Extensions/DelugeExtensions.cs b/code/Infrastructure/Extensions/DelugeExtensions.cs new file mode 100644 index 0000000..0c20a45 --- /dev/null +++ b/code/Infrastructure/Extensions/DelugeExtensions.cs @@ -0,0 +1,29 @@ +using Domain.Models.Deluge.Response; + +namespace Infrastructure.Extensions; + +public static class DelugeExtensions +{ + public static bool ShouldIgnore(this TorrentStatus download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.Hash?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.Label?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.Trackers.Any(x => x.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))) + { + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/code/Infrastructure/Extensions/QBitExtensions.cs b/code/Infrastructure/Extensions/QBitExtensions.cs new file mode 100644 index 0000000..1a6f020 --- /dev/null +++ b/code/Infrastructure/Extensions/QBitExtensions.cs @@ -0,0 +1,42 @@ +using QBittorrent.Client; + +namespace Infrastructure.Extensions; + +public static class QBitExtensions +{ + public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.Hash.Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + if (download.Category.Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + if (download.Tags.Contains(value, StringComparer.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } + + public static bool ShouldIgnore(this TorrentTracker tracker, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (tracker.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/code/Infrastructure/Extensions/TransmissionExtensions.cs b/code/Infrastructure/Extensions/TransmissionExtensions.cs new file mode 100644 index 0000000..c599945 --- /dev/null +++ b/code/Infrastructure/Extensions/TransmissionExtensions.cs @@ -0,0 +1,42 @@ +using Transmission.API.RPC.Entity; + +namespace Infrastructure.Extensions; + +public static class TransmissionExtensions +{ + public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.HashString?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.GetCategory().Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + bool? hasIgnoredTracker = download.Trackers? + .Any(x => new Uri(x.Announce!).Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)); + + if (hasIgnoredTracker is true) + { + return true; + } + } + + return false; + } + + public static string GetCategory(this TorrentInfo download) + { + if (string.IsNullOrEmpty(download.DownloadDir)) + { + return string.Empty; + } + + return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir)); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Helpers/CacheKeys.cs b/code/Infrastructure/Helpers/CacheKeys.cs index 61b1728..2f0d726 100644 --- a/code/Infrastructure/Helpers/CacheKeys.cs +++ b/code/Infrastructure/Helpers/CacheKeys.cs @@ -11,4 +11,6 @@ public static class CacheKeys public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes"; public static string Item(string hash) => $"item_{hash}"; + + public static string IgnoredDownloads(string name) => $"{name}_ignored"; } \ No newline at end of file diff --git a/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs b/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs new file mode 100644 index 0000000..fe093b1 --- /dev/null +++ b/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs @@ -0,0 +1,82 @@ +using Common.Configuration; +using Infrastructure.Helpers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Infrastructure.Providers; + +public sealed class IgnoredDownloadsProvider + where T : IIgnoredDownloadsConfig +{ + private readonly ILogger> _logger; + private IIgnoredDownloadsConfig _config; + private readonly IMemoryCache _cache; + private DateTime _lastModified = DateTime.MinValue; + + public IgnoredDownloadsProvider(ILogger> logger, IOptionsMonitor config, IMemoryCache cache) + { + _config = config.CurrentValue; + config.OnChange((newValue) => _config = newValue); + _logger = logger; + _cache = cache; + + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return; + } + + if (!File.Exists(_config.IgnoredDownloadsPath)) + { + throw new FileNotFoundException("file not found", _config.IgnoredDownloadsPath); + } + } + + public async Task> GetIgnoredDownloads() + { + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return Array.Empty(); + } + + FileInfo fileInfo = new(_config.IgnoredDownloadsPath); + + if (fileInfo.LastWriteTime > _lastModified || + !_cache.TryGetValue(CacheKeys.IgnoredDownloads(typeof(T).Name), out IReadOnlyList? ignoredDownloads) || + ignoredDownloads is null) + { + _lastModified = fileInfo.LastWriteTime; + + return await LoadFile(); + } + + return ignoredDownloads; + } + + private async Task> LoadFile() + { + try + { + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return Array.Empty(); + } + + string[] ignoredDownloads = (await File.ReadAllLinesAsync(_config.IgnoredDownloadsPath)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToArray(); + + _cache.Set(CacheKeys.IgnoredDownloads(typeof(T).Name), ignoredDownloads); + + _logger.LogInformation("ignored downloads reloaded"); + + return ignoredDownloads; + } + catch (Exception exception) + { + _logger.LogError(exception, "error while reading ignored downloads file | {file}", _config.IgnoredDownloadsPath); + } + + return Array.Empty(); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index 7eff718..857510f 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -6,6 +6,7 @@ using Common.Configuration.DownloadClient; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; @@ -22,7 +23,8 @@ public sealed class ContentBlocker : GenericHandler { private readonly ContentBlockerConfig _config; private readonly BlocklistProvider _blocklistProvider; - + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; + public ContentBlocker( ILogger logger, IOptions config, @@ -36,7 +38,8 @@ public sealed class ContentBlocker : GenericHandler ArrQueueIterator arrArrQueueIterator, BlocklistProvider blocklistProvider, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -47,6 +50,7 @@ public sealed class ContentBlocker : GenericHandler { _config = config.Value; _blocklistProvider = blocklistProvider; + _ignoredDownloadsProvider = ignoredDownloadsProvider; } public override async Task ExecuteAsync() @@ -73,6 +77,8 @@ public sealed class ContentBlocker : GenericHandler protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) { + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); HashSet itemsToBeRefreshed = []; @@ -106,13 +112,19 @@ public sealed class ContentBlocker : GenericHandler continue; } + if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) + { + _logger.LogInformation("skip | {title} | ignored", record.Title); + continue; + } + // push record to context ContextProvider.Set(nameof(QueueRecord), record); _logger.LogDebug("searching unwanted files for {title}", record.Title); BlockFilesResult result = await _downloadService - .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes); + .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads); if (!result.ShouldRemove) { diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index 2732694..e5f767c 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -3,6 +3,7 @@ using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Domain.Enums; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.DownloadClient; @@ -17,6 +18,7 @@ namespace Infrastructure.Verticals.DownloadCleaner; public sealed class DownloadCleaner : GenericHandler { private readonly DownloadCleanerConfig _config; + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; private readonly HashSet _excludedHashes = []; public DownloadCleaner( @@ -31,7 +33,8 @@ public sealed class DownloadCleaner : GenericHandler LidarrClient lidarrClient, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -42,6 +45,7 @@ public sealed class DownloadCleaner : GenericHandler { _config = config.Value; _config.Validate(); + _ignoredDownloadsProvider = ignoredDownloadsProvider; } public override async Task ExecuteAsync() @@ -58,6 +62,8 @@ public sealed class DownloadCleaner : GenericHandler return; } + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + await _downloadService.LoginAsync(); List? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); @@ -75,7 +81,7 @@ public sealed class DownloadCleaner : GenericHandler await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); - await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes); + await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads); } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index dd29288..e7eac07 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -16,6 +16,20 @@ public sealed class DelugeClient private readonly DelugeConfig _config; private readonly HttpClient _httpClient; + private static readonly IReadOnlyList Fields = + [ + "hash", + "state", + "name", + "eta", + "private", + "total_done", + "label", + "seeding_time", + "ratio", + "trackers" + ]; + public DelugeClient(IOptions config, IHttpClientFactory httpClientFactory) { _config = config.Value; @@ -68,7 +82,7 @@ public sealed class DelugeClient return await SendRequest( "web.get_torrent_status", hash, - new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } + Fields ); } @@ -77,7 +91,7 @@ public sealed class DelugeClient Dictionary? downloads = await SendRequest?>( "core.get_torrents_status", "", - new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } + Fields ); return downloads?.Values.ToList(); diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 47cfc3f..d104b04 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Domain.Enums; using Domain.Models.Deluge.Response; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -49,20 +50,26 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { hash = hash.ToLowerInvariant(); DelugeContents? contents = null; StalledResult result = new(); - TorrentStatus? status = await _client.GetTorrentStatus(hash); + TorrentStatus? download = await _client.GetTorrentStatus(hash); - if (status?.Hash is null) + if (download?.Hash is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } try { @@ -88,8 +95,8 @@ public class DelugeService : DownloadService, IDelugeService result.DeleteReason = DeleteReason.AllFilesBlocked; } - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(status); - result.IsPrivate = status.Private; + result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); + result.IsPrivate = download.Private; if (!shouldRemove && result.ShouldRemove) { @@ -100,30 +107,37 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ) + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { hash = hash.ToLowerInvariant(); - TorrentStatus? status = await _client.GetTorrentStatus(hash); + TorrentStatus? download = await _client.GetTorrentStatus(hash); BlockFilesResult result = new(); - if (status?.Hash is null) + if (download?.Hash is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } - - result.IsPrivate = status.Private; - if (_contentBlockerConfig.IgnorePrivate && status.Private) + var ceva = await _client.GetTorrentExtended(hash); + + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + + result.IsPrivate = download.Private; + + if (_contentBlockerConfig.IgnorePrivate && download.Private) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", status.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -205,7 +219,8 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentStatus download in downloads) { @@ -213,6 +228,12 @@ public class DelugeService : DownloadService, IDelugeService { continue; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } Category? category = categoriesToClean .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index 3f528d3..587fd53 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -60,15 +60,13 @@ public abstract class DownloadService : IDownloadService public abstract Task LoginAsync(); - public abstract Task ShouldRemoveFromArrQueueAsync(string hash); + public abstract Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// - public abstract Task BlockUnwantedFilesAsync( - string hash, + public abstract Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ); + ConcurrentBag regexes, IReadOnlyList ignoredDownloads); /// public abstract Task DeleteDownload(string hash); @@ -77,7 +75,8 @@ public abstract class DownloadService : IDownloadService public abstract Task?> GetAllDownloadsToBeCleaned(List categories); /// - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads); protected void ResetStrikesOnProgress(string hash, long downloaded) { @@ -131,7 +130,7 @@ public abstract class DownloadService : IDownloadService return new(); } - + private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) { if (category.MaxRatio < 0) diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 9126328..1d0e87e 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -28,12 +28,13 @@ public class DummyDownloadService : DownloadService return Task.CompletedTask; } - public override Task ShouldRemoveFromArrQueueAsync(string hash) + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } - public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, ConcurrentBag regexes) + public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } @@ -43,7 +44,8 @@ public class DummyDownloadService : DownloadService throw new NotImplementedException(); } - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index c2d446f..5ac9db7 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -14,7 +14,8 @@ public interface IDownloadService : IDisposable /// Checks whether the download should be removed from the *arr queue. /// /// The download hash. - public Task ShouldRemoveFromArrQueueAsync(string hash); + /// Downloads to ignore from processing. + public Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// /// Blocks unwanted files from being fully downloaded. @@ -23,12 +24,13 @@ public interface IDownloadService : IDisposable /// The . /// The patterns to test the files against. /// The regexes to test the files against. + /// Downloads to ignore from processing. /// True if all files have been blocked; otherwise false. - public Task BlockUnwantedFilesAsync( - string hash, + public Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes + ConcurrentBag regexes, + IReadOnlyList ignoredDownloads ); /// @@ -37,14 +39,16 @@ public interface IDownloadService : IDisposable /// The categories by which to filter the downloads. /// A list of downloads for the provided categories. Task?> GetAllDownloadsToBeCleaned(List categories); - + /// /// Cleans the downloads. /// /// /// The categories that should be cleaned. /// The hashes that should not be cleaned. - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + /// Downloads to ignore from processing. + public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads); /// /// Deletes a download item. diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 0e14776..8061fe2 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Common.Helpers; using Domain.Enums; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -58,18 +59,27 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { StalledResult result = new(); - TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) + TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + IReadOnlyList trackers = await GetTrackersAsync(hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); if (torrentProperties is null) @@ -83,7 +93,7 @@ public class QBitService : DownloadService, IQBitService && boolValue; // if all files were blocked by qBittorrent - if (torrent is { CompletionOn: not null, Downloaded: null or 0 }) + if (download is { CompletionOn: not null, Downloaded: null or 0 }) { result.ShouldRemove = true; result.DeleteReason = DeleteReason.AllFilesBlocked; @@ -100,7 +110,7 @@ public class QBitService : DownloadService, IQBitService return result; } - result.ShouldRemove = await IsItemStuckAndShouldRemove(torrent, result.IsPrivate); + result.ShouldRemove = await IsItemStuckAndShouldRemove(download, result.IsPrivate); if (result.ShouldRemove) { @@ -111,23 +121,32 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes + ConcurrentBag regexes, + IReadOnlyList ignoredDownloads ) { - TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) + TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); BlockFilesResult result = new(); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + IReadOnlyList trackers = await GetTrackersAsync(hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); if (torrentProperties is null) @@ -145,7 +164,7 @@ public class QBitService : DownloadService, IQBitService if (_contentBlockerConfig.IgnorePrivate && isPrivate) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", torrent.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -218,7 +237,8 @@ public class QBitService : DownloadService, IQBitService .ToList(); /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentInfo download in downloads) { @@ -227,6 +247,15 @@ public class QBitService : DownloadService, IQBitService continue; } + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + Category? category = categoriesToClean .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); @@ -303,7 +332,7 @@ public class QBitService : DownloadService, IQBitService { _client.Dispose(); } - + private async Task IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) { if (_queueCleanerConfig.StalledMaxStrikes is 0) @@ -329,4 +358,11 @@ public class QBitService : DownloadService, IQBitService return await StrikeAndCheckLimit(torrent.Hash, torrent.Name); } + + private async Task> GetTrackersAsync(string hash) + { + return (await _client.GetTorrentTrackersAsync(hash)) + .Where(x => !x.Url.ToString().Contains("**")) + .ToList(); + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 80abfa7..5160569 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Common.Helpers; using Domain.Enums; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -27,6 +28,23 @@ public class TransmissionService : DownloadService, ITransmissionService private readonly Client _client; private TorrentInfo[]? _torrentsCache; + private static readonly string[] Fields = + [ + TorrentFields.FILES, + TorrentFields.FILE_STATS, + TorrentFields.HASH_STRING, + TorrentFields.ID, + TorrentFields.ETA, + TorrentFields.NAME, + TorrentFields.STATUS, + TorrentFields.IS_PRIVATE, + TorrentFields.DOWNLOADED_EVER, + TorrentFields.DOWNLOAD_DIR, + TorrentFields.SECONDS_SEEDING, + TorrentFields.UPLOAD_RATIO, + TorrentFields.TRACKERS + ]; + public TransmissionService( IHttpClientFactory httpClientFactory, ILogger logger, @@ -60,21 +78,27 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { StalledResult result = new(); - TorrentInfo? torrent = await GetTorrentAsync(hash); + TorrentInfo? download = await GetTorrentAsync(hash); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } - bool shouldRemove = torrent.FileStats?.Length > 0; - result.IsPrivate = torrent.IsPrivate ?? false; + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + return result; + } + + bool shouldRemove = download.FileStats?.Length > 0; + result.IsPrivate = download.IsPrivate ?? false; - foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? []) + foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? []) { if (!stats.Wanted.HasValue) { @@ -95,7 +119,7 @@ public class TransmissionService : DownloadService, ITransmissionService } // remove if all files are unwanted or download is stuck - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(torrent); + result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); if (!shouldRemove && result.ShouldRemove) { @@ -106,28 +130,32 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ) + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { - TorrentInfo? torrent = await GetTorrentAsync(hash); + TorrentInfo? download = await GetTorrentAsync(hash); BlockFilesResult result = new(); - if (torrent?.FileStats is null || torrent.Files is null) + if (download?.FileStats is null || download.Files is null) { return result; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + return result; + } - bool isPrivate = torrent.IsPrivate ?? false; + bool isPrivate = download.IsPrivate ?? false; result.IsPrivate = isPrivate; if (_contentBlockerConfig.IgnorePrivate && isPrivate) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", torrent.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -135,27 +163,27 @@ public class TransmissionService : DownloadService, ITransmissionService long totalFiles = 0; long totalUnwantedFiles = 0; - for (int i = 0; i < torrent.Files.Length; i++) + for (int i = 0; i < download.Files.Length; i++) { - if (torrent.FileStats?[i].Wanted == null) + if (download.FileStats?[i].Wanted == null) { continue; } totalFiles++; - if (!torrent.FileStats[i].Wanted.Value) + if (!download.FileStats[i].Wanted.Value) { totalUnwantedFiles++; continue; } - if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes)) + if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes)) { continue; } - _logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name); + _logger.LogInformation("unwanted file found | {file}", download.Files[i].Name); unwantedFiles.Add(i); totalUnwantedFiles++; } @@ -175,7 +203,7 @@ public class TransmissionService : DownloadService, ITransmissionService _logger.LogDebug("changing priorities | torrent {hash}", hash); - await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, torrent.Id, unwantedFiles.ToArray()); + await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray()); return result; } @@ -183,22 +211,7 @@ public class TransmissionService : DownloadService, ITransmissionService /// public override async Task?> GetAllDownloadsToBeCleaned(List categories) { - string[] fields = [ - TorrentFields.FILES, - TorrentFields.FILE_STATS, - TorrentFields.HASH_STRING, - TorrentFields.ID, - TorrentFields.ETA, - TorrentFields.NAME, - TorrentFields.STATUS, - TorrentFields.IS_PRIVATE, - TorrentFields.DOWNLOADED_EVER, - TorrentFields.DOWNLOAD_DIR, - TorrentFields.SECONDS_SEEDING, - TorrentFields.UPLOAD_RATIO - ]; - - return (await _client.TorrentGetAsync(fields)) + return (await _client.TorrentGetAsync(Fields)) ?.Torrents ?.Where(x => !string.IsNullOrEmpty(x.HashString)) .Where(x => x.Status is 5 or 6) @@ -219,7 +232,8 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentInfo download in downloads) { @@ -227,6 +241,12 @@ public class TransmissionService : DownloadService, ITransmissionService { continue; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + continue; + } Category? category = categoriesToClean .FirstOrDefault(x => @@ -351,20 +371,8 @@ public class TransmissionService : DownloadService, ITransmissionService if (_torrentsCache is null || torrent is null) { - string[] fields = [ - TorrentFields.FILES, - TorrentFields.FILE_STATS, - TorrentFields.HASH_STRING, - TorrentFields.ID, - TorrentFields.ETA, - TorrentFields.NAME, - TorrentFields.STATUS, - TorrentFields.IS_PRIVATE, - TorrentFields.DOWNLOADED_EVER - ]; - // refresh cache - _torrentsCache = (await _client.TorrentGetAsync(fields)) + _torrentsCache = (await _client.TorrentGetAsync(Fields)) ?.Torrents; } diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 0bba5d1..561b38c 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -4,6 +4,7 @@ using Common.Configuration.QueueCleaner; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; @@ -19,7 +20,8 @@ namespace Infrastructure.Verticals.QueueCleaner; public sealed class QueueCleaner : GenericHandler { private readonly QueueCleanerConfig _config; - + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; + public QueueCleaner( ILogger logger, IOptions config, @@ -32,7 +34,8 @@ public sealed class QueueCleaner : GenericHandler LidarrClient lidarrClient, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -42,10 +45,13 @@ public sealed class QueueCleaner : GenericHandler ) { _config = config.Value; + _ignoredDownloadsProvider = ignoredDownloadsProvider; } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) { + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); HashSet itemsToBeRefreshed = []; @@ -75,6 +81,12 @@ public sealed class QueueCleaner : GenericHandler continue; } + if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) + { + _logger.LogInformation("skip | {title} | ignored", record.Title); + continue; + } + // push record to context ContextProvider.Set(nameof(QueueRecord), record); @@ -83,7 +95,7 @@ public sealed class QueueCleaner : GenericHandler if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent") { // stalled download check - stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); + stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); } // failed import check diff --git a/code/test/data/cleanuperr/ignored_downloads b/code/test/data/cleanuperr/ignored_downloads new file mode 100644 index 0000000..5537770 --- /dev/null +++ b/code/test/data/cleanuperr/ignored_downloads @@ -0,0 +1 @@ +ignored \ No newline at end of file diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 010531d..fb8c186 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -191,6 +191,7 @@ services: - TRIGGERS__DOWNLOADCLEANER=0/30 * * * * ? - QUEUECLEANER__ENABLED=true + - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true @@ -201,10 +202,12 @@ services: - QUEUECLEANER__STALLED_DELETE_PRIVATE=false - CONTENTBLOCKER__ENABLED=true + - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored - CONTENTBLOCKER__IGNORE_PRIVATE=true - CONTENTBLOCKER__DELETE_PRIVATE=false - DOWNLOADCLEANER__ENABLED=true + - DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored - DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 @@ -256,6 +259,7 @@ services: # - NOTIFIARR__CHANNEL_ID=discord_channel_id volumes: - ./data/cleanuperr/logs:/var/logs + - ./data/cleanuperr/ignored_downloads:/ignored restart: unless-stopped depends_on: - qbittorrent diff --git a/variables.md b/variables.md index dcd07ce..dbc0aaf 100644 --- a/variables.md +++ b/variables.md @@ -76,6 +76,31 @@ - Default: `true` - Required: No. +**`QUEUECLEANER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`QUEUECLEANER__RUNSEQUENTIALLY`** - Controls whether queue cleaner runs after content blocker instead of in parallel. - When `true`, streamlines the cleaning process by running immediately after content blocker. @@ -178,6 +203,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. +**`CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`CONTENTBLOCKER__IGNORE_PRIVATE`** - Controls whether to ignore downloads from private trackers. - Type: Boolean @@ -217,6 +267,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. +**`DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`DOWNLOADCLEANER__DELETE_PRIVATE`** - Controls whether to delete private downloads. - Type: Boolean. @@ -237,7 +312,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > The category name must match the category that was set in the *arr. > For qBittorrent, the category name is the name of the download category. > For Deluge, the category name is the name of the label. -> For Transmission, the category name is the name of the download location. +> For Transmission, the category name is the last directory from the save location. **`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`** - Maximum ratio to reach before removing a download. From 89a6eaf0ce3c49ccc1188bb4c4369ba3a35d9bd5 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Mon, 10 Mar 2025 00:13:40 +0200 Subject: [PATCH 04/16] Disable cleanup on torrent items if download client is not configured (#85) --- .../Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 561b38c..e422a5c 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -92,8 +92,14 @@ public sealed class QueueCleaner : GenericHandler StalledResult stalledCheckResult = new(); - if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent") + if (record.Protocol is "torrent") { + if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None) + { + _logger.LogWarning("skip | download client is not configured | {title}", record.Title); + continue; + } + // stalled download check stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); } From 3a9d5d9085648dc557159f106da7c8092bc0512b Mon Sep 17 00:00:00 2001 From: Flaminel Date: Tue, 11 Mar 2025 23:18:34 +0200 Subject: [PATCH 05/16] Fix patterns being loaded for disabled arrs (#80) --- .../ContentBlocker/BlocklistProvider.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs index 90d91e3..f0f5dbb 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs @@ -49,9 +49,9 @@ public sealed class BlocklistProvider try { - await LoadPatternsAndRegexesAsync(_sonarrConfig.Block.Type, _sonarrConfig.Block.Path, InstanceType.Sonarr); - await LoadPatternsAndRegexesAsync(_radarrConfig.Block.Type, _radarrConfig.Block.Path, InstanceType.Radarr); - await LoadPatternsAndRegexesAsync(_lidarrConfig.Block.Type, _lidarrConfig.Block.Path, InstanceType.Lidarr); + await LoadPatternsAndRegexesAsync(_sonarrConfig, InstanceType.Sonarr); + await LoadPatternsAndRegexesAsync(_radarrConfig, InstanceType.Radarr); + await LoadPatternsAndRegexesAsync(_lidarrConfig, InstanceType.Lidarr); _initialized = true; } @@ -83,14 +83,19 @@ public sealed class BlocklistProvider return regexes ?? []; } - private async Task LoadPatternsAndRegexesAsync(BlocklistType blocklistType, string? blocklistPath, InstanceType instanceType) + private async Task LoadPatternsAndRegexesAsync(ArrConfig arrConfig, InstanceType instanceType) { - if (string.IsNullOrEmpty(blocklistPath)) + if (!arrConfig.Enabled) { return; } - string[] filePatterns = await ReadContentAsync(blocklistPath); + if (string.IsNullOrEmpty(arrConfig.Block.Path)) + { + return; + } + + string[] filePatterns = await ReadContentAsync(arrConfig.Block.Path); long startTime = Stopwatch.GetTimestamp(); ParallelOptions options = new() { MaxDegreeOfParallelism = 5 }; @@ -121,13 +126,13 @@ public sealed class BlocklistProvider TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime); - _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType); + _cache.Set(CacheKeys.BlocklistType(instanceType), arrConfig.Block.Type); _cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns); _cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes); _logger.LogDebug("loaded {count} patterns", patterns.Count); _logger.LogDebug("loaded {count} regexes", regexes.Count); - _logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistPath); + _logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, arrConfig.Block.Path); } private async Task ReadContentAsync(string path) From 324c3ace8fe65f9a54819dadd8e4721534d49c09 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Tue, 11 Mar 2025 23:34:27 +0200 Subject: [PATCH 06/16] Fix multiple runs on queue cleaner when download cleaner is enabled (#90) --- code/Executable/DependencyInjection/QuartzDI.cs | 2 +- code/Infrastructure/Verticals/Jobs/JobChainingListener.cs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/code/Executable/DependencyInjection/QuartzDI.cs b/code/Executable/DependencyInjection/QuartzDI.cs index 73315fa..31b1ea4 100644 --- a/code/Executable/DependencyInjection/QuartzDI.cs +++ b/code/Executable/DependencyInjection/QuartzDI.cs @@ -55,7 +55,7 @@ public static class QuartzDI if (contentBlockerConfig?.Enabled is true && queueCleanerConfig is { Enabled: true, RunSequentially: true }) { q.AddJob(queueCleanerConfig, string.Empty); - q.AddJobListener(new JobChainingListener(nameof(QueueCleaner))); + q.AddJobListener(new JobChainingListener(nameof(ContentBlocker), nameof(QueueCleaner))); } else { diff --git a/code/Infrastructure/Verticals/Jobs/JobChainingListener.cs b/code/Infrastructure/Verticals/Jobs/JobChainingListener.cs index b9f5c8c..273f3b4 100644 --- a/code/Infrastructure/Verticals/Jobs/JobChainingListener.cs +++ b/code/Infrastructure/Verticals/Jobs/JobChainingListener.cs @@ -4,10 +4,12 @@ namespace Infrastructure.Verticals.Jobs; public class JobChainingListener : IJobListener { + private readonly string _firstJobName; private readonly string _nextJobName; - public JobChainingListener(string nextJobName) + public JobChainingListener(string firstJobName, string nextJobName) { + _firstJobName = firstJobName; _nextJobName = nextJobName; } @@ -19,7 +21,7 @@ public class JobChainingListener : IJobListener public async Task JobWasExecuted(IJobExecutionContext context, JobExecutionException? jobException, CancellationToken cancellationToken) { - if (string.IsNullOrEmpty(_nextJobName) || context.JobDetail.Key.Name == _nextJobName) + if (string.IsNullOrEmpty(_nextJobName) || context.JobDetail.Key.Name == _nextJobName || context.JobDetail.Key.Name != _firstJobName) { return; } From a68e13af35254a96a98f8a18ade83853a29d7ea0 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Tue, 11 Mar 2025 23:34:44 +0200 Subject: [PATCH 07/16] Fix notifications when poster is not found (#89) --- .../Verticals/Notifications/NotificationPublisher.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs index 3d7aed3..0577a96 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs @@ -128,9 +128,9 @@ public class NotificationPublisher : INotificationPublisher { Uri? image = instanceType switch { - InstanceType.Sonarr => record.Series!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, - InstanceType.Radarr => record.Movie!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, - InstanceType.Lidarr => record.Album!.Images.FirstOrDefault(x => x.CoverType == "cover")?.Url, + InstanceType.Sonarr => record.Series?.Images?.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, + InstanceType.Radarr => record.Movie?.Images?.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl, + InstanceType.Lidarr => record.Album?.Images?.FirstOrDefault(x => x.CoverType == "cover")?.Url, _ => throw new ArgumentOutOfRangeException(nameof(instanceType)) }; From e6d3929fc91e7e3da4e1db3b6c6882244cc8e2aa Mon Sep 17 00:00:00 2001 From: Flaminel Date: Tue, 11 Mar 2025 23:35:07 +0200 Subject: [PATCH 08/16] Restrict max strikes to a minimum value (#87) --- .../Configuration/QueueCleaner/QueueCleanerConfig.cs | 12 +++++++++++- .../Verticals/QueueCleaner/QueueCleaner.cs | 1 + variables.md | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index f0d7049..9006253 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.Configuration; +using Common.Exceptions; +using Microsoft.Extensions.Configuration; namespace Common.Configuration.QueueCleaner; @@ -39,5 +40,14 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig public void Validate() { + if (ImportFailedMaxStrikes is > 0 and < 3) + { + throw new ValidationException("the minimum value for IMPORT_FAILED_MAX_STRIKES must be 3"); + } + + if (StalledMaxStrikes is > 0 and < 3) + { + throw new ValidationException("the minimum value for STALLED_MAX_STRIKES must be 3"); + } } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index e422a5c..bc7df16 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -45,6 +45,7 @@ public sealed class QueueCleaner : GenericHandler ) { _config = config.Value; + _config.Validate(); _ignoredDownloadsProvider = ignoredDownloadsProvider; } diff --git a/variables.md b/variables.md index dbc0aaf..905aee1 100644 --- a/variables.md +++ b/variables.md @@ -114,7 +114,7 @@ - Set to `0` to never remove failed imports. - A strike is given when an item is stalled, stuck in metadata downloading, or failed to be imported. - Type: Integer -- Possible values: `0` or greater +- Possible values: `0` or a number greater than `2`. - Default: `0` - Required: No. @@ -153,7 +153,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Set to `0` to never remove stalled downloads. - A strike is given when download speed is 0. - Type: Integer -- Possible values: `0` or greater +- Possible values: `0` or a number greater than `2`. - Default: `0` - Required: No. From a92ebd75c277756a188cf964e7fc614b65cc61d5 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Tue, 11 Mar 2025 23:42:21 +0200 Subject: [PATCH 09/16] Update docs (#88) --- README.md | 111 ++++++++++----- variables.md | 376 +++++++++++++++++++++++++++------------------------ 2 files changed, 280 insertions(+), 207 deletions(-) diff --git a/README.md b/README.md index aad2468..a1523c4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ cleanuperr was created primarily to address malicious files, such as `*.lnk` or > - Trigger a search for downloads removed from the *arrs. > - Clean up downloads that have been seeding for a certain amount of time. > - Notify on strike or download removal. +> - Ignore certain torrent hashes, categories, tags or trackers from processing. cleanuperr supports both qBittorrent's built-in exclusion features and its own blocklist-based system. Binaries for all platforms are provided, along with Docker images for easy deployment. @@ -25,15 +26,21 @@ cleanuperr supports both qBittorrent's built-in exclusion features and its own b > https://discord.gg/sWggpnmGNY ## Table of contents: -- [Naming choice](README.md#naming-choice) -- [Quick Start](README.md#quick-start) -- [How it works](README.md#how-it-works) -- [Setup](README.md#setup) -- [Usage](README.md#usage) - - [Docker Compose](README.md#docker-compose-yaml) - - [Environment Variables](README.md#environment-variables) - - [Binaries](README.md#binaries-if-youre-not-using-docker) -- [Credits](README.md#credits) +- [Naming choice](#naming-choice) +- [Quick Start](#quick-start) +- [How it works](#how-it-works) + - [Content blocker](#1-content-blocker-will) + - [Queue cleaner](#2-queue-cleaner-will) + - [Download cleaner](#3-download-cleaner-will) +- [Setup](#setup-examples) +- [Usage](#usage) + - [Docker](#docker) + - [Environment Variables](#environment-variables) + - [Docker Compose](#docker-compose-example) + - [Windows](#windows) + - [Linux](#linux) + - [MacOS](#macos) +- [Credits](#credits) ## Naming choice @@ -52,10 +59,10 @@ I've seen a few discussions on this type of naming and I've decided that I didn' > Use the Unraid Community App. > > 3. **Manual Installation (if you're not using Docker)** -> More details [here](#binaries-if-youre-not-using-docker). +> Go to [Windows](#windows), [Linux](#linux) or [MacOS](#macos). > [!TIP] -> Refer to the [Environment variables](#Environment-variables) section for detailed configuration instructions and the [Setup](#Setup) section for an in-depth explanation of the cleanup process. +> Refer to the [Environment variables](#environment-variables) section for detailed configuration instructions and the [Setup examples](#setup-examples) section for an in-depth explanation of the cleanup process. > [!IMPORTANT] @@ -69,7 +76,7 @@ I've seen a few discussions on this type of naming and I've decided that I didn' # How it works -1. **Content blocker** will: +#### 1. **Content blocker** will: - Run every 5 minutes (or configured cron). - Process all items in the *arr queue. - Find the corresponding item from the download client for each queue item. @@ -80,7 +87,7 @@ I've seen a few discussions on this type of naming and I've decided that I didn' - It will be removed from the *arr's queue and blocked. - It will be deleted from the download client. - A new search will be triggered for the *arr item. -2. **Queue cleaner** will: +#### 2. **Queue cleaner** will: - Run every 5 minutes (or configured cron, or right after `content blocker`). - Process all items in the *arr queue. - Check each queue item if it is **stalled (download speed is 0)**, **stuck in metadata downloading** or **failed to be imported**. @@ -93,11 +100,11 @@ I've seen a few discussions on this type of naming and I've decided that I didn' - It will be removed from the *arr's queue and blocked. - It will be deleted from the download client. - A new search will be triggered for the *arr item. -3. **Download cleaner** will: +#### 3. **Download cleaner** will: - Run every hour (or configured cron). - Automatically clean up downloads that have been seeding for a certain amount of time. -# Setup +# Setup examples ## Using qBittorrent's built-in feature (works only with qBittorrent) @@ -129,7 +136,26 @@ I've seen a few discussions on this type of naming and I've decided that I didn' ## Usage -### Docker compose yaml +### Docker + + +### **Environment variables** + +**Jump to:** +- [General settings](variables.md#general-settings) +- [Queue Cleaner settings](variables.md#queue-cleaner-settings) +- [Content Blocker settings](variables.md#content-blocker-settings) +- [Download Cleaner settings](variables.md#download-cleaner-settings) +- [Download Client settings](variables.md#download-client-settings) +- [Arr settings](variables.md#arr-settings) +- [Notification settings](variables.md#notification-settings) +- [Advanced settings](variables.md#advanced-settings) + +### Docker compose example + +> [!NOTE] +> +> This example contains all settings and should be modified to fit your needs. ``` version: "3.3" @@ -232,28 +258,48 @@ services: - NOTIFIARR__CHANNEL_ID=discord_channel_id ``` -## Environment variables +### Windows -Jump to: -- [General settings](variables.md#general-settings) -- [Queue Cleaner settings](variables.md#queue-cleaner-settings) -- [Content Blocker settings](variables.md#content-blocker-settings) -- [Download Cleaner settings](variables.md#download-cleaner-settings) -- [Download Client settings](variables.md#download-client-settings) -- [Arr settings](variables.md#arr-settings) -- [Notification settings](variables.md#notification-settings) -- [Advanced settings](variables.md#advanced-settings) - -### Binaries (if you're not using Docker) - -1. Download the binaries from [releases](https://github.com/flmorg/cleanuperr/releases). -2. Extract them from the zip file. -3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [above](#environment-variables). +1. Download the zip file from [releases](https://github.com/flmorg/cleanuperr/releases). +2. Extract the zip file into `C:\example\directory`. +3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [here](#environment-variables). +4. Execute `cleanuperr.exe`. > [!TIP] > ### Run as a Windows Service > Check out this stackoverflow answer on how to do it: https://stackoverflow.com/a/15719678 +### Linux + +1. Download the zip file from [releases](https://github.com/flmorg/cleanuperr/releases). +2. Extract the zip file into `/example/directory`. +3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [here](#environment-variables). +4. Open a terminal and execute these commands: + ``` + cd /example/directory + chmod +x cleanuperr + ./cleanuperr + ``` + +### MacOS + +1. Download the zip file from [releases](https://github.com/flmorg/cleanuperr/releases). +2. Extract the zip file into `/example/directory`. +3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [here](#environment-variables). +4. Open a terminal and execute these commands: + ``` + cd /example/directory + chmod +x cleanuperr + ./cleanuperr + ``` + +> [!IMPORTANT] +> Some people have experienced problems when trying to execute cleanuperr on MacOS because the system actively blocked the file for not being signed. +> As per [this](), you may need to also execute this command: +> ``` +> codesign --sign - --force --preserve-metadata=entitlements,requirements,flags,runtime /example/directory/cleanuperr +> ``` + # Credits Special thanks for inspiration go to: - [ThijmenGThN/swaparr](https://github.com/ThijmenGThN/swaparr) @@ -265,4 +311,3 @@ Special thanks for inspiration go to: If I made your life just a tiny bit easier, consider buying me a coffee! Buy Me A Coffee - diff --git a/variables.md b/variables.md index 905aee1..e539035 100644 --- a/variables.md +++ b/variables.md @@ -1,52 +1,52 @@ ## Table of contents -- [General settings](variables.md#general-settings) -- [Queue Cleaner settings](variables.md#queue-cleaner-settings) -- [Content Blocker settings](variables.md#content-blocker-settings) -- [Download Cleaner settings](variables.md#download-cleaner-settings) -- [Download Client settings](variables.md#download-client-settings) -- [Arr settings](variables.md#arr-settings) -- [Notification settings](variables.md#notification-settings) -- [Advanced settings](variables.md#advanced-settings) +- [General settings](#general-settings) +- [Queue Cleaner settings](#queue-cleaner-settings) +- [Content Blocker settings](#content-blocker-settings) +- [Download Cleaner settings](#download-cleaner-settings) +- [Download Client settings](#download-client-settings) +- [Arr settings](#arr-settings) +- [Notification settings](#notification-settings) +- [Advanced settings](#advanced-settings) # ### General settings -**`TZ`** +#### **`TZ`** - The time zone to use. - Type: String. - Possible values: Any valid timezone. - Default: `UTC`. - Required: No. -**`DRY_RUN`** +#### **`DRY_RUN`** - When enabled, simulates irreversible operations (like deletions and notifications) without making actual changes. - Type: Boolean. - Possible values: `true`, `false`. - Default: `false`. - Required: No. -**`LOGGING__LOGLEVEL`** +#### **`LOGGING__LOGLEVEL`** - Controls the detail level of application logs. - Type: String. - Possible values: `Verbose`, `Debug`, `Information`, `Warning`, `Error`, `Fatal`. - Default: `Information`. - Required: No. -**`LOGGING__FILE__ENABLED`** +#### **`LOGGING__FILE__ENABLED`** - Enables logging to a file. - Type: Boolean. - Possible values: `true`, `false`. - Default: `false`. - Required: No. -**`LOGGING__FILE__PATH`** +#### **`LOGGING__FILE__PATH`** - Directory where log files will be saved. - Type: String. - Default: Empty. - Required: No. -**`LOGGING__ENHANCED`** +#### **`LOGGING__ENHANCED`** - Provides more detailed descriptions in logs whenever possible. - Type: Boolean. - Possible values: `true`, `false`. @@ -58,7 +58,7 @@ ### Queue Cleaner settings -**`TRIGGERS__QUEUECLEANER`** +#### **`TRIGGERS__QUEUECLEANER`** - Cron schedule for the queue cleaner job. - Type: String - [Quartz cron format](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). - Default: `0 0/5 * * * ?` (every 5 minutes). @@ -66,9 +66,9 @@ > [!NOTE] > - Maximum interval is 6 hours. -> - Ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`. +> - Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`. -**`QUEUECLEANER__ENABLED`** +#### **`QUEUECLEANER__ENABLED`** - Enables or disables the queue cleaning functionality. - When enabled, processes all items in the *arr queue. - Type: Boolean @@ -76,7 +76,7 @@ - Default: `true` - Required: No. -**`QUEUECLEANER__IGNORED_DOWNLOADS_PATH`** +#### **`QUEUECLEANER__IGNORED_DOWNLOADS_PATH`** - Local path to the file containing ignored downloads. - If the contents of the file are changed, they will be reloaded on the next job run. - Accepted values: @@ -101,7 +101,7 @@ >[!IMPORTANT] > Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. -**`QUEUECLEANER__RUNSEQUENTIALLY`** +#### **`QUEUECLEANER__RUNSEQUENTIALLY`** - Controls whether queue cleaner runs after content blocker instead of in parallel. - When `true`, streamlines the cleaning process by running immediately after content blocker. - Type: Boolean @@ -109,23 +109,24 @@ - Default: `true` - Required: No. -**`QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES`** +#### **`QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES`** - Number of strikes before removing a failed import. - Set to `0` to never remove failed imports. -- A strike is given when an item is stalled, stuck in metadata downloading, or failed to be imported. +- A strike is given when an item fails to be imported. - Type: Integer -- Possible values: `0` or a number greater than `2`. - Default: `0` - Required: No. +> [!NOTE] +> If not set to `0`, the minimum value is `3`. -**`QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE`** +#### **`QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE`** - Controls whether to ignore failed imports from private trackers. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE`** +#### **`QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE`** - Controls whether to delete failed imports from private trackers from the download client. - Has no effect if `QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE` is `true`. - Type: Boolean @@ -136,42 +137,43 @@ > [!WARNING] > Setting `QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. -**`QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS`** +#### **`QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS`** - Patterns to look for in failed import messages that should be ignored. - Multiple patterns can be specified using incrementing numbers starting from 0. - Type: String array - Default: Empty. - Required: No. - Example: -```yaml -QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0: "title mismatch" -QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" -``` + ```yaml + QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0: "title mismatch" + QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" + ``` -**`QUEUECLEANER__STALLED_MAX_STRIKES`** +#### **`QUEUECLEANER__STALLED_MAX_STRIKES`** - Number of strikes before removing a stalled download. - Set to `0` to never remove stalled downloads. -- A strike is given when download speed is 0. +- A strike is given when an item is stalled (not downloading) or stuck while downloading metadata. - Type: Integer -- Possible values: `0` or a number greater than `2`. - Default: `0` - Required: No. +> [!NOTE] +> If not set to `0`, the minimum value is `3`. -**`QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS`** +#### **`QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS`** - Controls whether to remove strikes if any download progress was made since last checked. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`QUEUECLEANER__STALLED_IGNORE_PRIVATE`** +#### **`QUEUECLEANER__STALLED_IGNORE_PRIVATE`** - Controls whether to ignore stalled downloads from private trackers. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`QUEUECLEANER__STALLED_DELETE_PRIVATE`** +#### **`QUEUECLEANER__STALLED_DELETE_PRIVATE`** - Controls whether to delete stalled private downloads from the download client. - Has no effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`. - Type: Boolean @@ -186,7 +188,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" ### Content Blocker settings -**`TRIGGERS__CONTENTBLOCKER`** +#### **`TRIGGERS__CONTENTBLOCKER`** - Cron schedule for the content blocker job. - Type: String - [Quartz cron format](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). - Default: `0 0/5 * * * ?` (every 5 minutes). @@ -195,7 +197,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > [!NOTE] > - Maximum interval is 6 hours. -**`CONTENTBLOCKER__ENABLED`** +#### **`CONTENTBLOCKER__ENABLED`** - Enables or disables the content blocker functionality. - When enabled, processes all items in the *arr queue and marks unwanted files. - Type: Boolean @@ -203,7 +205,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. -**`CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH`** +#### **`CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH`** - Local path to the file containing ignored downloads. - If the contents of the file are changed, they will be reloaded on the next job run. - Accepted values: @@ -228,14 +230,14 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" >[!IMPORTANT] > Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. -**`CONTENTBLOCKER__IGNORE_PRIVATE`** +#### **`CONTENTBLOCKER__IGNORE_PRIVATE`** - Controls whether to ignore downloads from private trackers. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`CONTENTBLOCKER__DELETE_PRIVATE`** +#### **`CONTENTBLOCKER__DELETE_PRIVATE`** - Controls whether to delete private downloads that have all files blocked from the download client. - Has no effect if `CONTENTBLOCKER__IGNORE_PRIVATE` is `true`. - Type: Boolean @@ -250,7 +252,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" ### Download Cleaner settings -**`TRIGGERS__DOWNLOADCLEANER`** +#### **`TRIGGERS__DOWNLOADCLEANER`** - Cron schedule for the download cleaner job. - Type: String - [Quartz cron format](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). - Default: `0 0 * * * ?` (every hour). @@ -259,7 +261,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > [!NOTE] > - Maximum interval is 6 hours. -**`DOWNLOADCLEANER__ENABLED`** +#### **`DOWNLOADCLEANER__ENABLED`** - Enables or disables the download cleaner functionality. - When enabled, automatically cleans up downloads that have been seeding for a certain amount of time. - Type: Boolean. @@ -267,7 +269,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. -**`DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH`** +#### **`DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH`** - Local path to the file containing ignored downloads. - If the contents of the file are changed, they will be reloaded on the next job run. - Accepted values: @@ -292,7 +294,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" >[!IMPORTANT] > Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. -**`DOWNLOADCLEANER__DELETE_PRIVATE`** +#### **`DOWNLOADCLEANER__DELETE_PRIVATE`** - Controls whether to delete private downloads. - Type: Boolean. - Possible values: `true`, `false` @@ -302,7 +304,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > [!WARNING] > Setting `DOWNLOADCLEANER__DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. -**`DOWNLOADCLEANER__CATEGORIES__0__NAME`** +#### **`DOWNLOADCLEANER__CATEGORIES__0__NAME`** - Name of the category to clean. - Type: String. - Default: Empty. @@ -310,18 +312,21 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > [!NOTE] > The category name must match the category that was set in the *arr. +> > For qBittorrent, the category name is the name of the download category. +> > For Deluge, the category name is the name of the label. +> > For Transmission, the category name is the last directory from the save location. -**`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`** +#### **`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`** - Maximum ratio to reach before removing a download. - Type: Decimal. - Possible values: `-1` or greater (`-1` means no limit or disabled). - Default: `-1` - Required: No. -**`DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME`** +#### **`DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME`** - Minimum number of hours to seed before removing a download, if the ratio has been met. - Used with `MAX_RATIO` to ensure a minimum seed time. - Type: Decimal. @@ -329,7 +334,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `0` - Required: No. -**`DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME`** +#### **`DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME`** - Maximum number of hours to seed before removing a download. - Type: Decimal. - Possible values: `-1` or greater (`-1` means no limit or disabled). @@ -352,7 +357,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" ### Download Client settings -**`DOWNLOAD_CLIENT`** +#### **`DOWNLOAD_CLIENT`** - Specifies which download client is used by *arrs. - Type: String. - Possible values: `none`, `qbittorrent`, `deluge`, `transmission`. @@ -362,49 +367,49 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > [!NOTE] > Only one download client can be enabled at a time. If you have more than one download client, you should deploy multiple instances of cleanuperr. -**`QBITTORRENT__URL`** +#### **`QBITTORRENT__URL`** - URL of the qBittorrent instance. - Type: String. - Default: `http://localhost:8080`. - Required: No. -**`QBITTORRENT__USERNAME`** +#### **`QBITTORRENT__USERNAME`** - Username for qBittorrent authentication. - Type: String. - Default: Empty. - Required: No. -**`QBITTORRENT__PASSWORD`** +#### **`QBITTORRENT__PASSWORD`** - Password for qBittorrent authentication. - Type: String. - Default: Empty. - Required: No. -**`DELUGE__URL`** +#### **`DELUGE__URL`** - URL of the Deluge instance. - Type: String. - Default: `http://localhost:8112`. - Required: No. -**`DELUGE__PASSWORD`** +#### **`DELUGE__PASSWORD`** - Password for Deluge authentication. - Type: String. - Default: Empty. - Required: No. -**`TRANSMISSION__URL`** +#### **`TRANSMISSION__URL`** - URL of the Transmission instance. - Type: String. - Default: `http://localhost:9091`. - Required: No. -**`TRANSMISSION__USERNAME`** +#### **`TRANSMISSION__USERNAME`** - Username for Transmission authentication. - Type: String. - Default: Empty. - Required: No. -**`TRANSMISSION__PASSWORD`** +#### **`TRANSMISSION__PASSWORD`** - Password for Transmission authentication. - Type: String. - Default: Empty. @@ -414,112 +419,6 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" ### Arr settings -**`SONARR__ENABLED`** -- Enables or disables Sonarr cleanup. -- Type: Boolean -- Possible values: `true`, `false` -- Default: `false` -- Required: No. - -**`SONARR__BLOCK__TYPE`** -- Determines how file blocking works for Sonarr. -- Type: String -- Possible values: `blacklist`, `whitelist` -- Default: `blacklist` -- Required: No. - -**`SONARR__BLOCK__PATH`** -- Path to the blocklist file (local file or URL). -- Must be JSON compatible. -- Type: String -- Default: Empty. -- Required: No. - -**`SONARR__SEARCHTYPE`** -- Determines what to search for after removing a queue item. -- Type: String -- Possible values: `Episode`, `Season`, `Series` -- Default: `Episode` -- Required: No. - -**`SONARR__INSTANCES__0__URL`** -- URL of the Sonarr instance. -- Type: String -- Default: `http://localhost:8989` -- Required: No. - -**`SONARR__INSTANCES__0__APIKEY`** -- API key for the Sonarr instance. -- Type: String -- Default: Empty. -- Required: No. - -**`RADARR__ENABLED`** -- Enables or disables Radarr cleanup. -- Type: Boolean -- Possible values: `true`, `false` -- Default: `false` -- Required: No. - -**`RADARR__BLOCK__TYPE`** -- Determines how file blocking works for Radarr. -- Type: String -- Possible values: `blacklist`, `whitelist` -- Default: `blacklist` -- Required: No. - -**`RADARR__BLOCK__PATH`** -- Path to the blocklist file (local file or URL). -- Must be JSON compatible. -- Type: String -- Default: Empty. -- Required: No. - -**`RADARR__INSTANCES__0__URL`** -- URL of the Radarr instance. -- Type: String -- Default: `http://localhost:7878` -- Required: No. - -**`RADARR__INSTANCES__0__APIKEY`** -- API key for the Radarr instance. -- Type: String -- Default: Empty. -- Required: No. - -**`LIDARR__ENABLED`** -- Enables or disables Lidarr cleanup. -- Type: Boolean -- Possible values: `true`, `false` -- Default: `false` -- Required: No. - -**`LIDARR__BLOCK__TYPE`** -- Determines how file blocking works for Lidarr. -- Type: String -- Possible values: `blacklist`, `whitelist` -- Default: `blacklist` -- Required: No. - -**`LIDARR__BLOCK__PATH`** -- Path to the blocklist file (local file or URL). -- Must be JSON compatible. -- Type: String -- Default: Empty. -- Required: No. - -**`LIDARR__INSTANCES__0__URL`** -- URL of the Lidarr instance. -- Type: String -- Default: `http://localhost:8686` -- Required: No. - -**`LIDARR__INSTANCES__0__APIKEY`** -- API key for the Lidarr instance. -- Type: String -- Default: Empty. -- Required: No. - > [!NOTE] > Multiple instances can be specified for each *arr using this format, where `` starts from 0: > ```yaml @@ -527,8 +426,29 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > __INSTANCES____APIKEY > ``` +#### **`SONARR__ENABLED`** +- Enables or disables Sonarr cleanup. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +#### **`SONARR__BLOCK__TYPE`** +- Determines how file blocking works for Sonarr. +- Type: String +- Possible values: `blacklist`, `whitelist` +- Default: `blacklist` +- Required: No. + +#### **`SONARR__BLOCK__PATH`** +- Path to the blocklist file (local file or URL). +- Must be JSON compatible. +- Type: String +- Default: Empty. +- Required: No. + > [!NOTE] -> The blocklists (blacklist/whitelist) support the following patterns: +> The blocklists support the following patterns: > ``` > *example // file name ends with "example" > example* // file name starts with "example" @@ -538,47 +458,155 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > ``` > [!NOTE] -> [This blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist) and [this whitelist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/whitelist) can be used for Sonarr and Radarr, but they are not suitable for other *arrs. +> [This blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist), [this permissive blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist_permissive) and [this whitelist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/whitelist) can be used for Sonarr and Radarr. + +#### **`SONARR__SEARCHTYPE`** +- Determines what to search for after removing a queue item. +- Type: String +- Possible values: `Episode`, `Season`, `Series` +- Default: `Episode` +- Required: No. + +#### **`SONARR__INSTANCES__0__URL`** +- URL of the Sonarr instance. +- Type: String +- Default: `http://localhost:8989` +- Required: No. + +#### **`SONARR__INSTANCES__0__APIKEY`** +- API key for the Sonarr instance. +- Type: String +- Default: Empty. +- Required: No. + +#### **`RADARR__ENABLED`** +- Enables or disables Radarr cleanup. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +#### **`RADARR__BLOCK__TYPE`** +- Determines how file blocking works for Radarr. +- Type: String +- Possible values: `blacklist`, `whitelist` +- Default: `blacklist` +- Required: No. + +#### **`RADARR__BLOCK__PATH`** +- Path to the blocklist file (local file or URL). +- Must be JSON compatible. +- Type: String +- Default: Empty. +- Required: No. + +> [!NOTE] +> The blocklists support the following patterns: +> ``` +> *example // file name ends with "example" +> example* // file name starts with "example" +> *example* // file name has "example" in the name +> example // file name is exactly the word "example" +> regex: // regex that needs to be marked at the start of the line with "regex:" +> ``` + +> [!NOTE] +> [This blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist), [this permissive blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist_permissive) and [this whitelist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/whitelist) can be used for Sonarr and Radarr. + +#### **`RADARR__INSTANCES__0__URL`** +- URL of the Radarr instance. +- Type: String +- Default: `http://localhost:7878` +- Required: No. + +#### **`RADARR__INSTANCES__0__APIKEY`** +- API key for the Radarr instance. +- Type: String +- Default: Empty. +- Required: No. + +#### **`LIDARR__ENABLED`** +- Enables or disables Lidarr cleanup. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +#### **`LIDARR__BLOCK__TYPE`** +- Determines how file blocking works for Lidarr. +- Type: String +- Possible values: `blacklist`, `whitelist` +- Default: `blacklist` +- Required: No. + +#### **`LIDARR__BLOCK__PATH`** +- Path to the blocklist file (local file or URL). +- Must be JSON compatible. +- Type: String +- Default: Empty. +- Required: No. + +> [!NOTE] +> The blocklists support the following patterns: +> ``` +> *example // file name ends with "example" +> example* // file name starts with "example" +> *example* // file name has "example" in the name +> example // file name is exactly the word "example" +> regex: // regex that needs to be marked at the start of the line with "regex:" +> ``` + +#### **`LIDARR__INSTANCES__0__URL`** +- URL of the Lidarr instance. +- Type: String +- Default: `http://localhost:8686` +- Required: No. + +#### **`LIDARR__INSTANCES__0__APIKEY`** +- API key for the Lidarr instance. +- Type: String +- Default: Empty. +- Required: No. # ### Notification settings -**`NOTIFIARR__API_KEY`** +#### **`NOTIFIARR__API_KEY`** - Notifiarr API key for sending notifications. - Requires Notifiarr's [`Passthrough`](https://notifiarr.wiki/en/Website/Integrations/Passthrough) integration to work. - Type: String - Default: Empty. - Required: No. -**`NOTIFIARR__CHANNEL_ID`** +#### **`NOTIFIARR__CHANNEL_ID`** - Discord channel ID where notifications will be sent. - Type: String - Default: Empty. - Required: No. -**`NOTIFIARR__ON_IMPORT_FAILED_STRIKE`** +#### **`NOTIFIARR__ON_IMPORT_FAILED_STRIKE`** - Controls whether to notify when an item receives a failed import strike. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`NOTIFIARR__ON_STALLED_STRIKE`** +#### **`NOTIFIARR__ON_STALLED_STRIKE`** - Controls whether to notify when an item receives a stalled download strike. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`NOTIFIARR__ON_QUEUE_ITEM_DELETED`** +#### **`NOTIFIARR__ON_QUEUE_ITEM_DELETED`** - Controls whether to notify when a queue item is deleted. - Type: Boolean - Possible values: `true`, `false` - Default: `false` - Required: No. -**`NOTIFIARR__ON_DOWNLOAD_CLEANED`** +#### **`NOTIFIARR__ON_DOWNLOAD_CLEANED`** - Controls whether to notify when a download is cleaned. - Type: Boolean - Possible values: `true`, `false` @@ -589,7 +617,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" ### Advanced settings -**`HTTP_MAX_RETRIES`** +#### **`HTTP_MAX_RETRIES`** - The number of times to retry a failed HTTP call. - Applies to calls to *arrs, download clients, and other services. - Type: Integer @@ -597,10 +625,10 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `0` - Required: No. -**`HTTP_TIMEOUT`** +#### **`HTTP_TIMEOUT`** - The number of seconds to wait before failing an HTTP call. - Applies to calls to *arrs, download clients, and other services. - Type: Integer -- Possible values: Greater than `0` +- Possible values: Greater than `0`. - Default: `100` - Required: No. \ No newline at end of file From b94ae21e119f69572bcc0c01f02fdbb757c8d98e Mon Sep 17 00:00:00 2001 From: Flaminel Date: Thu, 13 Mar 2025 10:16:52 +0200 Subject: [PATCH 10/16] update permissive blacklist --- blacklist_permissive | 1 + 1 file changed, 1 insertion(+) diff --git a/blacklist_permissive b/blacklist_permissive index 82a1014..9906b9f 100644 --- a/blacklist_permissive +++ b/blacklist_permissive @@ -1,4 +1,5 @@ *.apk +*.arj *.bat *.bin *.bmp From 32bcbab523029062075299b7ff469209b0965ce6 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Wed, 19 Mar 2025 01:25:31 +0200 Subject: [PATCH 11/16] added docs for FreeBSD --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/README.md b/README.md index a1523c4..c76f941 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ cleanuperr supports both qBittorrent's built-in exclusion features and its own b - [Windows](#windows) - [Linux](#linux) - [MacOS](#macos) + - [FreeBSD](#freebsd) - [Credits](#credits) ## Naming choice @@ -300,6 +301,54 @@ services: > codesign --sign - --force --preserve-metadata=entitlements,requirements,flags,runtime /example/directory/cleanuperr > ``` +### FreeBSD + +1. Installation: + ``` + # install dependencies + pkg install -y git icu libinotify libunwind wget + + # set up the dotnet SDK + cd ~ + wget -q https://github.com/Thefrank/dotnet-freebsd-crossbuild/releases/download/v9.0.104-amd64-freebsd-14/dotnet-sdk-9.0.104-freebsd-x64.tar.gz + export DOTNET_ROOT=$(pwd)/.dotnet + mkdir -p "$DOTNET_ROOT" && tar zxf dotnet-sdk-9.0.104-freebsd-x64.tar.gz -C "$DOTNET_ROOT" + export PATH=$PATH:$DOTNET_ROOT:$DOTNET_ROOT/tools + + # download NuGet dependencies + mkdir -p /tmp/nuget + wget -q -P /tmp/nuget/ https://github.com/Thefrank/dotnet-freebsd-crossbuild/releases/download/v9.0.104-amd64-freebsd-14/Microsoft.AspNetCore.App.Runtime.freebsd-x64.9.0.3.nupkg + wget -q -P /tmp/nuget/ https://github.com/Thefrank/dotnet-freebsd-crossbuild/releases/download/v9.0.104-amd64-freebsd-14/Microsoft.NETCore.App.Host.freebsd-x64.9.0.3.nupkg + wget -q -P /tmp/nuget/ https://github.com/Thefrank/dotnet-freebsd-crossbuild/releases/download/v9.0.104-amd64-freebsd-14/Microsoft.NETCore.App.Runtime.freebsd-x64.9.0.3.nupkg + + # add NuGet source + dotnet nuget add source /tmp/nuget --name tmp + + # add GitHub NuGet source + # a PAT (Personal Access Token) can be generated here https://github.com/settings/tokens + dotnet nuget add source --username --password --store-password-in-clear-text --name flmorg https://nuget.pkg.github.com/flmorg/index.json + ``` +2. Building: + ``` + # clone the project + git clone https://github.com/flmorg/cleanuperr.git + cd cleanuperr + + # build and publish the app + dotnet publish code/Executable/Executable.csproj -c Release --self-contained -o artifacts /p:PublishSingleFile=true + + # move the files to permanent destination + mv artifacts/cleanuperr /example/directory/ + mv artifacts/appsettings.json /example/directory/ + ``` +3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [here](#environment-variables). +4. Run the app: + ``` + cd /example/directory + chmod +x cleanuperr + ./cleanuperr + ``` + # Credits Special thanks for inspiration go to: - [ThijmenGThN/swaparr](https://github.com/ThijmenGThN/swaparr) From 4bc1c33e8169c3c09b286cbc357865aba36d72a2 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Wed, 19 Mar 2025 16:02:46 +0200 Subject: [PATCH 12/16] Add option to explicitly disable the download client (#93) --- code/Common/Enums/DownloadClient.cs | 3 ++- .../Verticals/ContentBlocker/ContentBlocker.cs | 4 ++-- .../Verticals/DownloadCleaner/DownloadCleaner.cs | 4 ++-- .../DownloadClient/DownloadServiceFactory.cs | 1 + .../Verticals/QueueCleaner/QueueCleaner.cs | 2 +- variables.md | 15 ++++++++++----- 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/code/Common/Enums/DownloadClient.cs b/code/Common/Enums/DownloadClient.cs index 6c72e83..7c21506 100644 --- a/code/Common/Enums/DownloadClient.cs +++ b/code/Common/Enums/DownloadClient.cs @@ -5,5 +5,6 @@ public enum DownloadClient QBittorrent, Deluge, Transmission, - None + None, + Disabled } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index 857510f..07ee792 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -55,9 +55,9 @@ public sealed class ContentBlocker : GenericHandler public override async Task ExecuteAsync() { - if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None) + if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled) { - _logger.LogWarning("download client is set to none"); + _logger.LogWarning("download client is not set"); return; } diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index e5f767c..25c79ea 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -50,9 +50,9 @@ public sealed class DownloadCleaner : GenericHandler public override async Task ExecuteAsync() { - if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None) + if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled) { - _logger.LogWarning("download client is set to none"); + _logger.LogWarning("download client is not set"); return; } diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs index d02c975..582b344 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs @@ -25,6 +25,7 @@ public sealed class DownloadServiceFactory Common.Enums.DownloadClient.Deluge => _serviceProvider.GetRequiredService(), Common.Enums.DownloadClient.Transmission => _serviceProvider.GetRequiredService(), Common.Enums.DownloadClient.None => _serviceProvider.GetRequiredService(), + Common.Enums.DownloadClient.Disabled => _serviceProvider.GetRequiredService(), _ => throw new ArgumentOutOfRangeException() }; } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index bc7df16..3991407 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -93,7 +93,7 @@ public sealed class QueueCleaner : GenericHandler StalledResult stalledCheckResult = new(); - if (record.Protocol is "torrent") + if (record.Protocol is "torrent" && _downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.Disabled) { if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None) { diff --git a/variables.md b/variables.md index e539035..6c64d31 100644 --- a/variables.md +++ b/variables.md @@ -135,7 +135,7 @@ - Required: No. > [!WARNING] -> Setting `QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. +> Setting `QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. #### **`QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS`** - Patterns to look for in failed import messages that should be ignored. @@ -182,7 +182,7 @@ - Required: No. > [!WARNING] -> Setting `QUEUECLEANER__STALLED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. +> Setting `QUEUECLEANER__STALLED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. # @@ -246,7 +246,7 @@ - Required: No. > [!WARNING] -> Setting `CONTENTBLOCKER__DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. +> Setting `CONTENTBLOCKER__DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. # @@ -302,7 +302,7 @@ - Required: No. > [!WARNING] -> Setting `DOWNLOADCLEANER__DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your tracker account. +> Setting `DOWNLOADCLEANER__DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. #### **`DOWNLOADCLEANER__CATEGORIES__0__NAME`** - Name of the category to clean. @@ -360,13 +360,18 @@ #### **`DOWNLOAD_CLIENT`** - Specifies which download client is used by *arrs. - Type: String. -- Possible values: `none`, `qbittorrent`, `deluge`, `transmission`. +- Possible values: `none`, `qbittorrent`, `deluge`, `transmission`, `disabled`. - Default: `none` - Required: No. > [!NOTE] > Only one download client can be enabled at a time. If you have more than one download client, you should deploy multiple instances of cleanuperr. +> [!IMPORTANT] +> When the download client is set to `disabled`, the queue cleaner will be able to remove items that are failed to be imported even if there is no download client configured. This means that all downloads, including private ones, will be completely removed. +> +> Setting `DOWNLOAD_CLIENT=disabled` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. + #### **`QBITTORRENT__URL`** - URL of the qBittorrent instance. - Type: String. From a1354f231a409dc66b81eb7521d5940169eee787 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Thu, 20 Mar 2025 00:08:51 +0200 Subject: [PATCH 13/16] Add base path support for arrs (#96) --- .../Infrastructure/Verticals/Arr/ArrClient.cs | 26 ++++++++---- .../Verticals/Arr/LidarrClient.cs | 38 ++++++++++++----- .../Verticals/Arr/RadarrClient.cs | 35 +++++++++++----- .../Verticals/Arr/SonarrClient.cs | 41 +++++++++++++------ 4 files changed, 96 insertions(+), 44 deletions(-) diff --git a/code/Infrastructure/Verticals/Arr/ArrClient.cs b/code/Infrastructure/Verticals/Arr/ArrClient.cs index 2af6d02..d5d87ce 100644 --- a/code/Infrastructure/Verticals/Arr/ArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/ArrClient.cs @@ -43,9 +43,11 @@ public abstract class ArrClient : IArrClient public virtual async Task GetQueueItemsAsync(ArrInstance arrInstance, int page) { - Uri uri = new(arrInstance.Url, GetQueueUrlPath(page)); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueUrlPath().TrimStart('/')}"; + uriBuilder.Query = GetQueueUrlQuery(page); - using HttpRequestMessage request = new(HttpMethod.Get, uri); + using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); using HttpResponseMessage response = await _httpClient.SendAsync(request); @@ -56,7 +58,7 @@ public abstract class ArrClient : IArrClient } catch { - _logger.LogError("queue list failed | {uri}", uri); + _logger.LogError("queue list failed | {uri}", uriBuilder.Uri); throw; } @@ -65,7 +67,7 @@ public abstract class ArrClient : IArrClient if (queueResponse is null) { - throw new Exception($"unrecognized queue list response | {uri} | {responseBody}"); + throw new Exception($"unrecognized queue list response | {uriBuilder.Uri} | {responseBody}"); } return queueResponse; @@ -114,11 +116,13 @@ public abstract class ArrClient : IArrClient public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient) { - Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id, removeFromClient)); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}"; + uriBuilder.Query = GetQueueDeleteUrlQuery(removeFromClient); try { - using HttpRequestMessage request = new(HttpMethod.Delete, uri); + using HttpRequestMessage request = new(HttpMethod.Delete, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync(SendRequestAsync, request); @@ -134,7 +138,7 @@ public abstract class ArrClient : IArrClient } catch { - _logger.LogError("queue delete failed | {uri} | {title}", uri, record.Title); + _logger.LogError("queue delete failed | {uri} | {title}", uriBuilder.Uri, record.Title); throw; } } @@ -152,9 +156,13 @@ public abstract class ArrClient : IArrClient return true; } - protected abstract string GetQueueUrlPath(int page); + protected abstract string GetQueueUrlPath(); - protected abstract string GetQueueDeleteUrlPath(long recordId, bool removeFromClient); + protected abstract string GetQueueUrlQuery(int page); + + protected abstract string GetQueueDeleteUrlPath(long recordId); + + protected abstract string GetQueueDeleteUrlQuery(bool removeFromClient); protected virtual void SetApiKey(HttpRequestMessage request, string apiKey) { diff --git a/code/Infrastructure/Verticals/Arr/LidarrClient.cs b/code/Infrastructure/Verticals/Arr/LidarrClient.cs index 08ee3f7..730615c 100644 --- a/code/Infrastructure/Verticals/Arr/LidarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/LidarrClient.cs @@ -27,29 +27,42 @@ public class LidarrClient : ArrClient, ILidarrClient { } - protected override string GetQueueUrlPath(int page) + protected override string GetQueueUrlPath() { - return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true"; + return "/api/v1/queue"; } - protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient) + protected override string GetQueueUrlQuery(int page) { - string path = $"/api/v1/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false"; + return $"page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true"; + } - path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + protected override string GetQueueDeleteUrlPath(long recordId) + { + return $"/api/v1/queue/{recordId}"; + } - return path; + protected override string GetQueueDeleteUrlQuery(bool removeFromClient) + { + string query = "blocklist=true&skipRedownload=true&changeCategory=false"; + query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + + return query; } public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) { - if (items?.Count is null or 0) return; + if (items?.Count is null or 0) + { + return; + } - Uri uri = new(arrInstance.Url, "/api/v1/command"); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/command"; foreach (var command in GetSearchCommands(items)) { - using HttpRequestMessage request = new(HttpMethod.Post, uri); + using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri); request.Content = new StringContent( JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }), Encoding.UTF8, @@ -132,8 +145,11 @@ public class LidarrClient : ArrClient, ILidarrClient private async Task?> GetAlbumsAsync(ArrInstance arrInstance, List albumIds) { - Uri uri = new(arrInstance.Url, $"api/v1/album?{string.Join('&', albumIds.Select(x => $"albumIds={x}"))}"); - using HttpRequestMessage request = new(HttpMethod.Get, uri); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/album"; + uriBuilder.Query = string.Join('&', albumIds.Select(x => $"albumIds={x}")); + + using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); using var response = await _httpClient.SendAsync(request); diff --git a/code/Infrastructure/Verticals/Arr/RadarrClient.cs b/code/Infrastructure/Verticals/Arr/RadarrClient.cs index 86add32..7b9e696 100644 --- a/code/Infrastructure/Verticals/Arr/RadarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/RadarrClient.cs @@ -27,18 +27,27 @@ public class RadarrClient : ArrClient, IRadarrClient { } - protected override string GetQueueUrlPath(int page) + protected override string GetQueueUrlPath() { - return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true"; + return "/api/v3/queue"; } - protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient) + protected override string GetQueueUrlQuery(int page) { - string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false"; - - path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + return $"page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true"; + } - return path; + protected override string GetQueueDeleteUrlPath(long recordId) + { + return $"/api/v3/queue/{recordId}"; + } + + protected override string GetQueueDeleteUrlQuery(bool removeFromClient) + { + string query = "blocklist=true&skipRedownload=true&changeCategory=false"; + query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + + return query; } public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) @@ -50,14 +59,16 @@ public class RadarrClient : ArrClient, IRadarrClient List ids = items.Select(item => item.Id).ToList(); - Uri uri = new(arrInstance.Url, "/api/v3/command"); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command"; + RadarrCommand command = new() { Name = "MoviesSearch", MovieIds = ids, }; - using HttpRequestMessage request = new(HttpMethod.Post, uri); + using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri); request.Content = new StringContent( JsonConvert.SerializeObject(command), Encoding.UTF8, @@ -135,8 +146,10 @@ public class RadarrClient : ArrClient, IRadarrClient private async Task GetMovie(ArrInstance arrInstance, long movieId) { - Uri uri = new(arrInstance.Url, $"api/v3/movie/{movieId}"); - using HttpRequestMessage request = new(HttpMethod.Get, uri); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/movie/{movieId}"; + + using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); using HttpResponseMessage response = await _httpClient.SendAsync(request); diff --git a/code/Infrastructure/Verticals/Arr/SonarrClient.cs b/code/Infrastructure/Verticals/Arr/SonarrClient.cs index bfba28a..2f5aa44 100644 --- a/code/Infrastructure/Verticals/Arr/SonarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/SonarrClient.cs @@ -28,18 +28,27 @@ public class SonarrClient : ArrClient, ISonarrClient { } - protected override string GetQueueUrlPath(int page) + protected override string GetQueueUrlPath() { - return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true&includeEpisode=true"; + return "/api/v3/queue"; } - - protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient) + + protected override string GetQueueUrlQuery(int page) { - string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false"; + return $"page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true&includeEpisode=true"; + } - path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + protected override string GetQueueDeleteUrlPath(long recordId) + { + return $"/api/v3/queue/{recordId}"; + } - return path; + protected override string GetQueueDeleteUrlQuery(bool removeFromClient) + { + string query = "blocklist=true&skipRedownload=true&changeCategory=false"; + query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false"; + + return query; } public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) @@ -49,11 +58,12 @@ public class SonarrClient : ArrClient, ISonarrClient return; } - Uri uri = new(arrInstance.Url, "/api/v3/command"); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command"; foreach (SonarrCommand command in GetSearchCommands(items.Cast().ToHashSet())) { - using HttpRequestMessage request = new(HttpMethod.Post, uri); + using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri); request.Content = new StringContent( JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }), Encoding.UTF8, @@ -199,8 +209,11 @@ public class SonarrClient : ArrClient, ISonarrClient private async Task?> GetEpisodesAsync(ArrInstance arrInstance, List episodeIds) { - Uri uri = new(arrInstance.Url, $"api/v3/episode?{string.Join('&', episodeIds.Select(x => $"episodeIds={x}"))}"); - using HttpRequestMessage request = new(HttpMethod.Get, uri); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/episode"; + uriBuilder.Query = string.Join('&', episodeIds.Select(x => $"episodeIds={x}")); + + using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); using HttpResponseMessage response = await _httpClient.SendAsync(request); @@ -212,8 +225,10 @@ public class SonarrClient : ArrClient, ISonarrClient private async Task GetSeriesAsync(ArrInstance arrInstance, long seriesId) { - Uri uri = new(arrInstance.Url, $"api/v3/series/{seriesId}"); - using HttpRequestMessage request = new(HttpMethod.Get, uri); + UriBuilder uriBuilder = new(arrInstance.Url); + uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/series/{seriesId}"; + + using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri); SetApiKey(request, arrInstance.ApiKey); using HttpResponseMessage response = await _httpClient.SendAsync(request); From f21f7388b73864f7ca0674fccc7f2e34a51e4e3d Mon Sep 17 00:00:00 2001 From: Flaminel Date: Thu, 20 Mar 2025 00:09:24 +0200 Subject: [PATCH 14/16] Add download client customizable url base (#43) --- README.md | 3 +++ .../DownloadClient/DelugeConfig.cs | 6 +++++- .../Configuration/DownloadClient/QBitConfig.cs | 6 +++++- .../DownloadClient/TransmissionConfig.cs | 6 +++++- code/Executable/appsettings.Development.json | 3 +++ code/Executable/appsettings.json | 3 +++ .../DownloadClient/Deluge/DelugeClient.cs | 9 +++++++-- .../DownloadClient/QBittorrent/QBitService.cs | 6 +++++- .../Transmission/TransmissionService.cs | 6 +++++- variables.md | 18 ++++++++++++++++++ 10 files changed, 59 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c76f941..f512537 100644 --- a/README.md +++ b/README.md @@ -214,15 +214,18 @@ services: # OR # - DOWNLOAD_CLIENT=qBittorrent # - QBITTORRENT__URL=http://localhost:8080 + # - QBITTORRENT__URL_BASE=myCustomPath # - QBITTORRENT__USERNAME=user # - QBITTORRENT__PASSWORD=pass # OR # - DOWNLOAD_CLIENT=deluge + # - DELUGE__URL_BASE=myCustomPath # - DELUGE__URL=http://localhost:8112 # - DELUGE__PASSWORD=testing # OR # - DOWNLOAD_CLIENT=transmission # - TRANSMISSION__URL=http://localhost:9091 + # - TRANSMISSION__URL_BASE=myCustomPath # - TRANSMISSION__USERNAME=test # - TRANSMISSION__PASSWORD=testing diff --git a/code/Common/Configuration/DownloadClient/DelugeConfig.cs b/code/Common/Configuration/DownloadClient/DelugeConfig.cs index be16ac0..59033a2 100644 --- a/code/Common/Configuration/DownloadClient/DelugeConfig.cs +++ b/code/Common/Configuration/DownloadClient/DelugeConfig.cs @@ -1,4 +1,5 @@ -using Common.Exceptions; +using Common.Exceptions; +using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadClient; @@ -8,6 +9,9 @@ public sealed record DelugeConfig : IConfig public Uri? Url { get; init; } + [ConfigurationKeyName("URL_BASE")] + public string UrlBase { get; init; } = string.Empty; + public string? Password { get; init; } public void Validate() diff --git a/code/Common/Configuration/DownloadClient/QBitConfig.cs b/code/Common/Configuration/DownloadClient/QBitConfig.cs index afdbc1e..7de0faf 100644 --- a/code/Common/Configuration/DownloadClient/QBitConfig.cs +++ b/code/Common/Configuration/DownloadClient/QBitConfig.cs @@ -1,4 +1,5 @@ -using Common.Exceptions; +using Common.Exceptions; +using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadClient; @@ -8,6 +9,9 @@ public sealed class QBitConfig : IConfig public Uri? Url { get; init; } + [ConfigurationKeyName("URL_BASE")] + public string UrlBase { get; init; } = string.Empty; + public string? Username { get; init; } public string? Password { get; init; } diff --git a/code/Common/Configuration/DownloadClient/TransmissionConfig.cs b/code/Common/Configuration/DownloadClient/TransmissionConfig.cs index c029b10..4d30b62 100644 --- a/code/Common/Configuration/DownloadClient/TransmissionConfig.cs +++ b/code/Common/Configuration/DownloadClient/TransmissionConfig.cs @@ -1,4 +1,5 @@ -using Common.Exceptions; +using Common.Exceptions; +using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadClient; @@ -8,6 +9,9 @@ public record TransmissionConfig : IConfig public Uri? Url { get; init; } + [ConfigurationKeyName("URL_BASE")] + public string UrlBase { get; init; } = "transmission"; + public string? Username { get; init; } public string? Password { get; init; } diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 26a69d1..f5d8720 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -52,15 +52,18 @@ "DOWNLOAD_CLIENT": "qbittorrent", "qBittorrent": { "Url": "http://localhost:8080", + "URL_BASE": "", "Username": "test", "Password": "testing" }, "Deluge": { "Url": "http://localhost:8112", + "URL_BASE": "", "Password": "testing" }, "Transmission": { "Url": "http://localhost:9091", + "URL_BASE": "transmission", "Username": "test", "Password": "testing" }, diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index 3b901e8..5fa1fc7 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -42,15 +42,18 @@ "DOWNLOAD_CLIENT": "none", "qBittorrent": { "Url": "http://localhost:8080", + "URL_BASE": "", "Username": "", "Password": "" }, "Deluge": { "Url": "http://localhost:8112", + "URL_BASE": "", "Password": "testing" }, "Transmission": { "Url": "http://localhost:9091", + "URL_BASE": "transmission", "Username": "test", "Password": "testing" }, diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index e7eac07..63048cb 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -33,6 +33,7 @@ public sealed class DelugeClient public DelugeClient(IOptions config, IHttpClientFactory httpClientFactory) { _config = config.Value; + _config.Validate(); _httpClient = httpClientFactory.CreateClient(nameof(DelugeService)); } @@ -121,8 +122,12 @@ public sealed class DelugeClient { StringContent content = new StringContent(json); content.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json"); - - var responseMessage = await _httpClient.PostAsync(new Uri(_config.Url, "/json"), content); + + UriBuilder uriBuilder = new(_config.Url); + uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase) + ? $"{uriBuilder.Path.TrimEnd('/')}/json" + : $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/json"; + var responseMessage = await _httpClient.PostAsync(uriBuilder.Uri, content); responseMessage.EnsureSuccessStatusCode(); var responseJson = await responseMessage.Content.ReadAsStringAsync(); diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 8061fe2..4d55d7c 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -45,7 +45,11 @@ public class QBitService : DownloadService, IQBitService { _config = config.Value; _config.Validate(); - _client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), _config.Url); + UriBuilder uriBuilder = new(_config.Url); + uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase) + ? uriBuilder.Path + : $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/')}"; + _client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), uriBuilder.Uri); } public override async Task LoginAsync() diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 5160569..7348959 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -64,9 +64,13 @@ public class TransmissionService : DownloadService, ITransmissionService { _config = config.Value; _config.Validate(); + UriBuilder uriBuilder = new(_config.Url); + uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase) + ? $"{uriBuilder.Path.TrimEnd('/')}/rpc" + : $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/rpc"; _client = new( httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), - new Uri(_config.Url, "/transmission/rpc").ToString(), + uriBuilder.Uri.ToString(), login: _config.Username, password: _config.Password ); diff --git a/variables.md b/variables.md index 6c64d31..e556705 100644 --- a/variables.md +++ b/variables.md @@ -378,6 +378,12 @@ - Default: `http://localhost:8080`. - Required: No. +#### **`QBITTORRENT__URL_BASE`** +- Adds a prefix to the qBittorrent url, such as `[QBITTORRENT__URL]/[QBITTORRENT__URL_BASE]/api`. +- Type: String. +- Default: Empty. +- Required: No. + #### **`QBITTORRENT__USERNAME`** - Username for qBittorrent authentication. - Type: String. @@ -396,6 +402,12 @@ - Default: `http://localhost:8112`. - Required: No. +#### **`DELUGE__URL_BASE`** +- Adds a prefix to the deluge json url, such as `[DELUGE__URL]/[DELUGE__URL_BASE]/json`. +- Type: String. +- Default: Empty. +- Required: No. + #### **`DELUGE__PASSWORD`** - Password for Deluge authentication. - Type: String. @@ -408,6 +420,12 @@ - Default: `http://localhost:9091`. - Required: No. +#### **`TRANSMISSION__URL_BASE`** +- Adds a prefix to the Transmission rpc url, such as `[TRANSMISSION__URL]/[TRANSMISSION__URL_BASE]/rpc`. +- Type: String. +- Default: `transmission`. +- Required: No. + #### **`TRANSMISSION__USERNAME`** - Username for Transmission authentication. - Type: String. From b8ce225ccce13fc2c57680ff36159dcfde3f2bac Mon Sep 17 00:00:00 2001 From: Flaminel Date: Thu, 20 Mar 2025 00:09:58 +0200 Subject: [PATCH 15/16] Fix Deluge service crashing when download is not found (#97) --- .../DownloadClient/Deluge/DelugeClient.cs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index 63048cb..195c08b 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -80,11 +80,24 @@ public sealed class DelugeClient public async Task GetTorrentStatus(string hash) { - return await SendRequest( - "web.get_torrent_status", - hash, - Fields - ); + try + { + return await SendRequest( + "web.get_torrent_status", + hash, + Fields + ); + } + catch (DelugeClientException e) + { + // Deluge returns an error when the torrent is not found + if (e.Message == "AttributeError: 'NoneType' object has no attribute 'call'") + { + return null; + } + + throw; + } } public async Task?> GetStatusForAllTorrents() From 5fe0f5750ab5ca1ec7c6915ae40c874551544b84 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Fri, 21 Mar 2025 23:06:31 +0200 Subject: [PATCH 16/16] Fix qBit queued items being processed (#102) --- code/Domain/Enums/DeleteReason.cs | 6 ++- code/Domain/Enums/StrikeType.cs | 1 + .../DownloadClient/DownloadServiceTests.cs | 5 ++- .../DownloadClient/TestDownloadService.cs | 3 +- .../Infrastructure/Verticals/Arr/ArrClient.cs | 12 ++++-- .../Verticals/Arr/Interfaces/IArrClient.cs | 2 +- .../ContentBlocker/ContentBlocker.cs | 2 +- .../DownloadClient/Deluge/DelugeService.cs | 29 +++++++------ .../DownloadClient/DownloadService.cs | 5 ++- .../DownloadClient/QBittorrent/QBitService.cs | 43 ++++++++++--------- .../Transmission/TransmissionService.cs | 24 +++++------ .../Verticals/QueueCleaner/QueueCleaner.cs | 2 +- 12 files changed, 74 insertions(+), 60 deletions(-) diff --git a/code/Domain/Enums/DeleteReason.cs b/code/Domain/Enums/DeleteReason.cs index e46cdd6..baeddd3 100644 --- a/code/Domain/Enums/DeleteReason.cs +++ b/code/Domain/Enums/DeleteReason.cs @@ -2,7 +2,11 @@ public enum DeleteReason { + None, Stalled, ImportFailed, - AllFilesBlocked + DownloadingMetadata, + AllFilesSkipped, + AllFilesSkippedByQBit, + AllFilesBlocked, } \ No newline at end of file diff --git a/code/Domain/Enums/StrikeType.cs b/code/Domain/Enums/StrikeType.cs index 1a04c15..234401d 100644 --- a/code/Domain/Enums/StrikeType.cs +++ b/code/Domain/Enums/StrikeType.cs @@ -3,5 +3,6 @@ public enum StrikeType { Stalled, + DownloadingMetadata, ImportFailed } \ No newline at end of file diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs index 9fcba54..d63fd3d 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs @@ -105,13 +105,14 @@ public class DownloadServiceTests : IClassFixture // Arrange const string hash = "test-hash"; const string itemName = "test-item"; - _fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, StrikeType.Stalled) + StrikeType strikeType = StrikeType.Stalled; + _fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, strikeType) .Returns(true); TestDownloadService sut = _fixture.CreateSut(); // Act - bool result = await sut.StrikeAndCheckLimit(hash, itemName); + bool result = await sut.StrikeAndCheckLimit(hash, itemName, strikeType); // Assert result.ShouldBeTrue(); diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index 60029be..fc0a125 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -3,6 +3,7 @@ using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; +using Domain.Enums; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadClient; @@ -45,6 +46,6 @@ public class TestDownloadService : DownloadService // Expose protected methods for testing public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); - public new Task StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName); + public new Task StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType) => base.StrikeAndCheckLimit(hash, itemName, strikeType); public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Arr/ArrClient.cs b/code/Infrastructure/Verticals/Arr/ArrClient.cs index d5d87ce..3574ed1 100644 --- a/code/Infrastructure/Verticals/Arr/ArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/ArrClient.cs @@ -114,7 +114,12 @@ public abstract class ArrClient : IArrClient return false; } - public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient) + public virtual async Task DeleteQueueItemAsync( + ArrInstance arrInstance, + QueueRecord record, + bool removeFromClient, + DeleteReason deleteReason + ) { UriBuilder uriBuilder = new(arrInstance.Url); uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}"; @@ -130,8 +135,9 @@ public abstract class ArrClient : IArrClient _logger.LogInformation( removeFromClient - ? "queue item deleted | {url} | {title}" - : "queue item removed from arr | {url} | {title}", + ? "queue item deleted with reason {reason} | {url} | {title}" + : "queue item removed from arr with reason {reason} | {url} | {title}", + deleteReason.ToString(), arrInstance.Url, record.Title ); diff --git a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs index 4435a05..30027a9 100644 --- a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs @@ -11,7 +11,7 @@ public interface IArrClient Task ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload); - Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient); + Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason); Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items); diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index 07ee792..a998180 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -142,7 +142,7 @@ public sealed class ContentBlocker : GenericHandler removeFromClient = false; } - await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient); + await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, DeleteReason.AllFilesBlocked); await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked); } }); diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index d104b04..689518e 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -65,6 +65,8 @@ public class DelugeService : DownloadService, IDelugeService return result; } + result.IsPrivate = download.Private; + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) { _logger.LogInformation("skip | download is ignored | {name}", download.Name); @@ -79,6 +81,7 @@ public class DelugeService : DownloadService, IDelugeService { _logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash); } + bool shouldRemove = contents?.Contents?.Count > 0; @@ -92,17 +95,15 @@ public class DelugeService : DownloadService, IDelugeService if (shouldRemove) { - result.DeleteReason = DeleteReason.AllFilesBlocked; + // remove if all files are unwanted + result.ShouldRemove = true; + result.DeleteReason = DeleteReason.AllFilesSkipped; + return result; } - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); - result.IsPrivate = download.Private; + // remove if download is stuck + (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download); - if (!shouldRemove && result.ShouldRemove) - { - result.DeleteReason = DeleteReason.Stalled; - } - return result; } @@ -295,33 +296,33 @@ public class DelugeService : DownloadService, IDelugeService await _client.ChangeFilesPriority(hash, sortedPriorities); } - private async Task IsItemStuckAndShouldRemove(TorrentStatus status) + private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentStatus status) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return false; + return (false, default); } if (_queueCleanerConfig.StalledIgnorePrivate && status.Private) { // ignore private trackers _logger.LogDebug("skip stalled check | download is private | {name}", status.Name); - return false; + return (false, default); } if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) { - return false; + return (false, default); } if (status.Eta > 0) { - return false; + return (false, default); } ResetStrikesOnProgress(status.Hash!, status.TotalDone); - return await StrikeAndCheckLimit(status.Hash!, status.Name!); + return (await StrikeAndCheckLimit(status.Hash!, status.Name!, StrikeType.Stalled), DeleteReason.Stalled); } private static void ProcessFiles(Dictionary? contents, Action processFile) diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index 587fd53..7b703bc 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -100,10 +100,11 @@ public abstract class DownloadService : IDownloadService /// /// The torrent hash. /// The name or title of the item. + /// /// True if the limit has been reached; otherwise, false. - protected async Task StrikeAndCheckLimit(string hash, string itemName) + protected async Task StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType) { - return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled); + return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, strikeType); } protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 4d55d7c..34c27d4 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -96,30 +96,26 @@ public class QBitService : DownloadService, IQBitService bool.TryParse(dictValue?.ToString(), out bool boolValue) && boolValue; - // if all files were blocked by qBittorrent - if (download is { CompletionOn: not null, Downloaded: null or 0 }) - { - result.ShouldRemove = true; - result.DeleteReason = DeleteReason.AllFilesBlocked; - return result; - } - IReadOnlyList? files = await _client.GetTorrentContentsAsync(hash); - // if all files are marked as skip if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip)) { result.ShouldRemove = true; - result.DeleteReason = DeleteReason.AllFilesBlocked; + + // if all files were blocked by qBittorrent + if (download is { CompletionOn: not null, Downloaded: null or 0 }) + { + result.DeleteReason = DeleteReason.AllFilesSkippedByQBit; + return result; + } + + // remove if all files are unwanted + result.DeleteReason = DeleteReason.AllFilesSkipped; return result; } - result.ShouldRemove = await IsItemStuckAndShouldRemove(download, result.IsPrivate); - - if (result.ShouldRemove) - { - result.DeleteReason = DeleteReason.Stalled; - } + // remove if download is stuck + (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download, result.IsPrivate); return result; } @@ -337,30 +333,35 @@ public class QBitService : DownloadService, IQBitService _client.Dispose(); } - private async Task IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) + private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return false; + return (false, default); } if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) { // ignore private trackers _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - return false; + return (false, default); } if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata and not TorrentState.ForcedFetchingMetadata) { // ignore other states - return false; + return (false, default); } ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); - return await StrikeAndCheckLimit(torrent.Hash, torrent.Name); + if (torrent.State is TorrentState.StalledDownload) + { + return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.Stalled), DeleteReason.Stalled); + } + + return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); } private async Task> GetTrackersAsync(string hash) diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 7348959..0503ca8 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -119,17 +119,15 @@ public class TransmissionService : DownloadService, ITransmissionService if (shouldRemove) { + // remove if all files are unwanted + result.ShouldRemove = true; result.DeleteReason = DeleteReason.AllFilesBlocked; + return result; } - // remove if all files are unwanted or download is stuck - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); + // remove if download is stuck + (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download); - if (!shouldRemove && result.ShouldRemove) - { - result.DeleteReason = DeleteReason.Stalled; - } - return result; } @@ -338,34 +336,34 @@ public class TransmissionService : DownloadService, ITransmissionService }); } - private async Task IsItemStuckAndShouldRemove(TorrentInfo torrent) + private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return false; + return (false, default); } if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false)) { // ignore private trackers _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - return false; + return (false, default); } if (torrent.Status is not 4) { // not in downloading state - return false; + return (false, default); } if (torrent.Eta > 0) { - return false; + return (false, default); } ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0); - return await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!); + return (await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!, StrikeType.Stalled), DeleteReason.Stalled); } private async Task GetTorrentAsync(string hash) diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 3991407..32aadca 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -132,7 +132,7 @@ public sealed class QueueCleaner : GenericHandler } } - await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient); + await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, deleteReason); await _notifier.NotifyQueueItemDeleted(removeFromClient, deleteReason); } });