Compare commits

...

2 Commits

Author SHA1 Message Date
Flaminel 027c4a0f4d Add option to ignore specific downloads (#79) 2025-03-09 23:38:27 +02:00
Flaminel 81990c6768 fixed missing README link 2025-03-03 22:37:22 +02:00
28 changed files with 553 additions and 122 deletions
+5 -1
View File
@@ -112,7 +112,7 @@ I've seen a few discussions on this type of naming and I've decided that I didn'
## Using cleanuperr's blocklist (works with all supported download clients) ## Using cleanuperr's blocklist (works with all supported download clients)
1. Set both `QUEUECLEANER__ENABLED` and `CONTENTBLOCKER__ENABLED` to `true` in your environment variables. 1. Set both `QUEUECLEANER__ENABLED` and `CONTENTBLOCKER__ENABLED` to `true` in your environment variables.
2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Arr variables](#Arr-variables) section. 2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Arr variables](variables.md#Arr-settings) section.
3. Once configured, cleanuperr will perform the following tasks: 3. Once configured, cleanuperr will perform the following tasks:
- Execute the **content blocker** job, as explained in the [How it works](#how-it-works) section. - Execute the **content blocker** job, as explained in the [How it works](#how-it-works) section.
- Execute the **queue cleaner** job, as explained in the [How it works](#how-it-works) section. - Execute the **queue cleaner** job, as explained in the [How it works](#how-it-works) section.
@@ -139,6 +139,7 @@ services:
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- ./cleanuperr/logs:/var/logs - ./cleanuperr/logs:/var/logs
- ./cleanuperr/ignored.txt:/ignored.txt
environment: environment:
- TZ=America/New_York - TZ=America/New_York
- DRY_RUN=false - DRY_RUN=false
@@ -153,6 +154,7 @@ services:
- TRIGGERS__DOWNLOADCLEANER=0 0 * * * ? - TRIGGERS__DOWNLOADCLEANER=0 0 * * * ?
- QUEUECLEANER__ENABLED=true - QUEUECLEANER__ENABLED=true
- QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt
- QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
@@ -165,10 +167,12 @@ services:
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false - QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored.txt
- CONTENTBLOCKER__IGNORE_PRIVATE=false - CONTENTBLOCKER__IGNORE_PRIVATE=false
- CONTENTBLOCKER__DELETE_PRIVATE=false - CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOADCLEANER__ENABLED=true - DOWNLOADCLEANER__ENABLED=true
- DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt
- DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__DELETE_PRIVATE=false
- DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr
- DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1
@@ -2,7 +2,7 @@
namespace Common.Configuration.ContentBlocker; namespace Common.Configuration.ContentBlocker;
public sealed record ContentBlockerConfig : IJobConfig public sealed record ContentBlockerConfig : IJobConfig, IIgnoredDownloadsConfig
{ {
public const string SectionName = "ContentBlocker"; public const string SectionName = "ContentBlocker";
@@ -13,6 +13,9 @@ public sealed record ContentBlockerConfig : IJobConfig
[ConfigurationKeyName("DELETE_PRIVATE")] [ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; init; } public bool DeletePrivate { get; init; }
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
public string? IgnoredDownloadsPath { get; init; }
public void Validate() public void Validate()
{ {
@@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration;
namespace Common.Configuration.DownloadCleaner; namespace Common.Configuration.DownloadCleaner;
public sealed record DownloadCleanerConfig : IJobConfig public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
{ {
public const string SectionName = "DownloadCleaner"; public const string SectionName = "DownloadCleaner";
@@ -12,7 +12,10 @@ public sealed record DownloadCleanerConfig : IJobConfig
public List<Category>? Categories { get; init; } public List<Category>? Categories { get; init; }
[ConfigurationKeyName("DELETE_PRIVATE")] [ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; set; } public bool DeletePrivate { get; init; }
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
public string? IgnoredDownloadsPath { get; init; }
public void Validate() public void Validate()
{ {
@@ -0,0 +1,6 @@
namespace Common.Configuration;
public interface IIgnoredDownloadsConfig
{
string? IgnoredDownloadsPath { get; }
}
@@ -2,7 +2,7 @@
namespace Common.Configuration.QueueCleaner; namespace Common.Configuration.QueueCleaner;
public sealed record QueueCleanerConfig : IJobConfig public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
{ {
public const string SectionName = "QueueCleaner"; public const string SectionName = "QueueCleaner";
@@ -10,6 +10,9 @@ public sealed record QueueCleanerConfig : IJobConfig
public required bool RunSequentially { get; init; } public required bool RunSequentially { get; init; }
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
public string? IgnoredDownloadsPath { get; init; }
[ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")] [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")]
public ushort ImportFailedMaxStrikes { get; init; } public ushort ImportFailedMaxStrikes { get; init; }
@@ -23,4 +23,11 @@ public sealed record TorrentStatus
public long SeedingTime { get; init; } public long SeedingTime { get; init; }
public float Ratio { get; init; } public float Ratio { get; init; }
public required IReadOnlyList<Tracker> Trackers { get; init; }
}
public sealed record Tracker
{
public required Uri Url { get; init; }
} }
@@ -1,4 +1,8 @@
using Infrastructure.Interceptors; using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadCleaner; using Infrastructure.Verticals.DownloadCleaner;
@@ -30,5 +34,8 @@ public static class ServicesDI
.AddTransient<ArrQueueIterator>() .AddTransient<ArrQueueIterator>()
.AddTransient<DownloadServiceFactory>() .AddTransient<DownloadServiceFactory>()
.AddSingleton<BlocklistProvider>() .AddSingleton<BlocklistProvider>()
.AddSingleton<IStriker, Striker>(); .AddSingleton<IStriker, Striker>()
.AddSingleton<IgnoredDownloadsProvider<QueueCleanerConfig>>()
.AddSingleton<IgnoredDownloadsProvider<ContentBlockerConfig>>()
.AddSingleton<IgnoredDownloadsProvider<DownloadCleanerConfig>>();
} }
+5 -2
View File
@@ -18,11 +18,13 @@
"ContentBlocker": { "ContentBlocker": {
"Enabled": true, "Enabled": true,
"IGNORE_PRIVATE": true, "IGNORE_PRIVATE": true,
"DELETE_PRIVATE": false "DELETE_PRIVATE": false,
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads"
}, },
"QueueCleaner": { "QueueCleaner": {
"Enabled": true, "Enabled": true,
"RunSequentially": true, "RunSequentially": true,
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads",
"IMPORT_FAILED_MAX_STRIKES": 5, "IMPORT_FAILED_MAX_STRIKES": 5,
"IMPORT_FAILED_IGNORE_PRIVATE": true, "IMPORT_FAILED_IGNORE_PRIVATE": true,
"IMPORT_FAILED_DELETE_PRIVATE": false, "IMPORT_FAILED_DELETE_PRIVATE": false,
@@ -44,7 +46,8 @@
"MIN_SEED_TIME": 0, "MIN_SEED_TIME": 0,
"MAX_SEED_TIME": -1 "MAX_SEED_TIME": -1
} }
] ],
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads"
}, },
"DOWNLOAD_CLIENT": "qbittorrent", "DOWNLOAD_CLIENT": "qbittorrent",
"qBittorrent": { "qBittorrent": {
+5 -2
View File
@@ -17,11 +17,13 @@
}, },
"ContentBlocker": { "ContentBlocker": {
"Enabled": false, "Enabled": false,
"IGNORE_PRIVATE": false "IGNORE_PRIVATE": false,
"IGNORED_DOWNLOADS_PATH": ""
}, },
"QueueCleaner": { "QueueCleaner": {
"Enabled": true, "Enabled": true,
"RunSequentially": true, "RunSequentially": true,
"IGNORED_DOWNLOADS_PATH": "",
"IMPORT_FAILED_MAX_STRIKES": 0, "IMPORT_FAILED_MAX_STRIKES": 0,
"IMPORT_FAILED_IGNORE_PRIVATE": false, "IMPORT_FAILED_IGNORE_PRIVATE": false,
"IMPORT_FAILED_DELETE_PRIVATE": false, "IMPORT_FAILED_DELETE_PRIVATE": false,
@@ -34,7 +36,8 @@
"DownloadCleaner": { "DownloadCleaner": {
"Enabled": false, "Enabled": false,
"DELETE_PRIVATE": false, "DELETE_PRIVATE": false,
"CATEGORIES": [] "CATEGORIES": [],
"IGNORED_DOWNLOADS_PATH": ""
}, },
"DOWNLOAD_CLIENT": "none", "DOWNLOAD_CLIENT": "none",
"qBittorrent": { "qBittorrent": {
@@ -35,12 +35,13 @@ public class TestDownloadService : DownloadService
public override void Dispose() { } public override void Dispose() { }
public override Task LoginAsync() => Task.CompletedTask; public override Task LoginAsync() => Task.CompletedTask;
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult()); public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads) => Task.FromResult(new StalledResult());
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) => Task.FromResult(new BlockFilesResult()); ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads) => Task.FromResult(new BlockFilesResult());
public override Task DeleteDownload(string hash) => Task.CompletedTask; public override Task DeleteDownload(string hash) => Task.CompletedTask;
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => Task.FromResult<List<object>?>(null); public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => Task.FromResult<List<object>?>(null);
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) => Task.CompletedTask; public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
// Expose protected methods for testing // Expose protected methods for testing
public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded);
@@ -0,0 +1,29 @@
using Domain.Models.Deluge.Response;
namespace Infrastructure.Extensions;
public static class DelugeExtensions
{
public static bool ShouldIgnore(this TorrentStatus download, IReadOnlyList<string> ignoredDownloads)
{
foreach (string value in ignoredDownloads)
{
if (download.Hash?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
{
return true;
}
if (download.Label?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
{
return true;
}
if (download.Trackers.Any(x => x.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)))
{
return true;
}
}
return false;
}
}
@@ -0,0 +1,42 @@
using QBittorrent.Client;
namespace Infrastructure.Extensions;
public static class QBitExtensions
{
public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList<string> ignoredDownloads)
{
foreach (string value in ignoredDownloads)
{
if (download.Hash.Equals(value, StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
if (download.Category.Equals(value, StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
if (download.Tags.Contains(value, StringComparer.InvariantCultureIgnoreCase))
{
return true;
}
}
return false;
}
public static bool ShouldIgnore(this TorrentTracker tracker, IReadOnlyList<string> ignoredDownloads)
{
foreach (string value in ignoredDownloads)
{
if (tracker.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
}
return false;
}
}
@@ -0,0 +1,42 @@
using Transmission.API.RPC.Entity;
namespace Infrastructure.Extensions;
public static class TransmissionExtensions
{
public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList<string> ignoredDownloads)
{
foreach (string value in ignoredDownloads)
{
if (download.HashString?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
{
return true;
}
if (download.GetCategory().Equals(value, StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
bool? hasIgnoredTracker = download.Trackers?
.Any(x => new Uri(x.Announce!).Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase));
if (hasIgnoredTracker is true)
{
return true;
}
}
return false;
}
public static string GetCategory(this TorrentInfo download)
{
if (string.IsNullOrEmpty(download.DownloadDir))
{
return string.Empty;
}
return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir));
}
}
+2
View File
@@ -11,4 +11,6 @@ public static class CacheKeys
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes"; public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
public static string Item(string hash) => $"item_{hash}"; public static string Item(string hash) => $"item_{hash}";
public static string IgnoredDownloads(string name) => $"{name}_ignored";
} }
@@ -0,0 +1,82 @@
using Common.Configuration;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Providers;
public sealed class IgnoredDownloadsProvider<T>
where T : IIgnoredDownloadsConfig
{
private readonly ILogger<IgnoredDownloadsProvider<T>> _logger;
private IIgnoredDownloadsConfig _config;
private readonly IMemoryCache _cache;
private DateTime _lastModified = DateTime.MinValue;
public IgnoredDownloadsProvider(ILogger<IgnoredDownloadsProvider<T>> logger, IOptionsMonitor<T> config, IMemoryCache cache)
{
_config = config.CurrentValue;
config.OnChange((newValue) => _config = newValue);
_logger = logger;
_cache = cache;
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
{
return;
}
if (!File.Exists(_config.IgnoredDownloadsPath))
{
throw new FileNotFoundException("file not found", _config.IgnoredDownloadsPath);
}
}
public async Task<IReadOnlyList<string>> GetIgnoredDownloads()
{
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
{
return Array.Empty<string>();
}
FileInfo fileInfo = new(_config.IgnoredDownloadsPath);
if (fileInfo.LastWriteTime > _lastModified ||
!_cache.TryGetValue(CacheKeys.IgnoredDownloads(typeof(T).Name), out IReadOnlyList<string>? ignoredDownloads) ||
ignoredDownloads is null)
{
_lastModified = fileInfo.LastWriteTime;
return await LoadFile();
}
return ignoredDownloads;
}
private async Task<IReadOnlyList<string>> LoadFile()
{
try
{
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
{
return Array.Empty<string>();
}
string[] ignoredDownloads = (await File.ReadAllLinesAsync(_config.IgnoredDownloadsPath))
.Where(x => !string.IsNullOrWhiteSpace(x))
.ToArray();
_cache.Set(CacheKeys.IgnoredDownloads(typeof(T).Name), ignoredDownloads);
_logger.LogInformation("ignored downloads reloaded");
return ignoredDownloads;
}
catch (Exception exception)
{
_logger.LogError(exception, "error while reading ignored downloads file | {file}", _config.IgnoredDownloadsPath);
}
return Array.Empty<string>();
}
}
@@ -6,6 +6,7 @@ using Common.Configuration.DownloadClient;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Arr; using Domain.Models.Arr;
using Domain.Models.Arr.Queue; using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
@@ -22,7 +23,8 @@ public sealed class ContentBlocker : GenericHandler
{ {
private readonly ContentBlockerConfig _config; private readonly ContentBlockerConfig _config;
private readonly BlocklistProvider _blocklistProvider; private readonly BlocklistProvider _blocklistProvider;
private readonly IgnoredDownloadsProvider<ContentBlockerConfig> _ignoredDownloadsProvider;
public ContentBlocker( public ContentBlocker(
ILogger<ContentBlocker> logger, ILogger<ContentBlocker> logger,
IOptions<ContentBlockerConfig> config, IOptions<ContentBlockerConfig> config,
@@ -36,7 +38,8 @@ public sealed class ContentBlocker : GenericHandler
ArrQueueIterator arrArrQueueIterator, ArrQueueIterator arrArrQueueIterator,
BlocklistProvider blocklistProvider, BlocklistProvider blocklistProvider,
DownloadServiceFactory downloadServiceFactory, DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier INotificationPublisher notifier,
IgnoredDownloadsProvider<ContentBlockerConfig> ignoredDownloadsProvider
) : base( ) : base(
logger, downloadClientConfig, logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig, sonarrConfig, radarrConfig, lidarrConfig,
@@ -47,6 +50,7 @@ public sealed class ContentBlocker : GenericHandler
{ {
_config = config.Value; _config = config.Value;
_blocklistProvider = blocklistProvider; _blocklistProvider = blocklistProvider;
_ignoredDownloadsProvider = ignoredDownloadsProvider;
} }
public override async Task ExecuteAsync() public override async Task ExecuteAsync()
@@ -73,6 +77,8 @@ public sealed class ContentBlocker : GenericHandler
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{ {
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = []; HashSet<SearchItem> itemsToBeRefreshed = [];
@@ -106,13 +112,19 @@ public sealed class ContentBlocker : GenericHandler
continue; continue;
} }
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
{
_logger.LogInformation("skip | {title} | ignored", record.Title);
continue;
}
// push record to context // push record to context
ContextProvider.Set(nameof(QueueRecord), record); ContextProvider.Set(nameof(QueueRecord), record);
_logger.LogDebug("searching unwanted files for {title}", record.Title); _logger.LogDebug("searching unwanted files for {title}", record.Title);
BlockFilesResult result = await _downloadService BlockFilesResult result = await _downloadService
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes); .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads);
if (!result.ShouldRemove) if (!result.ShouldRemove)
{ {
@@ -3,6 +3,7 @@ using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient; using Common.Configuration.DownloadClient;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Arr.Queue; using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient;
@@ -17,6 +18,7 @@ namespace Infrastructure.Verticals.DownloadCleaner;
public sealed class DownloadCleaner : GenericHandler public sealed class DownloadCleaner : GenericHandler
{ {
private readonly DownloadCleanerConfig _config; private readonly DownloadCleanerConfig _config;
private readonly IgnoredDownloadsProvider<DownloadCleanerConfig> _ignoredDownloadsProvider;
private readonly HashSet<string> _excludedHashes = []; private readonly HashSet<string> _excludedHashes = [];
public DownloadCleaner( public DownloadCleaner(
@@ -31,7 +33,8 @@ public sealed class DownloadCleaner : GenericHandler
LidarrClient lidarrClient, LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator, ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory, DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier INotificationPublisher notifier,
IgnoredDownloadsProvider<DownloadCleanerConfig> ignoredDownloadsProvider
) : base( ) : base(
logger, downloadClientConfig, logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig, sonarrConfig, radarrConfig, lidarrConfig,
@@ -42,6 +45,7 @@ public sealed class DownloadCleaner : GenericHandler
{ {
_config = config.Value; _config = config.Value;
_config.Validate(); _config.Validate();
_ignoredDownloadsProvider = ignoredDownloadsProvider;
} }
public override async Task ExecuteAsync() public override async Task ExecuteAsync()
@@ -58,6 +62,8 @@ public sealed class DownloadCleaner : GenericHandler
return; return;
} }
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
await _downloadService.LoginAsync(); await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories);
@@ -75,7 +81,7 @@ public sealed class DownloadCleaner : GenericHandler
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes); await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads);
} }
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
@@ -16,6 +16,20 @@ public sealed class DelugeClient
private readonly DelugeConfig _config; private readonly DelugeConfig _config;
private readonly HttpClient _httpClient; private readonly HttpClient _httpClient;
private static readonly IReadOnlyList<string> Fields =
[
"hash",
"state",
"name",
"eta",
"private",
"total_done",
"label",
"seeding_time",
"ratio",
"trackers"
];
public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory) public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory)
{ {
_config = config.Value; _config = config.Value;
@@ -68,7 +82,7 @@ public sealed class DelugeClient
return await SendRequest<TorrentStatus?>( return await SendRequest<TorrentStatus?>(
"web.get_torrent_status", "web.get_torrent_status",
hash, hash,
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } Fields
); );
} }
@@ -77,7 +91,7 @@ public sealed class DelugeClient
Dictionary<string, TorrentStatus>? downloads = await SendRequest<Dictionary<string, TorrentStatus>?>( Dictionary<string, TorrentStatus>? downloads = await SendRequest<Dictionary<string, TorrentStatus>?>(
"core.get_torrents_status", "core.get_torrents_status",
"", "",
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } Fields
); );
return downloads?.Values.ToList(); return downloads?.Values.ToList();
@@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Deluge.Response; using Domain.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
@@ -49,20 +50,26 @@ public class DelugeService : DownloadService, IDelugeService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{ {
hash = hash.ToLowerInvariant(); hash = hash.ToLowerInvariant();
DelugeContents? contents = null; DelugeContents? contents = null;
StalledResult result = new(); StalledResult result = new();
TorrentStatus? status = await _client.GetTorrentStatus(hash); TorrentStatus? download = await _client.GetTorrentStatus(hash);
if (status?.Hash is null) if (download?.Hash is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result; return result;
} }
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
try try
{ {
@@ -88,8 +95,8 @@ public class DelugeService : DownloadService, IDelugeService
result.DeleteReason = DeleteReason.AllFilesBlocked; result.DeleteReason = DeleteReason.AllFilesBlocked;
} }
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(status); result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download);
result.IsPrivate = status.Private; result.IsPrivate = download.Private;
if (!shouldRemove && result.ShouldRemove) if (!shouldRemove && result.ShouldRemove)
{ {
@@ -100,30 +107,37 @@ public class DelugeService : DownloadService, IDelugeService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
)
{ {
hash = hash.ToLowerInvariant(); hash = hash.ToLowerInvariant();
TorrentStatus? status = await _client.GetTorrentStatus(hash); TorrentStatus? download = await _client.GetTorrentStatus(hash);
BlockFilesResult result = new(); BlockFilesResult result = new();
if (status?.Hash is null) if (download?.Hash is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result; return result;
} }
result.IsPrivate = status.Private;
if (_contentBlockerConfig.IgnorePrivate && status.Private) var ceva = await _client.GetTorrentExtended(hash);
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
result.IsPrivate = download.Private;
if (_contentBlockerConfig.IgnorePrivate && download.Private)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", status.Name); _logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result; return result;
} }
@@ -205,7 +219,8 @@ public class DelugeService : DownloadService, IDelugeService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{ {
foreach (TorrentStatus download in downloads) foreach (TorrentStatus download in downloads)
{ {
@@ -213,6 +228,12 @@ public class DelugeService : DownloadService, IDelugeService
{ {
continue; continue;
} }
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
Category? category = categoriesToClean Category? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
@@ -60,15 +60,13 @@ public abstract class DownloadService : IDownloadService
public abstract Task LoginAsync(); public abstract Task LoginAsync();
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash); public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync( public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads);
);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task DeleteDownload(string hash); public abstract Task DeleteDownload(string hash);
@@ -77,7 +75,8 @@ public abstract class DownloadService : IDownloadService
public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories); public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes); public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
protected void ResetStrikesOnProgress(string hash, long downloaded) protected void ResetStrikesOnProgress(string hash, long downloaded)
{ {
@@ -131,7 +130,7 @@ public abstract class DownloadService : IDownloadService
return new(); return new();
} }
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category)
{ {
if (category.MaxRatio < 0) if (category.MaxRatio < 0)
@@ -28,12 +28,13 @@ public class DummyDownloadService : DownloadService
return Task.CompletedTask; return Task.CompletedTask;
} }
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
@@ -43,7 +44,8 @@ public class DummyDownloadService : DownloadService
throw new NotImplementedException(); throw new NotImplementedException();
} }
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
@@ -14,7 +14,8 @@ public interface IDownloadService : IDisposable
/// Checks whether the download should be removed from the *arr queue. /// Checks whether the download should be removed from the *arr queue.
/// </summary> /// </summary>
/// <param name="hash">The download hash.</param> /// <param name="hash">The download hash.</param>
public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash); /// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <summary> /// <summary>
/// Blocks unwanted files from being fully downloaded. /// Blocks unwanted files from being fully downloaded.
@@ -23,12 +24,13 @@ public interface IDownloadService : IDisposable
/// <param name="blocklistType">The <see cref="BlocklistType"/>.</param> /// <param name="blocklistType">The <see cref="BlocklistType"/>.</param>
/// <param name="patterns">The patterns to test the files against.</param> /// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param> /// <param name="regexes">The regexes to test the files against.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns> /// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<BlockFilesResult> BlockUnwantedFilesAsync( public Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
); );
/// <summary> /// <summary>
@@ -37,14 +39,16 @@ public interface IDownloadService : IDisposable
/// <param name="categories">The categories by which to filter the downloads.</param> /// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns> /// <returns>A list of downloads for the provided categories.</returns>
Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories); Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
/// <summary> /// <summary>
/// Cleans the downloads. /// Cleans the downloads.
/// </summary> /// </summary>
/// <param name="downloads"></param> /// <param name="downloads"></param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param> /// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param> /// <param name="excludedHashes">The hashes that should not be cleaned.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes); /// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
/// <summary> /// <summary>
/// Deletes a download item. /// Deletes a download item.
@@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Common.Helpers; using Common.Helpers;
using Domain.Enums; using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
@@ -58,18 +59,27 @@ public class QBitService : DownloadService, IQBitService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{ {
StalledResult result = new(); StalledResult result = new();
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault(); .FirstOrDefault();
if (torrent is null) if (download is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result; return result;
} }
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null) if (torrentProperties is null)
@@ -83,7 +93,7 @@ public class QBitService : DownloadService, IQBitService
&& boolValue; && boolValue;
// if all files were blocked by qBittorrent // if all files were blocked by qBittorrent
if (torrent is { CompletionOn: not null, Downloaded: null or 0 }) if (download is { CompletionOn: not null, Downloaded: null or 0 })
{ {
result.ShouldRemove = true; result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked; result.DeleteReason = DeleteReason.AllFilesBlocked;
@@ -100,7 +110,7 @@ public class QBitService : DownloadService, IQBitService
return result; return result;
} }
result.ShouldRemove = await IsItemStuckAndShouldRemove(torrent, result.IsPrivate); result.ShouldRemove = await IsItemStuckAndShouldRemove(download, result.IsPrivate);
if (result.ShouldRemove) if (result.ShouldRemove)
{ {
@@ -111,23 +121,32 @@ public class QBitService : DownloadService, IQBitService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
) )
{ {
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault(); .FirstOrDefault();
BlockFilesResult result = new(); BlockFilesResult result = new();
if (torrent is null) if (download is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result; return result;
} }
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null) if (torrentProperties is null)
@@ -145,7 +164,7 @@ public class QBitService : DownloadService, IQBitService
if (_contentBlockerConfig.IgnorePrivate && isPrivate) if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name); _logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result; return result;
} }
@@ -218,7 +237,8 @@ public class QBitService : DownloadService, IQBitService
.ToList(); .ToList();
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{ {
foreach (TorrentInfo download in downloads) foreach (TorrentInfo download in downloads)
{ {
@@ -227,6 +247,15 @@ public class QBitService : DownloadService, IQBitService
continue; continue;
} }
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
Category? category = categoriesToClean Category? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
@@ -303,7 +332,7 @@ public class QBitService : DownloadService, IQBitService
{ {
_client.Dispose(); _client.Dispose();
} }
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
{ {
if (_queueCleanerConfig.StalledMaxStrikes is 0) if (_queueCleanerConfig.StalledMaxStrikes is 0)
@@ -329,4 +358,11 @@ public class QBitService : DownloadService, IQBitService
return await StrikeAndCheckLimit(torrent.Hash, torrent.Name); return await StrikeAndCheckLimit(torrent.Hash, torrent.Name);
} }
private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
{
return (await _client.GetTorrentTrackersAsync(hash))
.Where(x => !x.Url.ToString().Contains("**"))
.ToList();
}
} }
@@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Common.Helpers; using Common.Helpers;
using Domain.Enums; using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
@@ -27,6 +28,23 @@ public class TransmissionService : DownloadService, ITransmissionService
private readonly Client _client; private readonly Client _client;
private TorrentInfo[]? _torrentsCache; private TorrentInfo[]? _torrentsCache;
private static readonly string[] Fields =
[
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER,
TorrentFields.DOWNLOAD_DIR,
TorrentFields.SECONDS_SEEDING,
TorrentFields.UPLOAD_RATIO,
TorrentFields.TRACKERS
];
public TransmissionService( public TransmissionService(
IHttpClientFactory httpClientFactory, IHttpClientFactory httpClientFactory,
ILogger<TransmissionService> logger, ILogger<TransmissionService> logger,
@@ -60,21 +78,27 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{ {
StalledResult result = new(); StalledResult result = new();
TorrentInfo? torrent = await GetTorrentAsync(hash); TorrentInfo? download = await GetTorrentAsync(hash);
if (torrent is null) if (download is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result; return result;
} }
bool shouldRemove = torrent.FileStats?.Length > 0; if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
result.IsPrivate = torrent.IsPrivate ?? false; {
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool shouldRemove = download.FileStats?.Length > 0;
result.IsPrivate = download.IsPrivate ?? false;
foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? []) foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? [])
{ {
if (!stats.Wanted.HasValue) if (!stats.Wanted.HasValue)
{ {
@@ -95,7 +119,7 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
// remove if all files are unwanted or download is stuck // remove if all files are unwanted or download is stuck
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(torrent); result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download);
if (!shouldRemove && result.ShouldRemove) if (!shouldRemove && result.ShouldRemove)
{ {
@@ -106,28 +130,32 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
)
{ {
TorrentInfo? torrent = await GetTorrentAsync(hash); TorrentInfo? download = await GetTorrentAsync(hash);
BlockFilesResult result = new(); BlockFilesResult result = new();
if (torrent?.FileStats is null || torrent.Files is null) if (download?.FileStats is null || download.Files is null)
{ {
return result; return result;
} }
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool isPrivate = torrent.IsPrivate ?? false; bool isPrivate = download.IsPrivate ?? false;
result.IsPrivate = isPrivate; result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate) if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name); _logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result; return result;
} }
@@ -135,27 +163,27 @@ public class TransmissionService : DownloadService, ITransmissionService
long totalFiles = 0; long totalFiles = 0;
long totalUnwantedFiles = 0; long totalUnwantedFiles = 0;
for (int i = 0; i < torrent.Files.Length; i++) for (int i = 0; i < download.Files.Length; i++)
{ {
if (torrent.FileStats?[i].Wanted == null) if (download.FileStats?[i].Wanted == null)
{ {
continue; continue;
} }
totalFiles++; totalFiles++;
if (!torrent.FileStats[i].Wanted.Value) if (!download.FileStats[i].Wanted.Value)
{ {
totalUnwantedFiles++; totalUnwantedFiles++;
continue; continue;
} }
if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes)) if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
{ {
continue; continue;
} }
_logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name); _logger.LogInformation("unwanted file found | {file}", download.Files[i].Name);
unwantedFiles.Add(i); unwantedFiles.Add(i);
totalUnwantedFiles++; totalUnwantedFiles++;
} }
@@ -175,7 +203,7 @@ public class TransmissionService : DownloadService, ITransmissionService
_logger.LogDebug("changing priorities | torrent {hash}", hash); _logger.LogDebug("changing priorities | torrent {hash}", hash);
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, torrent.Id, unwantedFiles.ToArray()); await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());
return result; return result;
} }
@@ -183,22 +211,7 @@ public class TransmissionService : DownloadService, ITransmissionService
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
{ {
string[] fields = [ return (await _client.TorrentGetAsync(Fields))
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER,
TorrentFields.DOWNLOAD_DIR,
TorrentFields.SECONDS_SEEDING,
TorrentFields.UPLOAD_RATIO
];
return (await _client.TorrentGetAsync(fields))
?.Torrents ?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString)) ?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6) .Where(x => x.Status is 5 or 6)
@@ -219,7 +232,8 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{ {
foreach (TorrentInfo download in downloads) foreach (TorrentInfo download in downloads)
{ {
@@ -227,6 +241,12 @@ public class TransmissionService : DownloadService, ITransmissionService
{ {
continue; continue;
} }
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
Category? category = categoriesToClean Category? category = categoriesToClean
.FirstOrDefault(x => .FirstOrDefault(x =>
@@ -351,20 +371,8 @@ public class TransmissionService : DownloadService, ITransmissionService
if (_torrentsCache is null || torrent is null) if (_torrentsCache is null || torrent is null)
{ {
string[] fields = [
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER
];
// refresh cache // refresh cache
_torrentsCache = (await _client.TorrentGetAsync(fields)) _torrentsCache = (await _client.TorrentGetAsync(Fields))
?.Torrents; ?.Torrents;
} }
@@ -4,6 +4,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Arr; using Domain.Models.Arr;
using Domain.Models.Arr.Queue; using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
@@ -19,7 +20,8 @@ namespace Infrastructure.Verticals.QueueCleaner;
public sealed class QueueCleaner : GenericHandler public sealed class QueueCleaner : GenericHandler
{ {
private readonly QueueCleanerConfig _config; private readonly QueueCleanerConfig _config;
private readonly IgnoredDownloadsProvider<QueueCleanerConfig> _ignoredDownloadsProvider;
public QueueCleaner( public QueueCleaner(
ILogger<QueueCleaner> logger, ILogger<QueueCleaner> logger,
IOptions<QueueCleanerConfig> config, IOptions<QueueCleanerConfig> config,
@@ -32,7 +34,8 @@ public sealed class QueueCleaner : GenericHandler
LidarrClient lidarrClient, LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator, ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory, DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier INotificationPublisher notifier,
IgnoredDownloadsProvider<QueueCleanerConfig> ignoredDownloadsProvider
) : base( ) : base(
logger, downloadClientConfig, logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig, sonarrConfig, radarrConfig, lidarrConfig,
@@ -42,10 +45,13 @@ public sealed class QueueCleaner : GenericHandler
) )
{ {
_config = config.Value; _config = config.Value;
_ignoredDownloadsProvider = ignoredDownloadsProvider;
} }
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{ {
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = []; HashSet<SearchItem> itemsToBeRefreshed = [];
@@ -75,6 +81,12 @@ public sealed class QueueCleaner : GenericHandler
continue; continue;
} }
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
{
_logger.LogInformation("skip | {title} | ignored", record.Title);
continue;
}
// push record to context // push record to context
ContextProvider.Set(nameof(QueueRecord), record); ContextProvider.Set(nameof(QueueRecord), record);
@@ -83,7 +95,7 @@ public sealed class QueueCleaner : GenericHandler
if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent") if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent")
{ {
// stalled download check // stalled download check
stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads);
} }
// failed import check // failed import check
@@ -0,0 +1 @@
ignored
+4
View File
@@ -191,6 +191,7 @@ services:
- TRIGGERS__DOWNLOADCLEANER=0/30 * * * * ? - TRIGGERS__DOWNLOADCLEANER=0/30 * * * * ?
- QUEUECLEANER__ENABLED=true - QUEUECLEANER__ENABLED=true
- QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored
- QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true
@@ -201,10 +202,12 @@ services:
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false - QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored
- CONTENTBLOCKER__IGNORE_PRIVATE=true - CONTENTBLOCKER__IGNORE_PRIVATE=true
- CONTENTBLOCKER__DELETE_PRIVATE=false - CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOADCLEANER__ENABLED=true - DOWNLOADCLEANER__ENABLED=true
- DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored
- DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__DELETE_PRIVATE=false
- DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr
- DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1
@@ -256,6 +259,7 @@ services:
# - NOTIFIARR__CHANNEL_ID=discord_channel_id # - NOTIFIARR__CHANNEL_ID=discord_channel_id
volumes: volumes:
- ./data/cleanuperr/logs:/var/logs - ./data/cleanuperr/logs:/var/logs
- ./data/cleanuperr/ignored_downloads:/ignored
restart: unless-stopped restart: unless-stopped
depends_on: depends_on:
- qbittorrent - qbittorrent
+76 -1
View File
@@ -76,6 +76,31 @@
- Default: `true` - Default: `true`
- Required: No. - Required: No.
**`QUEUECLEANER__IGNORED_DOWNLOADS_PATH`**
- Local path to the file containing ignored downloads.
- If the contents of the file are changed, they will be reloaded on the next job run.
- Accepted values:
- torrent hash
- qBitTorrent tag or category
- Deluge label
- Transmission category (last directory from the save location)
- torrent tracker domain
- Each value needs to be on a new line.
- Type: String.
- Default: Empty.
- Required: No.
- Example: `/ignored.txt`.
- Example of file contents:
```
fa800a7d7c443a2c3561d1f8f393c089036dade1
tv-sonarr
qbit-tag
mytracker.com
...
```
>[!IMPORTANT]
> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr.
**`QUEUECLEANER__RUNSEQUENTIALLY`** **`QUEUECLEANER__RUNSEQUENTIALLY`**
- Controls whether queue cleaner runs after content blocker instead of in parallel. - Controls whether queue cleaner runs after content blocker instead of in parallel.
- When `true`, streamlines the cleaning process by running immediately after content blocker. - When `true`, streamlines the cleaning process by running immediately after content blocker.
@@ -178,6 +203,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required"
- Default: `false` - Default: `false`
- Required: No. - Required: No.
**`CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH`**
- Local path to the file containing ignored downloads.
- If the contents of the file are changed, they will be reloaded on the next job run.
- Accepted values:
- torrent hash
- qBitTorrent tag or category
- Deluge label
- Transmission category (last directory from the save location)
- torrent tracker domain
- Each value needs to be on a new line.
- Type: String.
- Default: Empty.
- Required: No.
- Example: `/ignored.txt`.
- Example of file contents:
```
fa800a7d7c443a2c3561d1f8f393c089036dade1
tv-sonarr
qbit-tag
mytracker.com
...
```
>[!IMPORTANT]
> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr.
**`CONTENTBLOCKER__IGNORE_PRIVATE`** **`CONTENTBLOCKER__IGNORE_PRIVATE`**
- Controls whether to ignore downloads from private trackers. - Controls whether to ignore downloads from private trackers.
- Type: Boolean - Type: Boolean
@@ -217,6 +267,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required"
- Default: `false` - Default: `false`
- Required: No. - Required: No.
**`DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH`**
- Local path to the file containing ignored downloads.
- If the contents of the file are changed, they will be reloaded on the next job run.
- Accepted values:
- torrent hash
- qBitTorrent tag or category
- Deluge label
- Transmission category (last directory from the save location)
- torrent tracker domain
- Each value needs to be on a new line.
- Type: String.
- Default: Empty.
- Required: No.
- Example: `/ignored.txt`.
- Example of file contents:
```
fa800a7d7c443a2c3561d1f8f393c089036dade1
tv-sonarr
qbit-tag
mytracker.com
...
```
>[!IMPORTANT]
> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr.
**`DOWNLOADCLEANER__DELETE_PRIVATE`** **`DOWNLOADCLEANER__DELETE_PRIVATE`**
- Controls whether to delete private downloads. - Controls whether to delete private downloads.
- Type: Boolean. - Type: Boolean.
@@ -237,7 +312,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required"
> The category name must match the category that was set in the *arr. > The category name must match the category that was set in the *arr.
> For qBittorrent, the category name is the name of the download category. > For qBittorrent, the category name is the name of the download category.
> For Deluge, the category name is the name of the label. > For Deluge, the category name is the name of the label.
> For Transmission, the category name is the name of the download location. > For Transmission, the category name is the last directory from the save location.
**`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`** **`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`**
- Maximum ratio to reach before removing a download. - Maximum ratio to reach before removing a download.