added category change for downloads with no additional hardlinks

This commit is contained in:
Flaminel
2025-02-19 23:46:12 +02:00
parent 5adbdbd920
commit 017e25fb06
27 changed files with 458 additions and 56 deletions
@@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration;
namespace Common.Configuration.DownloadCleaner; namespace Common.Configuration.DownloadCleaner;
public sealed record Category : IConfig public sealed record CleanCategory : IConfig
{ {
public required string Name { get; init; } public required string Name { get; init; }
@@ -9,10 +9,16 @@ public sealed record DownloadCleanerConfig : IJobConfig
public bool Enabled { get; init; } public bool Enabled { get; init; }
public List<Category>? Categories { get; init; } public List<CleanCategory>? Categories { get; init; }
[ConfigurationKeyName("DELETE_PRIVATE")] [ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; set; } public bool DeletePrivate { get; init; }
[ConfigurationKeyName("NO_HARDLINKS_CATEGORY")]
public string NoHardlinksCategory { get; init; } = "";
[ConfigurationKeyName("HARDLINK_CATEGORIES")]
public List<string>? HardlinkCategories { get; init; }
public void Validate() public void Validate()
{ {
@@ -32,5 +38,25 @@ public sealed record DownloadCleanerConfig : IJobConfig
} }
Categories?.ForEach(x => x.Validate()); Categories?.ForEach(x => x.Validate());
if (string.IsNullOrEmpty(NoHardlinksCategory))
{
return;
}
if (HardlinkCategories?.Count is null or 0)
{
throw new ValidationException("no categories configured");
}
if (HardlinkCategories.Contains(NoHardlinksCategory))
{
throw new ValidationException("NO_HARDLINKS_CATEGORY is present in the list of filtered categories");
}
if (HardlinkCategories.Any(string.IsNullOrEmpty))
{
throw new ValidationException("empty hardlink filter category found");
}
} }
} }
@@ -16,7 +16,15 @@ public abstract record NotificationConfig
[ConfigurationKeyName("ON_DOWNLOAD_CLEANED")] [ConfigurationKeyName("ON_DOWNLOAD_CLEANED")]
public bool OnDownloadCleaned { get; init; } public bool OnDownloadCleaned { get; init; }
public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnQueueItemDeleted || OnDownloadCleaned; [ConfigurationKeyName("ON_CATEGORY_CHANGED")]
public bool OnCategoryChanged { get; init; }
public bool IsEnabled =>
OnImportFailedStrike ||
OnStalledStrike ||
OnQueueItemDeleted ||
OnDownloadCleaned ||
OnCategoryChanged;
public abstract bool IsValid(); public abstract bool IsValid();
} }
@@ -6,6 +6,7 @@ using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.DownloadClient.Deluge; using Infrastructure.Verticals.DownloadClient.Deluge;
using Infrastructure.Verticals.DownloadClient.QBittorrent; using Infrastructure.Verticals.DownloadClient.QBittorrent;
using Infrastructure.Verticals.DownloadClient.Transmission; using Infrastructure.Verticals.DownloadClient.Transmission;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.QueueCleaner; using Infrastructure.Verticals.QueueCleaner;
@@ -23,6 +24,7 @@ public static class ServicesDI
.AddTransient<ContentBlocker>() .AddTransient<ContentBlocker>()
.AddTransient<DownloadCleaner>() .AddTransient<DownloadCleaner>()
.AddTransient<IFilenameEvaluator, FilenameEvaluator>() .AddTransient<IFilenameEvaluator, FilenameEvaluator>()
.AddTransient<IHardlinkFileService, HardlinkFileService>()
.AddTransient<DummyDownloadService>() .AddTransient<DummyDownloadService>()
.AddTransient<QBitService>() .AddTransient<QBitService>()
.AddTransient<DelugeService>() .AddTransient<DelugeService>()
@@ -44,6 +44,11 @@
"MIN_SEED_TIME": 0, "MIN_SEED_TIME": 0,
"MAX_SEED_TIME": -1 "MAX_SEED_TIME": -1
} }
],
"NO_HARDLINKS_CATEGORY": "nohardlinks",
"HARDLINK_CATEGORIES": [
"tv-sonarr",
"radarr"
] ]
}, },
"DOWNLOAD_CLIENT": "qbittorrent", "DOWNLOAD_CLIENT": "qbittorrent",
@@ -1,9 +1,10 @@
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -56,6 +57,7 @@ public class DownloadServiceFixture : IDisposable
var filenameEvaluator = Substitute.For<IFilenameEvaluator>(); var filenameEvaluator = Substitute.For<IFilenameEvaluator>();
var notifier = Substitute.For<INotificationPublisher>(); var notifier = Substitute.For<INotificationPublisher>();
var dryRunInterceptor = Substitute.For<IDryRunInterceptor>(); var dryRunInterceptor = Substitute.For<IDryRunInterceptor>();
var hardlinkFileService = Substitute.For<IHardlinkFileService>();
return new TestDownloadService( return new TestDownloadService(
Logger, Logger,
@@ -66,7 +68,8 @@ public class DownloadServiceFixture : IDisposable
filenameEvaluator, filenameEvaluator,
Striker, Striker,
notifier, notifier,
dryRunInterceptor dryRunInterceptor,
hardlinkFileService
); );
} }
@@ -132,7 +132,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue() public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue()
{ {
// Arrange // Arrange
Category category = new() CleanCategory category = new()
{ {
Name = "test", Name = "test",
MaxRatio = 1.0, MaxRatio = 1.0,
@@ -158,7 +158,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse() public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse()
{ {
// Arrange // Arrange
Category category = new() CleanCategory category = new()
{ {
Name = "test", Name = "test",
MaxRatio = 1.0, MaxRatio = 1.0,
@@ -184,7 +184,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenMaxSeedTimeReached_ShouldReturnTrue() public void WhenMaxSeedTimeReached_ShouldReturnTrue()
{ {
// Arrange // Arrange
Category category = new() CleanCategory category = new()
{ {
Name = "test", Name = "test",
MaxRatio = -1, MaxRatio = -1,
@@ -210,7 +210,7 @@ public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
public void WhenNeitherConditionMet_ShouldReturnFalse() public void WhenNeitherConditionMet_ShouldReturnFalse()
{ {
// Arrange // Arrange
Category category = new() CleanCategory category = new()
{ {
Name = "test", Name = "test",
MaxRatio = 2.0, MaxRatio = 2.0,
@@ -1,4 +1,4 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadCleaner;
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -25,10 +26,11 @@ public class TestDownloadService : DownloadService
IFilenameEvaluator filenameEvaluator, IFilenameEvaluator filenameEvaluator,
IStriker striker, IStriker striker,
INotificationPublisher notifier, INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) : base( ) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService
) )
{ {
} }
@@ -38,12 +40,15 @@ public class TestDownloadService : DownloadService
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult()); public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult());
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) => Task.FromResult(new BlockFilesResult()); ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) => Task.FromResult(new BlockFilesResult());
public override Task DeleteDownload(string hash) => Task.CompletedTask; public override Task DeleteDownload(string hash) => Task.CompletedTask;
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => Task.FromResult<List<object>?>(null); public override Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories) => Task.FromResult<List<object>?>(null);
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) => Task.CompletedTask; public override Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories) => Task.FromResult<List<object>?>(null);
public override Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes) => Task.CompletedTask;
public override Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes) => Task.CompletedTask;
// Expose protected methods for testing // Expose protected methods for testing
public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded);
public new Task<bool> StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName); public new Task<bool> StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName);
public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category); public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) => base.ShouldCleanDownload(ratio, seedingTime, category);
} }
@@ -60,11 +60,20 @@ public sealed class DownloadCleaner : GenericHandler
await _downloadService.LoginAsync(); await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); List<object>? downloadsToBeCleaned = await _downloadService.GetDownloadsToBeCleaned(_config.Categories);
List<object>? downloadsToChangeCategory = null;
if (downloads?.Count is null or 0) if (!string.IsNullOrEmpty(_config.NoHardlinksCategory) && _config.HardlinkCategories?.Count > 0)
{ {
_logger.LogDebug("no downloads found in the download client"); downloadsToChangeCategory = await _downloadService.GetDownloadsToChangeCategory(_config.HardlinkCategories);
}
bool hasDownloadsToClean = downloadsToBeCleaned?.Count > 0;
bool hasDownloadsToChange = downloadsToChangeCategory?.Count > 0;
if (!hasDownloadsToClean && !hasDownloadsToChange)
{
_logger.LogDebug("no downloads to process");
return; return;
} }
@@ -75,7 +84,23 @@ public sealed class DownloadCleaner : GenericHandler
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes); if (hasDownloadsToChange)
{
await _downloadService.ChangeCategoryForNoHardlinksAsync(downloadsToChangeCategory, _excludedHashes);
}
else
{
_logger.LogDebug("no downloads found to change category");
}
if (hasDownloadsToClean)
{
await _downloadService.CleanDownloads(downloadsToBeCleaned, _config.Categories, _excludedHashes);
}
else
{
_logger.LogDebug("no downloads found to be cleaned");
}
} }
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
@@ -10,6 +10,7 @@ using Domain.Models.Deluge.Response;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -33,10 +34,11 @@ public class DelugeService : DownloadService, IDelugeService
IFilenameEvaluator filenameEvaluator, IFilenameEvaluator filenameEvaluator,
IStriker striker, IStriker striker,
INotificationPublisher notifier, INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) : base( ) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService
) )
{ {
config.Value.Validate(); config.Value.Validate();
@@ -194,7 +196,7 @@ public class DelugeService : DownloadService, IDelugeService
return result; return result;
} }
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) public override async Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories)
{ {
return (await _client.GetStatusForAllTorrents()) return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash)) ?.Where(x => !string.IsNullOrEmpty(x.Hash))
@@ -204,8 +206,13 @@ public class DelugeService : DownloadService, IDelugeService
.ToList(); .ToList();
} }
public override Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories)
{
throw new NotImplementedException();
}
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes)
{ {
foreach (TorrentStatus download in downloads) foreach (TorrentStatus download in downloads)
{ {
@@ -214,7 +221,7 @@ public class DelugeService : DownloadService, IDelugeService
continue; continue;
} }
Category? category = categoriesToClean CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null) if (category is null)
@@ -259,6 +266,11 @@ public class DelugeService : DownloadService, IDelugeService
} }
} }
public override Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes)
{
throw new NotImplementedException();
}
/// <inheritdoc/> /// <inheritdoc/>
[DryRunSafeguard] [DryRunSafeguard]
public override async Task DeleteDownload(string hash) public override async Task DeleteDownload(string hash)
@@ -1,4 +1,4 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadCleaner;
@@ -10,6 +10,7 @@ using Infrastructure.Helpers;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -30,6 +31,7 @@ public abstract class DownloadService : IDownloadService
protected readonly MemoryCacheEntryOptions _cacheOptions; protected readonly MemoryCacheEntryOptions _cacheOptions;
protected readonly INotificationPublisher _notifier; protected readonly INotificationPublisher _notifier;
protected readonly IDryRunInterceptor _dryRunInterceptor; protected readonly IDryRunInterceptor _dryRunInterceptor;
protected readonly IHardlinkFileService _hardlinkFileService;
protected DownloadService( protected DownloadService(
ILogger<DownloadService> logger, ILogger<DownloadService> logger,
@@ -40,7 +42,8 @@ public abstract class DownloadService : IDownloadService
IFilenameEvaluator filenameEvaluator, IFilenameEvaluator filenameEvaluator,
IStriker striker, IStriker striker,
INotificationPublisher notifier, INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) )
{ {
_logger = logger; _logger = logger;
@@ -52,6 +55,7 @@ public abstract class DownloadService : IDownloadService
_striker = striker; _striker = striker;
_notifier = notifier; _notifier = notifier;
_dryRunInterceptor = dryRunInterceptor; _dryRunInterceptor = dryRunInterceptor;
_hardlinkFileService = hardlinkFileService;
_cacheOptions = new MemoryCacheEntryOptions() _cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
} }
@@ -74,10 +78,16 @@ public abstract class DownloadService : IDownloadService
public abstract Task DeleteDownload(string hash); public abstract Task DeleteDownload(string hash);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories); public abstract Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes); public abstract Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories);
/// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes);
/// <inheritdoc/>
public abstract Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes);
protected void ResetStrikesOnProgress(string hash, long downloaded) protected void ResetStrikesOnProgress(string hash, long downloaded)
{ {
@@ -107,7 +117,7 @@ public abstract class DownloadService : IDownloadService
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled); return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
} }
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category)
{ {
// check ratio // check ratio
if (DownloadReachedRatio(ratio, seedingTime, category)) if (DownloadReachedRatio(ratio, seedingTime, category))
@@ -132,7 +142,7 @@ public abstract class DownloadService : IDownloadService
return new(); return new();
} }
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category)
{ {
if (category.MaxRatio < 0) if (category.MaxRatio < 0)
{ {
@@ -158,7 +168,7 @@ public abstract class DownloadService : IDownloadService
return true; return true;
} }
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category) private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category)
{ {
if (category.MaxSeedTime < 0) if (category.MaxSeedTime < 0)
{ {
@@ -1,10 +1,11 @@
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -15,7 +16,21 @@ namespace Infrastructure.Verticals.DownloadClient;
public class DummyDownloadService : DownloadService public class DummyDownloadService : DownloadService
{ {
public DummyDownloadService(ILogger<DownloadService> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IOptions<DownloadCleanerConfig> downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier, dryRunInterceptor) public DummyDownloadService(
ILogger<DownloadService> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig,
cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService
)
{ {
} }
@@ -38,12 +53,22 @@ public class DummyDownloadService : DownloadService
throw new NotImplementedException(); throw new NotImplementedException();
} }
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) public override Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories)
{
throw new NotImplementedException();
}
public override Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes)
{
throw new NotImplementedException();
}
public override Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
@@ -36,15 +36,24 @@ public interface IDownloadService : IDisposable
/// </summary> /// </summary>
/// <param name="categories">The categories by which to filter the downloads.</param> /// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns> /// <returns>A list of downloads for the provided categories.</returns>
Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories); Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories);
Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories);
/// <summary> /// <summary>
/// Cleans the downloads. /// Cleans the downloads.
/// </summary> /// </summary>
/// <param name="downloads"></param> /// <param name="downloads">The downloads to clean.</param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param> /// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param> /// <param name="excludedHashes">The hashes that should not be cleaned.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes); Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes);
/// <summary>
/// Changes the category for downloads that have no hardlinks.
/// </summary>
/// <param name="downloads">The downloads to change.</param>
/// <param name="excludedHashes"></param>
Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes);
/// <summary> /// <summary>
/// Deletes a download item. /// Deletes a download item.
@@ -10,13 +10,13 @@ using Domain.Enums;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using QBittorrent.Client; using QBittorrent.Client;
using Category = Common.Configuration.DownloadCleaner.Category;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent; namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
@@ -36,10 +36,11 @@ public class QBitService : DownloadService, IQBitService
IFilenameEvaluator filenameEvaluator, IFilenameEvaluator filenameEvaluator,
IStriker striker, IStriker striker,
INotificationPublisher notifier, INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) : base( ) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService
) )
{ {
_config = config.Value; _config = config.Value;
@@ -207,7 +208,7 @@ public class QBitService : DownloadService, IQBitService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => public override async Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories) =>
(await _client.GetTorrentListAsync(new() (await _client.GetTorrentListAsync(new()
{ {
Filter = TorrentListFilter.Seeding Filter = TorrentListFilter.Seeding
@@ -217,8 +218,19 @@ public class QBitService : DownloadService, IQBitService
.Cast<object>() .Cast<object>()
.ToList(); .ToList();
public override async Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories)
{
return (await _client.GetTorrentListAsync(new()
{
Filter = TorrentListFilter.Seeding
}))
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Cast<object>()
.ToList();
}
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes)
{ {
foreach (TorrentInfo download in downloads) foreach (TorrentInfo download in downloads)
{ {
@@ -227,7 +239,7 @@ public class QBitService : DownloadService, IQBitService
continue; continue;
} }
Category? category = categoriesToClean CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null) if (category is null)
@@ -286,6 +298,52 @@ public class QBitService : DownloadService, IQBitService
} }
} }
public override async Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes)
{
// TODO account for cross-seed
foreach (TorrentInfo download in downloads)
{
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
return;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
bool hasHardlinks = false;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
_logger.LogDebug("skip | file index is null for {name}", download.Name);
return;
}
if (_hardlinkFileService.GetHardLinkCount(file.Name) > 1)
{
hasHardlinks = true;
};
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await ((QBitService)Proxy).ChangeCategory(download.Hash, _downloadCleanerConfig.NoHardlinksCategory);
await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.NoHardlinksCategory);
}
}
/// <inheritdoc/> /// <inheritdoc/>
[DryRunSafeguard] [DryRunSafeguard]
public override async Task DeleteDownload(string hash) public override async Task DeleteDownload(string hash)
@@ -299,6 +357,12 @@ public class QBitService : DownloadService, IQBitService
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
} }
[DryRunSafeguard]
protected virtual async Task ChangeCategory(string hash, string newCategory)
{
await _client.SetTorrentCategoryAsync([hash], newCategory);
}
public override void Dispose() public override void Dispose()
{ {
_client.Dispose(); _client.Dispose();
@@ -10,6 +10,7 @@ using Domain.Enums;
using Infrastructure.Interceptors; using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications; using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
@@ -38,10 +39,11 @@ public class TransmissionService : DownloadService, ITransmissionService
IFilenameEvaluator filenameEvaluator, IFilenameEvaluator filenameEvaluator,
IStriker striker, IStriker striker,
INotificationPublisher notifier, INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor IDryRunInterceptor dryRunInterceptor,
IHardlinkFileService hardlinkFileService
) : base( ) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor filenameEvaluator, striker, notifier, dryRunInterceptor, hardlinkFileService
) )
{ {
_config = config.Value; _config = config.Value;
@@ -181,7 +183,7 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) public override async Task<List<object>?> GetDownloadsToBeCleaned(List<CleanCategory> categories)
{ {
string[] fields = [ string[] fields = [
TorrentFields.FILES, TorrentFields.FILES,
@@ -218,8 +220,13 @@ public class TransmissionService : DownloadService, ITransmissionService
.ToList(); .ToList();
} }
public override Task<List<object>?> GetDownloadsToChangeCategory(List<string> categories)
{
throw new NotImplementedException();
}
/// <inheritdoc/> /// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) public override async Task CleanDownloads(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes)
{ {
foreach (TorrentInfo download in downloads) foreach (TorrentInfo download in downloads)
{ {
@@ -228,7 +235,7 @@ public class TransmissionService : DownloadService, ITransmissionService
continue; continue;
} }
Category? category = categoriesToClean CleanCategory? category = categoriesToClean
.FirstOrDefault(x => .FirstOrDefault(x =>
{ {
if (download.DownloadDir is null) if (download.DownloadDir is null)
@@ -282,6 +289,11 @@ public class TransmissionService : DownloadService, ITransmissionService
} }
} }
public override Task ChangeCategoryForNoHardlinksAsync(List<object> downloads, HashSet<string> excludedHashes)
{
throw new NotImplementedException();
}
public override async Task DeleteDownload(string hash) public override async Task DeleteDownload(string hash)
{ {
TorrentInfo? torrent = await GetTorrentAsync(hash); TorrentInfo? torrent = await GetTorrentAsync(hash);
@@ -0,0 +1,99 @@
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
namespace Infrastructure.Verticals.Files;
public class HardlinkFileService : IHardlinkFileService
{
private readonly ILogger<HardlinkFileService> _logger;
public HardlinkFileService(ILogger<HardlinkFileService> logger)
{
_logger = logger;
}
public uint GetHardLinkCount(string filePath)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return GetWindowsHardLinkCount(filePath);
}
return GetUnixHardLinkCount(filePath);
}
private uint GetWindowsHardLinkCount(string filePath)
{
try
{
using SafeFileHandle fileStream = File.OpenHandle(filePath);
if (GetFileInformationByHandle(fileStream, out var file))
{
return file.NumberOfLinks;
}
}
catch (Exception exception)
{
// TODO log download name?
_logger.LogError(exception, "failed to stat Windows file {file}", filePath);
}
return default;
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool GetFileInformationByHandle(
SafeFileHandle hFile,
out BY_HANDLE_FILE_INFORMATION lpFileInformation
);
private struct BY_HANDLE_FILE_INFORMATION
{
public uint FileAttributes;
public System.Runtime.InteropServices.ComTypes.FILETIME CreationTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastAccessTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastWriteTime;
public uint VolumeSerialNumber;
public uint FileSizeHigh;
public uint FileSizeLow;
public uint NumberOfLinks;
public uint FileIndexHigh;
public uint FileIndexLow;
}
private uint GetUnixHardLinkCount(string filePath)
{
try
{
if (stat64(filePath, out Stat stat) == 0)
{
return stat.st_nlink;
}
}
catch (Exception exception)
{
// TODO log download name?
_logger.LogError(exception, "failed to stat Unix file {file}", filePath);
}
return 0;
}
[DllImport("libc", SetLastError = true)]
private static extern int stat64(string path, out Stat stat);
[StructLayout(LayoutKind.Sequential)]
private struct Stat
{
public uint st_dev;
public ulong st_ino;
public uint st_mode;
public uint st_nlink; // Hard link count
public uint st_uid;
public uint st_gid;
public uint st_rdev;
public long st_size;
}
}
@@ -0,0 +1,6 @@
namespace Infrastructure.Verticals.Files;
public interface IHardlinkFileService
{
uint GetHardLinkCount(string filePath);
}
@@ -33,6 +33,9 @@ public sealed class NotificationConsumer<T> : IConsumer<T> where T : Notificatio
case DownloadCleanedNotification downloadCleanedNotification: case DownloadCleanedNotification downloadCleanedNotification:
await _notificationService.Notify(downloadCleanedNotification); await _notificationService.Notify(downloadCleanedNotification);
break; break;
case CategoryChangedNotification categoryChangedNotification:
await _notificationService.Notify(categoryChangedNotification);
break;
default: default:
throw new NotImplementedException(); throw new NotImplementedException();
} }
@@ -9,4 +9,6 @@ public interface INotificationFactory
List<INotificationProvider> OnQueueItemDeletedEnabled(); List<INotificationProvider> OnQueueItemDeletedEnabled();
List<INotificationProvider> OnDownloadCleanedEnabled(); List<INotificationProvider> OnDownloadCleanedEnabled();
List<INotificationProvider> OnCategoryChangedEnabled();
} }
@@ -16,4 +16,6 @@ public interface INotificationProvider
Task OnQueueItemDeleted(QueueItemDeletedNotification notification); Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
Task OnDownloadCleaned(DownloadCleanedNotification notification); Task OnDownloadCleaned(DownloadCleanedNotification notification);
Task OnCategoryChanged(CategoryChangedNotification notification);
} }
@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record CategoryChangedNotification : Notification
{
}
@@ -42,6 +42,11 @@ public class NotifiarrProvider : NotificationProvider
await _proxy.SendNotification(BuildPayload(notification), _config); await _proxy.SendNotification(BuildPayload(notification), _config);
} }
public override async Task OnCategoryChanged(CategoryChangedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
private NotifiarrPayload BuildPayload(ArrNotification notification, string color) private NotifiarrPayload BuildPayload(ArrNotification notification, string color)
{ {
NotifiarrPayload payload = new() NotifiarrPayload payload = new()
@@ -105,4 +110,32 @@ public class NotifiarrProvider : NotificationProvider
return payload; return payload;
} }
private NotifiarrPayload BuildPayload(CategoryChangedNotification notification)
{
NotifiarrPayload payload = new()
{
Discord = new()
{
Color = WarningColor,
Text = new()
{
Title = notification.Title,
Icon = Logo,
Description = notification.Description,
Fields = notification.Fields?.Adapt<List<Field>>() ?? []
},
Ids = new Ids
{
Channel = _config.ChannelId
},
Images = new()
{
Thumbnail = new Uri(Logo)
}
}
};
return payload;
}
} }
@@ -34,4 +34,9 @@ public class NotificationFactory : INotificationFactory
ActiveProviders() ActiveProviders()
.Where(n => n.Config.OnDownloadCleaned) .Where(n => n.Config.OnDownloadCleaned)
.ToList(); .ToList();
public List<INotificationProvider> OnCategoryChangedEnabled() =>
ActiveProviders()
.Where(n => n.Config.OnCategoryChanged)
.ToList();
} }
@@ -22,4 +22,6 @@ public abstract class NotificationProvider : INotificationProvider
public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification); public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification); public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification);
public abstract Task OnCategoryChanged(CategoryChangedNotification notification);
} }
@@ -107,6 +107,25 @@ public class NotificationPublisher : INotificationPublisher
return _messageBus.Publish(message); return _messageBus.Publish(message);
} }
[DryRunSafeguard]
public virtual async Task NotifyCategoryChanged(string oldCategory, string newCategory)
{
CategoryChangedNotification notification = new()
{
Title = "Category changed",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Old category", Text = oldCategory },
new() { Title = "New category", Text = newCategory }
],
Level = NotificationLevel.Important
};
await _messageBus.Publish(notification);
}
private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) => private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) =>
instanceType switch instanceType switch
{ {
@@ -73,4 +73,19 @@ public class NotificationService
} }
} }
} }
public async Task Notify(CategoryChangedNotification notification)
{
foreach (INotificationProvider provider in _notificationFactory.OnCategoryChangedEnabled())
{
try
{
await provider.OnCategoryChanged(notification);
}
catch (Exception exception)
{
_logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name);
}
}
}
} }
+7 -2
View File
@@ -209,11 +209,14 @@ services:
- DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr
- DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1
- DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0 - DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0
- DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=0.01 - DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=-1
- DOWNLOADCLEANER__CATEGORIES__1__NAME=radarr - DOWNLOADCLEANER__CATEGORIES__1__NAME=radarr
- DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1
- DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0 - DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0
- DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=0.01 - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=-1
- DOWNLOADCLEANER__HARDLINK_CATEGORIES__0=tv-sonarr
- DOWNLOADCLEANER__HARDLINK_CATEGORIES__1=radarr
- DOWNLOAD_CLIENT=qbittorrent - DOWNLOAD_CLIENT=qbittorrent
- QBITTORRENT__URL=http://qbittorrent:8080 - QBITTORRENT__URL=http://qbittorrent:8080
@@ -252,10 +255,12 @@ services:
# - NOTIFIARR__ON_STALLED_STRIKE=true # - NOTIFIARR__ON_STALLED_STRIKE=true
# - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true # - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true
# - NOTIFIARR__ON_DOWNLOAD_CLEANED=true # - NOTIFIARR__ON_DOWNLOAD_CLEANED=true
# - NOTIFIARR__ON_CATEGORY_CHANGED=true
# - NOTIFIARR__API_KEY=notifiarr_secret # - NOTIFIARR__API_KEY=notifiarr_secret
# - NOTIFIARR__CHANNEL_ID=discord_channel_id # - NOTIFIARR__CHANNEL_ID=discord_channel_id
volumes: volumes:
- ./data/cleanuperr/logs:/var/logs - ./data/cleanuperr/logs:/var/logs
- ./data/qbittorrent/downloads:/downloads
restart: unless-stopped restart: unless-stopped
depends_on: depends_on:
- qbittorrent - qbittorrent