public/cleanuperr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Lidarr support (#30)

Browse Source
This commit is contained in:
Marius Nechifor
2025-01-15 23:55:34 +02:00
committed by GitHub
parent 2bc8e445ce
commit 922f586706
63 changed files with 943 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/Infrastructure/Verticals/Arr/ArrClient.cs
+7 -5
View File
@@ -101,9 +101,9 @@ public abstract class ArrClient
return false;
}
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord queueRecord)
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record)
{
Uri uri = new(arrInstance.Url, $"/api/v3/queue/{queueRecord.Id}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false");
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id));
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
SetApiKey(request, arrInstance.ApiKey);
@@ -114,16 +114,16 @@ public abstract class ArrClient
{
response.EnsureSuccessStatusCode();
_logger.LogInformation("queue item deleted | {url} | {title}", arrInstance.Url, queueRecord.Title);
_logger.LogInformation("queue item deleted | {url} | {title}", arrInstance.Url, record.Title);
}
catch
{
_logger.LogError("queue delete failed | {uri} | {title}", uri, queueRecord.Title);
_logger.LogError("queue delete failed | {uri} | {title}", uri, record.Title);
throw;
}
}
public abstract Task RefreshItemsAsync(ArrInstance arrInstance, ArrConfig config, HashSet<SearchItem>? items);
public abstract Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);
public virtual bool IsRecordValid(QueueRecord record)
{
@@ -143,6 +143,8 @@ public abstract class ArrClient
}
protected abstract string GetQueueUrlPath(int page);
protected abstract string GetQueueDeleteUrlPath(long recordId);
protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
{
code/Infrastructure/Verticals/Arr/LidarrClient.cs
+145
View File
@@ -0,0 +1,145 @@
using System.Text;
using Common.Configuration.Arr;
using Common.Configuration.Logging;
using Common.Configuration.QueueCleaner;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Lidarr;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
namespace Infrastructure.Verticals.Arr;
public sealed class LidarrClient : ArrClient
{
public LidarrClient(
ILogger<LidarrClient> logger,
IHttpClientFactory httpClientFactory,
IOptions<LoggingConfig> loggingConfig,
IOptions<QueueCleanerConfig> queueCleanerConfig,
Striker striker
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
{
}
protected override string GetQueueUrlPath(int page)
{
return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
}
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v1/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
{
if (items?.Count is null or 0) return;
Uri uri = new(arrInstance.Url, "/api/v1/command");
foreach (var command in GetSearchCommands(items))
{
using HttpRequestMessage request = new(HttpMethod.Post, uri);
request.Content = new StringContent(
JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }),
Encoding.UTF8,
"application/json"
);
SetApiKey(request, arrInstance.ApiKey);
using var response = await _httpClient.SendAsync(request);
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command);
try
{
response.EnsureSuccessStatusCode();
_logger.LogInformation("{log}", GetSearchLog(arrInstance.Url, command, true, logContext));
}
catch
{
_logger.LogError("{log}", GetSearchLog(arrInstance.Url, command, false, logContext));
throw;
}
}
}
public override bool IsRecordValid(QueueRecord record)
{
if (record.ArtistId is 0 || record.AlbumId is 0)
{
_logger.LogDebug("skip | artist id and/or album id missing | {title}", record.Title);
return false;
}
return base.IsRecordValid(record);
}
private static string GetSearchLog(
Uri instanceUrl,
LidarrCommand command,
bool success,
string? logContext
)
{
string status = success ? "triggered" : "failed";
return $"album search {status} | {instanceUrl} | {logContext ?? $"albums: {string.Join(',', command.AlbumIds)}"}";
}
private async Task<string?> ComputeCommandLogContextAsync(ArrInstance arrInstance, LidarrCommand command)
{
try
{
if (!_loggingConfig.Enhanced) return null;
StringBuilder log = new();
var albums = await GetAlbumsAsync(arrInstance, command.AlbumIds);
if (albums?.Count is null or 0) return null;
var groups = albums
.GroupBy(x => x.Artist.Id)
.ToList();
foreach (var group in groups)
{
var first = group.First();
log.Append($"[{first.Artist.ArtistName} albums {string.Join(',', group.Select(x => x.Title).ToList())}]");
}
return log.ToString();
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to compute log context");
}
return null;
}
private async Task<List<Album>?> GetAlbumsAsync(ArrInstance arrInstance, List<long> albumIds)
{
Uri uri = new(arrInstance.Url, $"api/v1/album?{string.Join('&', albumIds.Select(x => $"albumIds={x}"))}");
using HttpRequestMessage request = new(HttpMethod.Get, uri);
SetApiKey(request, arrInstance.ApiKey);
using var response = await _httpClient.SendAsync(request);
response.EnsureSuccessStatusCode();
string responseBody = await response.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<List<Album>>(responseBody);
}
private List<LidarrCommand> GetSearchCommands(HashSet<SearchItem> items)
{
const string albumSearch = "AlbumSearch";
return [new LidarrCommand { Name = albumSearch, AlbumIds = items.Select(i => i.Id).ToList() }];
}
}
code/Infrastructure/Verticals/Arr/RadarrClient.cs
+7 -3
View File
@@ -6,7 +6,6 @@ using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Radarr;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
@@ -30,7 +29,12 @@ public sealed class RadarrClient : ArrClient
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, ArrConfig config, HashSet<SearchItem>? items)
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
{
if (items?.Count is null or 0)
{
@@ -74,7 +78,7 @@ public sealed class RadarrClient : ArrClient
{
if (record.MovieId is 0)
{
_logger.LogDebug("skip | item information missing | {title}", record.Title);
_logger.LogDebug("skip | movie id missing | {title}", record.Title);
return false;
}
code/Infrastructure/Verticals/Arr/SonarrClient.cs
+7 -3
View File
@@ -6,7 +6,6 @@ using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Sonarr;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
@@ -29,8 +28,13 @@ public sealed class SonarrClient : ArrClient
{
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true";
}
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, ArrConfig config, HashSet<SearchItem>? items)
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
{
if (items?.Count is null or 0)
{
@@ -70,7 +74,7 @@ public sealed class SonarrClient : ArrClient
{
if (record.EpisodeId is 0 || record.SeriesId is 0)
{
_logger.LogDebug("skip | item information missing | {title}", record.Title);
_logger.LogDebug("skip | episode id and/or series id missing | {title}", record.Title);
return false;
}
code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs
+67 -41
View File
@@ -1,9 +1,11 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text.RegularExpressions;
using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Helpers;
using Domain.Enums;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -12,78 +14,98 @@ namespace Infrastructure.Verticals.ContentBlocker;
public sealed class BlocklistProvider
{
private readonly ILogger<BlocklistProvider> _logger;
private readonly ContentBlockerConfig _config;
private readonly SonarrConfig _sonarrConfig;
private readonly RadarrConfig _radarrConfig;
private readonly LidarrConfig _lidarrConfig;
private readonly HttpClient _httpClient;
public BlocklistType BlocklistType { get; }
private readonly IMemoryCache _cache;
private bool _initialized;
public ConcurrentBag<string> Patterns { get; } = [];
public ConcurrentBag<Regex> Regexes { get; } = [];
private const string Type = "type";
private const string Patterns = "patterns";
private const string Regexes = "regexes";
public BlocklistProvider(
ILogger<BlocklistProvider> logger,
IOptions<ContentBlockerConfig> config,
IHttpClientFactory httpClientFactory)
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
IMemoryCache cache,
IHttpClientFactory httpClientFactory
)
{
_logger = logger;
_config = config.Value;
_sonarrConfig = sonarrConfig.Value;
_radarrConfig = radarrConfig.Value;
_lidarrConfig = lidarrConfig.Value;
_cache = cache;
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
_config.Validate();
if (_config.Blacklist?.Enabled is true)
{
BlocklistType = BlocklistType.Blacklist;
}
if (_config.Whitelist?.Enabled is true)
{
BlocklistType = BlocklistType.Whitelist;
}
}
public async Task LoadBlocklistAsync()
public async Task LoadBlocklistsAsync()
{
if (Patterns.Count > 0 || Regexes.Count > 0)
if (_initialized)
{
_logger.LogDebug("blocklist already loaded");
_logger.LogDebug("blocklists already loaded");
return;
}
try
{
await LoadPatternsAndRegexesAsync();
await LoadPatternsAndRegexesAsync(_sonarrConfig.Block.Type, _sonarrConfig.Block.Path, InstanceType.Sonarr);
await LoadPatternsAndRegexesAsync(_radarrConfig.Block.Type, _radarrConfig.Block.Path, InstanceType.Radarr);
await LoadPatternsAndRegexesAsync(_lidarrConfig.Block.Type, _lidarrConfig.Block.Path, InstanceType.Lidarr);
_initialized = true;
}
catch
{
_logger.LogError("failed to load {type}", BlocklistType.ToString());
_logger.LogError("failed to load blocklists");
throw;
}
}
private async Task LoadPatternsAndRegexesAsync()
public BlocklistType GetBlocklistType(InstanceType instanceType)
{
string[] patterns;
_cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
return blocklistType ?? BlocklistType.Blacklist;
}
public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
return patterns ?? [];
}
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
if (BlocklistType is BlocklistType.Blacklist)
return regexes ?? [];
}
private async Task LoadPatternsAndRegexesAsync(BlocklistType blocklistType, string? blocklistPath, InstanceType instanceType)
{
if (string.IsNullOrEmpty(blocklistPath))
{
patterns = await ReadContentAsync(_config.Blacklist.Path);
}
else
{
patterns = await ReadContentAsync(_config.Whitelist.Path);
return;
}
string[] filePatterns = await ReadContentAsync(blocklistPath);
long startTime = Stopwatch.GetTimestamp();
ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
const string regexId = "regex:";
ConcurrentBag<string> patterns = [];
ConcurrentBag<Regex> regexes = [];
Parallel.ForEach(patterns, options, pattern =>
Parallel.ForEach(filePatterns, options, pattern =>
{
if (!pattern.StartsWith(regexId))
{
Patterns.Add(pattern);
patterns.Add(pattern);
return;
}
@@ -92,7 +114,7 @@ public sealed class BlocklistProvider
try
{
Regex regex = new(pattern, RegexOptions.Compiled);
Regexes.Add(regex);
regexes.Add(regex);
}
catch (ArgumentException)
{
@@ -101,10 +123,14 @@ public sealed class BlocklistProvider
});
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
_cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
_cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
_logger.LogDebug("loaded {count} patterns", Patterns.Count);
_logger.LogDebug("loaded {count} regexes", Regexes.Count);
_logger.LogDebug("blocklist loaded in {elapsed} ms", elapsed.TotalMilliseconds);
_logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count);
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistPath);
}
private async Task<string[]> ReadContentAsync(string path)
code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs
+50 -5
View File
@@ -1,12 +1,17 @@
using Common.Configuration.Arr;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadClient;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Serilog.Context;
namespace Infrastructure.Verticals.ContentBlocker;
@@ -19,12 +24,19 @@ public sealed class ContentBlocker : GenericHandler
IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
SonarrClient sonarrClient,
RadarrClient radarrClient,
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
BlocklistProvider blocklistProvider,
DownloadServiceFactory downloadServiceFactory
) : base(logger, downloadClientConfig, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
sonarrClient, radarrClient, lidarrClient,
arrArrQueueIterator, downloadServiceFactory
)
{
_blocklistProvider = blocklistProvider;
}
@@ -37,18 +49,40 @@ public sealed class ContentBlocker : GenericHandler
return;
}
await _blocklistProvider.LoadBlocklistAsync();
bool blocklistIsConfigured = _sonarrConfig.Enabled && !string.IsNullOrEmpty(_sonarrConfig.Block.Path) ||
_radarrConfig.Enabled && !string.IsNullOrEmpty(_radarrConfig.Block.Path) ||
_lidarrConfig.Enabled && !string.IsNullOrEmpty(_lidarrConfig.Block.Path);
if (!blocklistIsConfigured)
{
_logger.LogWarning("no blocklist is configured");
return;
}
await _blocklistProvider.LoadBlocklistsAsync();
await base.ExecuteAsync();
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
ArrClient arrClient = GetClient(instanceType);
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
{
foreach (QueueRecord record in items)
var groups = items
.GroupBy(x => x.DownloadId)
.ToList();
foreach (var group in groups)
{
QueueRecord record = group.First();
if (record.Protocol is not "torrent")
{
continue;
@@ -61,8 +95,19 @@ public sealed class ContentBlocker : GenericHandler
}
_logger.LogDebug("searching unwanted files for {title}", record.Title);
await _downloadService.BlockUnwantedFilesAsync(record.DownloadId);
if (!await _downloadService.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
}
});
await arrClient.RefreshItemsAsync(instance, itemsToBeRefreshed);
}
}
code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs
+16 -16
View File
@@ -1,4 +1,6 @@
using Domain.Enums;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.ContentBlocker;
@@ -6,46 +8,44 @@ namespace Infrastructure.Verticals.ContentBlocker;
public sealed class FilenameEvaluator
{
private readonly ILogger<FilenameEvaluator> _logger;
private readonly BlocklistProvider _blocklistProvider;
public FilenameEvaluator(ILogger<FilenameEvaluator> logger, BlocklistProvider blocklistProvider)
public FilenameEvaluator(ILogger<FilenameEvaluator> logger)
{
_logger = logger;
_blocklistProvider = blocklistProvider;
}
// TODO create unit tests
public bool IsValid(string filename)
public bool IsValid(string filename, BlocklistType type, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
{
return IsValidAgainstPatterns(filename) && IsValidAgainstRegexes(filename);
return IsValidAgainstPatterns(filename, type, patterns) && IsValidAgainstRegexes(filename, type, regexes);
}
private bool IsValidAgainstPatterns(string filename)
private static bool IsValidAgainstPatterns(string filename, BlocklistType type, ConcurrentBag<string> patterns)
{
if (_blocklistProvider.Patterns.Count is 0)
if (patterns.Count is 0)
{
return true;
}
return _blocklistProvider.BlocklistType switch
return type switch
{
BlocklistType.Blacklist => !_blocklistProvider.Patterns.Any(pattern => MatchesPattern(filename, pattern)),
BlocklistType.Whitelist => _blocklistProvider.Patterns.Any(pattern => MatchesPattern(filename, pattern)),
BlocklistType.Blacklist => !patterns.Any(pattern => MatchesPattern(filename, pattern)),
BlocklistType.Whitelist => patterns.Any(pattern => MatchesPattern(filename, pattern)),
_ => true
};
}
private bool IsValidAgainstRegexes(string filename)
private static bool IsValidAgainstRegexes(string filename, BlocklistType type, ConcurrentBag<Regex> regexes)
{
if (_blocklistProvider.Regexes.Count is 0)
if (regexes.Count is 0)
{
return true;
}
return _blocklistProvider.BlocklistType switch
return type switch
{
BlocklistType.Blacklist => !_blocklistProvider.Regexes.Any(regex => regex.IsMatch(filename)),
BlocklistType.Whitelist => _blocklistProvider.Regexes.Any(regex => regex.IsMatch(filename)),
BlocklistType.Blacklist => !regexes.Any(regex => regex.IsMatch(filename)),
BlocklistType.Whitelist => regexes.Any(regex => regex.IsMatch(filename)),
_ => true
};
}
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs
+5
View File
@@ -78,6 +78,11 @@ public sealed class DelugeClient
await SendRequest<DelugeResponse<object>>("core.set_torrent_options", hash, filePriorities);
}
public async Task<DelugeResponse<object>> DeleteTorrent(string hash)
{
return await SendRequest<DelugeResponse<object>>("core.remove_torrents", new List<string> { hash }, true);
}
private async Task<String> PostJson(String json)
{
StringContent content = new StringContent(json);
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
+47 -7
View File
@@ -1,3 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Domain.Models.Deluge.Response;
@@ -30,6 +33,7 @@ public sealed class DelugeService : DownloadServiceBase
await _client.LoginAsync();
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
{
hash = hash.ToLowerInvariant();
@@ -70,7 +74,13 @@ public sealed class DelugeService : DownloadServiceBase
return result;
}
public override async Task BlockUnwantedFilesAsync(string hash)
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
)
{
hash = hash.ToLowerInvariant();
@@ -79,14 +89,14 @@ public sealed class DelugeService : DownloadServiceBase
if (status?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return;
return false;
}
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", status.Name);
return;
return false;
}
DelugeContents? contents = null;
@@ -102,18 +112,27 @@ public sealed class DelugeService : DownloadServiceBase
if (contents is null)
{
return;
return false;
}
Dictionary<int, int> priorities = [];
bool hasPriorityUpdates = false;
long totalFiles = 0;
long totalUnwantedFiles = 0;
ProcessFiles(contents.Contents, (name, file) =>
{
totalFiles++;
int priority = file.Priority;
if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name))
if (file.Priority is 0)
{
totalUnwantedFiles++;
}
if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name, blocklistType, patterns, regexes))
{
totalUnwantedFiles++;
priority = 0;
hasPriorityUpdates = true;
_logger.LogInformation("unwanted file found | {file}", file.Path);
@@ -124,7 +143,7 @@ public sealed class DelugeService : DownloadServiceBase
if (!hasPriorityUpdates)
{
return;
return false;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -134,7 +153,23 @@ public sealed class DelugeService : DownloadServiceBase
.Select(x => x.Value)
.ToList();
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
}
await _client.ChangeFilesPriority(hash, sortedPriorities);
return false;
}
/// <inheritdoc/>
public override async Task Delete(string hash)
{
hash = hash.ToLowerInvariant();
await _client.DeleteTorrent(hash);
}
private bool IsItemStuckAndShouldRemove(TorrentStatus status)
@@ -173,8 +208,13 @@ public sealed class DelugeService : DownloadServiceBase
);
}
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory> contents, Action<string, DelugeFileOrDirectory> processFile)
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
{
if (contents is null)
{
return;
}
foreach (var (name, data) in contents)
{
switch (data.Type)
code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs
+20 -2
View File
@@ -1,4 +1,7 @@
using Common.Configuration.QueueCleaner;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
@@ -33,8 +36,23 @@ public abstract class DownloadServiceBase : IDownloadService
public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
public abstract Task BlockUnwantedFilesAsync(string hash);
/// <inheritdoc/>
public abstract Task<bool> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
);
/// <inheritdoc/>
public abstract Task Delete(string hash);
/// <summary>
/// Strikes an item and checks if the limit has been reached.
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="itemName">The name or title of the item.</param>
/// <returns>True if the limit has been reached; otherwise, false.</returns>
protected bool StrikeAndCheckLimit(string hash, string itemName)
{
return _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs
+10 -2
View File
@@ -1,4 +1,7 @@
using Common.Configuration.QueueCleaner;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
@@ -26,7 +29,12 @@ public sealed class DummyDownloadService : DownloadServiceBase
throw new NotImplementedException();
}
public override Task BlockUnwantedFilesAsync(string hash)
public override Task<bool> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
{
throw new NotImplementedException();
}
public override Task Delete(string hash)
{
throw new NotImplementedException();
}
code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs
+28 -2
View File
@@ -1,10 +1,36 @@
namespace Infrastructure.Verticals.DownloadClient;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
namespace Infrastructure.Verticals.DownloadClient;
public interface IDownloadService : IDisposable
{
public Task LoginAsync();
/// <summary>
/// Checks whether the download should be removed from the *arr queue.
/// </summary>
/// <param name="hash">The download hash.</param>
public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
public Task BlockUnwantedFilesAsync(string hash);
/// <summary>
/// Blocks unwanted files from being fully downloaded.
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="blocklistType">The <see cref="BlocklistType"/>.</param>
/// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<bool> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
);
/// <summary>
/// Deletes a download item.
/// </summary>
public Task Delete(string hash);
}
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
+54 -7
View File
@@ -1,3 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
@@ -38,6 +41,7 @@ public sealed class QBitService : DownloadServiceBase
await _client.LoginAsync(_config.Username, _config.Password);
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
{
RemoveResult result = new();
@@ -83,7 +87,13 @@ public sealed class QBitService : DownloadServiceBase
return result;
}
public override async Task BlockUnwantedFilesAsync(string hash)
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
)
{
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
@@ -91,7 +101,7 @@ public sealed class QBitService : DownloadServiceBase
if (torrent is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return;
return false;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
@@ -99,7 +109,7 @@ public sealed class QBitService : DownloadServiceBase
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return;
return false;
}
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
@@ -110,15 +120,19 @@ public sealed class QBitService : DownloadServiceBase
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return;
return false;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files is null)
{
return;
return false;
}
List<int> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
foreach (TorrentContent file in files)
{
@@ -127,14 +141,47 @@ public sealed class QBitService : DownloadServiceBase
continue;
}
if (file.Priority is TorrentContentPriority.Skip || _filenameEvaluator.IsValid(file.Name))
totalFiles++;
if (file.Priority is TorrentContentPriority.Skip)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", file.Name);
await _client.SetFilePriorityAsync(hash, file.Index.Value, TorrentContentPriority.Skip);
unwantedFiles.Add(file.Index.Value);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return false;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
}
foreach (int fileIndex in unwantedFiles)
{
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
return false;
}
/// <inheritdoc/>
public override async Task Delete(string hash)
{
await _client.DeleteAsync(hash, deleteDownloadedData: true);
}
public override void Dispose()
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
+47 -6
View File
@@ -1,4 +1,7 @@
using Common.Configuration.DownloadClient;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker;
@@ -41,6 +44,7 @@ public sealed class TransmissionService : DownloadServiceBase
await _client.GetSessionInformationAsync();
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
{
RemoveResult result = new();
@@ -76,23 +80,31 @@ public sealed class TransmissionService : DownloadServiceBase
return result;
}
public override async Task BlockUnwantedFilesAsync(string hash)
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent?.FileStats is null || torrent.Files is null)
{
return;
return false;
}
if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return;
return false;
}
List<long> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
for (int i = 0; i < torrent.Files.Length; i++)
{
@@ -100,19 +112,34 @@ public sealed class TransmissionService : DownloadServiceBase
{
continue;
}
totalFiles++;
if (!torrent.FileStats[i].Wanted.Value || _filenameEvaluator.IsValid(torrent.Files[i].Name))
if (!torrent.FileStats[i].Wanted.Value)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name);
unwantedFiles.Add(i);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return;
return false;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -122,6 +149,20 @@ public sealed class TransmissionService : DownloadServiceBase
Ids = [ torrent.Id ],
FilesUnwanted = unwantedFiles.ToArray(),
});
return false;
}
public override async Task Delete(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent is null)
{
return;
}
await _client.TorrentRemoveAsync([torrent.Id], true);
}
public override void Dispose()
code/Infrastructure/Verticals/ItemStriker/Striker.cs
+1 -1
View File
@@ -37,7 +37,7 @@ public class Striker
++strikeCount;
}
_logger.LogDebug("item on strike number {strike} | reason {reason} | {name}", strikeCount, strikeType.ToString(), itemName);
_logger.LogInformation("item on strike number {strike} | reason {reason} | {name}", strikeCount, strikeType.ToString(), itemName);
_cache.Set(key, strikeCount, _cacheOptions);
if (strikeCount < maxStrikes)
code/Infrastructure/Verticals/Jobs/GenericHandler.cs
+19 -15
View File
@@ -1,4 +1,4 @@
using Common.Configuration.Arr;
using Common.Configuration.Arr;
using Common.Configuration.DownloadClient;
using Domain.Enums;
using Domain.Models.Arr;
@@ -16,28 +16,34 @@ public abstract class GenericHandler : IDisposable
protected readonly DownloadClientConfig _downloadClientConfig;
protected readonly SonarrConfig _sonarrConfig;
protected readonly RadarrConfig _radarrConfig;
protected readonly LidarrConfig _lidarrConfig;
protected readonly SonarrClient _sonarrClient;
protected readonly RadarrClient _radarrClient;
protected readonly LidarrClient _lidarrClient;
protected readonly ArrQueueIterator _arrArrQueueIterator;
protected readonly IDownloadService _downloadService;
protected GenericHandler(
ILogger<GenericHandler> logger,
IOptions<DownloadClientConfig> downloadClientConfig,
SonarrConfig sonarrConfig,
RadarrConfig radarrConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
SonarrClient sonarrClient,
RadarrClient radarrClient,
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory
)
{
_logger = logger;
_downloadClientConfig = downloadClientConfig.Value;
_sonarrConfig = sonarrConfig;
_radarrConfig = radarrConfig;
_sonarrConfig = sonarrConfig.Value;
_radarrConfig = radarrConfig.Value;
_lidarrConfig = lidarrConfig.Value;
_sonarrClient = sonarrClient;
_radarrClient = radarrClient;
_lidarrClient = lidarrClient;
_arrArrQueueIterator = arrArrQueueIterator;
_downloadService = downloadServiceFactory.CreateDownloadClient();
}
@@ -48,6 +54,7 @@ public abstract class GenericHandler : IDisposable
await ProcessArrConfigAsync(_sonarrConfig, InstanceType.Sonarr);
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr);
}
public virtual void Dispose()
@@ -82,17 +89,10 @@ public abstract class GenericHandler : IDisposable
{
InstanceType.Sonarr => _sonarrClient,
InstanceType.Radarr => _radarrClient,
InstanceType.Lidarr => _lidarrClient,
_ => throw new NotImplementedException($"instance type {type} is not yet supported")
};
protected ArrConfig GetConfig(InstanceType type) =>
type switch
{
InstanceType.Sonarr => _sonarrConfig,
InstanceType.Radarr => _radarrConfig,
_ => throw new NotImplementedException($"instance type {type} is not yet supported")
};
protected SearchItem GetRecordSearchItem(InstanceType type, QueueRecord record, bool isPack = false)
{
return type switch
@@ -117,11 +117,15 @@ public abstract class GenericHandler : IDisposable
},
InstanceType.Sonarr when _sonarrConfig.SearchType is SonarrSearchType.Series => new SonarrSearchItem
{
Id = record.SeriesId,
Id = record.SeriesId
},
InstanceType.Radarr => new SearchItem
{
Id = record.MovieId,
Id = record.MovieId
},
InstanceType.Lidarr => new SearchItem
{
Id = record.AlbumId
},
_ => throw new NotImplementedException($"instance type {type} is not yet supported")
};
code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs
+12 -4
View File
@@ -8,6 +8,7 @@ using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Serilog.Context;
namespace Infrastructure.Verticals.QueueCleaner;
@@ -18,19 +19,27 @@ public sealed class QueueCleaner : GenericHandler
IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
SonarrClient sonarrClient,
RadarrClient radarrClient,
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory
) : base(logger, downloadClientConfig, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
sonarrClient, radarrClient, lidarrClient,
arrArrQueueIterator, downloadServiceFactory
)
{
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
ArrClient arrClient = GetClient(instanceType);
ArrConfig arrConfig = GetConfig(instanceType);
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
{
@@ -73,11 +82,10 @@ public sealed class QueueCleaner : GenericHandler
}
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
}
});
await arrClient.RefreshItemsAsync(instance, arrConfig, itemsToBeRefreshed);
await arrClient.RefreshItemsAsync(instance, itemsToBeRefreshed);
}
}