public/cleanuperr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into add_cleanup_on_no_hardlinks

Browse Source
This commit is contained in:
Flaminel
2025-03-24 16:34:11 +02:00
parent d993cd30a7 5fe0f5750a
commit a83809eef7
46 changed files with 1226 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/Infrastructure/Verticals/Arr/ArrClient.cs
+26 -12
View File
@@ -43,9 +43,11 @@ public abstract class ArrClient : IArrClient
public virtual async Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page)
{
Uri uri = new(arrInstance.Url, GetQueueUrlPath(page));
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueUrlPath().TrimStart('/')}";
uriBuilder.Query = GetQueueUrlQuery(page);
using HttpRequestMessage request = new(HttpMethod.Get, uri);
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
@@ -56,7 +58,7 @@ public abstract class ArrClient : IArrClient
}
catch
{
_logger.LogError("queue list failed | {uri}", uri);
_logger.LogError("queue list failed | {uri}", uriBuilder.Uri);
throw;
}
@@ -65,7 +67,7 @@ public abstract class ArrClient : IArrClient
if (queueResponse is null)
{
throw new Exception($"unrecognized queue list response | {uri} | {responseBody}");
throw new Exception($"unrecognized queue list response | {uriBuilder.Uri} | {responseBody}");
}
return queueResponse;
@@ -112,13 +114,20 @@ public abstract class ArrClient : IArrClient
return false;
}
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient)
public virtual async Task DeleteQueueItemAsync(
ArrInstance arrInstance,
QueueRecord record,
bool removeFromClient,
DeleteReason deleteReason
)
{
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id, removeFromClient));
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}";
uriBuilder.Query = GetQueueDeleteUrlQuery(removeFromClient);
try
{
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
using HttpRequestMessage request = new(HttpMethod.Delete, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync<HttpResponseMessage>(SendRequestAsync, request);
@@ -126,15 +135,16 @@ public abstract class ArrClient : IArrClient
_logger.LogInformation(
removeFromClient
? "queue item deleted | {url} | {title}"
: "queue item removed from arr | {url} | {title}",
? "queue item deleted with reason {reason} | {url} | {title}"
: "queue item removed from arr with reason {reason} | {url} | {title}",
deleteReason.ToString(),
arrInstance.Url,
record.Title
);
}
catch
{
_logger.LogError("queue delete failed | {uri} | {title}", uri, record.Title);
_logger.LogError("queue delete failed | {uri} | {title}", uriBuilder.Uri, record.Title);
throw;
}
}
@@ -152,9 +162,13 @@ public abstract class ArrClient : IArrClient
return true;
}
protected abstract string GetQueueUrlPath(int page);
protected abstract string GetQueueUrlPath();
protected abstract string GetQueueDeleteUrlPath(long recordId, bool removeFromClient);
protected abstract string GetQueueUrlQuery(int page);
protected abstract string GetQueueDeleteUrlPath(long recordId);
protected abstract string GetQueueDeleteUrlQuery(bool removeFromClient);
protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
{
code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs
+1 -1
View File
@@ -11,7 +11,7 @@ public interface IArrClient
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason);
Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);
code/Infrastructure/Verticals/Arr/LidarrClient.cs
+27 -11
View File
@@ -27,29 +27,42 @@ public class LidarrClient : ArrClient, ILidarrClient
{
}
protected override string GetQueueUrlPath(int page)
protected override string GetQueueUrlPath()
{
return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
return "/api/v1/queue";
}
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
protected override string GetQueueUrlQuery(int page)
{
string path = $"/api/v1/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
return $"page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
}
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v1/queue/{recordId}";
}
return path;
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
{
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return query;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
{
if (items?.Count is null or 0) return;
if (items?.Count is null or 0)
{
return;
}
Uri uri = new(arrInstance.Url, "/api/v1/command");
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/command";
foreach (var command in GetSearchCommands(items))
{
using HttpRequestMessage request = new(HttpMethod.Post, uri);
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
request.Content = new StringContent(
JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }),
Encoding.UTF8,
@@ -132,8 +145,11 @@ public class LidarrClient : ArrClient, ILidarrClient
private async Task<List<Album>?> GetAlbumsAsync(ArrInstance arrInstance, List<long> albumIds)
{
Uri uri = new(arrInstance.Url, $"api/v1/album?{string.Join('&', albumIds.Select(x => $"albumIds={x}"))}");
using HttpRequestMessage request = new(HttpMethod.Get, uri);
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/album";
uriBuilder.Query = string.Join('&', albumIds.Select(x => $"albumIds={x}"));
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
using var response = await _httpClient.SendAsync(request);
code/Infrastructure/Verticals/Arr/RadarrClient.cs
+24 -11
View File
@@ -27,18 +27,27 @@ public class RadarrClient : ArrClient, IRadarrClient
{
}
protected override string GetQueueUrlPath(int page)
protected override string GetQueueUrlPath()
{
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
return "/api/v3/queue";
}
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
protected override string GetQueueUrlQuery(int page)
{
string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return $"page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
}
return path;
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v3/queue/{recordId}";
}
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
{
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return query;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
@@ -50,14 +59,16 @@ public class RadarrClient : ArrClient, IRadarrClient
List<long> ids = items.Select(item => item.Id).ToList();
Uri uri = new(arrInstance.Url, "/api/v3/command");
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command";
RadarrCommand command = new()
{
Name = "MoviesSearch",
MovieIds = ids,
};
using HttpRequestMessage request = new(HttpMethod.Post, uri);
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
request.Content = new StringContent(
JsonConvert.SerializeObject(command),
Encoding.UTF8,
@@ -135,8 +146,10 @@ public class RadarrClient : ArrClient, IRadarrClient
private async Task<Movie?> GetMovie(ArrInstance arrInstance, long movieId)
{
Uri uri = new(arrInstance.Url, $"api/v3/movie/{movieId}");
using HttpRequestMessage request = new(HttpMethod.Get, uri);
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/movie/{movieId}";
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
code/Infrastructure/Verticals/Arr/SonarrClient.cs
+28 -13
View File
@@ -28,18 +28,27 @@ public class SonarrClient : ArrClient, ISonarrClient
{
}
protected override string GetQueueUrlPath(int page)
protected override string GetQueueUrlPath()
{
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true&includeEpisode=true";
return "/api/v3/queue";
}
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
protected override string GetQueueUrlQuery(int page)
{
string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
return $"page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true&includeEpisode=true";
}
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
protected override string GetQueueDeleteUrlPath(long recordId)
{
return $"/api/v3/queue/{recordId}";
}
return path;
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
{
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return query;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
@@ -49,11 +58,12 @@ public class SonarrClient : ArrClient, ISonarrClient
return;
}
Uri uri = new(arrInstance.Url, "/api/v3/command");
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command";
foreach (SonarrCommand command in GetSearchCommands(items.Cast<SonarrSearchItem>().ToHashSet()))
{
using HttpRequestMessage request = new(HttpMethod.Post, uri);
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
request.Content = new StringContent(
JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }),
Encoding.UTF8,
@@ -199,8 +209,11 @@ public class SonarrClient : ArrClient, ISonarrClient
private async Task<List<Episode>?> GetEpisodesAsync(ArrInstance arrInstance, List<long> episodeIds)
{
Uri uri = new(arrInstance.Url, $"api/v3/episode?{string.Join('&', episodeIds.Select(x => $"episodeIds={x}"))}");
using HttpRequestMessage request = new(HttpMethod.Get, uri);
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/episode";
uriBuilder.Query = string.Join('&', episodeIds.Select(x => $"episodeIds={x}"));
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
@@ -212,8 +225,10 @@ public class SonarrClient : ArrClient, ISonarrClient
private async Task<Series?> GetSeriesAsync(ArrInstance arrInstance, long seriesId)
{
Uri uri = new(arrInstance.Url, $"api/v3/series/{seriesId}");
using HttpRequestMessage request = new(HttpMethod.Get, uri);
UriBuilder uriBuilder = new(arrInstance.Url);
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/series/{seriesId}";
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs
+13 -8
View File
@@ -49,9 +49,9 @@ public sealed class BlocklistProvider
try
{
await LoadPatternsAndRegexesAsync(_sonarrConfig.Block.Type, _sonarrConfig.Block.Path, InstanceType.Sonarr);
await LoadPatternsAndRegexesAsync(_radarrConfig.Block.Type, _radarrConfig.Block.Path, InstanceType.Radarr);
await LoadPatternsAndRegexesAsync(_lidarrConfig.Block.Type, _lidarrConfig.Block.Path, InstanceType.Lidarr);
await LoadPatternsAndRegexesAsync(_sonarrConfig, InstanceType.Sonarr);
await LoadPatternsAndRegexesAsync(_radarrConfig, InstanceType.Radarr);
await LoadPatternsAndRegexesAsync(_lidarrConfig, InstanceType.Lidarr);
_initialized = true;
}
@@ -83,14 +83,19 @@ public sealed class BlocklistProvider
return regexes ?? [];
}
private async Task LoadPatternsAndRegexesAsync(BlocklistType blocklistType, string? blocklistPath, InstanceType instanceType)
private async Task LoadPatternsAndRegexesAsync(ArrConfig arrConfig, InstanceType instanceType)
{
if (string.IsNullOrEmpty(blocklistPath))
if (!arrConfig.Enabled)
{
return;
}
string[] filePatterns = await ReadContentAsync(blocklistPath);
if (string.IsNullOrEmpty(arrConfig.Block.Path))
{
return;
}
string[] filePatterns = await ReadContentAsync(arrConfig.Block.Path);
long startTime = Stopwatch.GetTimestamp();
ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
@@ -121,13 +126,13 @@ public sealed class BlocklistProvider
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType);
_cache.Set(CacheKeys.BlocklistType(instanceType), arrConfig.Block.Type);
_cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
_cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
_logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count);
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistPath);
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, arrConfig.Block.Path);
}
private async Task<string[]> ReadContentAsync(string path)
code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs
+18 -6
View File
@@ -6,6 +6,7 @@ using Common.Configuration.DownloadClient;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context;
@@ -22,7 +23,8 @@ public sealed class ContentBlocker : GenericHandler
{
private readonly ContentBlockerConfig _config;
private readonly BlocklistProvider _blocklistProvider;
private readonly IgnoredDownloadsProvider<ContentBlockerConfig> _ignoredDownloadsProvider;
public ContentBlocker(
ILogger<ContentBlocker> logger,
IOptions<ContentBlockerConfig> config,
@@ -36,7 +38,8 @@ public sealed class ContentBlocker : GenericHandler
ArrQueueIterator arrArrQueueIterator,
BlocklistProvider blocklistProvider,
DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier
INotificationPublisher notifier,
IgnoredDownloadsProvider<ContentBlockerConfig> ignoredDownloadsProvider
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
@@ -47,13 +50,14 @@ public sealed class ContentBlocker : GenericHandler
{
_config = config.Value;
_blocklistProvider = blocklistProvider;
_ignoredDownloadsProvider = ignoredDownloadsProvider;
}
public override async Task ExecuteAsync()
{
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None)
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled)
{
_logger.LogWarning("download client is set to none");
_logger.LogWarning("download client is not set");
return;
}
@@ -73,6 +77,8 @@ public sealed class ContentBlocker : GenericHandler
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
@@ -106,13 +112,19 @@ public sealed class ContentBlocker : GenericHandler
continue;
}
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
{
_logger.LogInformation("skip | {title} | ignored", record.Title);
continue;
}
// push record to context
ContextProvider.Set(nameof(QueueRecord), record);
_logger.LogDebug("searching unwanted files for {title}", record.Title);
BlockFilesResult result = await _downloadService
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes);
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads);
if (!result.ShouldRemove)
{
@@ -130,7 +142,7 @@ public sealed class ContentBlocker : GenericHandler
removeFromClient = false;
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, DeleteReason.AllFilesBlocked);
await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked);
}
});
code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs
+9 -3
View File
@@ -3,6 +3,7 @@ using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Domain.Enums;
using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.DownloadClient;
@@ -17,6 +18,7 @@ namespace Infrastructure.Verticals.DownloadCleaner;
public sealed class DownloadCleaner : GenericHandler
{
private readonly DownloadCleanerConfig _config;
private readonly IgnoredDownloadsProvider<DownloadCleanerConfig> _ignoredDownloadsProvider;
private readonly HashSet<string> _excludedHashes = [];
private static bool _hardLinkCategoryCreated;
@@ -33,7 +35,8 @@ public sealed class DownloadCleaner : GenericHandler
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier
INotificationPublisher notifier,
IgnoredDownloadsProvider<DownloadCleanerConfig> ignoredDownloadsProvider
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
@@ -44,13 +47,14 @@ public sealed class DownloadCleaner : GenericHandler
{
_config = config.Value;
_config.Validate();
_ignoredDownloadsProvider = ignoredDownloadsProvider;
}
public override async Task ExecuteAsync()
{
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None)
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled)
{
_logger.LogWarning("download client is set to none");
_logger.LogWarning("download client is not set");
return;
}
@@ -60,6 +64,8 @@ public sealed class DownloadCleaner : GenericHandler
return;
}
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetSeedingDownloads();
List<object>? downloadsToChangeCategory = null;
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs
+40 -8
View File
@@ -16,9 +16,24 @@ public sealed class DelugeClient
private readonly DelugeConfig _config;
private readonly HttpClient _httpClient;
private static readonly IReadOnlyList<string> Fields =
[
"hash",
"state",
"name",
"eta",
"private",
"total_done",
"label",
"seeding_time",
"ratio",
"trackers"
];
public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory)
{
_config = config.Value;
_config.Validate();
_httpClient = httpClientFactory.CreateClient(nameof(DelugeService));
}
@@ -65,11 +80,24 @@ public sealed class DelugeClient
public async Task<TorrentStatus?> GetTorrentStatus(string hash)
{
return await SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" }
);
try
{
return await SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
Fields
);
}
catch (DelugeClientException e)
{
// Deluge returns an error when the torrent is not found
if (e.Message == "AttributeError: 'NoneType' object has no attribute 'call'")
{
return null;
}
throw;
}
}
public async Task<List<TorrentStatus>?> GetStatusForAllTorrents()
@@ -77,7 +105,7 @@ public sealed class DelugeClient
Dictionary<string, TorrentStatus>? downloads = await SendRequest<Dictionary<string, TorrentStatus>?>(
"core.get_torrents_status",
"",
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" }
Fields
);
return downloads?.Values.ToList();
@@ -107,8 +135,12 @@ public sealed class DelugeClient
{
StringContent content = new StringContent(json);
content.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
var responseMessage = await _httpClient.PostAsync(new Uri(_config.Url, "/json"), content);
UriBuilder uriBuilder = new(_config.Url);
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
? $"{uriBuilder.Path.TrimEnd('/')}/json"
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/json";
var responseMessage = await _httpClient.PostAsync(uriBuilder.Uri, content);
responseMessage.EnsureSuccessStatusCode();
var responseJson = await responseMessage.Content.ReadAsStringAsync();
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
+50 -29
View File
@@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Domain.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
@@ -51,20 +52,28 @@ public class DelugeService : DownloadService, IDelugeService
}
/// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
hash = hash.ToLowerInvariant();
DelugeContents? contents = null;
StalledResult result = new();
TorrentStatus? status = await _client.GetTorrentStatus(hash);
TorrentStatus? download = await _client.GetTorrentStatus(hash);
if (status?.Hash is null)
if (download?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.IsPrivate = download.Private;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
try
{
@@ -74,6 +83,7 @@ public class DelugeService : DownloadService, IDelugeService
{
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
}
bool shouldRemove = contents?.Contents?.Count > 0;
@@ -87,45 +97,50 @@ public class DelugeService : DownloadService, IDelugeService
if (shouldRemove)
{
result.DeleteReason = DeleteReason.AllFilesBlocked;
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(status);
result.IsPrivate = status.Private;
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download);
if (!shouldRemove && result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
)
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{
hash = hash.ToLowerInvariant();
TorrentStatus? status = await _client.GetTorrentStatus(hash);
TorrentStatus? download = await _client.GetTorrentStatus(hash);
BlockFilesResult result = new();
if (status?.Hash is null)
if (download?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.IsPrivate = status.Private;
if (_contentBlockerConfig.IgnorePrivate && status.Private)
var ceva = await _client.GetTorrentExtended(hash);
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
result.IsPrivate = download.Private;
if (_contentBlockerConfig.IgnorePrivate && download.Private)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", status.Name);
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
@@ -220,8 +235,8 @@ public class DelugeService : DownloadService, IDelugeService
}
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes)
public override async Task CleanDownloadsAsync(List<object> downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{
foreach (TorrentStatus download in downloads)
{
@@ -229,6 +244,12 @@ public class DelugeService : DownloadService, IDelugeService
{
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
@@ -407,33 +428,33 @@ public class DelugeService : DownloadService, IDelugeService
await _client.SetTorrentLabel(hash, newLabel);
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentStatus status)
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentStatus status)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
return false;
return (false, default);
}
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return false;
return (false, default);
}
if (status.Eta > 0)
{
return false;
return (false, default);
}
ResetStrikesOnProgress(status.Hash!, status.TotalDone);
return await StrikeAndCheckLimit(status.Hash!, status.Name!);
return (await StrikeAndCheckLimit(status.Hash!, status.Name!, StrikeType.Stalled), DeleteReason.Stalled);
}
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs
+7 -8
View File
@@ -64,15 +64,13 @@ public abstract class DownloadService : IDownloadService
public abstract Task LoginAsync();
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
);
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task DeleteDownload(string hash);
@@ -87,7 +85,7 @@ public abstract class DownloadService : IDownloadService
public abstract List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
/// <inheritdoc/>
public abstract Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes);
public abstract Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes);
@@ -117,10 +115,11 @@ public abstract class DownloadService : IDownloadService
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="itemName">The name or title of the item.</param>
/// <param name="strikeType"></param>
/// <returns>True if the limit has been reached; otherwise, false.</returns>
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName)
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType)
{
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, strikeType);
}
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category)
code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs
+1
View File
@@ -25,6 +25,7 @@ public sealed class DownloadServiceFactory
Common.Enums.DownloadClient.Deluge => _serviceProvider.GetRequiredService<DelugeService>(),
Common.Enums.DownloadClient.Transmission => _serviceProvider.GetRequiredService<TransmissionService>(),
Common.Enums.DownloadClient.None => _serviceProvider.GetRequiredService<DummyDownloadService>(),
Common.Enums.DownloadClient.Disabled => _serviceProvider.GetRequiredService<DummyDownloadService>(),
_ => throw new ArgumentOutOfRangeException()
};
}
code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs
+4 -3
View File
@@ -43,12 +43,13 @@ public class DummyDownloadService : DownloadService
return Task.CompletedTask;
}
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
@@ -68,7 +69,7 @@ public class DummyDownloadService : DownloadService
throw new NotImplementedException();
}
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes)
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs
+7 -5
View File
@@ -14,7 +14,8 @@ public interface IDownloadService : IDisposable
/// Checks whether the download should be removed from the *arr queue.
/// </summary>
/// <param name="hash">The download hash.</param>
public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Blocks unwanted files from being fully downloaded.
@@ -23,12 +24,13 @@ public interface IDownloadService : IDisposable
/// <param name="blocklistType">The <see cref="BlocklistType"/>.</param>
/// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
public Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
);
/// <summary>
@@ -59,7 +61,7 @@ public interface IDownloadService : IDisposable
/// <param name="downloads">The downloads to clean.</param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes);
Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Changes the category for downloads that have no hardlinks.
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
+73 -33
View File
@@ -8,6 +8,7 @@ using Common.Configuration.QueueCleaner;
using Common.Exceptions;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
@@ -46,7 +47,11 @@ public class QBitService : DownloadService, IQBitService
{
_config = config.Value;
_config.Validate();
_client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), _config.Url);
UriBuilder uriBuilder = new(_config.Url);
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
? uriBuilder.Path
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/')}";
_client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), uriBuilder.Uri);
}
public override async Task LoginAsync()
@@ -60,18 +65,27 @@ public class QBitService : DownloadService, IQBitService
}
/// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
StalledResult result = new();
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
if (torrent is null)
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null)
@@ -84,52 +98,57 @@ public class QBitService : DownloadService, IQBitService
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
// if all files were blocked by qBittorrent
if (torrent is { CompletionOn: not null, Downloaded: null or 0 })
{
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
return result;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
// if all files are marked as skip
if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
{
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
return result;
}
// remove if all files are unwanted
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
result.ShouldRemove = await IsItemStuckAndShouldRemove(torrent, result.IsPrivate);
if (result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download, result.IsPrivate);
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
)
{
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
BlockFilesResult result = new();
if (torrent is null)
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null)
@@ -147,7 +166,7 @@ public class QBitService : DownloadService, IQBitService
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
@@ -238,7 +257,7 @@ public class QBitService : DownloadService, IQBitService
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes)
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
@@ -252,6 +271,15 @@ public class QBitService : DownloadService, IQBitService
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
@@ -458,30 +486,42 @@ public class QBitService : DownloadService, IQBitService
{
_client.Dispose();
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
return false;
return (false, default);
}
if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
and not TorrentState.ForcedFetchingMetadata)
{
// ignore other states
return false;
return (false, default);
}
ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return await StrikeAndCheckLimit(torrent.Hash, torrent.Name);
if (torrent.State is TorrentState.StalledDownload)
{
return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.Stalled), DeleteReason.Stalled);
}
return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
}
private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
{
return (await _client.GetTorrentTrackersAsync(hash))
.Where(x => !x.Url.ToString().Contains("**"))
.ToList();
}
}
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
+72 -48
View File
@@ -8,6 +8,7 @@ using Common.Configuration.QueueCleaner;
using Common.Exceptions;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
@@ -29,6 +30,23 @@ public class TransmissionService : DownloadService, ITransmissionService
private readonly Client _client;
private TorrentInfo[]? _torrentsCache;
private static readonly string[] Fields =
[
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER,
TorrentFields.DOWNLOAD_DIR,
TorrentFields.SECONDS_SEEDING,
TorrentFields.UPLOAD_RATIO,
TorrentFields.TRACKERS
];
public TransmissionService(
IHttpClientFactory httpClientFactory,
ILogger<TransmissionService> logger,
@@ -49,9 +67,13 @@ public class TransmissionService : DownloadService, ITransmissionService
{
_config = config.Value;
_config.Validate();
UriBuilder uriBuilder = new(_config.Url);
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
? $"{uriBuilder.Path.TrimEnd('/')}/rpc"
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/rpc";
_client = new(
httpClientFactory.CreateClient(Constants.HttpClientWithRetryName),
new Uri(_config.Url, "/transmission/rpc").ToString(),
uriBuilder.Uri.ToString(),
login: _config.Username,
password: _config.Password
);
@@ -63,21 +85,27 @@ public class TransmissionService : DownloadService, ITransmissionService
}
/// <inheritdoc/>
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
StalledResult result = new();
TorrentInfo? torrent = await GetTorrentAsync(hash);
TorrentInfo? download = await GetTorrentAsync(hash);
if (torrent is null)
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
bool shouldRemove = torrent.FileStats?.Length > 0;
result.IsPrivate = torrent.IsPrivate ?? false;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool shouldRemove = download.FileStats?.Length > 0;
result.IsPrivate = download.IsPrivate ?? false;
foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? [])
foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? [])
{
if (!stats.Wanted.HasValue)
{
@@ -94,43 +122,45 @@ public class TransmissionService : DownloadService, ITransmissionService
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
return result;
}
// remove if all files are unwanted or download is stuck
result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(torrent);
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download);
if (!shouldRemove && result.ShouldRemove)
{
result.DeleteReason = DeleteReason.Stalled;
}
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
)
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
TorrentInfo? download = await GetTorrentAsync(hash);
BlockFilesResult result = new();
if (torrent?.FileStats is null || torrent.Files is null)
if (download?.FileStats is null || download.Files is null)
{
return result;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool isPrivate = torrent.IsPrivate ?? false;
bool isPrivate = download.IsPrivate ?? false;
result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
@@ -138,27 +168,27 @@ public class TransmissionService : DownloadService, ITransmissionService
long totalFiles = 0;
long totalUnwantedFiles = 0;
for (int i = 0; i < torrent.Files.Length; i++)
for (int i = 0; i < download.Files.Length; i++)
{
if (torrent.FileStats?[i].Wanted == null)
if (download.FileStats?[i].Wanted == null)
{
continue;
}
totalFiles++;
if (!torrent.FileStats[i].Wanted.Value)
if (!download.FileStats[i].Wanted.Value)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes))
if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name);
_logger.LogInformation("unwanted file found | {file}", download.Files[i].Name);
unwantedFiles.Add(i);
totalUnwantedFiles++;
}
@@ -178,7 +208,7 @@ public class TransmissionService : DownloadService, ITransmissionService
_logger.LogDebug("changing priorities | torrent {hash}", hash);
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, torrent.Id, unwantedFiles.ToArray());
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());
return result;
}
@@ -237,7 +267,7 @@ public class TransmissionService : DownloadService, ITransmissionService
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes)
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
foreach (TorrentInfo download in downloads)
{
@@ -245,6 +275,12 @@ public class TransmissionService : DownloadService, ITransmissionService
{
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x =>
@@ -461,34 +497,34 @@ public class TransmissionService : DownloadService, ITransmissionService
});
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent)
private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return false;
return (false, default);
}
if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
return false;
return (false, default);
}
if (torrent.Status is not 4)
{
// not in downloading state
return false;
return (false, default);
}
if (torrent.Eta > 0)
{
return false;
return (false, default);
}
ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
return await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
return (await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!, StrikeType.Stalled), DeleteReason.Stalled);
}
private async Task<TorrentInfo?> GetTorrentAsync(string hash)
@@ -498,20 +534,8 @@ public class TransmissionService : DownloadService, ITransmissionService
if (_torrentsCache is null || torrent is null)
{
string[] fields = [
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER
];
// refresh cache
_torrentsCache = (await _client.TorrentGetAsync(fields))
_torrentsCache = (await _client.TorrentGetAsync(Fields))
?.Torrents;
}
code/Infrastructure/Verticals/Jobs/JobChainingListener.cs
+4 -2
View File
@@ -4,10 +4,12 @@ namespace Infrastructure.Verticals.Jobs;
public class JobChainingListener : IJobListener
{
private readonly string _firstJobName;
private readonly string _nextJobName;
public JobChainingListener(string nextJobName)
public JobChainingListener(string firstJobName, string nextJobName)
{
_firstJobName = firstJobName;
_nextJobName = nextJobName;
}
@@ -19,7 +21,7 @@ public class JobChainingListener : IJobListener
public async Task JobWasExecuted(IJobExecutionContext context, JobExecutionException? jobException, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(_nextJobName) || context.JobDetail.Key.Name == _nextJobName)
if (string.IsNullOrEmpty(_nextJobName) || context.JobDetail.Key.Name == _nextJobName || context.JobDetail.Key.Name != _firstJobName)
{
return;
}
code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs
+63 -37
View File
@@ -1,4 +1,4 @@
using System.Globalization;
using System.Globalization;
using Common.Attributes;
using Common.Configuration.Arr;
using Domain.Enums;
@@ -32,7 +32,7 @@ public class NotificationPublisher : INotificationPublisher
QueueRecord record = ContextProvider.Get<QueueRecord>(nameof(QueueRecord));
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
Uri instanceUrl = ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url));
Uri imageUrl = GetImageFromContext(record, instanceType);
Uri? imageUrl = GetImageFromContext(record, instanceType);
ArrNotification notification = new()
{
@@ -63,42 +63,59 @@ public class NotificationPublisher : INotificationPublisher
public virtual async Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason)
{
QueueRecord record = ContextProvider.Get<QueueRecord>(nameof(QueueRecord));
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
Uri instanceUrl = ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url));
Uri imageUrl = GetImageFromContext(record, instanceType);
QueueItemDeletedNotification notification = new()
try
{
InstanceType = instanceType,
InstanceUrl = instanceUrl,
Hash = record.DownloadId.ToLowerInvariant(),
Title = $"Deleting item from queue with reason: {reason}",
Description = record.Title,
Image = imageUrl,
Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }]
};
await NotifyInternal(notification);
QueueRecord record = ContextProvider.Get<QueueRecord>(nameof(QueueRecord));
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
Uri instanceUrl = ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url));
Uri? imageUrl = GetImageFromContext(record, instanceType);
QueueItemDeletedNotification notification = new()
{
InstanceType = instanceType,
InstanceUrl = instanceUrl,
Hash = record.DownloadId.ToLowerInvariant(),
Title = $"Deleting item from queue with reason: {reason}",
Description = record.Title,
Image = imageUrl,
Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }]
};
await NotifyInternal(notification);
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to notify queue item deleted");
}
}
public virtual async Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason)
{
DownloadCleanedNotification notification = new()
try
{
Title = $"Cleaned item from download client with reason: {reason}",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Category", Text = categoryName.ToLowerInvariant() },
new() { Title = "Ratio", Text = $"{ratio.ToString(CultureInfo.InvariantCulture)}%" },
new() { Title = "Seeding hours", Text = $"{Math.Round(seedingTime.TotalHours, 0).ToString(CultureInfo.InvariantCulture)}h" }
],
Level = NotificationLevel.Important
};
DownloadCleanedNotification notification = new()
{
Title = $"Cleaned item from download client with reason: {reason}",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Category", Text = categoryName.ToLowerInvariant() },
new() { Title = "Ratio", Text = $"{ratio.ToString(CultureInfo.InvariantCulture)}%" },
new()
{
Title = "Seeding hours", Text = $"{Math.Round(seedingTime.TotalHours, 0).ToString(CultureInfo.InvariantCulture)}h"
}
],
Level = NotificationLevel.Important
};
await NotifyInternal(notification);
await NotifyInternal(notification);
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to notify download cleaned");
}
}
[DryRunSafeguard]
@@ -131,12 +148,21 @@ public class NotificationPublisher : INotificationPublisher
return _messageBus.Publish(message);
}
private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) =>
instanceType switch
private Uri? GetImageFromContext(QueueRecord record, InstanceType instanceType)
{
Uri? image = instanceType switch
{
InstanceType.Sonarr => record.Series!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl,
InstanceType.Radarr => record.Movie!.Images.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl,
InstanceType.Lidarr => record.Album!.Images.FirstOrDefault(x => x.CoverType == "cover")?.Url,
InstanceType.Sonarr => record.Series?.Images?.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl,
InstanceType.Radarr => record.Movie?.Images?.FirstOrDefault(x => x.CoverType == "poster")?.RemoteUrl,
InstanceType.Lidarr => record.Album?.Images?.FirstOrDefault(x => x.CoverType == "cover")?.Url,
_ => throw new ArgumentOutOfRangeException(nameof(instanceType))
} ?? throw new Exception("failed to get image url from context");
};
if (image is null)
{
_logger.LogWarning("no poster found for {title}", record.Title);
}
return image;
}
}
code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs
+24 -5
View File
@@ -4,6 +4,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Providers;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context;
@@ -19,7 +20,8 @@ namespace Infrastructure.Verticals.QueueCleaner;
public sealed class QueueCleaner : GenericHandler
{
private readonly QueueCleanerConfig _config;
private readonly IgnoredDownloadsProvider<QueueCleanerConfig> _ignoredDownloadsProvider;
public QueueCleaner(
ILogger<QueueCleaner> logger,
IOptions<QueueCleanerConfig> config,
@@ -32,7 +34,8 @@ public sealed class QueueCleaner : GenericHandler
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory,
INotificationPublisher notifier
INotificationPublisher notifier,
IgnoredDownloadsProvider<QueueCleanerConfig> ignoredDownloadsProvider
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
@@ -42,10 +45,14 @@ public sealed class QueueCleaner : GenericHandler
)
{
_config = config.Value;
_config.Validate();
_ignoredDownloadsProvider = ignoredDownloadsProvider;
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
@@ -75,15 +82,27 @@ public sealed class QueueCleaner : GenericHandler
continue;
}
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
{
_logger.LogInformation("skip | {title} | ignored", record.Title);
continue;
}
// push record to context
ContextProvider.Set(nameof(QueueRecord), record);
StalledResult stalledCheckResult = new();
if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent")
if (record.Protocol is "torrent" && _downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.Disabled)
{
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None)
{
_logger.LogWarning("skip | download client is not configured | {title}", record.Title);
continue;
}
// stalled download check
stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId);
stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads);
}
// failed import check
@@ -113,7 +132,7 @@ public sealed class QueueCleaner : GenericHandler
}
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, deleteReason);
await _notifier.NotifyQueueItemDeleted(removeFromClient, deleteReason);
}
});