Compare commits

...

12 Commits

Author SHA1 Message Date
Marius Nechifor e738ba2334 Fix queue items with no title not being processed (#54) 2025-02-02 18:20:42 +02:00
Marius Nechifor c813215f3e Add more Lidarr checks for failed imports (#48) 2025-01-28 19:10:07 +02:00
Flaminel 0f63a2d271 updated README 2025-01-26 01:36:08 +02:00
Marius Nechifor 133c34de53 Add option to reset stalled strikes on download progress (#50) 2025-01-25 03:27:40 +02:00
Flaminel a3ca735b12 updated deployment 2025-01-25 01:18:03 +02:00
Marius Nechifor 519ab6a0cd Fix strike defaults (#49) 2025-01-22 22:18:31 +02:00
Marius Nechifor 0c691a540a Add missing failed import status (#47) 2025-01-21 00:14:55 +02:00
Marius Nechifor 209f78717f Fix usenet usage (#46) 2025-01-18 19:12:28 +02:00
Flaminel a02be80ac1 updated README 2025-01-18 17:25:15 +02:00
Marius Nechifor 8a8b906b6f Add option to not remove private downloads from the download client (#45) 2025-01-18 17:20:23 +02:00
Marius Nechifor b88ddde417 Fix content blocker env var usage (#44) 2025-01-18 16:23:34 +02:00
Flaminel 666c2656ec added svg logo 2025-01-17 22:11:05 +02:00
27 changed files with 330 additions and 114 deletions
File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 112 KiB

+26 -16
View File
@@ -1,3 +1,5 @@
_Love this project? Give it a ⭐️ and let others know!_
# <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> cleanuperr # <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> cleanuperr
cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, cleanuperr can also trigger a search to replace the deleted shows/movies. cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, cleanuperr can also trigger a search to replace the deleted shows/movies.
@@ -106,13 +108,17 @@ services:
- QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
- QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
- QUEUECLEANER__STALLED_MAX_STRIKES=5 - QUEUECLEANER__STALLED_MAX_STRIKES=5
- QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=false
- QUEUECLEANER__STALLED_IGNORE_PRIVATE=false - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORE_PRIVATE=true - CONTENTBLOCKER__IGNORE_PRIVATE=false
- CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOAD_CLIENT=none - DOWNLOAD_CLIENT=none
# OR # OR
@@ -166,24 +172,28 @@ services:
| Variable | Required | Description | Default value | | Variable | Required | Description | Default value |
|---|---|---|---| |---|---|---|---|
| LOGGING__LOGLEVEL | No | Can be `Verbose`, `Debug`, `Information`, `Warning`, `Error` or `Fatal` | `Information` | | LOGGING__LOGLEVEL | No | Can be `Verbose`, `Debug`, `Information`, `Warning`, `Error` or `Fatal`. | `Information` |
| LOGGING__FILE__ENABLED | No | Enable or disable logging to file | false | | LOGGING__FILE__ENABLED | No | Enable or disable logging to file. | false |
| LOGGING__FILE__PATH | No | Directory where to save the log files | empty | | LOGGING__FILE__PATH | No | Directory where to save the log files. | empty |
| LOGGING__ENHANCED | No | Enhance logs whenever possible<br>A more detailed description is provided [here](variables.md#LOGGING__ENHANCED) | true | | LOGGING__ENHANCED | No | Enhance logs whenever possible.<br>A more detailed description is provided. [here](variables.md#LOGGING__ENHANCED) | true |
||||| |||||
| TRIGGERS__QUEUECLEANER | Yes if queue cleaner is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval<br>**Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`** | 0 0/5 * * * ? | | TRIGGERS__QUEUECLEANER | Yes if queue cleaner is enabled | - [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).<br>- Can be a max of 6h interval.<br>- **Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`**. | 0 0/5 * * * ? |
| TRIGGERS__CONTENTBLOCKER | Yes if content blocker is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval | 0 0/5 * * * ? | | TRIGGERS__CONTENTBLOCKER | Yes if content blocker is enabled | - [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).<br>- Can be a max of 6h interval. | 0 0/5 * * * ? |
||||| |||||
| QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true | | QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner. | true |
| QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true | | QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process. | true |
| QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed<br>0 means never | 0 | | QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | - After how many strikes should a failed import be removed.<br>- 0 means never. | 0 |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false | | QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers. | false |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed<br>If the specified message pattern is found, the item is skipped | empty | | QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE | No | - Whether to delete failed imports of private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
| QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed<br>0 means never | 0 | | QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | - First pattern to look for when an import is failed.<br>- If the specified message pattern is found, the item is skipped. | empty |
| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false | | QUEUECLEANER__STALLED_MAX_STRIKES | No | - After how many strikes should a stalled download be removed.<br>- 0 means never. | 0 |
| QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS | No | Whether to remove strikes if any download progress was made since last checked. | false |
| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers. | false |
| QUEUECLEANER__STALLED_DELETE_PRIVATE | No | - Whether to delete stalled private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
||||| |||||
| CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false | | CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker. | false |
| CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers | false | | CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers. | false |
| CONTENTBLOCKER__DELETE_PRIVATE | No | - Whether to delete private downloads that have all files blocked from the download client.<br>- Does not have any effect if `CONTENTBLOCKER__IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
</details> </details>
### Download client variables ### Download client variables
+25 -4
View File
@@ -11,7 +11,7 @@ deployment:
tag: latest tag: latest
env: env:
- name: LOGGING__LOGLEVEL - name: LOGGING__LOGLEVEL
value: Information value: Debug
- name: LOGGING__FILE__ENABLED - name: LOGGING__FILE__ENABLED
value: "true" value: "true"
- name: LOGGING__FILE__PATH - name: LOGGING__FILE__PATH
@@ -22,34 +22,55 @@ deployment:
value: 0 0/5 * * * ? value: 0 0/5 * * * ?
- name: TRIGGERS__CONTENTBLOCKER - name: TRIGGERS__CONTENTBLOCKER
value: 0 0/5 * * * ? value: 0 0/5 * * * ?
- name: QUEUECLEANER__ENABLED - name: QUEUECLEANER__ENABLED
value: "true" value: "true"
- name: QUEUECLEANER__RUNSEQUENTIALLY - name: QUEUECLEANER__RUNSEQUENTIALLY
value: "true" value: "true"
- name: QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES - name: QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES
value: "3" value: "3"
- name: QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE
value: "false"
- name: QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE
value: "false"
- name: QUEUECLEANER__STALLED_MAX_STRIKES - name: QUEUECLEANER__STALLED_MAX_STRIKES
value: "3" value: "3"
- name: QUEUECLEANER__STALLED_IGNORE_PRIVATE
value: "false"
- name: QUEUECLEANER__STALLED_DELETE_PRIVATE
value: "false"
- name: CONTENTBLOCKER__ENABLED - name: CONTENTBLOCKER__ENABLED
value: "true" value: "true"
- name: CONTENTBLOCKER__BLACKLIST__ENABLED - name: CONTENTBLOCKER__IGNORE_PRIVATE
value: "true" value: "true"
- name: CONTENTBLOCKER__BLACKLIST__PATH - name: CONTENTBLOCKER__DELETE_PRIVATE
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist value: "false"
- name: DOWNLOAD_CLIENT - name: DOWNLOAD_CLIENT
value: qbittorrent value: qbittorrent
- name: QBITTORRENT__URL - name: QBITTORRENT__URL
value: http://service.qbittorrent-videos.svc.cluster.local value: http://service.qbittorrent-videos.svc.cluster.local
- name: SONARR__ENABLED - name: SONARR__ENABLED
value: "true" value: "true"
- name: SONARR__SEARCHTYPE - name: SONARR__SEARCHTYPE
value: Episode value: Episode
- name: SONARR__BLOCK__TYPE
value: blacklist
- name: SONARR__BLOCK__PATH
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
- name: SONARR__INSTANCES__0__URL - name: SONARR__INSTANCES__0__URL
value: http://service.sonarr-low-res.svc.cluster.local value: http://service.sonarr-low-res.svc.cluster.local
- name: SONARR__INSTANCES__1__URL - name: SONARR__INSTANCES__1__URL
value: http://service.sonarr-high-res.svc.cluster.local value: http://service.sonarr-high-res.svc.cluster.local
- name: RADARR__ENABLED - name: RADARR__ENABLED
value: "true" value: "true"
- name: RADARR__BLOCK__TYPE
value: blacklist
- name: RADARR__BLOCK__PATH
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
- name: RADARR__INSTANCES__0__URL - name: RADARR__INSTANCES__0__URL
value: http://service.radarr-low-res.svc.cluster.local value: http://service.radarr-low-res.svc.cluster.local
- name: RADARR__INSTANCES__1__URL - name: RADARR__INSTANCES__1__URL
@@ -1,4 +1,4 @@
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
namespace Common.Configuration.ContentBlocker; namespace Common.Configuration.ContentBlocker;
@@ -11,6 +11,9 @@ public sealed record ContentBlockerConfig : IJobConfig
[ConfigurationKeyName("IGNORE_PRIVATE")] [ConfigurationKeyName("IGNORE_PRIVATE")]
public bool IgnorePrivate { get; init; } public bool IgnorePrivate { get; init; }
[ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; init; }
public void Validate() public void Validate()
{ {
} }
@@ -16,14 +16,23 @@ public sealed record QueueCleanerConfig : IJobConfig
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")] [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")]
public bool ImportFailedIgnorePrivate { get; init; } public bool ImportFailedIgnorePrivate { get; init; }
[ConfigurationKeyName("IMPORT_FAILED_DELETE_PRIVATE")]
public bool ImportFailedDeletePrivate { get; init; }
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")] [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")]
public List<string>? ImportFailedIgnorePatterns { get; init; } public List<string>? ImportFailedIgnorePatterns { get; init; }
[ConfigurationKeyName("STALLED_MAX_STRIKES")] [ConfigurationKeyName("STALLED_MAX_STRIKES")]
public ushort StalledMaxStrikes { get; init; } public ushort StalledMaxStrikes { get; init; }
[ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")]
public bool StalledResetStrikesOnProgress { get; init; }
[ConfigurationKeyName("STALLED_IGNORE_PRIVATE")] [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
public bool StalledIgnorePrivate { get; init; } public bool StalledIgnorePrivate { get; init; }
[ConfigurationKeyName("STALLED_DELETE_PRIVATE")]
public bool StalledDeletePrivate { get; init; }
public void Validate() public void Validate()
{ {
+9
View File
@@ -0,0 +1,9 @@
namespace Domain.Models.Cache;
public sealed record CacheItem
{
/// <summary>
/// The amount of bytes that have been downloaded.
/// </summary>
public long Downloaded { get; set; }
}
@@ -1,4 +1,6 @@
namespace Domain.Models.Deluge.Response; using Newtonsoft.Json;
namespace Domain.Models.Deluge.Response;
public sealed record TorrentStatus public sealed record TorrentStatus
{ {
@@ -11,4 +13,7 @@ public sealed record TorrentStatus
public ulong Eta { get; init; } public ulong Eta { get; init; }
public bool Private { get; init; } public bool Private { get; init; }
[JsonProperty("total_done")]
public long TotalDone { get; init; }
} }
+6 -2
View File
@@ -15,18 +15,22 @@
}, },
"ContentBlocker": { "ContentBlocker": {
"Enabled": true, "Enabled": true,
"IGNORE_PRIVATE": true "IGNORE_PRIVATE": true,
"DELETE_PRIVATE": false
}, },
"QueueCleaner": { "QueueCleaner": {
"Enabled": true, "Enabled": true,
"RunSequentially": true, "RunSequentially": true,
"IMPORT_FAILED_MAX_STRIKES": 5, "IMPORT_FAILED_MAX_STRIKES": 5,
"IMPORT_FAILED_IGNORE_PRIVATE": true, "IMPORT_FAILED_IGNORE_PRIVATE": true,
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [ "IMPORT_FAILED_IGNORE_PATTERNS": [
"file is a sample" "file is a sample"
], ],
"STALLED_MAX_STRIKES": 5, "STALLED_MAX_STRIKES": 5,
"STALLED_IGNORE_PRIVATE": true "STALLED_RESET_STRIKES_ON_PROGRESS": true,
"STALLED_IGNORE_PRIVATE": true,
"STALLED_DELETE_PRIVATE": false
}, },
"DOWNLOAD_CLIENT": "qbittorrent", "DOWNLOAD_CLIENT": "qbittorrent",
"qBittorrent": { "qBittorrent": {
+6 -3
View File
@@ -20,11 +20,14 @@
"QueueCleaner": { "QueueCleaner": {
"Enabled": true, "Enabled": true,
"RunSequentially": true, "RunSequentially": true,
"IMPORT_FAILED_MAX_STRIKES": 5, "IMPORT_FAILED_MAX_STRIKES": 0,
"IMPORT_FAILED_IGNORE_PRIVATE": false, "IMPORT_FAILED_IGNORE_PRIVATE": false,
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [], "IMPORT_FAILED_IGNORE_PATTERNS": [],
"STALLED_MAX_STRIKES": 5, "STALLED_MAX_STRIKES": 0,
"STALLED_IGNORE_PRIVATE": false "STALLED_RESET_STRIKES_ON_PROGRESS": false,
"STALLED_IGNORE_PRIVATE": false,
"STALLED_DELETE_PRIVATE": false
}, },
"DOWNLOAD_CLIENT": "none", "DOWNLOAD_CLIENT": "none",
"qBittorrent": { "qBittorrent": {
+14
View File
@@ -0,0 +1,14 @@
using Domain.Enums;
namespace Infrastructure.Helpers;
public static class CacheKeys
{
public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}";
public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type";
public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
public static string Item(string hash) => $"item_{hash}";
}
+20 -14
View File
@@ -1,4 +1,4 @@
using Common.Configuration.Arr; using Common.Configuration.Arr;
using Common.Configuration.Logging; using Common.Configuration.Logging;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Common.Helpers; using Common.Helpers;
@@ -66,7 +66,7 @@ public abstract class ArrClient
return queueResponse; return queueResponse;
} }
public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload) public virtual bool ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload)
{ {
if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload) if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
{ {
@@ -81,8 +81,14 @@ public abstract class ArrClient
.Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase); .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
bool isImportPending() => record.TrackedDownloadState bool isImportPending() => record.TrackedDownloadState
.Equals("importPending", StringComparison.InvariantCultureIgnoreCase); .Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
bool isImportFailed() => record.TrackedDownloadState
.Equals("importFailed", StringComparison.InvariantCultureIgnoreCase);
bool isFailedLidarr() => instanceType is InstanceType.Lidarr &&
(record.Status.Equals("failed", StringComparison.InvariantCultureIgnoreCase) ||
record.Status.Equals("completed", StringComparison.InvariantCultureIgnoreCase)) &&
hasWarn();
if (hasWarn() && (isImportBlocked() || isImportPending())) if (hasWarn() && (isImportBlocked() || isImportPending() || isImportFailed()) || isFailedLidarr())
{ {
if (HasIgnoredPatterns(record)) if (HasIgnoredPatterns(record))
{ {
@@ -101,9 +107,9 @@ public abstract class ArrClient
return false; return false;
} }
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record) public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient)
{ {
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id)); Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id, removeFromClient));
using HttpRequestMessage request = new(HttpMethod.Delete, uri); using HttpRequestMessage request = new(HttpMethod.Delete, uri);
SetApiKey(request, arrInstance.ApiKey); SetApiKey(request, arrInstance.ApiKey);
@@ -113,8 +119,14 @@ public abstract class ArrClient
try try
{ {
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
_logger.LogInformation("queue item deleted | {url} | {title}", arrInstance.Url, record.Title); _logger.LogInformation(
removeFromClient
? "queue item deleted | {url} | {title}"
: "queue item removed from arr | {url} | {title}",
arrInstance.Url,
record.Title
);
} }
catch catch
{ {
@@ -133,18 +145,12 @@ public abstract class ArrClient
return false; return false;
} }
if (record.DownloadId.Equals(record.Title, StringComparison.InvariantCultureIgnoreCase))
{
_logger.LogDebug("skip | item is not ready yet | {title}", record.Title);
return false;
}
return true; return true;
} }
protected abstract string GetQueueUrlPath(int page); protected abstract string GetQueueUrlPath(int page);
protected abstract string GetQueueDeleteUrlPath(long recordId); protected abstract string GetQueueDeleteUrlPath(long recordId, bool removeFromClient);
protected virtual void SetApiKey(HttpRequestMessage request, string apiKey) protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
{ {
@@ -29,9 +29,13 @@ public sealed class LidarrClient : ArrClient
return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true"; return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
} }
protected override string GetQueueDeleteUrlPath(long recordId) protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{ {
return $"/api/v1/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false"; string path = $"/api/v1/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
} }
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items) public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
@@ -29,9 +29,13 @@ public sealed class RadarrClient : ArrClient
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true"; return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
} }
protected override string GetQueueDeleteUrlPath(long recordId) protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{ {
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false"; string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
} }
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items) public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
@@ -29,9 +29,13 @@ public sealed class SonarrClient : ArrClient
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true"; return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true";
} }
protected override string GetQueueDeleteUrlPath(long recordId) protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{ {
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false"; string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
} }
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items) public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
@@ -5,6 +5,7 @@ using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Helpers; using Common.Helpers;
using Domain.Enums; using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -21,10 +22,6 @@ public sealed class BlocklistProvider
private readonly IMemoryCache _cache; private readonly IMemoryCache _cache;
private bool _initialized; private bool _initialized;
private const string Type = "type";
private const string Patterns = "patterns";
private const string Regexes = "regexes";
public BlocklistProvider( public BlocklistProvider(
ILogger<BlocklistProvider> logger, ILogger<BlocklistProvider> logger,
IOptions<SonarrConfig> sonarrConfig, IOptions<SonarrConfig> sonarrConfig,
@@ -67,21 +64,21 @@ public sealed class BlocklistProvider
public BlocklistType GetBlocklistType(InstanceType instanceType) public BlocklistType GetBlocklistType(InstanceType instanceType)
{ {
_cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType); _cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType);
return blocklistType ?? BlocklistType.Blacklist; return blocklistType ?? BlocklistType.Blacklist;
} }
public ConcurrentBag<string> GetPatterns(InstanceType instanceType) public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
{ {
_cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns); _cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag<string>? patterns);
return patterns ?? []; return patterns ?? [];
} }
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType) public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
{ {
_cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes); _cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
return regexes ?? []; return regexes ?? [];
} }
@@ -124,9 +121,9 @@ public sealed class BlocklistProvider
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime); TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set($"{instanceType.ToString()}_{Type}", blocklistType); _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType);
_cache.Set($"{instanceType.ToString()}_{Patterns}", patterns); _cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
_cache.Set($"{instanceType.ToString()}_{Regexes}", regexes); _cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
_logger.LogDebug("loaded {count} patterns", patterns.Count); _logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count); _logger.LogDebug("loaded {count} regexes", regexes.Count);
@@ -17,10 +17,12 @@ namespace Infrastructure.Verticals.ContentBlocker;
public sealed class ContentBlocker : GenericHandler public sealed class ContentBlocker : GenericHandler
{ {
private readonly ContentBlockerConfig _config;
private readonly BlocklistProvider _blocklistProvider; private readonly BlocklistProvider _blocklistProvider;
public ContentBlocker( public ContentBlocker(
ILogger<ContentBlocker> logger, ILogger<ContentBlocker> logger,
IOptions<ContentBlockerConfig> config,
IOptions<DownloadClientConfig> downloadClientConfig, IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig, IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig, IOptions<RadarrConfig> radarrConfig,
@@ -38,6 +40,7 @@ public sealed class ContentBlocker : GenericHandler
arrArrQueueIterator, downloadServiceFactory arrArrQueueIterator, downloadServiceFactory
) )
{ {
_config = config.Value;
_blocklistProvider = blocklistProvider; _blocklistProvider = blocklistProvider;
} }
@@ -96,7 +99,10 @@ public sealed class ContentBlocker : GenericHandler
_logger.LogDebug("searching unwanted files for {title}", record.Title); _logger.LogDebug("searching unwanted files for {title}", record.Title);
if (!await _downloadService.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes)) BlockFilesResult result = await _downloadService
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes);
if (!result.ShouldRemove)
{ {
continue; continue;
} }
@@ -104,7 +110,15 @@ public sealed class ContentBlocker : GenericHandler
_logger.LogDebug("all files are marked as unwanted | {hash}", record.Title); _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1)); itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
bool removeFromClient = true;
if (result.IsPrivate && !_config.DeletePrivate)
{
removeFromClient = false;
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
} }
}); });
@@ -0,0 +1,14 @@
namespace Infrastructure.Verticals.DownloadClient;
public sealed record BlockFilesResult
{
/// <summary>
/// True if the download should be removed; otherwise false.
/// </summary>
public bool ShouldRemove { get; set; }
/// <summary>
/// True if the download is private; otherwise false.
/// </summary>
public bool IsPrivate { get; set; }
}
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Deluge.Response; using Domain.Models.Deluge.Response;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -20,9 +21,11 @@ public sealed class DelugeService : DownloadServiceBase
IOptions<DelugeConfig> config, IOptions<DelugeConfig> config,
IHttpClientFactory httpClientFactory, IHttpClientFactory httpClientFactory,
IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator, FilenameEvaluator filenameEvaluator,
Striker striker Striker striker
) : base(logger, queueCleanerConfig, filenameEvaluator, striker) ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{ {
config.Value.Validate(); config.Value.Validate();
_client = new (config, httpClientFactory); _client = new (config, httpClientFactory);
@@ -34,12 +37,12 @@ public sealed class DelugeService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{ {
hash = hash.ToLowerInvariant(); hash = hash.ToLowerInvariant();
DelugeContents? contents = null; DelugeContents? contents = null;
RemoveResult result = new(); StalledResult result = new();
TorrentStatus? status = await GetTorrentStatus(hash); TorrentStatus? status = await GetTorrentStatus(hash);
@@ -75,7 +78,7 @@ public sealed class DelugeService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash, string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
@@ -85,18 +88,21 @@ public sealed class DelugeService : DownloadServiceBase
hash = hash.ToLowerInvariant(); hash = hash.ToLowerInvariant();
TorrentStatus? status = await GetTorrentStatus(hash); TorrentStatus? status = await GetTorrentStatus(hash);
BlockFilesResult result = new();
if (status?.Hash is null) if (status?.Hash is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return false; return result;
} }
result.IsPrivate = status.Private;
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private) if (_contentBlockerConfig.IgnorePrivate && status.Private)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", status.Name); _logger.LogDebug("skip files check | download is private | {name}", status.Name);
return false; return result;
} }
DelugeContents? contents = null; DelugeContents? contents = null;
@@ -112,7 +118,7 @@ public sealed class DelugeService : DownloadServiceBase
if (contents is null) if (contents is null)
{ {
return false; return result;
} }
Dictionary<int, int> priorities = []; Dictionary<int, int> priorities = [];
@@ -143,7 +149,7 @@ public sealed class DelugeService : DownloadServiceBase
if (!hasPriorityUpdates) if (!hasPriorityUpdates)
{ {
return false; return result;
} }
_logger.LogDebug("changing priorities | torrent {hash}", hash); _logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -156,12 +162,14 @@ public sealed class DelugeService : DownloadServiceBase
if (totalUnwantedFiles == totalFiles) if (totalUnwantedFiles == totalFiles)
{ {
// Skip marking files as unwanted. The download will be removed completely. // Skip marking files as unwanted. The download will be removed completely.
return true; result.ShouldRemove = true;
return result;
} }
await _client.ChangeFilesPriority(hash, sortedPriorities); await _client.ChangeFilesPriority(hash, sortedPriorities);
return false; return result;
} }
/// <inheritdoc/> /// <inheritdoc/>
@@ -195,6 +203,8 @@ public sealed class DelugeService : DownloadServiceBase
{ {
return false; return false;
} }
ResetStrikesOnProgress(status.Hash!, status.TotalDone);
return StrikeAndCheckLimit(status.Hash!, status.Name!); return StrikeAndCheckLimit(status.Hash!, status.Name!);
} }
@@ -204,7 +214,7 @@ public sealed class DelugeService : DownloadServiceBase
return await _client.SendRequest<TorrentStatus?>( return await _client.SendRequest<TorrentStatus?>(
"web.get_torrent_status", "web.get_torrent_status",
hash, hash,
new[] { "hash", "state", "name", "eta", "private" } new[] { "hash", "state", "name", "eta", "private", "total_done" }
); );
} }
@@ -2,9 +2,13 @@
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker; using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -14,30 +18,39 @@ public abstract class DownloadServiceBase : IDownloadService
{ {
protected readonly ILogger<DownloadServiceBase> _logger; protected readonly ILogger<DownloadServiceBase> _logger;
protected readonly QueueCleanerConfig _queueCleanerConfig; protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly IMemoryCache _cache;
protected readonly FilenameEvaluator _filenameEvaluator; protected readonly FilenameEvaluator _filenameEvaluator;
protected readonly Striker _striker; protected readonly Striker _striker;
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected DownloadServiceBase( protected DownloadServiceBase(
ILogger<DownloadServiceBase> logger, ILogger<DownloadServiceBase> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator, FilenameEvaluator filenameEvaluator,
Striker striker Striker striker
) )
{ {
_logger = logger; _logger = logger;
_queueCleanerConfig = queueCleanerConfig.Value; _queueCleanerConfig = queueCleanerConfig.Value;
_contentBlockerConfig = contentBlockerConfig.Value;
_cache = cache;
_filenameEvaluator = filenameEvaluator; _filenameEvaluator = filenameEvaluator;
_striker = striker; _striker = striker;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
} }
public abstract void Dispose(); public abstract void Dispose();
public abstract Task LoginAsync(); public abstract Task LoginAsync();
public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash); public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task<bool> BlockUnwantedFilesAsync( public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash, string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
@@ -47,6 +60,23 @@ public abstract class DownloadServiceBase : IDownloadService
/// <inheritdoc/> /// <inheritdoc/>
public abstract Task Delete(string hash); public abstract Task Delete(string hash);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
{
return;
}
if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
{
// cache item found
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
_logger.LogDebug("resetting strikes for {hash} due to progress", hash);
}
_cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
}
/// <summary> /// <summary>
/// Strikes an item and checks if the limit has been reached. /// Strikes an item and checks if the limit has been reached.
/// </summary> /// </summary>
@@ -4,6 +4,7 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner; using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -11,7 +12,7 @@ namespace Infrastructure.Verticals.DownloadClient;
public sealed class DummyDownloadService : DownloadServiceBase public sealed class DummyDownloadService : DownloadServiceBase
{ {
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, filenameEvaluator, striker) public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{ {
} }
@@ -24,12 +25,12 @@ public sealed class DummyDownloadService : DownloadServiceBase
return Task.CompletedTask; return Task.CompletedTask;
} }
public override Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash) public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
public override Task<bool> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }
@@ -12,7 +12,7 @@ public interface IDownloadService : IDisposable
/// Checks whether the download should be removed from the *arr queue. /// Checks whether the download should be removed from the *arr queue.
/// </summary> /// </summary>
/// <param name="hash">The download hash.</param> /// <param name="hash">The download hash.</param>
public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash); public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <summary> /// <summary>
/// Blocks unwanted files from being fully downloaded. /// Blocks unwanted files from being fully downloaded.
@@ -22,7 +22,7 @@ public interface IDownloadService : IDisposable
/// <param name="patterns">The patterns to test the files against.</param> /// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param> /// <param name="regexes">The regexes to test the files against.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns> /// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<bool> BlockUnwantedFilesAsync( public Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash, string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers; using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using QBittorrent.Client; using QBittorrent.Client;
@@ -22,9 +23,11 @@ public sealed class QBitService : DownloadServiceBase
IHttpClientFactory httpClientFactory, IHttpClientFactory httpClientFactory,
IOptions<QBitConfig> config, IOptions<QBitConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator, FilenameEvaluator filenameEvaluator,
Striker striker Striker striker
) : base(logger, queueCleanerConfig, filenameEvaluator, striker) ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{ {
_config = config.Value; _config = config.Value;
_config.Validate(); _config.Validate();
@@ -42,9 +45,9 @@ public sealed class QBitService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{ {
RemoveResult result = new(); StalledResult result = new();
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault(); .FirstOrDefault();
@@ -88,7 +91,7 @@ public sealed class QBitService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash, string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
@@ -97,11 +100,12 @@ public sealed class QBitService : DownloadServiceBase
{ {
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault(); .FirstOrDefault();
BlockFilesResult result = new();
if (torrent is null) if (torrent is null)
{ {
_logger.LogDebug("failed to find torrent {hash} in the download client", hash); _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return false; return result;
} }
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
@@ -109,25 +113,27 @@ public sealed class QBitService : DownloadServiceBase
if (torrentProperties is null) if (torrentProperties is null)
{ {
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash); _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return false; return result;
} }
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue) bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue; && boolValue;
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name); _logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return false; return result;
} }
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash); IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files is null) if (files is null)
{ {
return false; return result;
} }
List<int> unwantedFiles = []; List<int> unwantedFiles = [];
@@ -161,13 +167,15 @@ public sealed class QBitService : DownloadServiceBase
if (unwantedFiles.Count is 0) if (unwantedFiles.Count is 0)
{ {
return false; return result;
} }
if (totalUnwantedFiles == totalFiles) if (totalUnwantedFiles == totalFiles)
{ {
// Skip marking files as unwanted. The download will be removed completely. // Skip marking files as unwanted. The download will be removed completely.
return true; result.ShouldRemove = true;
return result;
} }
foreach (int fileIndex in unwantedFiles) foreach (int fileIndex in unwantedFiles)
@@ -175,7 +183,7 @@ public sealed class QBitService : DownloadServiceBase
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
} }
return false; return result;
} }
/// <inheritdoc/> /// <inheritdoc/>
@@ -210,6 +218,8 @@ public sealed class QBitService : DownloadServiceBase
return false; return false;
} }
ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return StrikeAndCheckLimit(torrent.Hash, torrent.Name); return StrikeAndCheckLimit(torrent.Hash, torrent.Name);
} }
} }
@@ -1,6 +1,6 @@
namespace Infrastructure.Verticals.DownloadClient; namespace Infrastructure.Verticals.DownloadClient;
public sealed record RemoveResult public sealed record StalledResult
{ {
/// <summary> /// <summary>
/// True if the download should be removed; otherwise false. /// True if the download should be removed; otherwise false.
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers; using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Transmission.API.RPC; using Transmission.API.RPC;
@@ -25,9 +26,11 @@ public sealed class TransmissionService : DownloadServiceBase
ILogger<TransmissionService> logger, ILogger<TransmissionService> logger,
IOptions<TransmissionConfig> config, IOptions<TransmissionConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator, FilenameEvaluator filenameEvaluator,
Striker striker Striker striker
) : base(logger, queueCleanerConfig, filenameEvaluator, striker) ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{ {
_config = config.Value; _config = config.Value;
_config.Validate(); _config.Validate();
@@ -45,9 +48,9 @@ public sealed class TransmissionService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash) public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{ {
RemoveResult result = new(); StalledResult result = new();
TorrentInfo? torrent = await GetTorrentAsync(hash); TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent is null) if (torrent is null)
@@ -81,7 +84,7 @@ public sealed class TransmissionService : DownloadServiceBase
} }
/// <inheritdoc/> /// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync( public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash, string hash,
BlocklistType blocklistType, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<string> patterns,
@@ -89,17 +92,21 @@ public sealed class TransmissionService : DownloadServiceBase
) )
{ {
TorrentInfo? torrent = await GetTorrentAsync(hash); TorrentInfo? torrent = await GetTorrentAsync(hash);
BlockFilesResult result = new();
if (torrent?.FileStats is null || torrent.Files is null) if (torrent?.FileStats is null || torrent.Files is null)
{ {
return false; return result;
} }
bool isPrivate = torrent.IsPrivate ?? false;
result.IsPrivate = isPrivate;
if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false)) if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{ {
// ignore private trackers // ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name); _logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return false; return result;
} }
List<long> unwantedFiles = []; List<long> unwantedFiles = [];
@@ -133,13 +140,15 @@ public sealed class TransmissionService : DownloadServiceBase
if (unwantedFiles.Count is 0) if (unwantedFiles.Count is 0)
{ {
return false; return result;
} }
if (totalUnwantedFiles == totalFiles) if (totalUnwantedFiles == totalFiles)
{ {
// Skip marking files as unwanted. The download will be removed completely. // Skip marking files as unwanted. The download will be removed completely.
return true; result.ShouldRemove = true;
return result;
} }
_logger.LogDebug("changing priorities | torrent {hash}", hash); _logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -150,7 +159,7 @@ public sealed class TransmissionService : DownloadServiceBase
FilesUnwanted = unwantedFiles.ToArray(), FilesUnwanted = unwantedFiles.ToArray(),
}); });
return false; return result;
} }
public override async Task Delete(string hash) public override async Task Delete(string hash)
@@ -193,6 +202,8 @@ public sealed class TransmissionService : DownloadServiceBase
{ {
return false; return false;
} }
ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!); return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
} }
@@ -212,7 +223,8 @@ public sealed class TransmissionService : DownloadServiceBase
TorrentFields.ETA, TorrentFields.ETA,
TorrentFields.NAME, TorrentFields.NAME,
TorrentFields.STATUS, TorrentFields.STATUS,
TorrentFields.IS_PRIVATE TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER
]; ];
// refresh cache // refresh cache
@@ -1,5 +1,6 @@
using Common.Helpers; using Common.Helpers;
using Domain.Enums; using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
@@ -26,7 +27,7 @@ public class Striker
return false; return false;
} }
string key = $"{strikeType.ToString()}_{hash}"; string key = CacheKeys.Strike(strikeType, hash);
if (!_cache.TryGetValue(key, out int? strikeCount)) if (!_cache.TryGetValue(key, out int? strikeCount))
{ {
@@ -1,5 +1,6 @@
using Common.Configuration.Arr; using Common.Configuration.Arr;
using Common.Configuration.DownloadClient; using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Domain.Enums; using Domain.Enums;
using Domain.Models.Arr; using Domain.Models.Arr;
using Domain.Models.Arr.Queue; using Domain.Models.Arr.Queue;
@@ -14,8 +15,11 @@ namespace Infrastructure.Verticals.QueueCleaner;
public sealed class QueueCleaner : GenericHandler public sealed class QueueCleaner : GenericHandler
{ {
private readonly QueueCleanerConfig _config;
public QueueCleaner( public QueueCleaner(
ILogger<QueueCleaner> logger, ILogger<QueueCleaner> logger,
IOptions<QueueCleanerConfig> config,
IOptions<DownloadClientConfig> downloadClientConfig, IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig, IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig, IOptions<RadarrConfig> radarrConfig,
@@ -32,6 +36,7 @@ public sealed class QueueCleaner : GenericHandler
arrArrQueueIterator, downloadServiceFactory arrArrQueueIterator, downloadServiceFactory
) )
{ {
_config = config.Value;
} }
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
@@ -56,33 +61,46 @@ public sealed class QueueCleaner : GenericHandler
QueueRecord record = group.First(); QueueRecord record = group.First();
if (record.Protocol is not "torrent")
{
continue;
}
if (!arrClient.IsRecordValid(record)) if (!arrClient.IsRecordValid(record))
{ {
continue; continue;
} }
RemoveResult removeResult = new(); StalledResult stalledCheckResult = new();
if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None) if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent")
{ {
removeResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); // stalled download check
stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId);
} }
bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record, removeResult.IsPrivate); // failed import check
bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(instanceType, record, stalledCheckResult.IsPrivate);
if (!shouldRemoveFromArr && !removeResult.ShouldRemove) if (!shouldRemoveFromArr && !stalledCheckResult.ShouldRemove)
{ {
_logger.LogInformation("skip | {title}", record.Title); _logger.LogInformation("skip | {title}", record.Title);
continue; continue;
} }
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1)); itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
bool removeFromClient = true;
if (stalledCheckResult.IsPrivate)
{
if (stalledCheckResult.ShouldRemove && !_config.StalledDeletePrivate)
{
removeFromClient = false;
}
if (shouldRemoveFromArr && !_config.ImportFailedDeletePrivate)
{
removeFromClient = false;
}
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
} }
}); });
+3
View File
@@ -186,12 +186,15 @@ services:
- QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true
- QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample
- QUEUECLEANER__STALLED_MAX_STRIKES=5 - QUEUECLEANER__STALLED_MAX_STRIKES=5
- QUEUECLEANER__STALLED_IGNORE_PRIVATE=true - QUEUECLEANER__STALLED_IGNORE_PRIVATE=true
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORE_PRIVATE=true - CONTENTBLOCKER__IGNORE_PRIVATE=true
- CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOAD_CLIENT=qbittorrent - DOWNLOAD_CLIENT=qbittorrent
- QBITTORRENT__URL=http://qbittorrent:8080 - QBITTORRENT__URL=http://qbittorrent:8080