public/cleanuperr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add option to reset stalled strikes on download progress (#50)

Browse Source
This commit is contained in:
Marius Nechifor
2025-01-25 03:27:40 +02:00
committed by GitHub
parent a3ca735b12
commit 133c34de53
14 changed files with 92 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs
+3
View File
@@ -25,6 +25,9 @@ public sealed record QueueCleanerConfig : IJobConfig
[ConfigurationKeyName("STALLED_MAX_STRIKES")]
public ushort StalledMaxStrikes { get; init; }
[ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")]
public bool StalledResetStrikesOnProgress { get; init; }
[ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
public bool StalledIgnorePrivate { get; init; }
code/Domain/Models/Cache/CacheItem.cs
+9
View File
@@ -0,0 +1,9 @@
namespace Domain.Models.Cache;
public sealed record CacheItem
{
/// <summary>
/// The amount of bytes that have been downloaded.
/// </summary>
public long Downloaded { get; set; }
}
code/Domain/Models/Deluge/Response/TorrentStatus.cs
+6 -1
View File
@@ -1,4 +1,6 @@
namespace Domain.Models.Deluge.Response;
using Newtonsoft.Json;
namespace Domain.Models.Deluge.Response;
public sealed record TorrentStatus
{
@@ -11,4 +13,7 @@ public sealed record TorrentStatus
public ulong Eta { get; init; }
public bool Private { get; init; }
[JsonProperty("total_done")]
public long TotalDone { get; init; }
}
code/Executable/appsettings.Development.json
+1
View File
@@ -28,6 +28,7 @@
"file is a sample"
],
"STALLED_MAX_STRIKES": 5,
"STALLED_RESET_STRIKES_ON_PROGRESS": true,
"STALLED_IGNORE_PRIVATE": true,
"STALLED_DELETE_PRIVATE": false
},
code/Executable/appsettings.json
+1
View File
@@ -25,6 +25,7 @@
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [],
"STALLED_MAX_STRIKES": 0,
"STALLED_RESET_STRIKES_ON_PROGRESS": false,
"STALLED_IGNORE_PRIVATE": false,
"STALLED_DELETE_PRIVATE": false
},
code/Infrastructure/Helpers/CacheKeys.cs
+14
View File
@@ -0,0 +1,14 @@
using Domain.Enums;
namespace Infrastructure.Helpers;
public static class CacheKeys
{
public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}";
public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type";
public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
public static string Item(string hash) => $"item_{hash}";
}
code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs
+7 -10
View File
@@ -5,6 +5,7 @@ using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -21,10 +22,6 @@ public sealed class BlocklistProvider
private readonly IMemoryCache _cache;
private bool _initialized;
private const string Type = "type";
private const string Patterns = "patterns";
private const string Regexes = "regexes";
public BlocklistProvider(
ILogger<BlocklistProvider> logger,
IOptions<SonarrConfig> sonarrConfig,
@@ -67,21 +64,21 @@ public sealed class BlocklistProvider
public BlocklistType GetBlocklistType(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
_cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType);
return blocklistType ?? BlocklistType.Blacklist;
}
public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
_cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag<string>? patterns);
return patterns ?? [];
}
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
_cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
return regexes ?? [];
}
@@ -124,9 +121,9 @@ public sealed class BlocklistProvider
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
_cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
_cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
_cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType);
_cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
_cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
_logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count);
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
+6 -2
View File
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Deluge.Response;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -21,9 +22,10 @@ public sealed class DelugeService : DownloadServiceBase
IHttpClientFactory httpClientFactory,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
config.Value.Validate();
_client = new (config, httpClientFactory);
@@ -201,6 +203,8 @@ public sealed class DelugeService : DownloadServiceBase
{
return false;
}
ResetStrikesOnProgress(status.Hash!, status.TotalDone);
return StrikeAndCheckLimit(status.Hash!, status.Name!);
}
@@ -210,7 +214,7 @@ public sealed class DelugeService : DownloadServiceBase
return await _client.SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
new[] { "hash", "state", "name", "eta", "private" }
new[] { "hash", "state", "name", "eta", "private", "total_done" }
);
}
code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs
+27
View File
@@ -2,9 +2,13 @@
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -15,13 +19,16 @@ public abstract class DownloadServiceBase : IDownloadService
protected readonly ILogger<DownloadServiceBase> _logger;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly IMemoryCache _cache;
protected readonly FilenameEvaluator _filenameEvaluator;
protected readonly Striker _striker;
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected DownloadServiceBase(
ILogger<DownloadServiceBase> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
)
@@ -29,8 +36,11 @@ public abstract class DownloadServiceBase : IDownloadService
_logger = logger;
_queueCleanerConfig = queueCleanerConfig.Value;
_contentBlockerConfig = contentBlockerConfig.Value;
_cache = cache;
_filenameEvaluator = filenameEvaluator;
_striker = striker;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
public abstract void Dispose();
@@ -50,6 +60,23 @@ public abstract class DownloadServiceBase : IDownloadService
/// <inheritdoc/>
public abstract Task Delete(string hash);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
{
return;
}
if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
{
// cache item found
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
_logger.LogDebug("resetting strikes for {hash} due to progress", hash);
}
_cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
}
/// <summary>
/// Strikes an item and checks if the limit has been reached.
/// </summary>
code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs
+2 -1
View File
@@ -4,6 +4,7 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -11,7 +12,7 @@ namespace Infrastructure.Verticals.DownloadClient;
public sealed class DummyDownloadService : DownloadServiceBase
{
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
}
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
+5 -1
View File
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using QBittorrent.Client;
@@ -23,9 +24,10 @@ public sealed class QBitService : DownloadServiceBase
IOptions<QBitConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
_config = config.Value;
_config.Validate();
@@ -216,6 +218,8 @@ public sealed class QBitService : DownloadServiceBase
return false;
}
ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return StrikeAndCheckLimit(torrent.Hash, torrent.Name);
}
}
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
+7 -2
View File
@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Transmission.API.RPC;
@@ -26,9 +27,10 @@ public sealed class TransmissionService : DownloadServiceBase
IOptions<TransmissionConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
_config = config.Value;
_config.Validate();
@@ -200,6 +202,8 @@ public sealed class TransmissionService : DownloadServiceBase
{
return false;
}
ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
}
@@ -219,7 +223,8 @@ public sealed class TransmissionService : DownloadServiceBase
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER
];
// refresh cache
code/Infrastructure/Verticals/ItemStriker/Striker.cs
+2 -1
View File
@@ -1,5 +1,6 @@
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
@@ -26,7 +27,7 @@ public class Striker
return false;
}
string key = $"{strikeType.ToString()}_{hash}";
string key = CacheKeys.Strike(strikeType, hash);
if (!_cache.TryGetValue(key, out int? strikeCount))
{