Add Lidarr support (#30)

2025-01-15 23:55:34 +02:00
parent 2bc8e445ce
commit 922f586706
63 changed files with 943 additions and 243 deletions
@@ -1,9 +1,11 @@
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.Text.RegularExpressions;
+using Common.Configuration.Arr;
 using Common.Configuration.ContentBlocker;
 using Common.Helpers;
 using Domain.Enums;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;

@@ -12,78 +14,98 @@ namespace Infrastructure.Verticals.ContentBlocker;
 public sealed class BlocklistProvider
 {
    private readonly ILogger<BlocklistProvider> _logger;
-    private readonly ContentBlockerConfig _config;
+    private readonly SonarrConfig _sonarrConfig;
+    private readonly RadarrConfig _radarrConfig;
+    private readonly LidarrConfig _lidarrConfig;
    private readonly HttpClient _httpClient;
-    
-    public BlocklistType BlocklistType { get; }
+    private readonly IMemoryCache _cache;
+    private bool _initialized;

-    public ConcurrentBag<string> Patterns { get; } = [];
-
-    public ConcurrentBag<Regex> Regexes { get; } = [];
+    private const string Type = "type";
+    private const string Patterns = "patterns";
+    private const string Regexes = "regexes";

    public BlocklistProvider(
        ILogger<BlocklistProvider> logger,
-        IOptions<ContentBlockerConfig> config,
-        IHttpClientFactory httpClientFactory)
+        IOptions<SonarrConfig> sonarrConfig,
+        IOptions<RadarrConfig> radarrConfig,
+        IOptions<LidarrConfig> lidarrConfig,
+        IMemoryCache cache,
+        IHttpClientFactory httpClientFactory
+    )
    {
        _logger = logger;
-        _config = config.Value;
+        _sonarrConfig = sonarrConfig.Value;
+        _radarrConfig = radarrConfig.Value;
+        _lidarrConfig = lidarrConfig.Value;
+        _cache = cache;
        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
-        
-        _config.Validate();
-        
-        if (_config.Blacklist?.Enabled is true)
-        {
-            BlocklistType = BlocklistType.Blacklist;
-        }
-
-        if (_config.Whitelist?.Enabled is true)
-        {
-            BlocklistType = BlocklistType.Whitelist;
-        }
    }

-    public async Task LoadBlocklistAsync()
+    public async Task LoadBlocklistsAsync()
    {
-        if (Patterns.Count > 0 || Regexes.Count > 0)
+        if (_initialized)
        {
-            _logger.LogDebug("blocklist already loaded");
+            _logger.LogDebug("blocklists already loaded");
            return;
        }
        
        try
        {
-            await LoadPatternsAndRegexesAsync();
+            await LoadPatternsAndRegexesAsync(_sonarrConfig.Block.Type, _sonarrConfig.Block.Path, InstanceType.Sonarr);
+            await LoadPatternsAndRegexesAsync(_radarrConfig.Block.Type, _radarrConfig.Block.Path, InstanceType.Radarr);
+            await LoadPatternsAndRegexesAsync(_lidarrConfig.Block.Type, _lidarrConfig.Block.Path, InstanceType.Lidarr);
+            
+            _initialized = true;
        }
        catch
        {
-            _logger.LogError("failed to load {type}", BlocklistType.ToString());
+            _logger.LogError("failed to load blocklists");
            throw;
        }
    }

-    private async Task LoadPatternsAndRegexesAsync()
+    public BlocklistType GetBlocklistType(InstanceType instanceType)
    {
-        string[] patterns;
+        _cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
+
+        return blocklistType ?? BlocklistType.Blacklist;
+    }
+    
+    public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
+    {
+        _cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
+
+        return patterns ?? [];
+    }
+
+    public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
+    {
+        _cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
        
-        if (BlocklistType is BlocklistType.Blacklist)
+        return regexes ?? [];
+    }
+
+    private async Task LoadPatternsAndRegexesAsync(BlocklistType blocklistType, string? blocklistPath, InstanceType instanceType)
+    {
+        if (string.IsNullOrEmpty(blocklistPath))
        {
-            patterns = await ReadContentAsync(_config.Blacklist.Path);
-        }
-        else
-        {
-            patterns = await ReadContentAsync(_config.Whitelist.Path);
+            return;
        }
        
+        string[] filePatterns = await ReadContentAsync(blocklistPath);
+        
        long startTime = Stopwatch.GetTimestamp();
        ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
        const string regexId = "regex:";
+        ConcurrentBag<string> patterns = [];
+        ConcurrentBag<Regex> regexes = [];
        
-        Parallel.ForEach(patterns, options, pattern =>
+        Parallel.ForEach(filePatterns, options, pattern =>
        {
            if (!pattern.StartsWith(regexId))
            {
-                Patterns.Add(pattern);
+                patterns.Add(pattern);
                return;
            }
            
@@ -92,7 +114,7 @@ public sealed class BlocklistProvider
            try
            {
                Regex regex = new(pattern, RegexOptions.Compiled);
-                Regexes.Add(regex);
+                regexes.Add(regex);
            }
            catch (ArgumentException)
            {
@@ -101,10 +123,14 @@ public sealed class BlocklistProvider
        });

        TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
+
+        _cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
+        _cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
+        _cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
        
-        _logger.LogDebug("loaded {count} patterns", Patterns.Count);
-        _logger.LogDebug("loaded {count} regexes", Regexes.Count);
-        _logger.LogDebug("blocklist loaded in {elapsed} ms", elapsed.TotalMilliseconds);
+        _logger.LogDebug("loaded {count} patterns", patterns.Count);
+        _logger.LogDebug("loaded {count} regexes", regexes.Count);
+        _logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistPath);
    }
    
    private async Task<string[]> ReadContentAsync(string path)
@@ -1,12 +1,17 @@
-using Common.Configuration.Arr;
+using System.Collections.Concurrent;
+using System.Text.RegularExpressions;
+using Common.Configuration.Arr;
+using Common.Configuration.ContentBlocker;
 using Common.Configuration.DownloadClient;
 using Domain.Enums;
+using Domain.Models.Arr;
 using Domain.Models.Arr.Queue;
 using Infrastructure.Verticals.Arr;
 using Infrastructure.Verticals.DownloadClient;
 using Infrastructure.Verticals.Jobs;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Serilog.Context;

 namespace Infrastructure.Verticals.ContentBlocker;

@@ -19,12 +24,19 @@ public sealed class ContentBlocker : GenericHandler
        IOptions<DownloadClientConfig> downloadClientConfig,
        IOptions<SonarrConfig> sonarrConfig,
        IOptions<RadarrConfig> radarrConfig,
+        IOptions<LidarrConfig> lidarrConfig,
        SonarrClient sonarrClient,
        RadarrClient radarrClient,
+        LidarrClient lidarrClient,
        ArrQueueIterator arrArrQueueIterator,
        BlocklistProvider blocklistProvider,
        DownloadServiceFactory downloadServiceFactory
-    ) : base(logger, downloadClientConfig, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
+    ) : base(
+        logger, downloadClientConfig,
+        sonarrConfig, radarrConfig, lidarrConfig,
+        sonarrClient, radarrClient, lidarrClient,
+        arrArrQueueIterator, downloadServiceFactory
+    )
    {
        _blocklistProvider = blocklistProvider;
    }
@@ -37,18 +49,40 @@ public sealed class ContentBlocker : GenericHandler
            return;
        }
        
-        await _blocklistProvider.LoadBlocklistAsync();
+        bool blocklistIsConfigured = _sonarrConfig.Enabled && !string.IsNullOrEmpty(_sonarrConfig.Block.Path) ||
+                                     _radarrConfig.Enabled && !string.IsNullOrEmpty(_radarrConfig.Block.Path) ||
+                                     _lidarrConfig.Enabled && !string.IsNullOrEmpty(_lidarrConfig.Block.Path);
+
+        if (!blocklistIsConfigured)
+        {
+            _logger.LogWarning("no blocklist is configured");
+            return;
+        }
+        
+        await _blocklistProvider.LoadBlocklistsAsync();
        await base.ExecuteAsync();
    }

    protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
    {
+        using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
+
+        HashSet<SearchItem> itemsToBeRefreshed = [];
        ArrClient arrClient = GetClient(instanceType);
+        BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
+        ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
+        ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);

        await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
        {
-            foreach (QueueRecord record in items)
+            var groups = items
+                .GroupBy(x => x.DownloadId)
+                .ToList();
+            
+            foreach (var group in groups)
            {
+                QueueRecord record = group.First();
+                
                if (record.Protocol is not "torrent")
                {
                    continue;
@@ -61,8 +95,19 @@ public sealed class ContentBlocker : GenericHandler
                }
                
                _logger.LogDebug("searching unwanted files for {title}", record.Title);
-                await _downloadService.BlockUnwantedFilesAsync(record.DownloadId);
+
+                if (!await _downloadService.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes))
+                {
+                    continue;
+                }
+                
+                _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
+                
+                itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
+                await arrClient.DeleteQueueItemAsync(instance, record);
            }
        });
+        
+        await arrClient.RefreshItemsAsync(instance, itemsToBeRefreshed);
    }
 }
@@ -1,4 +1,6 @@
-using Domain.Enums;
+using System.Collections.Concurrent;
+using System.Text.RegularExpressions;
+using Common.Configuration.ContentBlocker;
 using Microsoft.Extensions.Logging;

 namespace Infrastructure.Verticals.ContentBlocker;
@@ -6,46 +8,44 @@ namespace Infrastructure.Verticals.ContentBlocker;
 public sealed class FilenameEvaluator
 {
    private readonly ILogger<FilenameEvaluator> _logger;
-    private readonly BlocklistProvider _blocklistProvider;
    
-    public FilenameEvaluator(ILogger<FilenameEvaluator> logger, BlocklistProvider blocklistProvider)
+    public FilenameEvaluator(ILogger<FilenameEvaluator> logger)
    {
        _logger = logger;
-        _blocklistProvider = blocklistProvider;
    }
    
    // TODO create unit tests
-    public bool IsValid(string filename)
+    public bool IsValid(string filename, BlocklistType type, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
    {
-        return IsValidAgainstPatterns(filename) && IsValidAgainstRegexes(filename);
+        return IsValidAgainstPatterns(filename, type, patterns) && IsValidAgainstRegexes(filename, type, regexes);
    }

-    private bool IsValidAgainstPatterns(string filename)
+    private static bool IsValidAgainstPatterns(string filename, BlocklistType type, ConcurrentBag<string> patterns)
    {
-        if (_blocklistProvider.Patterns.Count is 0)
+        if (patterns.Count is 0)
        {
            return true;
        }

-        return _blocklistProvider.BlocklistType switch
+        return type switch
        {
-            BlocklistType.Blacklist => !_blocklistProvider.Patterns.Any(pattern => MatchesPattern(filename, pattern)),
-            BlocklistType.Whitelist => _blocklistProvider.Patterns.Any(pattern => MatchesPattern(filename, pattern)),
+            BlocklistType.Blacklist => !patterns.Any(pattern => MatchesPattern(filename, pattern)),
+            BlocklistType.Whitelist => patterns.Any(pattern => MatchesPattern(filename, pattern)),
            _ => true
        };
    }

-    private bool IsValidAgainstRegexes(string filename)
+    private static bool IsValidAgainstRegexes(string filename, BlocklistType type, ConcurrentBag<Regex> regexes)
    {
-        if (_blocklistProvider.Regexes.Count is 0)
+        if (regexes.Count is 0)
        {
            return true;
        }
        
-        return _blocklistProvider.BlocklistType switch
+        return type switch
        {
-            BlocklistType.Blacklist => !_blocklistProvider.Regexes.Any(regex => regex.IsMatch(filename)),
-            BlocklistType.Whitelist => _blocklistProvider.Regexes.Any(regex => regex.IsMatch(filename)),
+            BlocklistType.Blacklist => !regexes.Any(regex => regex.IsMatch(filename)),
+            BlocklistType.Whitelist => regexes.Any(regex => regex.IsMatch(filename)),
            _ => true
        };
    }