Fixed: Don't purge xem scene mapping cache when new series gets added.

src/NzbDrone.Common.Test/CacheTests/CachedDictionaryFixture.cs

@@ -0,0 +1,102 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Common.Cache;
+
+namespace NzbDrone.Common.Test.CacheTests
+{
+    [TestFixture]
+    public class CachedDictionaryFixture
+    {
+        private CachedDictionary<string> _cachedString;
+        private DictionaryWorker _worker;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _worker = new DictionaryWorker();
+            _cachedString = new CachedDictionary<string>(_worker.GetDict, TimeSpan.FromMilliseconds(100));
+        }
+
+        [Test]
+        public void should_not_fetch_on_create()
+        {
+            _worker.HitCount.Should().Be(0);
+        }
+
+        [Test]
+        public void should_fetch_on_first_call()
+        {
+            var result = _cachedString.Get("Hi");
+
+            _worker.HitCount.Should().Be(1);
+
+            result.Should().Be("Value");
+        }
+
+        [Test]
+        public void should_fetch_once()
+        {
+            var result1 = _cachedString.Get("Hi");
+            var result2 = _cachedString.Get("HitCount");
+
+            _worker.HitCount.Should().Be(1);
+        }
+
+        [Test]
+        public void should_auto_refresh_after_lifetime()
+        {
+            var result1 = _cachedString.Get("Hi");
+
+            Thread.Sleep(200);
+
+            var result2 = _cachedString.Get("Hi");
+
+            _worker.HitCount.Should().Be(2);
+        }
+
+        [Test]
+        public void should_refresh_early_if_requested()
+        {
+            var result1 = _cachedString.Get("Hi");
+
+            Thread.Sleep(10);
+
+            _cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(1));
+
+            var result2 = _cachedString.Get("Hi");
+
+            _worker.HitCount.Should().Be(2);
+        }
+
+        [Test]
+        public void should_not_refresh_early_if_not_expired()
+        {
+            var result1 = _cachedString.Get("Hi");
+
+            _cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(50));
+
+            var result2 = _cachedString.Get("Hi");
+
+            _worker.HitCount.Should().Be(1);
+        }
+    }
+
+    public class DictionaryWorker
+    {
+        public int HitCount { get; private set; }
+
+        public Dictionary<string, string> GetDict()
+        {
+            HitCount++;
+
+            var result = new Dictionary<string, string>();
+            result["Hi"] = "Value";
+            result["HitCount"] = "Hit count is " + HitCount;
+
+            return result;
+        }
+    }
+}

src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj

@@ -66,6 +66,7 @@
     </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="CacheTests\CachedDictionaryFixture.cs" />
     <Compile Include="CacheTests\CachedFixture.cs" />
     <Compile Include="CacheTests\CachedManagerFixture.cs" />
     <Compile Include="ConfigFileProviderTest.cs" />

src/NzbDrone.Common/Cache/CacheManager.cs

@@ -6,8 +6,9 @@ namespace NzbDrone.Common.Cache
 {
     public interface ICacheManager
     {
-        ICached<T> GetCache<T>(Type host, string name);
         ICached<T> GetCache<T>(Type host);
+        ICached<T> GetCache<T>(Type host, string name);
+        ICachedDictionary<T> GetCacheDictionary<T>(Type host, string name, Func<IDictionary<string, T>> fetchFunc = null, TimeSpan? lifeTime = null);
         void Clear();
         ICollection<ICached> Caches { get; }
     }
@@ -22,12 +23,6 @@ public CacheManager()
 
         }
 
-        public ICached<T> GetCache<T>(Type host)
-        {
-            Ensure.That(host, () => host).IsNotNull();
-            return GetCache<T>(host, host.FullName);
-        }
-
         public void Clear()
         {
             _cache.Clear();
@@ -35,6 +30,12 @@ public void Clear()
 
         public ICollection<ICached> Caches { get { return _cache.Values; } }
 
+        public ICached<T> GetCache<T>(Type host)
+        {
+            Ensure.That(host, () => host).IsNotNull();
+            return GetCache<T>(host, host.FullName);
+        }
+
         public ICached<T> GetCache<T>(Type host, string name)
         {
             Ensure.That(host, () => host).IsNotNull();
@@ -42,5 +43,13 @@ public ICached<T> GetCache<T>(Type host, string name)
 
             return (ICached<T>)_cache.Get(host.FullName + "_" + name, () => new Cached<T>());
         }
+
+        public ICachedDictionary<T> GetCacheDictionary<T>(Type host, string name, Func<IDictionary<string, T>> fetchFunc = null, TimeSpan? lifeTime = null)
+        {
+            Ensure.That(host, () => host).IsNotNull();
+            Ensure.That(name, () => name).IsNotNullOrWhiteSpace();
+
+            return (ICachedDictionary<T>)_cache.Get("dict_" + host.FullName + "_" + name, () => new CachedDictionary<T>(fetchFunc, lifeTime));
+        }
     }
 }

src/NzbDrone.Common/Cache/CachedDictionary.cs

@@ -0,0 +1,137 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+
+namespace NzbDrone.Common.Cache
+{
+
+    public class CachedDictionary<TValue> : ICachedDictionary<TValue>
+    {
+        private readonly Func<IDictionary<string, TValue>> _fetchFunc;
+        private readonly TimeSpan? _ttl;
+
+        private DateTime _lastRefreshed = DateTime.MinValue;
+        private ConcurrentDictionary<string, TValue> _items = new ConcurrentDictionary<string, TValue>();
+
+        public CachedDictionary(Func<IDictionary<string, TValue>> fetchFunc = null, TimeSpan? ttl = null)
+        {
+            _fetchFunc = fetchFunc;
+            _ttl = ttl;
+        }
+
+        public bool IsExpired(TimeSpan ttl)
+        {
+            return _lastRefreshed.Add(ttl) < DateTime.UtcNow;
+        }
+
+        public void RefreshIfExpired()
+        {
+            if (_ttl.HasValue && _fetchFunc != null)
+            {
+                RefreshIfExpired(_ttl.Value);
+            }
+        }
+
+        public void RefreshIfExpired(TimeSpan ttl)
+    {
+            if (IsExpired(ttl))
+            {
+                Refresh();
+            }
+        }
+
+        public void Refresh()
+        {
+            if (_fetchFunc == null)
+            {
+                throw new InvalidOperationException("Cannot update cache without data source.");
+            }
+
+            Update(_fetchFunc());
+            ExtendTTL();
+        }
+
+        public void Update(IDictionary<string, TValue> items)
+        {
+            _items = new ConcurrentDictionary<string, TValue>(items);
+            ExtendTTL();
+        }
+
+        public void ExtendTTL()
+        {
+            _lastRefreshed = DateTime.UtcNow;
+        }
+
+        [DebuggerBrowsable(DebuggerBrowsableState.Never)]
+        public ICollection<TValue> Values
+        {
+            get
+            {
+                RefreshIfExpired();
+                return _items.Values;
+            }
+        }
+
+        [DebuggerBrowsable(DebuggerBrowsableState.Never)]
+        public int Count
+        {
+            get
+            {
+                RefreshIfExpired();
+                return _items.Count;
+            }
+        }
+
+        public TValue Get(string key)
+        {
+            RefreshIfExpired();
+
+            TValue result;
+
+            if (!_items.TryGetValue(key, out result))
+            {
+                throw new KeyNotFoundException(string.Format("Item {0} not found in cache.", key));
+            }
+
+            return result;
+        }
+
+        public TValue Find(string key)
+        {
+            RefreshIfExpired();
+
+            TValue result;
+
+            _items.TryGetValue(key, out result);
+
+            return result;
+        }
+
+        public void Clear()
+        {
+            _items.Clear();
+            _lastRefreshed = DateTime.MinValue;
+        }
+
+        public void ClearExpired()
+        {
+            if (!_ttl.HasValue)
+            {
+                throw new InvalidOperationException("Checking expiry without ttl not possible.");
+            }
+
+            if (IsExpired(_ttl.Value))
+            {
+                Clear();
+            }
+        }
+
+        public void Remove(string key)
+        {
+            TValue item;
+            _items.TryRemove(key, out item);
+        }
+    }
+}

src/NzbDrone.Common/Cache/ICachedDictionary.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NzbDrone.Common.Cache
+{
+    public interface ICachedDictionary<TValue> : ICached
+    {
+        void RefreshIfExpired();
+        void RefreshIfExpired(TimeSpan ttl);
+        void Refresh();
+        void Update(IDictionary<string, TValue> items);
+        void ExtendTTL();
+        TValue Get(string key);
+        TValue Find(string key);
+        bool IsExpired(TimeSpan ttl);
+    }
+}

src/NzbDrone.Common/Http/HttpClient.cs

@@ -28,7 +28,6 @@ public class HttpClient : IHttpClient
         private readonly Logger _logger;
         private readonly IRateLimitService _rateLimitService;
         private readonly ICached<CookieContainer> _cookieContainerCache;
-        private readonly ICached<bool> _curlTLSFallbackCache;
         private readonly List<IHttpRequestInterceptor> _requestInterceptors;
         private readonly IHttpDispatcher _httpDispatcher;
 

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -64,7 +64,9 @@
     <Compile Include="ArchiveService.cs" />
     <Compile Include="Cache\Cached.cs" />
     <Compile Include="Cache\CacheManager.cs" />
+    <Compile Include="Cache\CachedDictionary.cs" />
     <Compile Include="Cache\ICached.cs" />
+    <Compile Include="Cache\ICachedDictionary.cs" />
     <Compile Include="Cloud\CloudClient.cs" />
     <Compile Include="Composition\Container.cs" />
     <Compile Include="Composition\ContainerBuilderBase.cs" />

src/NzbDrone.Core.Test/DataAugmentation/SceneNumbering/XemServiceFixture.cs

@@ -11,6 +11,7 @@
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Core.Tv;
 using NzbDrone.Core.Tv.Events;
+using NzbDrone.Test.Common;
 
 namespace NzbDrone.Core.Test.DataAugmentation.SceneNumbering
 {
@@ -144,6 +145,25 @@ public void should_not_clear_scenenumbering_if_no_results_at_all_from_thexem()
 
             Mocker.GetMock<ISeriesService>()
                   .Verify(v => v.UpdateSeries(It.IsAny<Series>()), Times.Never());
+
+            ExceptionVerification.ExpectedWarns(1);
+        }
+
+        [Test]
+        public void should_not_clear_scenenumbering_if_thexem_throws()
+        {
+            GivenExistingMapping();
+
+            Mocker.GetMock<IXemProxy>()
+                  .Setup(v => v.GetXemSeriesIds())
+                  .Throws(new InvalidOperationException());
+
+            Subject.Handle(new SeriesUpdatedEvent(_series));
+
+            Mocker.GetMock<ISeriesService>()
+                  .Verify(v => v.UpdateSeries(It.IsAny<Series>()), Times.Never());
+
+            ExceptionVerification.ExpectedWarns(1);
         }
 
         [Test]

src/NzbDrone.Core/DataAugmentation/Scene/SceneMappingService.cs

@@ -28,8 +28,8 @@ public class SceneMappingService : ISceneMappingService,
         private readonly IEnumerable<ISceneMappingProvider> _sceneMappingProviders;
         private readonly IEventAggregator _eventAggregator;
         private readonly Logger _logger;
-        private readonly ICached<List<SceneMapping>> _getTvdbIdCache;
-        private readonly ICached<List<SceneMapping>> _findByTvdbIdCache;
+        private readonly ICachedDictionary<List<SceneMapping>> _getTvdbIdCache;
+        private readonly ICachedDictionary<List<SceneMapping>> _findByTvdbIdCache;
 
         public SceneMappingService(ISceneMappingRepository repository,
                                    ICacheManager cacheManager,
@@ -40,10 +40,10 @@ public SceneMappingService(ISceneMappingRepository repository,
             _repository = repository;
             _sceneMappingProviders = sceneMappingProviders;
             _eventAggregator = eventAggregator;
-
-            _getTvdbIdCache = cacheManager.GetCache<List<SceneMapping>>(GetType(), "tvdb_id");
-            _findByTvdbIdCache = cacheManager.GetCache<List<SceneMapping>>(GetType(), "find_tvdb_id");
             _logger = logger;
+
+            _getTvdbIdCache = cacheManager.GetCacheDictionary<List<SceneMapping>>(GetType(), "tvdb_id");
+            _findByTvdbIdCache = cacheManager.GetCacheDictionary<List<SceneMapping>>(GetType(), "find_tvdb_id");
         }
 
         public List<string> GetSceneNames(int tvdbId, IEnumerable<int> seasonNumbers)
@@ -143,6 +143,7 @@ private void UpdateMappings()
             }
             
             RefreshCache();
+
             _eventAggregator.PublishEvent(new SceneMappingsUpdatedEvent());
         }
 
@@ -184,18 +185,8 @@ private void RefreshCache()
         {
             var mappings = _repository.All().ToList();
 
-            _getTvdbIdCache.Clear();
-            _findByTvdbIdCache.Clear();
-
-            foreach (var sceneMapping in mappings.GroupBy(v => v.ParseTerm))
-            {
-                _getTvdbIdCache.Set(sceneMapping.Key, sceneMapping.ToList());
-            }
-
-            foreach (var sceneMapping in mappings.GroupBy(x => x.TvdbId))
-            {
-                _findByTvdbIdCache.Set(sceneMapping.Key.ToString(), sceneMapping.ToList());
-            }
+            _getTvdbIdCache.Update(mappings.GroupBy(v => v.ParseTerm).ToDictionary(v => v.Key, v => v.ToList()));
+            _findByTvdbIdCache.Update(mappings.GroupBy(v => v.TvdbId).ToDictionary(v => v.Key.ToString(), v => v.ToList()));
         }
 
         private List<string> FilterNonEnglish(List<string> titles)
@@ -205,7 +196,10 @@ private List<string> FilterNonEnglish(List<string> titles)
 
         public void Handle(SeriesRefreshStartingEvent message)
         {
-            UpdateMappings();
+            if (message.ManualTrigger && _findByTvdbIdCache.IsExpired(TimeSpan.FromMinutes(1)))
+            {
+                UpdateMappings();
+            }
         }
 
         public void Execute(UpdateSceneMappingCommand message)

src/NzbDrone.Core/DataAugmentation/Xem/XemService.cs

@@ -16,7 +16,7 @@ public class XemService : ISceneMappingProvider, IHandle<SeriesUpdatedEvent>, IH
         private readonly IXemProxy _xemProxy;
         private readonly ISeriesService _seriesService;
         private readonly Logger _logger;
-        private readonly ICached<bool> _cache;
+        private readonly ICachedDictionary<bool> _cache;
 
         public XemService(IEpisodeService episodeService,
                            IXemProxy xemProxy,
@@ -26,7 +26,7 @@ public XemService(IEpisodeService episodeService,
             _xemProxy = xemProxy;
             _seriesService = seriesService;
             _logger = logger;
-            _cache = cacheManager.GetCache<bool>(GetType());
+            _cache = cacheManager.GetCacheDictionary<bool>(GetType(), "mappedTvdbid");
         }
 
         private void PerformUpdate(Series series)
@@ -40,7 +40,6 @@ private void PerformUpdate(Series series)
                 if (!mappings.Any() && !series.UseSceneNumbering)
                 {
                     _logger.Debug("Mappings for: {0} are empty, skipping", series);
-                    _cache.Remove(series.TvdbId.ToString());
                     return;
                 }
 
@@ -171,18 +170,25 @@ private void ExtrapolateMappings(Series series, List<Episode> episodes, List<Mod
             }
         }
 
-        private void RefreshCache()
+        private void UpdateXemSeriesIds()
         {
-            var ids = _xemProxy.GetXemSeriesIds();
-
-            if (ids.Any())
+            try
             {
-                _cache.Clear();
+                var ids = _xemProxy.GetXemSeriesIds();
+
+                if (ids.Any())
+                {
+                    _cache.Update(ids.ToDictionary(v => v.ToString(), v => true));
+                    return;
+                }
+
+                _cache.ExtendTTL();
+                _logger.Warn("Failed to update Xem series list.");
             }
-
-            foreach (var id in ids)
+            catch (Exception ex)
             {
-                _cache.Set(id.ToString(), true, TimeSpan.FromHours(1));
+                _cache.ExtendTTL();
+                _logger.Warn(ex, "Failed to update Xem series list.");
             }
         }
 
@@ -206,9 +212,9 @@ public List<SceneMapping> GetSceneMappings()
 
         public void Handle(SeriesUpdatedEvent message)
         {
-            if (_cache.Count == 0)
+            if (_cache.IsExpired(TimeSpan.FromHours(3)))
             {
-                RefreshCache();
+                UpdateXemSeriesIds();
             }
 
             if (_cache.Count == 0)
@@ -228,7 +234,10 @@ public void Handle(SeriesUpdatedEvent message)
 
         public void Handle(SeriesRefreshStartingEvent message)
         {
-            RefreshCache();
+            if (message.ManualTrigger && _cache.IsExpired(TimeSpan.FromMinutes(1)))
+            {
+                UpdateXemSeriesIds();
+            }
         }
     }
 }

src/NzbDrone.Core/Tv/Events/SeriesRefreshStartingEvent.cs

@@ -4,5 +4,11 @@ namespace NzbDrone.Core.Tv.Events
 {
     public class SeriesRefreshStartingEvent : IEvent
     {
+        public bool ManualTrigger { get; set; }
+
+        public SeriesRefreshStartingEvent(bool manualTrigger)
+        {
+            ManualTrigger = manualTrigger;
+        }
     }
 }

src/NzbDrone.Core/Tv/RefreshSeriesService.cs

@@ -144,7 +144,7 @@ private List<Season> UpdateSeasons(Series series, Series seriesInfo)
 
         public void Execute(RefreshSeriesCommand message)
         {
-            _eventAggregator.PublishEvent(new SeriesRefreshStartingEvent());
+            _eventAggregator.PublishEvent(new SeriesRefreshStartingEvent(message.Trigger == CommandTrigger.Manual));
 
             if (message.SeriesId.HasValue)
             {