mirror of
https://github.com/jellyfin/jellyfin
synced 2024-09-30 04:55:54 +00:00
Precache livetv program images (#11083)
* Precache livetv program images * return if cache hit * use EnsureSuccessStatusCode * Read proper bytes
This commit is contained in:
parent
3bd1a5c557
commit
f7f3ad9eb7
|
@ -1860,7 +1860,7 @@ namespace Emby.Server.Implementations.Library
|
|||
try
|
||||
{
|
||||
var index = item.GetImageIndex(img);
|
||||
image = await ConvertImageToLocal(item, img, index).ConfigureAwait(false);
|
||||
image = await ConvertImageToLocal(item, img, index, removeOnFailure: true).ConfigureAwait(false);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
|
@ -2787,7 +2787,7 @@ namespace Emby.Server.Implementations.Library
|
|||
await SavePeopleMetadataAsync(people, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex)
|
||||
public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure)
|
||||
{
|
||||
foreach (var url in image.Path.Split('|'))
|
||||
{
|
||||
|
@ -2806,6 +2806,7 @@ namespace Emby.Server.Implementations.Library
|
|||
if (ex.StatusCode.HasValue
|
||||
&& (ex.StatusCode.Value == HttpStatusCode.NotFound || ex.StatusCode.Value == HttpStatusCode.Forbidden))
|
||||
{
|
||||
_logger.LogDebug(ex, "Error downloading image {Url}", url);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -2813,11 +2814,14 @@ namespace Emby.Server.Implementations.Library
|
|||
}
|
||||
}
|
||||
|
||||
// Remove this image to prevent it from retrying over and over
|
||||
item.RemoveImage(image);
|
||||
await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false);
|
||||
if (removeOnFailure)
|
||||
{
|
||||
// Remove this image to prevent it from retrying over and over
|
||||
item.RemoveImage(image);
|
||||
await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException();
|
||||
throw new InvalidOperationException("Unable to convert any images to local");
|
||||
}
|
||||
|
||||
public async Task AddVirtualFolder(string name, CollectionTypeOptions? collectionType, LibraryOptions options, bool refreshLibrary)
|
||||
|
|
|
@ -517,8 +517,9 @@ namespace MediaBrowser.Controller.Library
|
|||
/// <param name="item">The item.</param>
|
||||
/// <param name="image">The image.</param>
|
||||
/// <param name="imageIndex">Index of the image.</param>
|
||||
/// <param name="removeOnFailure">Whether to remove the image from the item on failure.</param>
|
||||
/// <returns>Task.</returns>
|
||||
Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex);
|
||||
Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex, bool removeOnFailure = true);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the items.
|
||||
|
|
|
@ -9,6 +9,7 @@ using System.Net.Http;
|
|||
using System.Net.Mime;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AsyncKeyedLock;
|
||||
using Jellyfin.Data.Enums;
|
||||
using Jellyfin.Data.Events;
|
||||
using Jellyfin.Extensions;
|
||||
|
@ -30,6 +31,7 @@ using MediaBrowser.Model.Extensions;
|
|||
using MediaBrowser.Model.IO;
|
||||
using MediaBrowser.Model.Net;
|
||||
using MediaBrowser.Model.Providers;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Book = MediaBrowser.Controller.Entities.Book;
|
||||
using Episode = MediaBrowser.Controller.Entities.TV.Episode;
|
||||
|
@ -59,6 +61,13 @@ namespace MediaBrowser.Providers.Manager
|
|||
private readonly ConcurrentDictionary<Guid, double> _activeRefreshes = new();
|
||||
private readonly CancellationTokenSource _disposeCancellationTokenSource = new();
|
||||
private readonly PriorityQueue<(Guid ItemId, MetadataRefreshOptions RefreshOptions), RefreshPriority> _refreshQueue = new();
|
||||
private readonly IMemoryCache _memoryCache;
|
||||
|
||||
private readonly AsyncKeyedLocker<string> _imageSaveLock = new(o =>
|
||||
{
|
||||
o.PoolSize = 20;
|
||||
o.PoolInitialFill = 1;
|
||||
});
|
||||
|
||||
private IImageProvider[] _imageProviders = Array.Empty<IImageProvider>();
|
||||
private IMetadataService[] _metadataServices = Array.Empty<IMetadataService>();
|
||||
|
@ -81,6 +90,7 @@ namespace MediaBrowser.Providers.Manager
|
|||
/// <param name="libraryManager">The library manager.</param>
|
||||
/// <param name="baseItemManager">The BaseItem manager.</param>
|
||||
/// <param name="lyricManager">The lyric manager.</param>
|
||||
/// <param name="memoryCache">The memory cache.</param>
|
||||
public ProviderManager(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
ISubtitleManager subtitleManager,
|
||||
|
@ -91,7 +101,8 @@ namespace MediaBrowser.Providers.Manager
|
|||
IServerApplicationPaths appPaths,
|
||||
ILibraryManager libraryManager,
|
||||
IBaseItemManager baseItemManager,
|
||||
ILyricManager lyricManager)
|
||||
ILyricManager lyricManager,
|
||||
IMemoryCache memoryCache)
|
||||
{
|
||||
_logger = logger;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
|
@ -103,6 +114,7 @@ namespace MediaBrowser.Providers.Manager
|
|||
_subtitleManager = subtitleManager;
|
||||
_baseItemManager = baseItemManager;
|
||||
_lyricManager = lyricManager;
|
||||
_memoryCache = memoryCache;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
|
@ -150,52 +162,79 @@ namespace MediaBrowser.Providers.Manager
|
|||
/// <inheritdoc/>
|
||||
public async Task SaveImage(BaseItem item, string url, ImageType type, int? imageIndex, CancellationToken cancellationToken)
|
||||
{
|
||||
var httpClient = _httpClientFactory.CreateClient(NamedClient.Default);
|
||||
using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (response.StatusCode != HttpStatusCode.OK)
|
||||
using (await _imageSaveLock.LockAsync(url, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
throw new HttpRequestException("Invalid image received.", null, response.StatusCode);
|
||||
}
|
||||
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType;
|
||||
|
||||
// Workaround for tvheadend channel icons
|
||||
// TODO: Isolate this hack into the tvh plugin
|
||||
if (string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase))
|
||||
if (_memoryCache.TryGetValue(url, out (string ContentType, byte[] ImageContents)? cachedValue)
|
||||
&& cachedValue is not null)
|
||||
{
|
||||
contentType = "image/png";
|
||||
var imageContents = cachedValue.Value.ImageContents;
|
||||
var cacheStream = new MemoryStream(imageContents, 0, imageContents.Length, false);
|
||||
await using (cacheStream.ConfigureAwait(false))
|
||||
{
|
||||
await SaveImage(
|
||||
item,
|
||||
cacheStream,
|
||||
cachedValue.Value.ContentType,
|
||||
type,
|
||||
imageIndex,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
var httpClient = _httpClientFactory.CreateClient(NamedClient.Default);
|
||||
using var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType;
|
||||
|
||||
// Workaround for tvheadend channel icons
|
||||
// TODO: Isolate this hack into the tvh plugin
|
||||
if (string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode);
|
||||
if (url.Contains("/imagecache/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
contentType = MediaTypeNames.Image.Png;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new HttpRequestException("Invalid image received: contentType not set.", null, response.StatusCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons...
|
||||
if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound);
|
||||
}
|
||||
// TVDb will sometimes serve a rubbish 404 html page with a 200 OK code, because reasons...
|
||||
if (contentType.Equals(MediaTypeNames.Text.Html, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new HttpRequestException("Invalid image received.", null, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
// some iptv/epg providers don't correctly report media type, extract from url if no extension found
|
||||
if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType)))
|
||||
{
|
||||
contentType = MimeTypes.GetMimeType(url);
|
||||
}
|
||||
// some iptv/epg providers don't correctly report media type, extract from url if no extension found
|
||||
if (string.IsNullOrWhiteSpace(MimeTypes.ToExtension(contentType)))
|
||||
{
|
||||
// Strip query parameters from url to get actual path.
|
||||
contentType = MimeTypes.GetMimeType(new Uri(url).GetLeftPart(UriPartial.Path));
|
||||
}
|
||||
|
||||
var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using (stream.ConfigureAwait(false))
|
||||
{
|
||||
await SaveImage(
|
||||
item,
|
||||
stream,
|
||||
contentType,
|
||||
type,
|
||||
imageIndex,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (!contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new HttpRequestException($"Request returned {contentType} instead of an image type", null, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
var responseBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
var stream = new MemoryStream(responseBytes, 0, responseBytes.Length, false);
|
||||
await using (stream.ConfigureAwait(false))
|
||||
{
|
||||
_memoryCache.Set(url, (contentType, responseBytes), TimeSpan.FromSeconds(10));
|
||||
|
||||
await SaveImage(
|
||||
item,
|
||||
stream,
|
||||
contentType,
|
||||
type,
|
||||
imageIndex,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1115,6 +1154,7 @@ namespace MediaBrowser.Providers.Manager
|
|||
}
|
||||
|
||||
_disposeCancellationTokenSource.Dispose();
|
||||
_imageSaveLock.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AsyncKeyedLock" />
|
||||
<PackageReference Include="LrcParser" />
|
||||
<PackageReference Include="MetaBrainz.MusicBrainz" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
|
||||
|
|
|
@ -27,6 +27,8 @@ public class GuideManager : IGuideManager
|
|||
private const string EtagKey = "ProgramEtag";
|
||||
private const string ExternalServiceTag = "ExternalServiceId";
|
||||
|
||||
private static readonly ParallelOptions _cacheParallelOptions = new() { MaxDegreeOfParallelism = Math.Min(Environment.ProcessorCount, 10) };
|
||||
|
||||
private readonly ILogger<GuideManager> _logger;
|
||||
private readonly IConfigurationManager _config;
|
||||
private readonly IFileSystem _fileSystem;
|
||||
|
@ -209,6 +211,7 @@ public class GuideManager : IGuideManager
|
|||
|
||||
_logger.LogInformation("Refreshing guide with {0} days of guide data", guideDays);
|
||||
|
||||
var maxCacheDate = DateTime.UtcNow.AddDays(2);
|
||||
foreach (var currentChannel in list)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
@ -263,6 +266,7 @@ public class GuideManager : IGuideManager
|
|||
if (newPrograms.Count > 0)
|
||||
{
|
||||
_libraryManager.CreateItems(newPrograms, null, cancellationToken);
|
||||
await PrecacheImages(newPrograms, maxCacheDate).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (updatedPrograms.Count > 0)
|
||||
|
@ -272,6 +276,7 @@ public class GuideManager : IGuideManager
|
|||
currentChannel,
|
||||
ItemUpdateType.MetadataImport,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await PrecacheImages(updatedPrograms, maxCacheDate).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
currentChannel.IsMovie = isMovie;
|
||||
|
@ -708,4 +713,41 @@ public class GuideManager : IGuideManager
|
|||
|
||||
return (item, isNew, isUpdated);
|
||||
}
|
||||
|
||||
private async Task PrecacheImages(IReadOnlyList<BaseItem> programs, DateTime maxCacheDate)
|
||||
{
|
||||
await Parallel.ForEachAsync(
|
||||
programs
|
||||
.Where(p => p.EndDate.HasValue && p.EndDate.Value < maxCacheDate)
|
||||
.DistinctBy(p => p.Id),
|
||||
_cacheParallelOptions,
|
||||
async (program, cancellationToken) =>
|
||||
{
|
||||
for (var i = 0; i < program.ImageInfos.Length; i++)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var imageInfo = program.ImageInfos[i];
|
||||
if (!imageInfo.IsLocalFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
program.ImageInfos[i] = await _libraryManager.ConvertImageToLocal(
|
||||
program,
|
||||
imageInfo,
|
||||
imageIndex: 0,
|
||||
removeOnFailure: false)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Unable to precache {Url}", imageInfo.Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@ using MediaBrowser.Controller.Subtitles;
|
|||
using MediaBrowser.Model.Configuration;
|
||||
using MediaBrowser.Model.IO;
|
||||
using MediaBrowser.Providers.Manager;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
@ -572,7 +573,8 @@ namespace Jellyfin.Providers.Tests.Manager
|
|||
Mock.Of<IServerApplicationPaths>(),
|
||||
libraryManager.Object,
|
||||
baseItemManager!,
|
||||
Mock.Of<ILyricManager>());
|
||||
Mock.Of<ILyricManager>(),
|
||||
Mock.Of<IMemoryCache>());
|
||||
|
||||
return providerManager;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue