public async Task <bool> Render(
ThumbnailsRenderContext ctx,
ThumbnailsRenderFileInfo fileInfo,
ThumbnailsRenderOption option)
{
await _fileService.ReadFileStream(fileInfo.FileHandle, videoStream =>
{
using var formatContext = new FormatContext(videoStream);
var attachedPicStream = formatContext.FindAttachedPicStream();
if (attachedPicStream != null)
{
DrawAttachedPicture(ctx, attachedPicStream);
}
else
{
var audioStream = formatContext.FindBestAudioStream();
if (audioStream == null)
{
throw new InvalidDataException("No audio stream found");
}
DrawWaves(ctx, audioStream);
}
return(ValueTask.FromResult(true));
});
return(true);
}
protected override Task <bool> Render(
ThumbnailsRenderContext ctx,
ThumbnailsRenderFileInfo fileInfo,
ThumbnailsRenderOption option)
{
Assert.AreEqual(MimeType.image_png, fileInfo.MimeType);
return(Task.FromResult(true));
}
public async Task <bool> Render(
ThumbnailsRenderContext ctx,
ThumbnailsRenderFileInfo fileInfo,
ThumbnailsRenderOption option)
{
await _fileService.ReadFileStream(
fileInfo.FileHandle,
videoStream => ValueTask.FromResult(DrawVideo(videoStream, ctx)));
return(true);
}
async Task <bool> IThumbnailsRenderer.Render(
ThumbnailsRenderContext ctx,
ThumbnailsRenderFileInfo fileInfo,
ThumbnailsRenderOption option)
{
if (!IsSupported(fileInfo))
{
return(false);
}
return(await Render(ctx, fileInfo, option));
}
/// <inheritdoc />
protected override async Task <bool> Render(
ThumbnailsRenderContext ctx,
ThumbnailsRenderFileInfo fileInfo,
ThumbnailsRenderOption option)
{
// use the following code maybe faster. https://github.com/kleisauke/net-vips/issues/128
// > sourceVipsImage = Image.Thumbnail(localPath, loadImageSize, loadImageSize, noRotate: false);
return(await _fileService.ReadFileStream(
fileInfo.FileHandle,
stream =>
{
var sourceVipsImage = Image.ThumbnailStream(
stream,
(int)(ThumbnailUtils.DefaultMaxWidth * ctx.Density),
height: (int)(ThumbnailUtils.DefaultMaxHeight * ctx.Density),
noRotate: false);
sourceVipsImage = sourceVipsImage.Colourspace(Enums.Interpretation.Srgb).Cast(Enums.BandFormat.Uchar);
if (!sourceVipsImage.HasAlpha())
{
sourceVipsImage = sourceVipsImage.Bandjoin(255);
}
var imageWidth = sourceVipsImage.Width;
var imageHeight = sourceVipsImage.Height;
var sourceImageDataPtr = sourceVipsImage.WriteToMemory(out _);
sourceVipsImage.Close();
try
{
using var colorspace = SKColorSpace.CreateSrgb();
var sourceImageInfo = new SKImageInfo(
imageWidth,
imageHeight,
SKColorType.Rgba8888,
SKAlphaType.Unpremul,
colorspace);
using var image =
SKImage.FromPixels(sourceImageInfo, sourceImageDataPtr, sourceImageInfo.RowBytes);
ThumbnailUtils.DrawShadowView(ctx, new SkImageView(image));
}
finally
{
NetVips.NetVips.Free(sourceImageDataPtr);
}
return ValueTask.FromResult(true);
}));
}
public async Task Setup()
{
var services = new ServiceCollection();
services.AddSingleton(typeof(ILogger <>), typeof(NullLogger <>));
services.AddSingleton <ILogger, NullLogger>();
services.AddSingleton <ILoggerFactory, NullLoggerFactory>();
services.TryAddSingletonFileService(builder => builder.TryAddMemoryFileSystem("memory"));
_serviceProvider = services.BuildServiceProvider();
_fileService = _serviceProvider.GetRequiredService <IFileService>();
var root = await _fileService.CreateFileHandle(Url.Parse("file://memory/"));
_exampleMp4FileHandle = await _fileService.CreateFile(
root,
"example.mp4",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.mp4"));
_exampleMp4FileStats = await _fileService.Stat(_exampleMp4FileHandle);
_exampleMp3FileHandle = await _fileService.CreateFile(
root,
"example.mp3",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.mp3"));
_exampleMp3FileStats = await _fileService.Stat(_exampleMp3FileHandle);
_exampleMp3WithCoverFileHandle = await _fileService.CreateFile(
root,
"example-with-cover.mp3",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example-with-cover.mp3"));
_exampleMp3WithCoverFileStats = await _fileService.Stat(_exampleMp3WithCoverFileHandle);
_examplePngFileHandle = await _fileService.CreateFile(
root,
"example.png",
Resources.ReadEmbeddedFile(typeof(RendererBenchmark).Assembly, "/Resources/example.png"));
_examplePngFileStats = await _fileService.Stat(_examplePngFileHandle);
_renderContext = new ThumbnailsRenderContext();
_audioFileRenderer = new AudioFileRenderer(_fileService);
_videoFileRenderer = new VideoFileRenderer(_fileService);
_renderOption = new ThumbnailsRenderOption {
Size = 512
};
}
public async Task ImplementationTest()
{
IThumbnailsRenderer testRenderer = new TestRenderer();
Assert.IsTrue(
testRenderer.IsSupported(
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 100, FileType.File, new FileHash("unavailable")),
MimeType.text_plain)));
Assert.IsTrue(
testRenderer.IsSupported(
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 100, FileType.File, new FileHash("unavailable")),
MimeType.image_png)));
Assert.IsFalse(
testRenderer.IsSupported(
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 1001, FileType.File, new FileHash("unavailable")),
MimeType.image_png)));
Assert.IsFalse(
testRenderer.IsSupported(
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 100, FileType.File, new FileHash("unavailable")),
MimeType.image_jpeg)));
using var testRenderContext = new ThumbnailsRenderContext();
Assert.IsTrue(
await testRenderer.Render(
testRenderContext,
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 100, FileType.File, new FileHash("unavailable")),
MimeType.image_png),
new ThumbnailsRenderOption()));
Assert.IsFalse(
await testRenderer.Render(
testRenderContext,
new ThumbnailsRenderFileInfo(
new FileHandle("test_file"),
new FileStats(DateTimeOffset.Now, DateTimeOffset.Now, 100, FileType.File, new FileHash("unavailable")),
MimeType.image_jpeg),
new ThumbnailsRenderOption()));
}
private static void DrawAttachedPicture(ThumbnailsRenderContext ctx, MediaStream attachedPicStream)
{
using var attachedPicture = attachedPicStream.ReadAttachedPicture();
var attachedPictureVipsImage = Image.ThumbnailStream(
attachedPicture,
(int)(ThumbnailUtils.DefaultMaxWidth * ctx.Density),
height: (int)(ThumbnailUtils.DefaultMaxHeight * ctx.Density),
noRotate: false);
attachedPictureVipsImage = attachedPictureVipsImage.Colourspace(Enums.Interpretation.Srgb).Cast(Enums.BandFormat.Uchar);
if (!attachedPictureVipsImage.HasAlpha())
{
attachedPictureVipsImage = attachedPictureVipsImage.Bandjoin(255);
}
var imageWidth = attachedPictureVipsImage.Width;
var imageHeight = attachedPictureVipsImage.Height;
var sourceImageDataPtr = attachedPictureVipsImage.WriteToMemory(out _);
attachedPictureVipsImage.Close();
try
{
using var colorspace = SKColorSpace.CreateSrgb();
var sourceImageInfo = new SKImageInfo(
imageWidth,
imageHeight,
SKColorType.Rgba8888,
SKAlphaType.Unpremul,
colorspace);
using var image =
SKImage.FromPixels(sourceImageInfo, sourceImageDataPtr, sourceImageInfo.RowBytes);
_cachedDecorationImage ??= SKImage.FromEncodedData(ReadDecorationImage());
ThumbnailUtils.DrawShadowView(
ctx,
new SkImageView(image),
_cachedDecorationImage,
new SKColor(0, 0, 0),
minSize: new SKSize(24, 24));
}
finally
{
NetVips.NetVips.Free(sourceImageDataPtr);
}
}
private static unsafe void DrawWaves(ThumbnailsRenderContext ctx, MediaStream audioStream)
{
var height = 88;
var width = 88;
using var recorder = new SKPictureRecorder();
using var canvas = recorder.BeginRecording(SKRect.Create(width, height));
var columnCount = 88 * 2;
var columnWidth = (float)width / columnCount;
using var wavePaint = new SKPaint { Color = new SKColor(128, 128, 128, 76), StrokeWidth = columnWidth };
var totalSample = audioStream.SampleRate * audioStream.Duration;
var columnMaxSample = (int)(totalSample / columnCount);
var columns = new short[columnCount];
using var decoder = audioStream.CreateStreamDecoder();
using var filter = new AudioFormatFilter("sample_fmts=s16:channel_layouts=mono", audioStream, decoder);
filter.Build();
long sum = 0;
var n = 0;
var c = 0;
while (filter.MoveNext())
{
var frame = filter.Current.Value;
var p = (short *)frame->data[0];
for (var i = 0; i < frame->nb_samples; i++)
{
sum += SampleAbs(p[i]);
n++;
if (n == columnMaxSample && c < columnCount)
{
columns[c] = (short)(sum / n);
n = 0;
sum = 0;
c++;
}
}
}
for (var i = 0; i < columnCount; i++)
{
var h = Math.Max((float)columns[i] * height / short.MaxValue, 0.5f);
var x = i * columnWidth;
canvas.DrawLine(x, (height - h) / 2.0f, x, (height + h) / 2.0f, wavePaint);
}
_cachedWaveformDecorationImage ??= SKImage.FromEncodedData(ReadWaveformDecorationImage());
canvas.DrawImage(_cachedWaveformDecorationImage, SKRect.Create((width - 28) / 2.0f, (height - 36) / 2.0f, 28, 36));
using var picture = recorder.EndRecording();
ThumbnailUtils.DrawShadowView(
ctx,
new SkPictureView(
picture,
new SKSize(88, 88)));
}
public Task <bool> Render(ThumbnailsRenderContext ctx, ThumbnailsRenderFileInfo fileInfo, ThumbnailsRenderOption option)
{
RenderCount++;
return(_wrappedThumbnailsRenderer.Render(ctx, fileInfo, option));
}
protected abstract Task <bool> Render(ThumbnailsRenderContext ctx, ThumbnailsRenderFileInfo fileInfo, ThumbnailsRenderOption option);
private static unsafe bool DrawVideo(Stream videoStream, ThumbnailsRenderContext ctx)
{
using var formatContext = new FormatContext(videoStream);
var stream = formatContext.FindBestVideoStream();
if (stream == null)
{
return(false);
}
using var videoStreamDecoder = stream.CreateStreamDecoder();
try
{
if (videoStreamDecoder.Duration <= 0)
{
videoStreamDecoder.SeekFrame(10 * 1000000);
}
if (videoStreamDecoder.Duration > 3)
{
videoStreamDecoder.SeekFrame(videoStreamDecoder.Duration / 3);
}
}
catch (FFmpegException err)
{
Console.WriteLine("Seek failed: " + err);
}
var destinationSize = ThumbnailUtils.ContainSize(
new SKSize(videoStreamDecoder.FrameWidth, videoStreamDecoder.FrameHeight),
new SKSize(ThumbnailUtils.DefaultMaxWidth * ctx.Density, ThumbnailUtils.DefaultMaxHeight * ctx.Density)).ToSizeI();
var sourcePixelFormat = videoStreamDecoder.PixelFormat;
if (!videoStreamDecoder.MoveNext())
{
throw new InvalidDataException("Can't decode the video.");
}
using var vfc =
new VideoFrameConverter(
videoStreamDecoder.FrameWidth,
videoStreamDecoder.FrameHeight,
sourcePixelFormat,
destinationSize.Width,
destinationSize.Height);
var convertedFrame = vfc.Convert(videoStreamDecoder.Current.Value);
using var colorspace = SKColorSpace.CreateSrgb();
var sourceImageInfo = new SKImageInfo(
convertedFrame.width,
convertedFrame.height,
SKColorType.Rgba8888,
SKAlphaType.Unpremul,
colorspace);
using var image =
SKImage.FromPixels(sourceImageInfo, (IntPtr)convertedFrame.data[0], sourceImageInfo.RowBytes);
_cachedDecorationImage ??= SKImage.FromEncodedData(ReadDecorationImage());
ThumbnailUtils.DrawShadowView(
ctx,
new SkImageView(image),
_cachedDecorationImage,
new SKColor(0, 0, 0),
minSize: new SKSize(24, 24));
return(true);
}
public static void DrawShadowView(
ThumbnailsRenderContext ctx,
ISkView view,
SKImage?decorationImage = null,
SKColor?backgroundColor = null,
SKSize?maxSize = null,
SKSize?minSize = null,
SKFilterQuality resizeQuality = SKFilterQuality.None)
{
// max width = 128 - 12 - 12
// max height = 128 - 20 - 20
maxSize ??= new SKSize(DefaultMaxWidth, DefaultMaxHeight);
minSize ??= new SKSize(0, 0);
backgroundColor ??= new SKColor(0xff, 0xff, 0xff);
var imageSize = ContainSize(new SKSize(view.Size.Width, view.Size.Height), maxSize.Value);
var imageRect = CentralRect(imageSize);
var borderSize = new SKSize(Math.Max(imageSize.Width, minSize.Value.Width), Math.Max(imageSize.Height, minSize.Value.Height));
var borderRect = CentralRect(new SKSize(borderSize.Width + 1, borderSize.Height + 1));
using (new SKAutoCanvasRestore(ctx.Canvas))
{
ctx.Canvas.Clear();
using (var rectFillPaint = new SKPaint
{
Style = SKPaintStyle.Fill,
Color = backgroundColor.Value,
ImageFilter = SKImageFilter.CreateDropShadow(0, 1, 4, 4, new SKColor(136, 136, 136, 128))
})
using (var rectStrokePaint = new SKPaint
{
Style = SKPaintStyle.Stroke, StrokeWidth = 1, Color = new SKColor(136, 136, 136, 64), BlendMode = SKBlendMode.Src
})
{
// draw border
using SKRoundRect roundRect = new(borderRect, 5);
ctx.Canvas.DrawRoundRect(roundRect, rectFillPaint);
ctx.Canvas.DrawRoundRect(roundRect, rectStrokePaint);
}
using (new SKAutoCanvasRestore(ctx.Canvas))
using (var imagePaint = new SKPaint {
FilterQuality = resizeQuality
})
using (var decorationImagePaint = new SKPaint {
FilterQuality = SKFilterQuality.Medium
})
{
using SKRoundRect roundRect = new(imageRect, 4.5f);
using (new SKAutoCanvasRestore(ctx.Canvas))
{
ctx.Canvas.ClipRoundRect(roundRect);
// draw image
view.Draw(ctx.Canvas, imageRect, imagePaint);
}
if (decorationImage != null)
{
var decorationRect = SKRect.Create(borderRect.Right - 24.5f, borderRect.Bottom - 22.5f, 36, 36);
ctx.Canvas.DrawImage(decorationImage, decorationRect, decorationImagePaint);
}
}
}
}
