private async Task <byte[]> LoadImage(HyperTag tag)
{
if (Context?.HyperStore == null)
{
return(null);
}
var ids = this.HyperTagId;
var geometry = tag.GetElement <HyperTagGeometry>();
var args2 = new RetrieveFragmentFramesArgs()
{
AssetId = ids.HyperId.AssetId.Value,
FragmentId = ids.HyperId.HasFullFragmentData ? ids.HyperId.FragmentId.Value : new HyperFragmentId(0),
SliceIds = new HyperSliceId[] { ids.HyperId.HasFullSliceData?ids.HyperId.SliceId.Value : new HyperSliceId(0) },
GeometryItem = this.Context?.ExtractMode == true ? geometry?.GeometryItem : null,
FabricServiceId = this.Context?.FabricServiceId,
};
var sliceResult = await Context.HyperStore.ExecuteAsync(args2);
if (sliceResult == null || sliceResult.Length == 0 || sliceResult[0].Image?.Data == null)
{
return(null);
}
byte[] imageData = sliceResult[0].Image.Data;
if (this.Context?.ExtractMode != true)
{
imageData = TagRenderHelper.RenderTag(tag, imageData);
}
return(imageData);
}
private async Task <UniImage> GetImageAsync(HyperId hyperId)
{
var args = new RetrieveFragmentFramesArgs
{
AssetId = hyperId.AssetId.Value,
FragmentId = hyperId.FragmentId.Value,
SliceIds = new HyperSliceId[] { hyperId.SliceId.Value },
FabricServiceId = this._settings.FabricServiceId
};
var res = await _hyperStore.ExecuteAsyncThrows(args);
return(res.First().Image);
}
private async Task <UniImage> GetFirstImageForMetadatasetAssetAsync(DateTime?from)
{
var retrieveAssetArgs = new RetrieveAssetsArgs
{
AssetsIds = _metadataSet.AssetIds
};
var assets = (await _hyperStore.ExecuteAsyncThrows(retrieveAssetArgs));
HyperAsset assetToGetSliceFrom = null;
foreach (var asset in assets)
{
var retrieveTrackArgs = new RetrieveTrackArgs
{
AssetId = asset.Id,
TrackId = asset.DefaultVideoTrackId.Value
};
asset.AddTrack(await _hyperStore.RetrieveTrackAsync(asset.Id, asset.DefaultVideoTrackId.Value, null, false));
if (from >= asset.TimeStampUTC && from <= asset.TimeStampUTC + asset.GetDuration().Value)
{
assetToGetSliceFrom = asset;
}
}
if (assetToGetSliceFrom == null)
{
assetToGetSliceFrom = assets[0];
from = assetToGetSliceFrom.TimeStampUTC;
}
var assetStartTime = assetToGetSliceFrom.TimeStampUTC;
from = from ?? assetStartTime;
var startingPosition = from - assetStartTime;
var beginningHyperId = await assetToGetSliceFrom.DefaultVideoTrack.FindAtAsync(_hyperStore, assetToGetSliceFrom.Id, (ulong)startingPosition.Value.TotalMilliseconds, HyperSeek.SeekModes.Estimated, false);
var args = new RetrieveFragmentFramesArgs
{
AssetId = _metadataSet.AssetIds.First(),
SliceIds = new HyperSliceId[] { beginningHyperId.Value.SliceId }
};
var res = await _hyperStore.ExecuteAsyncThrows(args);
return(res.First().Image);
}
/// <summary>
/// Generates a heatmap based on subset of tags using Fixed Camera Preset with homography.
/// </summary>
/// <param name="tags">Subset of tags to use.</param>
/// <param name="fixedCameraPresetId">Id of Fixed Camera Preset to take homography configuration from.</param>
/// <param name="renderingMode">If enabled, heatmap masks will be rendered, otherwise real pixels from corresponding frames will be rendered in overlap.</param>
/// <param name="cutOff">If we should cutoff the image by homography or just render a polygon line over.</param>
public async Task <UniImage> GenerateFromTagsAsync(List <HyperTag> tags, HyperDocumentId fixedCameraPresetId, bool cutOff = false, int tagGroupsLimit = 30)
{
if (tags == null || tags.Count < 1)
{
return(null);
}
var configuration = await RetrieveHyperDocumentArgs.RetrieveAsyncThrows <FixedCameraEnhancedData>(_hyperStore, fixedCameraPresetId);
if (configuration == null)
{
return(null);
}
var tagsWithHyperId = tags.Select(x => new
{
Tag = x,
HyperId = x.GetElement <IHyperTagHyperIds>().HyperId
}).ToArray();
tagsWithHyperId = tagsWithHyperId.OrderBy(x => x.HyperId).ToArray();
var hyperId = tagsWithHyperId.First().HyperId;
if (!_keepWorking || _ct.IsCancellationRequested)
{
return(null);
}
await UpdateImageDataAsync(hyperId);
if (!_keepWorking || _ct.IsCancellationRequested)
{
return(null);
}
SKImage maskedImage = null;
if (_settings.RenderingMode == RenderingMode.Masks)
{
_globalMatrix = new uint[_height, _width];
foreach (var tag in tags)
{
var tagExtraction = ProcessMaskedTag(tag);
if (tagExtraction != null)
{
ProcessMaskOverlapsIntoMatrix(tagExtraction.Image, tagExtraction.Rect);
}
}
maskedImage = RenderMask(_lastImage);
}
else if (_settings.RenderingMode == RenderingMode.LowerPointOfGeometry)
{
var framePoints = new List <HeatmapRenderHelper.HeatPoint>();
foreach (var tag in tags)
{
ProcessGeometryTag(tag, framePoints);
}
var mask = HeatmapRenderHelper.RenderToSkImage(framePoints, _width, _height);
maskedImage = BlendTwoImages(SKBitmap.Decode(_lastImage.Data), mask, SKBlendMode.Plus);
}
else if (_settings.RenderingMode == RenderingMode.RealImage)
{
var tagsBySlice = tagsWithHyperId.GroupBy(x => x.HyperId).OrderBy(x => x.Key).ToArray();
var sourceBitmap = SKBitmap.Decode(_lastImage.Data);
foreach (var tagsForSlice in tagsBySlice.Take(tagGroupsLimit))
{
if (!_keepWorking || _ct.IsCancellationRequested)
{
return(null);
}
var args = new RetrieveFragmentFramesArgs
{
AssetId = tagsForSlice.Key.AssetId.Value,
TrackId = tagsForSlice.Key.TrackId.Value,
FragmentId = tagsForSlice.Key.FragmentId.Value,
SliceIds = new HyperSliceId[] { tagsForSlice.Key.SliceId.Value },
FabricServiceId = this._settings.FabricServiceId
};
var result = await _hyperStore.ExecuteAsync(args) ?? new RetrieveFragmentFramesArgs.SliceResult[0];
if (result.Length == 0)
{
continue;
}
var frame = result[0].Image;
var frameSkBitmap = SKBitmap.Decode(frame.Data);
_globalMatrix = new uint[_height, _width];
foreach (var tag in tagsForSlice.Select(x => x.Tag))
{
if (!_keepWorking || _ct.IsCancellationRequested)
{
return(null);
}
var tagExtraction = ProcessMaskedTag(tag);
if (tagExtraction != null)
{
ProcessMaskTransparencyIntoMatrix(tagExtraction.Image, tagExtraction.Rect);
}
}
var sourcePixelsAddr = sourceBitmap.GetPixels();
var destPixelsAddr = frameSkBitmap.GetPixels();
byte MixColors(byte color1, byte color2, byte transparencyValue)
{
var transparency2 = (double)1 / 255 * transparencyValue;
var transparency1 = 1 - transparency2;
var result = (byte)((color1 * transparency1) + (color2 * transparency2));
return(result);
}
unsafe
{
byte *sourcePtr = (byte *)sourcePixelsAddr.ToPointer();
byte *destPtr = (byte *)destPixelsAddr.ToPointer();
for (int row = 0; row < sourceBitmap.Height; row++)
{
for (int col = 0; col < sourceBitmap.Width; col++)
{
var transpValue = _globalMatrix[row, col];
if (transpValue == 0) // Do not modify anything
{
sourcePtr += 4;
destPtr += 4;
}
else
{
var destBlue = *destPtr++; // blue
var destGreen = *destPtr++; // green
var destRed = *destPtr++; // red
destPtr++; // skip alpha
var sourceBlue = *sourcePtr;
var sourceGreen = *(sourcePtr + 1);
var sourceRed = *(sourcePtr + 2);
var mixedBlue = MixColors(sourceBlue, destBlue, (byte)transpValue);
var mixedGreen = MixColors(sourceGreen, destGreen, (byte)transpValue);
var mixedRed = MixColors(sourceRed, destRed, (byte)transpValue);
*sourcePtr++ = mixedBlue; // blue
*sourcePtr++ = mixedGreen; // green
*sourcePtr++ = mixedRed; // red
sourcePtr++; // skip alpha
}
}
}
}
}
maskedImage = SKImage.FromBitmap(sourceBitmap);
}
if (maskedImage == null)
{
return(null);
}
if (cutOff)
{
return(RenderWithCrop(maskedImage, configuration));
}
else
{
return(RenderWithNoCutoff(maskedImage, configuration));
}
}
