/// <summary>
/// Initializes the DataflowBlock
/// </summary>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="inputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InputBlock"/></param>
/// <param name="innerBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InnerBlock"/></param>
/// <param name="outputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="OutputBlock"/></param>
/// <param name="blockname">Name of this DataflowBlock (must be unique in a given chain of DataflowBlocks)</param>
/// <param name="isRestorePointAfterDeserialization">
/// <c>true</c>, if after deserialiization from disk, <see cref="PendingImportResourceNewGen"/>s are restored to
/// this block. If <c>false</c>, they are restored to the last passed DataflowBlock having this parameter set to <c>true</c>
/// </param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
protected ImporterWorkerDataflowBlockBase(ImportJobInformation importJobInformation, ExecutionDataflowBlockOptions inputBlockOptions, ExecutionDataflowBlockOptions innerBlockOptions, ExecutionDataflowBlockOptions outputBlockOptions, String blockname, bool isRestorePointAfterDeserialization, ImportJobController parentImportJobController, CancellationToken ct)
{
_blockName = blockname;
_isRestorePointAfterDeserialization = isRestorePointAfterDeserialization;
_ct = ct;
ImportJobInformation = importJobInformation;
InputBlockOptions = inputBlockOptions;
InnerBlockOptions = innerBlockOptions;
OutputBlockOptions = outputBlockOptions;
ParentImportJobController = parentImportJobController;
_stopWatch = new Stopwatch();
_tcs = new TaskCompletionSource <object>();
Activated = new AsyncManualResetEvent(InnerBlockOptions.CancellationToken);
InputBlock = new TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => InputBlockMethod(p), InputBlockOptions);
OutputBlock = new TransformManyBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => OutputBlockMethod(p), OutputBlockOptions);
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
InnerBlock = CreateInnerBlock();
InnerBlock.LinkTo(OutputBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
InputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
InnerBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
OutputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(InputBlock.Completion, InnerBlock.Completion, OutputBlock.Completion).ContinueWith(OnAllBlocksFinished);
}
/// <summary>
/// Initiates the MediaItemSaveBlock
/// </summary>
/// <remarks>
/// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 500 },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
BLOCK_NAME, false, parentImportJobController)
{
}
/// <summary>
/// Initiates the DirectorySaveBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
new ExecutionDataflowBlockOptions { CancellationToken = ct },
BLOCK_NAME, true, parentImportJobController)
{
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
/// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100 },
new ExecutionDataflowBlockOptions { CancellationToken = ct, BoundedCapacity = 1 },
forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
{
_forceQuickMode = forceQuickMode;
_mostRecentMiaCreationDate = new Lazy<Task<DateTime>>(GetMostRecentMiaCreationDate);
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding MediaItemLoadBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
BLOCK_NAME, true, parentImportJobController, ct)
{
_forceQuickMode = forceQuickMode;
}
/// <summary>
/// Initiates the DirectoryUnfoldBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectoryUnfoldBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController, ct)
{
}
/// <summary>
/// Initiates the MetadataExtractorBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 100,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
/// <param name="forceQuickMode"><c>true</c> if this is the MetadataExtractorBlock used for FirstPassImports, else <c>false</c></param>
public MetadataExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController, bool forceQuickMode)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 100
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
forceQuickMode ? BLOCK_NAME_QUICK : BLOCK_NAME_FULL, true, parentImportJobController)
{
_forceQuickMode = forceQuickMode;
_mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
}
/// <summary>
/// Initiates the RelationshipExtractorBlock
/// </summary>
/// <remarks>
/// The preceding MediaItemSaveBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this RelationshipExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 50,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public RelationshipExtractorBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, MaxDegreeOfParallelism = Environment.ProcessorCount * 5, BoundedCapacity = 50
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController, ct)
{
_relationshipCache = new LRURelationshipCache();
_cacheSync = new SemaphoreSlim(1, 1);
_processedMediaItemIds = new ConcurrentDictionary <Guid, byte>();
}
/// <summary>
/// Initiates the MediaItemLoadBlock
/// </summary>
/// <remarks>
/// The preceding FileUnfoldBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MetadataExtractorBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. For the reason mentioned before, we also have to
/// set the BoundedCapacity of the OutputBlock to 1.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemLoadBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 500
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
BLOCK_NAME, false, parentImportJobController, ct)
{
_mostRecentMiaCreationDate = new Lazy <Task <DateTime> >(GetMostRecentMiaCreationDate);
}
/// <summary>
/// Initiates the DirectorySaveBlock
/// </summary>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public DirectorySaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, true, parentImportJobController)
{
}
/// <summary>
/// Initiates the MediaItemSaveBlock
/// </summary>
/// <remarks>
/// The preceding MetadataExtractorBlock has a BoundedCapacity. To avoid that this limitation does not have any effect
/// because all the items are immediately passed to an unbounded InputBlock of this MediaItemSaveBlock, we
/// have to set the BoundedCapacity of the InputBlock to 1. The BoundedCapacity of the InnerBlock is set to 500,
/// which is a good trade-off between speed and memory usage. The OutputBlock disposes the PendingImportResources and
/// therefore does not need a BoundedCapacity.
/// </remarks>
/// <param name="ct">CancellationToken used to cancel this DataflowBlock</param>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
public MediaItemSaveBlock(CancellationToken ct, ImportJobInformation importJobInformation, ImportJobController parentImportJobController)
: base(importJobInformation,
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 1
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct, BoundedCapacity = 500
},
new ExecutionDataflowBlockOptions {
CancellationToken = ct
},
BLOCK_NAME, false, parentImportJobController)
{
}
