/// <summary>
/// Impeletemt Mirror mapping when destination columns similar ro source columns + prefix (for example)
/// </summary>
/// <param name="SourceAdapter"></param>
/// <param name="Prefix"></param>
/// <returns></returns>
public static List <ColumnAssignment> CreateMapingWithPrefix(ISourceAdapter SourceAdapter, string Prefix)
{
List <ColumnAssignment> result = new List <ColumnAssignment>();
foreach (var col in SourceAdapter.Fileds)
{
var columnAssignment = new ColumnAssignment()
{
FromColumn = new SimpleMapColumn()
{
InternalName = col.InternalName,
ReadOnly = false
},
ToColumn = new SimpleMapColumn()
{
InternalName = Prefix + col.InternalName,
ReadOnly = false
}
};
result.Add(columnAssignment);
}
return(result);
}
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source)
{
const string crLf = "\r\n";
base.Prepare(source);
if (FieldSizes == null || FieldSizes.Length <= 0)
{
InitializeFieldSizesWithDefaults();
}
if (FieldSizes.Length != Source.FieldCount)
{
throw new DataPipelineException("Provided FieldSizes array has a different length than result set column count");
}
if (OverflowStringFieldWidthBehaviors != null && OverflowStringFieldWidthBehaviors.Length != Source.FieldCount)
{
throw new DataPipelineException("When OverflowStringFieldWidthBehaviors is provided its length must match result set field count");
}
_rowSize = 0;
FormatString = "";
for (var i = 0; i < Source.FieldCount; i++)
{
_rowSize += Math.Abs(FieldSizes[i]);
FormatString += $"{{{i},{FieldSizes[i]}{ColumnFormatString(i)}}}";
}
_rowSize += crLf.Length;
WriteBuffer = new byte[_rowSize];
FormatString += crLf;
}
public virtual void Pump(ISourceAdapter <TRow> sourceAdapter,
IEnumerable <ITargetAdapter <TRow> > targetAdapters)
{
using var resetCurrentSourceAdapter = new Resettable <ISourceAdapter <TRow> >(CurrentSourceAdapter = sourceAdapter, adapter => CurrentSourceAdapter = null);
using var setEventFinishedResettable = new Resettable <ManualResetEvent>(FinishedEvent, evt => evt.Set());
SetupAndHealthCheckAdapters(sourceAdapter, targetAdapters, out var targetAdaptersCount);
using var unSetupAdaptersResettable = new Resettable <object>(this, obj => UnSetupAdapters(sourceAdapter, targetAdapters));
sourceAdapter.Prepare();
using var unPrepareSourceAdapterResettable = new Resettable <ISourceAdapter <TRow> >(sourceAdapter, adapter => adapter.UnPrepare());
foreach (var targetAdapter in targetAdapters)
{
targetAdapter.Prepare(sourceAdapter);
}
CheckDeadlockPotential(sourceAdapter, targetAdapters);
var targetAdapterIndex = 0;
var targetAdapterConveyors = new Conveyor <TRow> [targetAdaptersCount];
foreach (var targetAdapter in targetAdapters)
{
targetAdapterConveyors[targetAdapterIndex++] = new Conveyor <TRow>((row, context) =>
{
((ITargetAdapter <TRow>)context).Process(row);
sourceAdapter.ReleaseRow(row);
}, targetAdapter);
}
try
{
foreach (var targetAdapterConveyor in targetAdapterConveyors)
{
targetAdapterConveyor.Start();
}
var sourceRows = sourceAdapter.RowsEnumerable;
foreach (var row in sourceRows)
{
foreach (var targetAdapterConveyor in targetAdapterConveyors)
{
targetAdapterConveyor.InsertPacket(row);
}
}
foreach (var targetAdapterConveyor in targetAdapterConveyors)
{
targetAdapterConveyor.StopAndWait();
}
targetAdapters.ForEach(adapter => adapter.UnPrepare());
}
catch (Exception e)
{
targetAdapterConveyors.ForEach(conveyor => conveyor.Stop(), typeof(InvalidOperationException), false);
targetAdapters.ForEach(adapter => adapter.AbortedWithException(e));
throw;
}
}
public void Host(ObfuscationConfiguration obfuscationConfiguration, Action <long, bool, double> statusCallback)
{
IEnumerable <IDictionary <string, object> > sourceDataEnumerable;
IEnumerable <Message> messages;
if ((object)obfuscationConfiguration == null)
{
throw new ArgumentNullException("obfuscationConfiguration");
}
messages = obfuscationConfiguration.Validate();
if (messages.Any())
{
throw new ApplicationException(string.Format("Obfuscation configuration validation failed:\r\n{0}", string.Join("\r\n", messages.Select(m => m.Description).ToArray())));
}
using (IOxymoronEngine oxymoronEngine = new OxymoronEngine(this, obfuscationConfiguration))
{
using (DisposableList <IDictionaryAdapter> dictionaryAdapters = new DisposableList <IDictionaryAdapter>())
{
foreach (DictionaryConfiguration dictionaryConfiguration in obfuscationConfiguration.DictionaryConfigurations)
{
IDictionaryAdapter dictionaryAdapter;
dictionaryAdapter = dictionaryConfiguration.DictionaryAdapterConfiguration.GetAdapterInstance <IDictionaryAdapter>();
dictionaryAdapters.Add(dictionaryAdapter);
dictionaryAdapter.Initialize(dictionaryConfiguration.DictionaryAdapterConfiguration);
dictionaryAdapter.InitializePreloadCache(dictionaryConfiguration, oxymoronEngine.SubstitutionCacheRoot);
this.DictionaryConfigurationToAdapterMappings.Add(dictionaryConfiguration, dictionaryAdapter);
}
using (ISourceAdapter sourceAdapter = obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterInstance <ISourceAdapter>())
{
sourceAdapter.Initialize(obfuscationConfiguration.SourceAdapterConfiguration);
using (IDestinationAdapter destinationAdapter = obfuscationConfiguration.DestinationAdapterConfiguration.GetAdapterInstance <IDestinationAdapter>())
{
destinationAdapter.Initialize(obfuscationConfiguration.DestinationAdapterConfiguration);
destinationAdapter.UpstreamMetadata = sourceAdapter.UpstreamMetadata;
sourceDataEnumerable = sourceAdapter.PullData(obfuscationConfiguration.TableConfiguration);
sourceDataEnumerable = oxymoronEngine.GetObfuscatedValues(sourceDataEnumerable);
if ((object)statusCallback != null)
{
sourceDataEnumerable = WrapRecordCounter(sourceDataEnumerable, statusCallback);
}
destinationAdapter.PushData(obfuscationConfiguration.TableConfiguration, sourceDataEnumerable);
}
}
}
}
}
private static void CheckDeadlockPotential(ISourceAdapter <TRow> sourceAdapter, IEnumerable <ITargetAdapter <TRow> > targetAdapters)
{
var totalTargetBufferSize = targetAdapters.Sum(targetAdapter => targetAdapter.BufferSize);
if (sourceAdapter.RowsPoolSize > 0 && sourceAdapter.RowsPoolSize < totalTargetBufferSize)
{
throw new DataPipelineException("Source adapter rows pool size can't be lesser than the sum of all target adapters buffer sizes. Deadlock would occur upon call to Pump()");
}
}
private void UnSetupAdapters(ISourceAdapter <TRow> sourceAdapter,
IEnumerable <ITargetAdapter <TRow> > targetAdapters)
{
sourceAdapter.OnSourceAdapterRowReadError -= OnSourceAdapterRowReadError;
foreach (var targetAdapter in targetAdapters)
{
targetAdapter.BeforeTargetAdapterProcessRow -= BeforeTargetAdapterProcessRow;
targetAdapter.AfterTargetAdapterProcessRow -= AfterTargetAdapterProcessRow;
targetAdapter.OnTargetAdapterRowProcessError -= OnTargetAdapterRowProcessError;
}
}
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source)
{
base.Prepare(source);
if (ColumnFormatStrings != null && ColumnFormatStrings.Length != Source.FieldCount)
{
throw new DataPipelineException("When ColumnFormatStrings is provided its length must match result set field count");
}
if (ColumnFormatStrings == null)
{
BuildDefaultColumnFormatStrings(source);
}
}
private void BuildDefaultColumnFormatStrings(ISourceAdapter <PoolEntry <object[]> > source)
{
ColumnFormatStrings = new string [source.ColumnMetadatas.Count];
var index = 0;
// ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator
foreach (var meta in source.ColumnMetadatas)
{
if (meta.DataType == typeof(decimal) && (meta.NumericScale ?? 0) != 0)
{
ColumnFormatStrings[index] = "0." + new string('#', meta.NumericScale ?? DefaultNumericScale);
}
index++;
}
}
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source)
{
base.Prepare(source);
if (BatchSize == MaxPossibleBatchSize)
{
BatchSize = SqlClientUtils.MaxMSSQLParams / ColumnNameToMetadataIndexMap.Count;
}
if (ColumnNameToMetadataIndexMap.Count * BatchSize > SqlClientUtils.MaxMSSQLParams)
{
throw new TargetAdapterException(
$"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds MSSQL limit of {SqlClientUtils.MaxMSSQLParams} parameters in a query");
}
}
public override void Pump(ISourceAdapter <PoolEntry <object[]> > sourceAdapter,
IEnumerable <ITargetAdapter <PoolEntry <object[]> > > targetAdapters)
{
_blockingQueueSourceAdapter = sourceAdapter as BlockingQueueSourceAdapter ??
throw new InvalidOperationException($"sourceAdapter must be of class {nameof(BlockingQueueSourceAdapter)}");
_startedRunning.Set();
BeforeTargetAdapterProcessRow += AttemptFlush;
try
{
base.Pump(sourceAdapter, targetAdapters);
}
finally
{
BeforeTargetAdapterProcessRow -= AttemptFlush;
}
}
private void SetupAndHealthCheckAdapters(ISourceAdapter <TRow> sourceAdapter,
IEnumerable <ITargetAdapter <TRow> > targetAdapters,
out int targetAdaptersCount)
{
sourceAdapter.OnSourceAdapterRowReadError += OnSourceAdapterRowReadError;
sourceAdapter.AbortOnReadException = AbortOnSourceAdapterException;
targetAdaptersCount = 0;
foreach (var targetAdapter in targetAdapters)
{
targetAdapter.BeforeTargetAdapterProcessRow += BeforeTargetAdapterProcessRow;
targetAdapter.AfterTargetAdapterProcessRow += AfterTargetAdapterProcessRow;
targetAdapter.OnTargetAdapterRowProcessError += OnTargetAdapterRowProcessError;
targetAdapter.AbortOnProcessException ??= AbortOnTargetAdapterException;
targetAdaptersCount++;
}
sourceAdapter.ParallelLevel = targetAdaptersCount;
}
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source)
{
base.Prepare(source);
if (BatchSize == MaxPossibleBatchSize)
{
BatchSize = SqlClientUtils.MaxMSSQLParams / ColumnNameToMetadataIndexMap.Count;
}
if (!LiteralParamBinding && ColumnNameToMetadataIndexMap.Count * BatchSize > SqlClientUtils.MaxMSSQLParams)
{
throw new TargetAdapterException(
$"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds MSSQL limit of {SqlClientUtils.MaxMSSQLParams} parameters in a query");
}
if (LiteralParamBinding && BatchSize > SqlClientUtils.MaxMSSQLInsertRows)
{
throw new TargetAdapterException("BatchSize can't be greater than 1000, not supported by MSSQL");
}
}
public bool TryGetUpstreamMetadata(ObfuscationConfiguration obfuscationConfiguration, out IEnumerable <IMetaColumn> metaColumns)
{
if ((object)obfuscationConfiguration == null)
{
throw new ArgumentNullException("obfuscationConfiguration");
}
metaColumns = null;
if ((object)obfuscationConfiguration.SourceAdapterConfiguration != null &&
(object)obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterType() != null)
{
using (ISourceAdapter sourceAdapter = obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterInstance <ISourceAdapter>())
{
sourceAdapter.Initialize(obfuscationConfiguration.SourceAdapterConfiguration);
metaColumns = sourceAdapter.UpstreamMetadata;
}
}
return(true);
}
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source)
{
base.Prepare(source);
if (BatchSize == MaxPossibleBatchSize)
{
BatchSize = OracleUtils.DefaultMaxOracleParams / ColumnNameToMetadataIndexMap.Count;
}
if (ColumnNameToMetadataIndexMap.Count * BatchSize > OracleUtils.DefaultMaxOracleParams)
{
throw new TargetAdapterException(
$"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds ORACLE limit of {OracleUtils.DefaultMaxOracleParams} parameters in a query");
}
if (!UseArrayBinding)
{
return;
}
_oracleArrayBindingHelper = new OracleArrayBindingHelper(BatchSize, ColumnNameToMetadataIndexMap, SourceValueToParamValue)
{
UseNativeTypeConvertor = UseNativeTypeConvertor
};
}
/// <summary>
/// Routes the source to a target via mapper.
/// </summary>
/// <param name="source">The source adapter.</param>
/// <param name="mapper">The mapper.</param>
/// <param name="target">The target adapter.</param>
public static void Route(ISourceAdapter source, IMapper mapper, ITargetAdapter target)
{
target.Load(
mapper.Transform(
source.Extract(), source.SourceSpecification, target.TargetSpecification));
}
public virtual void Prepare(ISourceAdapter <TRow> source)
{
Source = source;
}
public ETLConfig SetSourceAdapter(ISourceAdapter SourceAdapter)
{
_sourceAdapter = SourceAdapter;
return(this);
}
/// <summary> /// Chain routes from a source to a series of targets. /// </summary> /// <param name="source">The starting source adapter.</param> /// <param name="chain">The series of mappers and target adapters.</param> /// <remarks> /// The chaining targets should be ISourceable. /// If the iterated target is not ISourceable, the "chain" breaks/stops. /// </remarks> public static void ChainRoute(ISourceAdapter source, IList <(IMapper mapper, ITargetAdapter target)> chain)
protected Property(Lifetime lf, string name = null)
{
Name = name;
_changed = new SourceAdapter <T>(lf);
}
