/// <summary> /// Impeletemt Mirror mapping when destination columns similar ro source columns + prefix (for example) /// </summary> /// <param name="SourceAdapter"></param> /// <param name="Prefix"></param> /// <returns></returns> public static List <ColumnAssignment> CreateMapingWithPrefix(ISourceAdapter SourceAdapter, string Prefix) { List <ColumnAssignment> result = new List <ColumnAssignment>(); foreach (var col in SourceAdapter.Fileds) { var columnAssignment = new ColumnAssignment() { FromColumn = new SimpleMapColumn() { InternalName = col.InternalName, ReadOnly = false }, ToColumn = new SimpleMapColumn() { InternalName = Prefix + col.InternalName, ReadOnly = false } }; result.Add(columnAssignment); } return(result); }
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source) { const string crLf = "\r\n"; base.Prepare(source); if (FieldSizes == null || FieldSizes.Length <= 0) { InitializeFieldSizesWithDefaults(); } if (FieldSizes.Length != Source.FieldCount) { throw new DataPipelineException("Provided FieldSizes array has a different length than result set column count"); } if (OverflowStringFieldWidthBehaviors != null && OverflowStringFieldWidthBehaviors.Length != Source.FieldCount) { throw new DataPipelineException("When OverflowStringFieldWidthBehaviors is provided its length must match result set field count"); } _rowSize = 0; FormatString = ""; for (var i = 0; i < Source.FieldCount; i++) { _rowSize += Math.Abs(FieldSizes[i]); FormatString += $"{{{i},{FieldSizes[i]}{ColumnFormatString(i)}}}"; } _rowSize += crLf.Length; WriteBuffer = new byte[_rowSize]; FormatString += crLf; }
public virtual void Pump(ISourceAdapter <TRow> sourceAdapter, IEnumerable <ITargetAdapter <TRow> > targetAdapters) { using var resetCurrentSourceAdapter = new Resettable <ISourceAdapter <TRow> >(CurrentSourceAdapter = sourceAdapter, adapter => CurrentSourceAdapter = null); using var setEventFinishedResettable = new Resettable <ManualResetEvent>(FinishedEvent, evt => evt.Set()); SetupAndHealthCheckAdapters(sourceAdapter, targetAdapters, out var targetAdaptersCount); using var unSetupAdaptersResettable = new Resettable <object>(this, obj => UnSetupAdapters(sourceAdapter, targetAdapters)); sourceAdapter.Prepare(); using var unPrepareSourceAdapterResettable = new Resettable <ISourceAdapter <TRow> >(sourceAdapter, adapter => adapter.UnPrepare()); foreach (var targetAdapter in targetAdapters) { targetAdapter.Prepare(sourceAdapter); } CheckDeadlockPotential(sourceAdapter, targetAdapters); var targetAdapterIndex = 0; var targetAdapterConveyors = new Conveyor <TRow> [targetAdaptersCount]; foreach (var targetAdapter in targetAdapters) { targetAdapterConveyors[targetAdapterIndex++] = new Conveyor <TRow>((row, context) => { ((ITargetAdapter <TRow>)context).Process(row); sourceAdapter.ReleaseRow(row); }, targetAdapter); } try { foreach (var targetAdapterConveyor in targetAdapterConveyors) { targetAdapterConveyor.Start(); } var sourceRows = sourceAdapter.RowsEnumerable; foreach (var row in sourceRows) { foreach (var targetAdapterConveyor in targetAdapterConveyors) { targetAdapterConveyor.InsertPacket(row); } } foreach (var targetAdapterConveyor in targetAdapterConveyors) { targetAdapterConveyor.StopAndWait(); } targetAdapters.ForEach(adapter => adapter.UnPrepare()); } catch (Exception e) { targetAdapterConveyors.ForEach(conveyor => conveyor.Stop(), typeof(InvalidOperationException), false); targetAdapters.ForEach(adapter => adapter.AbortedWithException(e)); throw; } }
public void Host(ObfuscationConfiguration obfuscationConfiguration, Action <long, bool, double> statusCallback) { IEnumerable <IDictionary <string, object> > sourceDataEnumerable; IEnumerable <Message> messages; if ((object)obfuscationConfiguration == null) { throw new ArgumentNullException("obfuscationConfiguration"); } messages = obfuscationConfiguration.Validate(); if (messages.Any()) { throw new ApplicationException(string.Format("Obfuscation configuration validation failed:\r\n{0}", string.Join("\r\n", messages.Select(m => m.Description).ToArray()))); } using (IOxymoronEngine oxymoronEngine = new OxymoronEngine(this, obfuscationConfiguration)) { using (DisposableList <IDictionaryAdapter> dictionaryAdapters = new DisposableList <IDictionaryAdapter>()) { foreach (DictionaryConfiguration dictionaryConfiguration in obfuscationConfiguration.DictionaryConfigurations) { IDictionaryAdapter dictionaryAdapter; dictionaryAdapter = dictionaryConfiguration.DictionaryAdapterConfiguration.GetAdapterInstance <IDictionaryAdapter>(); dictionaryAdapters.Add(dictionaryAdapter); dictionaryAdapter.Initialize(dictionaryConfiguration.DictionaryAdapterConfiguration); dictionaryAdapter.InitializePreloadCache(dictionaryConfiguration, oxymoronEngine.SubstitutionCacheRoot); this.DictionaryConfigurationToAdapterMappings.Add(dictionaryConfiguration, dictionaryAdapter); } using (ISourceAdapter sourceAdapter = obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterInstance <ISourceAdapter>()) { sourceAdapter.Initialize(obfuscationConfiguration.SourceAdapterConfiguration); using (IDestinationAdapter destinationAdapter = obfuscationConfiguration.DestinationAdapterConfiguration.GetAdapterInstance <IDestinationAdapter>()) { destinationAdapter.Initialize(obfuscationConfiguration.DestinationAdapterConfiguration); destinationAdapter.UpstreamMetadata = sourceAdapter.UpstreamMetadata; sourceDataEnumerable = sourceAdapter.PullData(obfuscationConfiguration.TableConfiguration); sourceDataEnumerable = oxymoronEngine.GetObfuscatedValues(sourceDataEnumerable); if ((object)statusCallback != null) { sourceDataEnumerable = WrapRecordCounter(sourceDataEnumerable, statusCallback); } destinationAdapter.PushData(obfuscationConfiguration.TableConfiguration, sourceDataEnumerable); } } } } }
private static void CheckDeadlockPotential(ISourceAdapter <TRow> sourceAdapter, IEnumerable <ITargetAdapter <TRow> > targetAdapters) { var totalTargetBufferSize = targetAdapters.Sum(targetAdapter => targetAdapter.BufferSize); if (sourceAdapter.RowsPoolSize > 0 && sourceAdapter.RowsPoolSize < totalTargetBufferSize) { throw new DataPipelineException("Source adapter rows pool size can't be lesser than the sum of all target adapters buffer sizes. Deadlock would occur upon call to Pump()"); } }
private void UnSetupAdapters(ISourceAdapter <TRow> sourceAdapter, IEnumerable <ITargetAdapter <TRow> > targetAdapters) { sourceAdapter.OnSourceAdapterRowReadError -= OnSourceAdapterRowReadError; foreach (var targetAdapter in targetAdapters) { targetAdapter.BeforeTargetAdapterProcessRow -= BeforeTargetAdapterProcessRow; targetAdapter.AfterTargetAdapterProcessRow -= AfterTargetAdapterProcessRow; targetAdapter.OnTargetAdapterRowProcessError -= OnTargetAdapterRowProcessError; } }
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source) { base.Prepare(source); if (ColumnFormatStrings != null && ColumnFormatStrings.Length != Source.FieldCount) { throw new DataPipelineException("When ColumnFormatStrings is provided its length must match result set field count"); } if (ColumnFormatStrings == null) { BuildDefaultColumnFormatStrings(source); } }
private void BuildDefaultColumnFormatStrings(ISourceAdapter <PoolEntry <object[]> > source) { ColumnFormatStrings = new string [source.ColumnMetadatas.Count]; var index = 0; // ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator foreach (var meta in source.ColumnMetadatas) { if (meta.DataType == typeof(decimal) && (meta.NumericScale ?? 0) != 0) { ColumnFormatStrings[index] = "0." + new string('#', meta.NumericScale ?? DefaultNumericScale); } index++; } }
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source) { base.Prepare(source); if (BatchSize == MaxPossibleBatchSize) { BatchSize = SqlClientUtils.MaxMSSQLParams / ColumnNameToMetadataIndexMap.Count; } if (ColumnNameToMetadataIndexMap.Count * BatchSize > SqlClientUtils.MaxMSSQLParams) { throw new TargetAdapterException( $"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds MSSQL limit of {SqlClientUtils.MaxMSSQLParams} parameters in a query"); } }
public override void Pump(ISourceAdapter <PoolEntry <object[]> > sourceAdapter, IEnumerable <ITargetAdapter <PoolEntry <object[]> > > targetAdapters) { _blockingQueueSourceAdapter = sourceAdapter as BlockingQueueSourceAdapter ?? throw new InvalidOperationException($"sourceAdapter must be of class {nameof(BlockingQueueSourceAdapter)}"); _startedRunning.Set(); BeforeTargetAdapterProcessRow += AttemptFlush; try { base.Pump(sourceAdapter, targetAdapters); } finally { BeforeTargetAdapterProcessRow -= AttemptFlush; } }
private void SetupAndHealthCheckAdapters(ISourceAdapter <TRow> sourceAdapter, IEnumerable <ITargetAdapter <TRow> > targetAdapters, out int targetAdaptersCount) { sourceAdapter.OnSourceAdapterRowReadError += OnSourceAdapterRowReadError; sourceAdapter.AbortOnReadException = AbortOnSourceAdapterException; targetAdaptersCount = 0; foreach (var targetAdapter in targetAdapters) { targetAdapter.BeforeTargetAdapterProcessRow += BeforeTargetAdapterProcessRow; targetAdapter.AfterTargetAdapterProcessRow += AfterTargetAdapterProcessRow; targetAdapter.OnTargetAdapterRowProcessError += OnTargetAdapterRowProcessError; targetAdapter.AbortOnProcessException ??= AbortOnTargetAdapterException; targetAdaptersCount++; } sourceAdapter.ParallelLevel = targetAdaptersCount; }
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source) { base.Prepare(source); if (BatchSize == MaxPossibleBatchSize) { BatchSize = SqlClientUtils.MaxMSSQLParams / ColumnNameToMetadataIndexMap.Count; } if (!LiteralParamBinding && ColumnNameToMetadataIndexMap.Count * BatchSize > SqlClientUtils.MaxMSSQLParams) { throw new TargetAdapterException( $"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds MSSQL limit of {SqlClientUtils.MaxMSSQLParams} parameters in a query"); } if (LiteralParamBinding && BatchSize > SqlClientUtils.MaxMSSQLInsertRows) { throw new TargetAdapterException("BatchSize can't be greater than 1000, not supported by MSSQL"); } }
public bool TryGetUpstreamMetadata(ObfuscationConfiguration obfuscationConfiguration, out IEnumerable <IMetaColumn> metaColumns) { if ((object)obfuscationConfiguration == null) { throw new ArgumentNullException("obfuscationConfiguration"); } metaColumns = null; if ((object)obfuscationConfiguration.SourceAdapterConfiguration != null && (object)obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterType() != null) { using (ISourceAdapter sourceAdapter = obfuscationConfiguration.SourceAdapterConfiguration.GetAdapterInstance <ISourceAdapter>()) { sourceAdapter.Initialize(obfuscationConfiguration.SourceAdapterConfiguration); metaColumns = sourceAdapter.UpstreamMetadata; } } return(true); }
public override void Prepare(ISourceAdapter <PoolEntry <object[]> > source) { base.Prepare(source); if (BatchSize == MaxPossibleBatchSize) { BatchSize = OracleUtils.DefaultMaxOracleParams / ColumnNameToMetadataIndexMap.Count; } if (ColumnNameToMetadataIndexMap.Count * BatchSize > OracleUtils.DefaultMaxOracleParams) { throw new TargetAdapterException( $"Number of columns in target adapter buffer size ({ColumnNameToMetadataIndexMap.Count * BatchSize}) exceeds ORACLE limit of {OracleUtils.DefaultMaxOracleParams} parameters in a query"); } if (!UseArrayBinding) { return; } _oracleArrayBindingHelper = new OracleArrayBindingHelper(BatchSize, ColumnNameToMetadataIndexMap, SourceValueToParamValue) { UseNativeTypeConvertor = UseNativeTypeConvertor }; }
/// <summary> /// Routes the source to a target via mapper. /// </summary> /// <param name="source">The source adapter.</param> /// <param name="mapper">The mapper.</param> /// <param name="target">The target adapter.</param> public static void Route(ISourceAdapter source, IMapper mapper, ITargetAdapter target) { target.Load( mapper.Transform( source.Extract(), source.SourceSpecification, target.TargetSpecification)); }
public virtual void Prepare(ISourceAdapter <TRow> source) { Source = source; }
public ETLConfig SetSourceAdapter(ISourceAdapter SourceAdapter) { _sourceAdapter = SourceAdapter; return(this); }
/// <summary> /// Chain routes from a source to a series of targets. /// </summary> /// <param name="source">The starting source adapter.</param> /// <param name="chain">The series of mappers and target adapters.</param> /// <remarks> /// The chaining targets should be ISourceable. /// If the iterated target is not ISourceable, the "chain" breaks/stops. /// </remarks> public static void ChainRoute(ISourceAdapter source, IList <(IMapper mapper, ITargetAdapter target)> chain)
protected Property(Lifetime lf, string name = null) { Name = name; _changed = new SourceAdapter <T>(lf); }