public void TestUnionAllPartials()
{
var unionAll = new ParallelUnionAllOperation()
.Add(new PartialProcessOperation()
.Register(
new FakeOperation(
new Row {
{ "k1", "v1" }, { "k2", "v2" }
},
new Row {
{ "k1", "v3" }, { "k2", "v4" }
}
)))
.Add(new PartialProcessOperation()
.Register(
new FakeOperation(
new Row {
{ "k1", "v1" }, { "k2", "v2" }
},
new Row {
{ "k1", "v3" }, { "k2", "v4" }
},
new Row {
{ "key1", "value5" }, { "key2", "value6" }
}
)));
var results = TestOperation(
unionAll
, new LogOperation()
);
Assert.AreEqual(5, results.Count);
}
public EntityKeysPartial(Process process, Entity entity) : base(process)
{
_process = process;
_entity = entity;
if (entity.Input.Count == 1)
{
var connection = entity.Input.First().Connection;
if (connection.IsDatabase && !entity.HasSqlOverride())
{
Register(ComposeInputOperation(process, connection));
}
}
else
{
var union = new ParallelUnionAllOperation();
foreach (var input in entity.Input)
{
if (input.Connection.IsDatabase && !_entity.HasSqlOverride())
{
union.Add(ComposeInputOperation(process, input.Connection));
}
}
Register(union);
}
Register(new EntityKeysDistinct(_entity));
}
public ServerCrawlProcess(string connectionString, string server)
{
var union = new ParallelUnionAllOperation(
new DefinitionProcess(connectionString),
new CachedSqlProcess(connectionString),
new SqlAgentJobExtract(connectionString),
new ReportingServicesProcess(connectionString)
);
Register(union);
RegisterLast(new AppendToRowOperation("server", server));
}
protected override void Initialize()
{
var firstConnection = _entity.Input.First().Connection;
var singleInput = _entity.Input.Count == 1;
if (singleInput)
{
Register(
firstConnection.Is.Internal() ?
_entity.InputOperation :
firstConnection.ExtractAllKeysFromInput(_process, _entity)
);
}
else
{
var multiInput = new ParallelUnionAllOperation();
foreach (var namedConnection in _entity.Input)
{
multiInput.Add(namedConnection.Connection.ExtractAllKeysFromInput(_process, _entity));
}
Register(multiInput);
}
//primary key and/or version may be calculated, so defaults and transformations should be run on them
if (!_entity.PrimaryKey.All(f => f.Input) || (_entity.Version != null && !_entity.Version.Input))
{
_logger.EntityWarn(_entity.Alias, "Using a calculated primary key or version to perform deletes requires setting default values and all transformations to run. The preferred method is to use an input field.");
Register(new ApplyDefaults(true, new Fields(_entity.Fields, _entity.CalculatedFields))
{
EntityName = _entity.Name
});
foreach (var transform in _entity.OperationsBeforeAggregation)
{
Register(transform);
}
foreach (var transform in _entity.OperationsAfterAggregation)
{
Register(transform);
}
}
Register(new EntityDetectDeletes(_process, _entity).Right(_process.OutputConnection.ExtractAllKeysFromOutput(_entity)));
Register(new EntityActionFilter(_process, _entity, EntityAction.Delete));
Register(_process.OutputConnection.Delete(_entity));
}
protected override void Initialize()
{
Register(new EntityKeysPartial(_process, _entity));
if (_entity.Input.Count == 1)
{
Register(_entity.Input.First().Connection.Extract(_process, _entity, _process.IsFirstRun));
}
else
{
var union = new ParallelUnionAllOperation();
foreach (var input in _entity.Input)
{
union.Add(input.Connection.Extract(_process, _entity, Process.IsFirstRun));
}
Register(union);
}
if (!_entity.Sampled && _entity.Sample > 0m && _entity.Sample < 100m)
{
Register(new SampleOperation(_entity.Sample)
{
EntityName = _entity.Name
});
}
Register(new ApplyDefaults(true, new Fields(_entity.Fields, _entity.CalculatedFields))
{
EntityName = _entity.Name
});
foreach (var transform in _entity.OperationsBeforeAggregation)
{
Register(transform);
}
if (_entity.HasSort())
{
Register(new SortOperation(_entity)
{
EntityName = _entity.Name
});
}
if (_entity.Group)
{
Register(new EntityAggregation(_entity));
}
foreach (var transform in _entity.OperationsAfterAggregation)
{
Register(transform);
}
Register(new TruncateOperation(_entity.Name, _entity.Fields, _entity.CalculatedFields));
var standardOutput = new NamedConnection {
Connection = _process.OutputConnection, Name = STANDARD_OUTPUT
};
if (_entity.Output.Count > 0)
{
var branch = new BranchingOperation()
.Add(PrepareOutputOperation(_process, standardOutput));
foreach (var output in _entity.Output)
{
_collectors[output.Name] = new CollectorOperation();
branch.Add(PrepareOutputOperation(_process, output));
}
Register(branch);
}
else
{
Register(PrepareOutputOperation(_process, standardOutput));
}
}
