public void LastIpLookedUpTest()
{
var target = new PrivateAccessor(new ConcreteService());
var result = target.CallMethod("LogUserInfo", "123.123.123.123");
Assert.AreEqual("123.123.123.123", target.GetField("_lastIpLookedUp"));
}
public void Ctor_ShouldOrderServices()
{
var services = (IEnumerable <ISupportsModelPersistence>)PrivateAccessor.GetField(this.subject, "databaseDependents");
var sequence = 0;
foreach (var service in services)
{
if (sequence > service.LoadSequence)
{
Assert.Fail("Services are not ordered in ascending order. This is important to ensure critical and root dependent services are evaluated first.");
}
sequence = service.LoadSequence;
}
}
public void ProcessPipeline_InFieldsButNotOutInConfig_FieldsToUseFromInFields()
{
//arrange
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
//act
_ = orchestrator.ExecuteAsync().Result;
var fieldsToUse = ((OutputProvider)orchestratorPA.GetField("_outputProvider")).FieldsToUse;
//assert
fieldsToUse.Count().Should().Be(3);
fieldsToUse[0].Should().Be("InFld1");
fieldsToUse[1].Should().Be("InFld2");
fieldsToUse[2].Should().Be("InFld3");
}
public void ProcessPipeline_OutputFieldsInConfig_FieldsToUseFromConfig()
{
//arrange
_config.OutputFields = "OutFLd1|6,OutFld2|14";
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
//act
_ = orchestrator.ExecuteAsync().Result;
var fieldsToUse = ((OutputProvider)orchestratorPA.GetField("_outputProvider")).FieldsToUse;
//assert
fieldsToUse.Should().HaveCount(2);
fieldsToUse[0].Should().Be("OutFLd1");
fieldsToUse[1].Should().Be("OutFld2");
}
public void ProcessPipeline_NoInOrOutFieldsInConfig_FieldsToUseFromHeaderRow()
{
//arrange
_config.InputFields = "|4,|7,|13"; //field widths are still needed here (FF intake), but field names are not specified, so they will be taken from header row
_config.TrimInputValues = true; //needed to remove spaces from field names
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
//act
_ = orchestrator.ExecuteAsync().Result;
var fieldsToUse = ((OutputProvider)orchestratorPA.GetField("_outputProvider")).FieldsToUse;
//assert
fieldsToUse.Count().Should().Be(3);
fieldsToUse[0].Should().Be("YEAR");
fieldsToUse[1].Should().Be("MONTH");
fieldsToUse[2].Should().Be("NUMERIC DATA");
}
public void When_less_than_reservoir_size_events_are_reported_number_events_seen_is_accurate()
{
var expectedAddAttempts = 99;
for (var i = 0; i < expectedAddAttempts; i++)
{
_errorEventAggregator.Collect(new ErrorEventWireModel(_attribValues, false, i * 0.001f));
}
// Access the private collection of events to get the number of add attempts.
var privateAccessor = new PrivateAccessor(_errorEventAggregator);
var errorEvents = privateAccessor.GetField("_errorEvents") as ConcurrentPriorityQueue <PrioritizedNode <ErrorEventWireModel> >;
if (errorEvents == null)
{
throw new ArgumentNullException(nameof(errorEvents));
}
var actualAddAttempts = errorEvents.GetAddAttemptsCount();
_harvestAction();
Assert.AreEqual(expectedAddAttempts, actualAddAttempts);
}
public void GlobalCache_VariousHandlers_SameInstance()
{
//arrange
_cacheInstance = null;
_callCount = 0;
bool cacheIsSameInstance = true;
_config.HeadersInFirstInputRow = true;
_config.ClusterMarker = (r, pr, n) =>
{
if (!VerifyInstance(r.GlobalCache))
{
cacheIsSameInstance = false;
}
if (!VerifyInstance(pr?.GlobalCache))
{
cacheIsSameInstance = false; //note that previous record is null for the 1st record
}
return(true);
};
_config.TransformerType = TransformerType.Recordbound;
_config.RecordboundTransformer = r =>
{
if (!VerifyInstance(r.GlobalCache))
{
cacheIsSameInstance = false;
}
return(r);
};
_config.RouterType = RouterType.PerCluster;
_config.ClusterRouter = c =>
{
if (!VerifyInstance(c.GlobalCache))
{
cacheIsSameInstance = false;
}
return(1);
};
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
var unclusteringBlock = (TransformManyBlock <KeyValCluster, KeyValRecord>)orchestratorPA.GetField("_unclusteringBlock");
unclusteringBlock.LinkTo(_resultsExtractor, new DataflowLinkOptions {
PropagateCompletion = true
});
//act
var result = orchestrator.ExecuteAsync().Result;
_resultsExtractor.Completion.Wait();
//assert
result.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
result.RowsRead.Should().Be(4);
result.ClustersRead.Should().Be(3);
result.ClustersWritten.Should().Be(3);
result.RowsWritten.Should().Be(0); //no output invoked during the test
result.GlobalCache.Should().BeSameAs(_cacheInstance);
_resultingRecords.Count.Should().Be(3);
cacheIsSameInstance.Should().BeTrue(); //meaning all instances were the same
_callCount.Should().Be(12); // 3 handlers called 3 times (one making 2 calls)
}
public void GlobalCache_ReplaceValueInConcurrentAccess_CorrectData(int iterationNumber, int totalRepeats)
{
var dummy = iterationNumber; // to clear xUnit1026 warning
dummy = totalRepeats;
//arrange
_config.HeadersInFirstInputRow = true;
_config.GlobalCacheElements = new string[] { "STR|Exes", "DT|5/12/2011" };
_config.ClusterMarker = (r, pr, n) =>
{
r.GlobalCache.ReplaceValue <string, string>("STR", s => s + "X");
return(true);
};
_config.TransformerType = TransformerType.ClusterFilter;
_config.ClusterFilterPredicate = c =>
{
c.GlobalCache.ReplaceValue <DateTime, DateTime>("DT", d => d.AddMonths(1));
return(true);
};
_config.RouterType = RouterType.PerRecord;
_config.RecordRouter = (r, c) =>
{
r.GlobalCache.ReplaceValue <string, string>("STR", s => s + "x");
r.GlobalCache.ReplaceValue <DateTime, DateTime>("DT", d => d.AddDays(1));
return(1);
};
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
var unclusteringBlock = (TransformManyBlock <KeyValCluster, KeyValRecord>)orchestratorPA.GetField("_unclusteringBlock");
unclusteringBlock.LinkTo(_resultsExtractor, new DataflowLinkOptions {
PropagateCompletion = true
});
//act
var result = orchestrator.ExecuteAsync().Result;
_resultsExtractor.Completion.Wait();
//assert
result.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
result.RowsRead.Should().Be(4);
result.ClustersRead.Should().Be(3);
result.ClustersWritten.Should().Be(3);
result.RowsWritten.Should().Be(0); //no output invoked during the test
((string)result.GlobalCache["STR"]).ToLower().Should().Be("exesxxxxxx");
((DateTime)result.GlobalCache["DT"]).Should().Be(new DateTime(2011, 8, 15));
}
public void GlobalCache_TryReplaceInConcurrentAccess_CorrectData(int iterationNumber, int totalRepeats)
{
var dummy = iterationNumber; // to clear xUnit1026 warning
dummy = totalRepeats;
//Note the following pattern for replacing GlobalCache values using TryReplace in a thread-safe manner:
// T prevVal; //, where T is one of: int, DateTime, decimal or string
// do { prevVal = (T)gc["key"]; }
// while (!gc.TryReplace("key", CalcNewVal(prevVal), prevVal));
//arrange
int attemptCnt = 0;
_config.HeadersInFirstInputRow = true;
_config.BufferSize = 1;
_config.GlobalCacheElements = new string[] { "STR|Exes", "DT|5/12/2011" };
_config.ClusterMarker = (r, pr, n) =>
{
int attempts = ReplaceVal <string, string>(r.GlobalCache, "STR", s => s + "X", 1); //3 Xs (upper case) appended (one per record)
for (int i = 0; i < attempts; i++)
{
Interlocked.Increment(ref attemptCnt);
}
return(true);
};
_config.RecordboundTransformer = r =>
{
int attempts = ReplaceVal <DateTime, DateTime>(r.GlobalCache, "DT", d => d.AddMonths(1), 2); //3 months added
for (int i = 0; i < attempts; i++)
{
Interlocked.Increment(ref attemptCnt);
}
return(r);
};
_config.RouterType = RouterType.PerCluster;
_config.ClusterRouter = c =>
{
int attempts = ReplaceVal <string, string>(c.GlobalCache, "STR", s => s + "x", 0) // 2 xs (lower case) appended
+ ReplaceVal <DateTime, DateTime>(c.GlobalCache, "DT", d => d.AddDays(1), 3); // 3 days added
for (int i = 0; i < attempts; i++)
{
Interlocked.Increment(ref attemptCnt);
}
return(1);
};
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
var unclusteringBlock = (TransformManyBlock <KeyValCluster, KeyValRecord>)orchestratorPA.GetField("_unclusteringBlock");
unclusteringBlock.LinkTo(_resultsExtractor, new DataflowLinkOptions {
PropagateCompletion = true
});
//act
var result = orchestrator.ExecuteAsync().Result;
_resultsExtractor.Completion.Wait();
//assert
result.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
result.RowsRead.Should().Be(4);
result.ClustersRead.Should().Be(3);
result.ClustersWritten.Should().Be(3);
result.RowsWritten.Should().Be(0); //no output invoked during the test
((string)result.GlobalCache["STR"]).ToLower().Should().Be("exesxxxxxx");
((DateTime)result.GlobalCache["DT"]).Should().Be(new DateTime(2011, 8, 15));
attemptCnt.Should().BeGreaterOrEqualTo(12); //3 + 3 + 6 + retries
}
public void GlobalCache_Elements_CorrectData()
{
//arrange
IReadOnlyList <KeyValuePair <string, object> > elemsAtInitRec1 = null;
IReadOnlyList <KeyValuePair <string, object> > elemsAtInitRec2 = null;
IReadOnlyList <KeyValuePair <string, object> > elemsAtInitRec3 = null;
IReadOnlyList <KeyValuePair <string, object> > elemsAtRouterRec1 = null;
IReadOnlyList <KeyValuePair <string, object> > elemsAtRouterRec2 = null;
IReadOnlyList <KeyValuePair <string, object> > elemsAtRouterRec3 = null;
_config.HeadersInFirstInputRow = true;
_config.GlobalCacheElements = new string[] { "key1", "key2|", "key3|24" };
_config.RecordInitiator = (r, tb) =>
{
var gc = r.GlobalCache;
if (r.RecNo == 1)
{
elemsAtInitRec1 = gc.Elements; gc.ReplaceValue <object, int>("key1", _ => 16);
}
if (r.RecNo == 2)
{
elemsAtInitRec2 = gc.Elements; gc.ReplaceValue <object, string>("key2", _ => "blah1");
}
if (r.RecNo == 3)
{
elemsAtInitRec3 = gc.Elements; gc.IncrementValue("key3");
}
return(true);
};
_config.RouterType = RouterType.PerCluster;
_config.ClusterRouter = c =>
{
var gc = c.GlobalCache;
if (c.StartRecNo == 1)
{
elemsAtRouterRec1 = gc.Elements; gc.IncrementValue("key3"); gc.ReplaceValue <object, string>("key2", _ => "blah2");
}
if (c.StartRecNo == 2)
{
elemsAtRouterRec2 = gc.Elements; gc.IncrementValue("key1");
}
if (c.StartRecNo == 3)
{
elemsAtRouterRec3 = gc.Elements;
}
return(1);
};
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
var unclusteringBlock = (TransformManyBlock <KeyValCluster, KeyValRecord>)orchestratorPA.GetField("_unclusteringBlock");
unclusteringBlock.LinkTo(_resultsExtractor, new DataflowLinkOptions {
PropagateCompletion = true
});
//act
var result = orchestrator.ExecuteAsync().Result;
_resultsExtractor.Completion.Wait();
//assert
result.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
result.RowsRead.Should().Be(4);
result.ClustersRead.Should().Be(3);
result.ClustersWritten.Should().Be(3);
result.RowsWritten.Should().Be(0); //no output invoked during the test
_resultingRecords.Count.Should().Be(3);
elemsAtInitRec1.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtInitRec1.Should().HaveCount(3);
elemsAtInitRec1.Single(e => e.Key == "key1").Value.Should().BeNull();
elemsAtInitRec1.Single(e => e.Key == "key2").Value.Should().Be(string.Empty);
elemsAtInitRec1.Single(e => e.Key == "key3").Value.Should().Be(24);
elemsAtInitRec2.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtInitRec2.Should().HaveCount(3);
elemsAtInitRec2.Single(e => e.Key == "key1").Value.Should().Be(16);
elemsAtInitRec2.Single(e => e.Key == "key2").Value.Should().Be(string.Empty);
elemsAtInitRec2.Single(e => e.Key == "key3").Value.Should().Be(24);
elemsAtInitRec3.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtInitRec3.Should().HaveCount(3);
elemsAtInitRec3.Single(e => e.Key == "key1").Value.Should().Be(16);
elemsAtInitRec3.Single(e => e.Key == "key2").Value.Should().Be("blah1");
elemsAtInitRec3.Single(e => e.Key == "key3").Value.Should().Be(24);
elemsAtRouterRec1.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtRouterRec1.Should().HaveCount(3);
elemsAtRouterRec1.Single(e => e.Key == "key1").Value.Should().Be(16);
elemsAtRouterRec1.Single(e => e.Key == "key2").Value.Should().Be("blah1");
elemsAtRouterRec1.Single(e => e.Key == "key3").Value.Should().Be(25);
elemsAtRouterRec2.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtRouterRec2.Should().HaveCount(3);
elemsAtRouterRec2.Single(e => e.Key == "key1").Value.Should().Be(16);
elemsAtRouterRec2.Single(e => e.Key == "key2").Value.Should().Be("blah2");
elemsAtRouterRec2.Single(e => e.Key == "key3").Value.Should().Be(26);
elemsAtRouterRec3.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
elemsAtRouterRec3.Should().HaveCount(3);
elemsAtRouterRec3.Single(e => e.Key == "key1").Value.Should().Be(17);
elemsAtRouterRec3.Single(e => e.Key == "key2").Value.Should().Be("blah2");
elemsAtRouterRec3.Single(e => e.Key == "key3").Value.Should().Be(26);
}
public void GlobalCache_ReplaceAndRetrieveValue_SameValueAtOtherHandler()
{
//arrange
object valueRetrieved = null; //value retrieved from cache
int valCntAtRec1 = -1; //number of values stored in cache at 1st record (by RecordInitiator)
string valueAtRec1 = null;
int valCntAtRec2 = -1; //number of values stored in cache at 2nd record (by RecordInitiator)
string valueAtRec2 = null;
int valCntAtRec3 = -1; //number of values stored in cache at 3rd record (by RecordInitiator)
string valueAtRec3 = null;
_config.HeadersInFirstInputRow = true;
_config.GlobalCacheElements = new string[] { "key|" };
_config.RecordInitiator = (r, tb) =>
{
//intake is single threaded, so no need for thread safety
var gc = r.GlobalCache;
if (r.RecNo == 1)
{
valCntAtRec1 = gc.Count; valueAtRec1 = gc["key"] as string;
}
if (r.RecNo == 2)
{
valCntAtRec2 = gc.Count; valueAtRec2 = gc["key"] as string;
}
if (r.RecNo == 3)
{
valCntAtRec3 = gc.Count; valueAtRec3 = gc["key"] as string;
}
var oldVal = gc["key"] as string;
gc.TryReplace("key", oldVal + "blah", oldVal);
return(true);
};
_config.RouterType = RouterType.PerCluster;
_config.ClusterRouter = c =>
{
var cache = c.GlobalCache;
cache.TryGet("key", out valueRetrieved);
return(1);
};
var orchestrator = new EtlOrchestrator(_config);
var orchestratorPA = new PrivateAccessor(orchestrator);
var unclusteringBlock = (TransformManyBlock <KeyValCluster, KeyValRecord>)orchestratorPA.GetField("_unclusteringBlock");
unclusteringBlock.LinkTo(_resultsExtractor, new DataflowLinkOptions {
PropagateCompletion = true
});
//act
var result = orchestrator.ExecuteAsync().Result;
_resultsExtractor.Completion.Wait();
//assert
result.CompletionStatus.Should().Be(CompletionStatus.IntakeDepleted);
result.RowsRead.Should().Be(4);
result.ClustersRead.Should().Be(3);
result.ClustersWritten.Should().Be(3);
result.RowsWritten.Should().Be(0); //no output invoked during the test
_resultingRecords.Count.Should().Be(3);
valCntAtRec1.Should().Be(1);
valueAtRec1.Should().BeOfType <string>();
valueAtRec1.Should().Be(string.Empty);
valCntAtRec2.Should().Be(1);
valueAtRec2.Should().BeOfType <string>();
valueAtRec2.Should().Be("blah");
valCntAtRec3.Should().Be(1);
valueAtRec3.Should().BeOfType <string>();
valueAtRec3.Should().Be("blahblah");
valueRetrieved.Should().BeOfType <string>();
valueRetrieved.Should().Be("blahblahblah");
result.GlobalCache.Count.Should().Be(1);
result.GlobalCache["key"].Should().BeOfType <string>();
result.GlobalCache["key"].Should().Be("blahblahblah");
var gcElems = result.GlobalCache.Elements;
gcElems.Should().BeOfType <ReadOnlyCollection <KeyValuePair <string, object> > >();
gcElems.Should().ContainSingle();
gcElems[0].Key.Should().Be("key");
gcElems[0].Value.Should().Be("blahblahblah");
}
public void ShouldSetIsNewToFalse()
{
Assert.IsFalse((bool)PrivateAccessor.GetField(Result, "isNew"));
}
