public void OnError_ProcessorResult_Is_Not_Changed_By_Execute()
{
MockNonGenericProcessor processor = new MockNonGenericProcessor();
Exception executionException = new Exception();
object[] executionOutput = new object[0];
processor.OnExecuteAction = input => new ProcessorResult
{
Status = ProcessorStatus.Error,
Output = executionOutput,
Error = executionException
};
Exception onErrorException = new Exception();
object[] onErrorOutput = new object[0];
processor.OnErrorAction = result =>
{
Assert.AreEqual(ProcessorStatus.Error, result.Status, "The result status should have been ProcessorStatus.Error.");
Assert.AreSame(result.Error, executionException, "The Error should have been the same instance as the one set in OnExecute.");
Assert.AreSame(result.Output, executionOutput, "The Output should have been the same instance as the one set in OnExecute.");
result.Error = onErrorException;
result.Output = onErrorOutput;
};
processor.Initialize();
ProcessorResult resultFromExecute = processor.Execute(new object[0]);
Assert.IsNotNull(resultFromExecute, "There should have been a non-null result.");
Assert.AreEqual(ProcessorStatus.Error, resultFromExecute.Status, "The result status should have been ProcessorStatus.Error.");
Assert.AreSame(resultFromExecute.Error, onErrorException, "The Error should have been the same instance as the one set in OnError.");
Assert.AreSame(resultFromExecute.Output, onErrorOutput, "The Output should have been the same instance as the one set in OnError.");
}
public override ProcessorResult <string> OnExecute(int intValueDerived)
{
ProcessorResult <string> result = base.OnExecute(intValueDerived);
result.Output = "Derived" + result.Output;
return(result);
}
public void PipelineBuilder_Build_Binds_By_Name_And_Executes()
{
MockPipelineBuilder pb = new MockPipelineBuilder();
MockProcessor1 processor1 = new MockProcessor1();
MockProcessorEchoString processor2 = new MockProcessorEchoString();
// Rename processor inputs and outputs to bind by name to pipeline arguments and to each other:
// pipeline[intInput] --> [intInput]p1[echoInput] --> [echoInput]p2[stringOutput] --> [stringOutput]pipeline
processor1.InArguments[0].Name = "intInput";
processor2.InArguments[0].Name = processor1.OutArguments[0].Name = "echoInput";
processor2.OutArguments[0].Name = "stringOutput";
Pipeline pipeline = pb.Build(
new Processor[] { processor1, processor2 },
new ProcessorArgument[] { new ProcessorArgument("intInput", typeof(int)) },
new ProcessorArgument[] { new ProcessorArgument("stringOutput", typeof(string)) }
);
Assert.IsNotNull(pipeline, "Build should produce non-null pipeline");
ProcessorResult result = pipeline.Execute(new object[] { 9 });
Assert.IsNotNull(result, "Null result from execute");
Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Expected OK result status");
Assert.IsNotNull(result.Output, "Expected non-null output");
Assert.AreEqual(1, result.Output.Length, "Expected 1 output value");
Assert.AreEqual(9.ToString(), result.Output[0], "Wrong output value");
}
private void AssertNullableProcessorResult <T>(params T[] values) where T : struct
{
ProcessorResult <T?> result = new ProcessorResult <T?>();
ProcessorResult baseResult = result as ProcessorResult;
foreach (T value in values)
{
baseResult.Output = new object[] { value };
Assert.IsTrue(result.Output.HasValue, "Expected HasValue==true for ProcessorType>" + typeof(T).Name + "> for " + value);
Assert.AreEqual(value, result.Output.Value, "Wrong value for ProcessorType>" + typeof(T).Name + "> for " + value);
}
baseResult.Output = new object[] { null };
T?nullableValue = result.Output;
Assert.IsFalse(nullableValue.HasValue, "ProcessorType>" + typeof(T).Name + "> should set HasValue == false for null");
T?unused;
ExceptionAssert.ThrowsInvalidOperation(
"Setting illegal value to ProcessorType>" + typeof(T).Name + "> should throw",
() =>
{
baseResult.Output = new object[] { "hello" };
unused = result.Output;
}
);
}
public void Pipeline_Execute_Throws_Wrong_Output_Length()
{
MockPipeline pipeline = TestPipelines.CreateMockProcessor1Pipeline();
ProcessorResult errorResult = null;
// Override the OnExecute to return a ProcessorResult
// which contains the wrong
((MockProcessor1)pipeline.Processors[1]).OnExecuteCalled =
i =>
{
ProcessorResult pr = new ProcessorResult <string>()
{
};
pr.Output = new object[] { "string1", "string2" };
return(pr as ProcessorResult <string>);
};
pipeline.OnErrorCalled = r => errorResult = r;
// The final pipeline result should reflect that error status
ProcessorResult result = pipeline.Execute(new object[] { 5 });
Assert.IsNotNull(result, "Failed to get a result");
Assert.AreEqual(ProcessorStatus.Error, result.Status, "Expected failure status");
Assert.IsNotNull(result.Error, "Expected error result");
Assert.AreEqual(typeof(InvalidOperationException), result.Error.GetType(), "Expected InvalidOperationException");
Assert.IsNotNull(errorResult, "Pipeline.OnError should have been called");
}
protected override void OnError(ProcessorResult result)
{
if (this.OnErrorCalled != null)
{
this.OnErrorCalled(result);
}
}
public void ProcessCrawledDomain(CrawlContext crawlContext)
{
string webhost = string.Empty;
try
{
webhost = Dig.Instance.GetWebHostName(crawlContext.RootUri.DnsSafeHost);
}
catch (Exception e)
{
_logger.ErrorFormat("Exception occurred getting webhost name for [{0}]", crawlContext.RootUri.DnsSafeHost, e);
}
ProcessorResult result = new ProcessorResult {
UniqueAttributeId = ATTRIB_TYPE_ID
}; //mask
result.IsAHit = webhost != "None";
result.Attributes.Add(ATTRIB_TYPE_ID.ToString(), webhost);
if (result.IsAHit)
{
DomainSave(crawlContext, result);
}
}
public void Pipeline_Executes_2_Processors_Common_In_Double_Out()
{
Pipeline pipeline = TestPipelines.CreateDoubleMockProcessor1Pipeline();
// Override the OnExecute of both so we see unique outputs
((MockProcessor1)pipeline.Processors[1]).OnExecuteCalled =
i => new ProcessorResult <string>()
{
Output = "result1 for " + i
};
((MockProcessor1)pipeline.Processors[2]).OnExecuteCalled =
i => new ProcessorResult <string>()
{
Output = "result2 for " + i
};
ProcessorResult result = pipeline.Execute(new object[] { 5 });
// This test has 2 MockProcessor1's, both bound to the single pipeline input 'intValue'.
// Each processor output is bound to one of the 2 pipeline outputs
object[] output = result.Output;
Assert.IsNotNull(output, "Processing output was null");
Assert.AreEqual(2, output.Length, "Should have received 1 output");
Assert.AreEqual("result1 for 5", output[0], "Should have seen this output[0]");
Assert.AreEqual("result2 for 5", output[1], "Should have seen this output[1]");
}
public void Execute_Allows_Null_Output_And_Null_Error_On_ProcessResult()
{
MockNonGenericProcessor processor = new MockNonGenericProcessor();
Exception exception = new Exception();
ProcessorResult resultFromOnExecute = null;
processor.OnExecuteAction = input =>
{
resultFromOnExecute = new ProcessorResult()
{
Status = ProcessorStatus.Ok, Output = null, Error = null
};
return(resultFromOnExecute);
};
processor.Initialize();
object[] inputObjArray = new object[1] {
"hello"
};
ProcessorResult result = processor.Execute(inputObjArray);
Assert.IsNotNull(result, "Processor.Execute should never return null.");
Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Processor.Execute should have returned the same value as returned from OnExecute.");
Assert.AreSame(resultFromOnExecute, result, "Processor.Execute should have returned the same instance returned from OnExecute.");
Assert.IsNull(result.Output, "Processor.Execute should have returned the same instance returned from OnExecute.");
Assert.IsNull(result.Error, "Processor.Execute should have returned the same instance returned from OnExecute.");
}
protected override ProcessorResult ProcessPage(CrawlContext crawlContext, CrawledPage crawledPage)
{
ProcessorResult result = new ProcessorResult
{
UniqueAttributeId = 222
};
Match regexResult = wordPressPattern.Match(crawledPage.RawContent);
if (regexResult.Success)
{
result.Attributes.Add("siteBuilder", "BlogWordPress");
result.IsAHit = true;
return(result);
}
HtmlNodeCollection listhref = crawledPage.HtmlDocument.DocumentNode.SelectNodes("//a[@href]") ?? new HtmlNodeCollection(null);
if (listhref.Select(node => node.GetAttributeValue("href", "")).Any(content => content.Contains("wordpress.org")))
{
result.Attributes.Add("siteBuilder", "BlogWordPress");
result.IsAHit = true;
return(result);
}
return(result);
}
public void ProcessorResult_Ctor()
{
ProcessorResult result = new ProcessorResult();
Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Default ctor should set status to ok");
Assert.IsNull(result.Output, "Default ctor should set output to null");
}
private void AssertValueTypeProcessorResult <T>(params T[] values) where T : struct
{
ProcessorResult <T> result = new ProcessorResult <T>();
ProcessorResult baseResult = result as ProcessorResult;
foreach (T value in values)
{
baseResult.Output = new object[] { value };
Assert.AreEqual(value, result.Output, "Wrong value for ProcessorType>" + typeof(T).Name + "> for " + value);
}
// Attempting read from a null value should give default(T)
baseResult.Output = new object[] { null };
object valueFromNull = result.Output;
Assert.AreEqual(default(T), valueFromNull, "Expected default(T) for ProcessorType>" + typeof(T).Name);
// Attempting read from an illegal value type should throw
T unused;
ExceptionAssert.ThrowsInvalidOperation(
"Setting illegal value to ProcessorType>" + typeof(T).Name + "> should throw",
() =>
{
baseResult.Output = new object[] { "hello" };
unused = result.Output;
}
);
}
public void PipelineContext_OnReadAllInputs_Overridable()
{
MockPipeline pipeline = TestPipelines.CreateMockProcessor1Pipeline();
MockPipelineContext context = pipeline.Context;
bool processorCalled = false;
context.OnReadAllInputsCalled = (inProc, inValues) =>
{
// Inject a '3' into the inputs for proc 1
if (inProc == pipeline.Processors[1])
{
processorCalled = true;
Assert.AreEqual(4, inValues[0], "Expected to see '4' as the pipeline input");
inValues[0] = 3;
}
};
ProcessorResult procResult = pipeline.Execute(new object[] { 4 });
Assert.IsNotNull(procResult.Output, "Expected output array");
Assert.AreEqual(1, procResult.Output.Length, "Output size mismatch");
Assert.AreEqual(3.ToString(), procResult.Output[0], "Pipeline did not use context we injected");
Assert.IsTrue(processorCalled, "Our ReadAllInputs override was not called");
}
// This takes a list of result objects and consolidates them into one.
// This keeps us from having a nasty memory leak.
public void ConsolidateResults(ProcessorResult final, List <ProcessorResult> results)
{
foreach (var node in results)
{
foreach (var sub in node.WordCountBySub)
{
final.WordCountBySub.TryAdd(sub.Key, new Dictionary <string, long>());
foreach (var word in node.WordCountBySub[sub.Key])
{
final.WordCountBySub[sub.Key].TryAdd(word.Key, 0);
final.WordCountBySub[sub.Key][word.Key] += word.Value;
}
}
foreach (var user in node.UniqueUsers)
{
final.UniqueUsers.TryAdd(user.Key, 0);
final.UniqueUsers[user.Key] += user.Value;
}
foreach (var sub in node.UniqueUsersBySub)
{
final.UniqueUsersBySub.TryAdd(sub.Key, new Dictionary <string, long>());
foreach (var user in sub.Value)
{
final.UniqueUsersBySub[sub.Key].TryAdd(user.Key, 0);
final.UniqueUsersBySub[sub.Key][user.Key] += user.Value;
}
}
}
}
protected override ProcessorResult OnExecute(object[] input)
{
ProcessorResult result = null;
if (input == null)
{
throw new ArgumentNullException("input");
}
if (this.ignore)
{
result = new ProcessorResult();
}
if (this.mode == MediaTypeProcessorMode.Request)
{
result = this.ExecuteRequest(input);
}
else
{
this.ExecuteResponse(input);
}
if (result == null)
{
result = new ProcessorResult();
}
return(result);
}
protected override ProcessorResult ProcessPage(CrawlContext crawlContext, CrawledPage crawledPage)
{
nodeQueryList.Add(new KeyValuePair <string, string>("login", "//a[contains(@href, 'login')]"));
nodeQueryList.Add(new KeyValuePair <string, string>("signin", "//a[contains(@href, 'signin')]"));
ProcessorResult result = new ProcessorResult {
UniqueAttributeId = 17
};
//<input type="password"
var pwdInputs = crawledPage.CsQueryDocument.Select("input[type='password']");
if (pwdInputs.Length > 0)
{
result.IsAHit = true;
}
//check links
if (!result.IsAHit)
{
result.IsAHit = FindTags(crawledPage, crawlContext.RootUri.DnsSafeHost.ToLower());
}
//if we found it, set it
if (result.IsAHit)
{
result.Attributes.Add(result.UniqueAttributeId.ToString(), "true");
}
return(result);
}
protected override void OnError(ProcessorResult result)
{
if (this.OnErrorAction != null)
{
this.OnErrorAction(result);
}
base.OnError(result);
}
protected override void Write(ContentWriter output, ProcessorResult <TileFile> value)
{
var result = value.Result;
output.Write(result.Type);
output.Write(result.Sprite);
output.Write(result.IsWallJumpable);
}
public void Constructor()
{
ProcessorResult uut = new ProcessorResult();
Assert.IsNotNull(uut.Attributes);
Assert.AreEqual(0, uut.Attributes.Count);
Assert.AreEqual(0, uut.UniqueAttributeId);
Assert.IsFalse(uut.IsAHit);
}
public void Pipeline_Executes_One_Input_Output_Processor()
{
Pipeline pipeline = TestPipelines.CreateMockProcessor1Pipeline();
ProcessorResult result = pipeline.Execute(new object[] { 5 });
object[] output = result.Output;
Assert.IsNotNull(output, "Processing output was null");
Assert.AreEqual(1, output.Length, "Should have received 1 output");
Assert.AreEqual(5.ToString(), output[0], "Should have seen this output");
}
public void Execute_Throws_If_Initialize_Has_Not_Been_Called()
{
MockNonGenericProcessor processor = new MockNonGenericProcessor();
ExceptionAssert.ThrowsInvalidOperation(
"Execute should have thrown since the input parameter is null.",
() =>
{
ProcessorResult resultFromExecute = processor.Execute(new object[0]);
});
}
private void Check_Tou_File(string fileName)
{
string fileNameWithPath = Path.Combine(path, fileName);
Reader reader = new Reader();
ProcessorResult processorResult = reader.ProcessFile(fileNameWithPath);
var tou_Data = Proc_Tou.Get_Data_Directly(fileNameWithPath);
Assert.IsTrue(processorResult.Result.SequenceEqual(tou_Data));
}
public void PipelineContext_ReadInput_Returns_Result_After_Execution()
{
MockPipeline pipeline = TestPipelines.CreateMockProcessor1Pipeline();
MockPipelineContext context = pipeline.Context;
object value = context.ReadInput(pipeline.Processors[1].InArguments[0]);
Assert.IsNull(value, "Expected null prior to execution");
ProcessorResult result = pipeline.Execute(new object[] { 7 });
value = context.ReadInput(pipeline.Processors[1].InArguments[0]);
Assert.AreEqual(7, value, "Expected processor 1 to have this input value after execution");
}
private Task BuildProcessorTask(string chunk, Action <string, ProcessorResult> executor, ProcessorResult fileResult, string name)
{
var tempChunk = chunk;
_taskTracker++;
var taskId = _taskTracker;
var task = new Task(() =>
{
var result = new ProcessorResult
{
Name = name,
WordCountBySub = new Dictionary <string, Dictionary <string, long> >(),
UniqueUsers = new Dictionary <string, long>(),
UniqueUsersBySub = new Dictionary <string, Dictionary <string, long> >()
};
executor.Invoke(tempChunk, result);
var results = new[]
{
result
}.ToList();
try
{
lock (_lock)
{
ConsolidateResults(fileResult, results);
}
if (taskId % 2000 == 0)
{
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
}
Console.WriteLine("End Thread " + taskId);
}
finally
{
if (_semaphore != null)
{
_semaphore.Release();
}
}
});
task.Start();
return(task);
}
public void Pipeline_Executes_When_Empty()
{
// Create a pipeline with only an entry and exit processor (no actual external processors).
// The pipeline input is bound to the pipeline output.
// This tests the zero-processor case as well as pipeline in/out binding
Pipeline pipeline = TestPipelines.CreateEmptyPipeline();
ProcessorResult result = pipeline.Execute(new object[] { "aString" });
object[] output = result.Output;
Assert.IsNotNull(output, "Processing output was null");
Assert.AreEqual(1, output.Length, "Should have received 1 output");
Assert.AreEqual("aString", output[0], "Should have seen this output");
}
public override ProcessorResult <string> OnExecute(int intValue, double doubleValue)
{
ProcessorResult <string> result = new ProcessorResult <string>()
{
Output = ((double)(intValue) + doubleValue).ToString()
};
if (this.OnAfterExecute != null)
{
this.OnAfterExecute(result);
}
return(result);
}
private ProcessorResult ExecuteRequest(object[] input)
{
var request = (HttpRequestMessage)input[0];
ProcessorResult result = null;
if (request.Content != null && request.Content.Headers.ContentType != null && this.SupportedMediaTypes.Contains(request.Content.Headers.ContentType.MediaType))
{
result = new ProcessorResult <object> {
Output = this.ReadFromStream(request.Content.ContentReadStream, request)
};
}
return(result);
}
private void CompileTotalCount(ProcessorResult result)
{
result.WordCounts = new Dictionary <string, long>();
foreach (var sub in result.WordCountBySub)
{
foreach (var word in sub.Value)
{
// Populate word counts
result.WordCounts.TryAdd(word.Key, 0);
result.WordCounts[word.Key] += word.Value;
}
}
}
public void ProcessCrawledDomain(CrawlContext crawlContext)
{
string parkedType = string.Empty;
IPAddress ip = null;
crawlContext.CrawlBag.NoCrawl = false;
try
{
ip = Dig.Instance.GetIPAddress(crawlContext.RootUri.DnsSafeHost);
if (!Object.Equals(null, ip))
{
parkedType = FindParkedType(ip);
}
}
catch (Exception e)
{
_logger.ErrorFormat("Exception occurred getting ipaddress name for [{0}] [{1}]", crawlContext.RootUri.DnsSafeHost, e);
}
ProcessorResult result = new ProcessorResult {
UniqueAttributeId = ATTRIB_TYPE_ID
};
result.IsAHit = !string.IsNullOrEmpty(parkedType);
//save isparkedpage
if (result.IsAHit)
{
result.Attributes.Add(ATTRIB_TYPE_ID.ToString(), parkedType);
crawlContext.CrawlBag.NoCrawl = true; //Park Forward?
DomainSave(crawlContext, result);
}
//save ip address
if (!Object.Equals(null, ip))
{
ProcessorResult resultIP = new ProcessorResult {
UniqueAttributeId = 21
};
resultIP.IsAHit = true;
resultIP.Attributes.Add("21", ip.ToString());
DomainSave(crawlContext, resultIP);
}
else
{
_logger.DebugFormat("Domain [{0}] has no A record.", crawlContext.RootUri.DnsSafeHost);
}
}
public override ProcessorResult <string> OnExecute(int intValue)
{
if (this.OnExecuteCalled != null)
{
return(this.OnExecuteCalled(intValue));
}
ProcessorResult <string> result = new ProcessorResult <string>()
{
Output = intValue.ToString()
};
return(result);
}
