public void OnError_ProcessorResult_Is_Not_Changed_By_Execute()
        {
            MockNonGenericProcessor processor = new MockNonGenericProcessor();

            Exception executionException = new Exception();

            object[] executionOutput = new object[0];
            processor.OnExecuteAction = input => new ProcessorResult
            {
                Status = ProcessorStatus.Error,
                Output = executionOutput,
                Error  = executionException
            };

            Exception onErrorException = new Exception();

            object[] onErrorOutput = new object[0];
            processor.OnErrorAction = result =>
            {
                Assert.AreEqual(ProcessorStatus.Error, result.Status, "The result status should have been ProcessorStatus.Error.");
                Assert.AreSame(result.Error, executionException, "The Error should have been the same instance as the one set in OnExecute.");
                Assert.AreSame(result.Output, executionOutput, "The Output should have been the same instance as the one set in OnExecute.");
                result.Error  = onErrorException;
                result.Output = onErrorOutput;
            };

            processor.Initialize();
            ProcessorResult resultFromExecute = processor.Execute(new object[0]);

            Assert.IsNotNull(resultFromExecute, "There should have been a non-null result.");
            Assert.AreEqual(ProcessorStatus.Error, resultFromExecute.Status, "The result status should have been ProcessorStatus.Error.");
            Assert.AreSame(resultFromExecute.Error, onErrorException, "The Error should have been the same instance as the one set in OnError.");
            Assert.AreSame(resultFromExecute.Output, onErrorOutput, "The Output should have been the same instance as the one set in OnError.");
        }
Пример #2
0
        public override ProcessorResult <string> OnExecute(int intValueDerived)
        {
            ProcessorResult <string> result = base.OnExecute(intValueDerived);

            result.Output = "Derived" + result.Output;
            return(result);
        }
Пример #3
0
        public void PipelineBuilder_Build_Binds_By_Name_And_Executes()
        {
            MockPipelineBuilder     pb         = new MockPipelineBuilder();
            MockProcessor1          processor1 = new MockProcessor1();
            MockProcessorEchoString processor2 = new MockProcessorEchoString();

            // Rename processor inputs and outputs to bind by name to pipeline arguments and to each other:
            // pipeline[intInput] --> [intInput]p1[echoInput] --> [echoInput]p2[stringOutput] --> [stringOutput]pipeline
            processor1.InArguments[0].Name  = "intInput";
            processor2.InArguments[0].Name  = processor1.OutArguments[0].Name = "echoInput";
            processor2.OutArguments[0].Name = "stringOutput";

            Pipeline pipeline = pb.Build(
                new Processor[] { processor1, processor2 },
                new ProcessorArgument[] { new ProcessorArgument("intInput", typeof(int)) },
                new ProcessorArgument[] { new ProcessorArgument("stringOutput", typeof(string)) }
                );

            Assert.IsNotNull(pipeline, "Build should produce non-null pipeline");

            ProcessorResult result = pipeline.Execute(new object[] { 9 });

            Assert.IsNotNull(result, "Null result from execute");
            Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Expected OK result status");
            Assert.IsNotNull(result.Output, "Expected non-null output");
            Assert.AreEqual(1, result.Output.Length, "Expected 1 output value");
            Assert.AreEqual(9.ToString(), result.Output[0], "Wrong output value");
        }
Пример #4
0
        private void AssertNullableProcessorResult <T>(params T[] values) where T : struct
        {
            ProcessorResult <T?> result     = new ProcessorResult <T?>();
            ProcessorResult      baseResult = result as ProcessorResult;

            foreach (T value in values)
            {
                baseResult.Output = new object[] { value };
                Assert.IsTrue(result.Output.HasValue, "Expected HasValue==true for ProcessorType>" + typeof(T).Name + "> for " + value);
                Assert.AreEqual(value, result.Output.Value, "Wrong value for ProcessorType>" + typeof(T).Name + "> for " + value);
            }

            baseResult.Output = new object[] { null };
            T?nullableValue = result.Output;

            Assert.IsFalse(nullableValue.HasValue, "ProcessorType>" + typeof(T).Name + "> should set HasValue == false for null");

            T?unused;

            ExceptionAssert.ThrowsInvalidOperation(
                "Setting illegal value to ProcessorType>" + typeof(T).Name + "> should throw",
                () =>
            {
                baseResult.Output = new object[] { "hello" };
                unused            = result.Output;
            }
                );
        }
Пример #5
0
        public void Pipeline_Execute_Throws_Wrong_Output_Length()
        {
            MockPipeline    pipeline    = TestPipelines.CreateMockProcessor1Pipeline();
            ProcessorResult errorResult = null;

            // Override the OnExecute to return a ProcessorResult
            // which contains the wrong
            ((MockProcessor1)pipeline.Processors[1]).OnExecuteCalled =
                i =>
            {
                ProcessorResult pr = new ProcessorResult <string>()
                {
                };
                pr.Output = new object[] { "string1", "string2" };
                return(pr as ProcessorResult <string>);
            };

            pipeline.OnErrorCalled = r => errorResult = r;

            // The final pipeline result should reflect that error status
            ProcessorResult result = pipeline.Execute(new object[] { 5 });

            Assert.IsNotNull(result, "Failed to get a result");
            Assert.AreEqual(ProcessorStatus.Error, result.Status, "Expected failure status");
            Assert.IsNotNull(result.Error, "Expected error result");
            Assert.AreEqual(typeof(InvalidOperationException), result.Error.GetType(), "Expected InvalidOperationException");

            Assert.IsNotNull(errorResult, "Pipeline.OnError should have been called");
        }
 protected override void OnError(ProcessorResult result)
 {
     if (this.OnErrorCalled != null)
     {
         this.OnErrorCalled(result);
     }
 }
Пример #7
0
        public void ProcessCrawledDomain(CrawlContext crawlContext)
        {
            string webhost = string.Empty;

            try
            {
                webhost = Dig.Instance.GetWebHostName(crawlContext.RootUri.DnsSafeHost);
            }
            catch (Exception e)
            {
                _logger.ErrorFormat("Exception occurred getting webhost name for [{0}]", crawlContext.RootUri.DnsSafeHost, e);
            }

            ProcessorResult result = new ProcessorResult {
                UniqueAttributeId = ATTRIB_TYPE_ID
            };                                                                                               //mask

            result.IsAHit = webhost != "None";
            result.Attributes.Add(ATTRIB_TYPE_ID.ToString(), webhost);

            if (result.IsAHit)
            {
                DomainSave(crawlContext, result);
            }
        }
Пример #8
0
        public void Pipeline_Executes_2_Processors_Common_In_Double_Out()
        {
            Pipeline pipeline = TestPipelines.CreateDoubleMockProcessor1Pipeline();

            // Override the OnExecute of both so we see unique outputs
            ((MockProcessor1)pipeline.Processors[1]).OnExecuteCalled =
                i => new ProcessorResult <string>()
            {
                Output = "result1 for " + i
            };

            ((MockProcessor1)pipeline.Processors[2]).OnExecuteCalled =
                i => new ProcessorResult <string>()
            {
                Output = "result2 for " + i
            };

            ProcessorResult result = pipeline.Execute(new object[] { 5 });

            // This test has 2 MockProcessor1's, both bound to the single pipeline input 'intValue'.
            // Each processor output is bound to one of the 2 pipeline outputs
            object[] output = result.Output;
            Assert.IsNotNull(output, "Processing output was null");
            Assert.AreEqual(2, output.Length, "Should have received 1 output");
            Assert.AreEqual("result1 for 5", output[0], "Should have seen this output[0]");
            Assert.AreEqual("result2 for 5", output[1], "Should have seen this output[1]");
        }
        public void Execute_Allows_Null_Output_And_Null_Error_On_ProcessResult()
        {
            MockNonGenericProcessor processor   = new MockNonGenericProcessor();
            Exception       exception           = new Exception();
            ProcessorResult resultFromOnExecute = null;

            processor.OnExecuteAction = input =>
            {
                resultFromOnExecute = new ProcessorResult()
                {
                    Status = ProcessorStatus.Ok, Output = null, Error = null
                };
                return(resultFromOnExecute);
            };

            processor.Initialize();

            object[] inputObjArray = new object[1] {
                "hello"
            };
            ProcessorResult result = processor.Execute(inputObjArray);

            Assert.IsNotNull(result, "Processor.Execute should never return null.");
            Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Processor.Execute should have returned the same value as returned from OnExecute.");
            Assert.AreSame(resultFromOnExecute, result, "Processor.Execute should have returned the same instance returned from OnExecute.");
            Assert.IsNull(result.Output, "Processor.Execute should have returned the same instance returned from OnExecute.");
            Assert.IsNull(result.Error, "Processor.Execute should have returned the same instance returned from OnExecute.");
        }
Пример #10
0
        protected override ProcessorResult ProcessPage(CrawlContext crawlContext, CrawledPage crawledPage)
        {
            ProcessorResult result = new ProcessorResult
            {
                UniqueAttributeId = 222
            };

            Match regexResult = wordPressPattern.Match(crawledPage.RawContent);

            if (regexResult.Success)
            {
                result.Attributes.Add("siteBuilder", "BlogWordPress");
                result.IsAHit = true;
                return(result);
            }

            HtmlNodeCollection listhref = crawledPage.HtmlDocument.DocumentNode.SelectNodes("//a[@href]") ?? new HtmlNodeCollection(null);

            if (listhref.Select(node => node.GetAttributeValue("href", "")).Any(content => content.Contains("wordpress.org")))
            {
                result.Attributes.Add("siteBuilder", "BlogWordPress");
                result.IsAHit = true;
                return(result);
            }

            return(result);
        }
Пример #11
0
        public void ProcessorResult_Ctor()
        {
            ProcessorResult result = new ProcessorResult();

            Assert.AreEqual(ProcessorStatus.Ok, result.Status, "Default ctor should set status to ok");
            Assert.IsNull(result.Output, "Default ctor should set output to null");
        }
Пример #12
0
        private void AssertValueTypeProcessorResult <T>(params T[] values) where T : struct
        {
            ProcessorResult <T> result     = new ProcessorResult <T>();
            ProcessorResult     baseResult = result as ProcessorResult;

            foreach (T value in values)
            {
                baseResult.Output = new object[] { value };
                Assert.AreEqual(value, result.Output, "Wrong value for ProcessorType>" + typeof(T).Name + "> for " + value);
            }

            // Attempting read from a null value should give default(T)
            baseResult.Output = new object[] { null };
            object valueFromNull = result.Output;

            Assert.AreEqual(default(T), valueFromNull, "Expected default(T) for ProcessorType>" + typeof(T).Name);

            // Attempting read from an illegal value type should throw
            T unused;

            ExceptionAssert.ThrowsInvalidOperation(
                "Setting illegal value to ProcessorType>" + typeof(T).Name + "> should throw",
                () =>
            {
                baseResult.Output = new object[] { "hello" };
                unused            = result.Output;
            }
                );
        }
        public void PipelineContext_OnReadAllInputs_Overridable()
        {
            MockPipeline        pipeline = TestPipelines.CreateMockProcessor1Pipeline();
            MockPipelineContext context  = pipeline.Context;
            bool processorCalled         = false;

            context.OnReadAllInputsCalled = (inProc, inValues) =>
            {
                // Inject a '3' into the inputs for proc 1
                if (inProc == pipeline.Processors[1])
                {
                    processorCalled = true;
                    Assert.AreEqual(4, inValues[0], "Expected to see '4' as the pipeline input");
                    inValues[0] = 3;
                }
            };

            ProcessorResult procResult = pipeline.Execute(new object[] { 4 });

            Assert.IsNotNull(procResult.Output, "Expected output array");
            Assert.AreEqual(1, procResult.Output.Length, "Output size mismatch");
            Assert.AreEqual(3.ToString(), procResult.Output[0], "Pipeline did not use context we injected");

            Assert.IsTrue(processorCalled, "Our ReadAllInputs override was not called");
        }
Пример #14
0
        // This takes a list of result objects and consolidates them into one.
        // This keeps us from having a nasty memory leak.
        public void ConsolidateResults(ProcessorResult final, List <ProcessorResult> results)
        {
            foreach (var node in results)
            {
                foreach (var sub in node.WordCountBySub)
                {
                    final.WordCountBySub.TryAdd(sub.Key, new Dictionary <string, long>());

                    foreach (var word in node.WordCountBySub[sub.Key])
                    {
                        final.WordCountBySub[sub.Key].TryAdd(word.Key, 0);
                        final.WordCountBySub[sub.Key][word.Key] += word.Value;
                    }
                }

                foreach (var user in node.UniqueUsers)
                {
                    final.UniqueUsers.TryAdd(user.Key, 0);
                    final.UniqueUsers[user.Key] += user.Value;
                }

                foreach (var sub in node.UniqueUsersBySub)
                {
                    final.UniqueUsersBySub.TryAdd(sub.Key, new Dictionary <string, long>());

                    foreach (var user in sub.Value)
                    {
                        final.UniqueUsersBySub[sub.Key].TryAdd(user.Key, 0);
                        final.UniqueUsersBySub[sub.Key][user.Key] += user.Value;
                    }
                }
            }
        }
Пример #15
0
        protected override ProcessorResult OnExecute(object[] input)
        {
            ProcessorResult result = null;

            if (input == null)
            {
                throw new ArgumentNullException("input");
            }

            if (this.ignore)
            {
                result = new ProcessorResult();
            }

            if (this.mode == MediaTypeProcessorMode.Request)
            {
                result = this.ExecuteRequest(input);
            }
            else
            {
                this.ExecuteResponse(input);
            }

            if (result == null)
            {
                result = new ProcessorResult();
            }

            return(result);
        }
Пример #16
0
        protected override ProcessorResult ProcessPage(CrawlContext crawlContext, CrawledPage crawledPage)
        {
            nodeQueryList.Add(new KeyValuePair <string, string>("login", "//a[contains(@href, 'login')]"));
            nodeQueryList.Add(new KeyValuePair <string, string>("signin", "//a[contains(@href, 'signin')]"));

            ProcessorResult result = new ProcessorResult {
                UniqueAttributeId = 17
            };

            //<input type="password"
            var pwdInputs = crawledPage.CsQueryDocument.Select("input[type='password']");

            if (pwdInputs.Length > 0)
            {
                result.IsAHit = true;
            }

            //check links
            if (!result.IsAHit)
            {
                result.IsAHit = FindTags(crawledPage, crawlContext.RootUri.DnsSafeHost.ToLower());
            }

            //if we found it, set it
            if (result.IsAHit)
            {
                result.Attributes.Add(result.UniqueAttributeId.ToString(), "true");
            }

            return(result);
        }
Пример #17
0
 protected override void OnError(ProcessorResult result)
 {
     if (this.OnErrorAction != null)
     {
         this.OnErrorAction(result);
     }
     base.OnError(result);
 }
Пример #18
0
        protected override void Write(ContentWriter output, ProcessorResult <TileFile> value)
        {
            var result = value.Result;

            output.Write(result.Type);
            output.Write(result.Sprite);
            output.Write(result.IsWallJumpable);
        }
Пример #19
0
        public void Constructor()
        {
            ProcessorResult uut = new ProcessorResult();

            Assert.IsNotNull(uut.Attributes);
            Assert.AreEqual(0, uut.Attributes.Count);
            Assert.AreEqual(0, uut.UniqueAttributeId);
            Assert.IsFalse(uut.IsAHit);
        }
Пример #20
0
        public void Pipeline_Executes_One_Input_Output_Processor()
        {
            Pipeline        pipeline = TestPipelines.CreateMockProcessor1Pipeline();
            ProcessorResult result   = pipeline.Execute(new object[] { 5 });

            object[] output = result.Output;
            Assert.IsNotNull(output, "Processing output was null");
            Assert.AreEqual(1, output.Length, "Should have received 1 output");
            Assert.AreEqual(5.ToString(), output[0], "Should have seen this output");
        }
        public void Execute_Throws_If_Initialize_Has_Not_Been_Called()
        {
            MockNonGenericProcessor processor = new MockNonGenericProcessor();

            ExceptionAssert.ThrowsInvalidOperation(
                "Execute should have thrown since the input parameter is null.",
                () =>
            {
                ProcessorResult resultFromExecute = processor.Execute(new object[0]);
            });
        }
Пример #22
0
        private void Check_Tou_File(string fileName)
        {
            string fileNameWithPath = Path.Combine(path, fileName);

            Reader          reader          = new Reader();
            ProcessorResult processorResult = reader.ProcessFile(fileNameWithPath);

            var tou_Data = Proc_Tou.Get_Data_Directly(fileNameWithPath);

            Assert.IsTrue(processorResult.Result.SequenceEqual(tou_Data));
        }
        public void PipelineContext_ReadInput_Returns_Result_After_Execution()
        {
            MockPipeline        pipeline = TestPipelines.CreateMockProcessor1Pipeline();
            MockPipelineContext context  = pipeline.Context;
            object value = context.ReadInput(pipeline.Processors[1].InArguments[0]);

            Assert.IsNull(value, "Expected null prior to execution");
            ProcessorResult result = pipeline.Execute(new object[] { 7 });

            value = context.ReadInput(pipeline.Processors[1].InArguments[0]);
            Assert.AreEqual(7, value, "Expected processor 1 to have this input value after execution");
        }
Пример #24
0
        private Task BuildProcessorTask(string chunk, Action <string, ProcessorResult> executor, ProcessorResult fileResult, string name)
        {
            var tempChunk = chunk;

            _taskTracker++;
            var taskId = _taskTracker;

            var task = new Task(() =>
            {
                var result = new ProcessorResult
                {
                    Name             = name,
                    WordCountBySub   = new Dictionary <string, Dictionary <string, long> >(),
                    UniqueUsers      = new Dictionary <string, long>(),
                    UniqueUsersBySub = new Dictionary <string, Dictionary <string, long> >()
                };

                executor.Invoke(tempChunk, result);

                var results = new[]
                {
                    result
                }.ToList();

                try
                {
                    lock (_lock)
                    {
                        ConsolidateResults(fileResult, results);
                    }

                    if (taskId % 2000 == 0)
                    {
                        GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
                        GC.Collect();
                    }

                    Console.WriteLine("End Thread " + taskId);
                }
                finally
                {
                    if (_semaphore != null)
                    {
                        _semaphore.Release();
                    }
                }
            });

            task.Start();

            return(task);
        }
Пример #25
0
        public void Pipeline_Executes_When_Empty()
        {
            // Create a pipeline with only an entry and exit processor (no actual external processors).
            // The pipeline input is bound to the pipeline output.
            // This tests the zero-processor case as well as pipeline in/out binding
            Pipeline        pipeline = TestPipelines.CreateEmptyPipeline();
            ProcessorResult result   = pipeline.Execute(new object[] { "aString" });

            object[] output = result.Output;
            Assert.IsNotNull(output, "Processing output was null");
            Assert.AreEqual(1, output.Length, "Should have received 1 output");
            Assert.AreEqual("aString", output[0], "Should have seen this output");
        }
Пример #26
0
        public override ProcessorResult <string> OnExecute(int intValue, double doubleValue)
        {
            ProcessorResult <string> result = new ProcessorResult <string>()
            {
                Output = ((double)(intValue) + doubleValue).ToString()
            };

            if (this.OnAfterExecute != null)
            {
                this.OnAfterExecute(result);
            }

            return(result);
        }
Пример #27
0
        private ProcessorResult ExecuteRequest(object[] input)
        {
            var             request = (HttpRequestMessage)input[0];
            ProcessorResult result  = null;

            if (request.Content != null && request.Content.Headers.ContentType != null && this.SupportedMediaTypes.Contains(request.Content.Headers.ContentType.MediaType))
            {
                result = new ProcessorResult <object> {
                    Output = this.ReadFromStream(request.Content.ContentReadStream, request)
                };
            }

            return(result);
        }
Пример #28
0
        private void CompileTotalCount(ProcessorResult result)
        {
            result.WordCounts = new Dictionary <string, long>();

            foreach (var sub in result.WordCountBySub)
            {
                foreach (var word in sub.Value)
                {
                    // Populate word counts
                    result.WordCounts.TryAdd(word.Key, 0);
                    result.WordCounts[word.Key] += word.Value;
                }
            }
        }
Пример #29
0
        public void ProcessCrawledDomain(CrawlContext crawlContext)
        {
            string    parkedType = string.Empty;
            IPAddress ip         = null;

            crawlContext.CrawlBag.NoCrawl = false;

            try
            {
                ip = Dig.Instance.GetIPAddress(crawlContext.RootUri.DnsSafeHost);
                if (!Object.Equals(null, ip))
                {
                    parkedType = FindParkedType(ip);
                }
            }
            catch (Exception e)
            {
                _logger.ErrorFormat("Exception occurred getting ipaddress name for [{0}] [{1}]", crawlContext.RootUri.DnsSafeHost, e);
            }

            ProcessorResult result = new ProcessorResult {
                UniqueAttributeId = ATTRIB_TYPE_ID
            };

            result.IsAHit = !string.IsNullOrEmpty(parkedType);

            //save isparkedpage
            if (result.IsAHit)
            {
                result.Attributes.Add(ATTRIB_TYPE_ID.ToString(), parkedType);
                crawlContext.CrawlBag.NoCrawl = true; //Park Forward?
                DomainSave(crawlContext, result);
            }

            //save ip address
            if (!Object.Equals(null, ip))
            {
                ProcessorResult resultIP = new ProcessorResult {
                    UniqueAttributeId = 21
                };
                resultIP.IsAHit = true;
                resultIP.Attributes.Add("21", ip.ToString());

                DomainSave(crawlContext, resultIP);
            }
            else
            {
                _logger.DebugFormat("Domain [{0}] has no A record.", crawlContext.RootUri.DnsSafeHost);
            }
        }
Пример #30
0
        public override ProcessorResult <string> OnExecute(int intValue)
        {
            if (this.OnExecuteCalled != null)
            {
                return(this.OnExecuteCalled(intValue));
            }

            ProcessorResult <string> result = new ProcessorResult <string>()
            {
                Output = intValue.ToString()
            };

            return(result);
        }