示例#1
0
        static void Main(string[] args)
        {
            Logger logger = Logger.GetRootLogger();
            ZeroMqReceiverComponent zmqRcv = new ZeroMqReceiverComponent(delegate(string key) {
                if (key == "MessageSendAddress" || key == "ReceiveLoadBalancingAdress" || key == "FinishPublish")
                {
                    return(null);
                }                                                                                                                  // ignore these settings
                return(ConfigurationManager.AppSettings.Get(key));
            });
            ZeroMqEmitterComponent zmqEmt = new ZeroMqEmitterComponent(delegate(string key) {
                if (key == "MessageReceiveAddress" || key == "SendLoadBalancingAddress" || key == "FinishReceive")
                {
                    return(null);
                }                                                                                                                   // ignore these settings
                return(ConfigurationManager.AppSettings.Get(key));
            });
            PassOnComponent oldBranch = new PassOnComponent(); // first branch (goes to WP4)

            oldBranch.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
            PassOnComponent bypass = new PassOnComponent(); // second branch ("bypass", writes to DB)

            bypass.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
            zmqRcv.Subscribe(oldBranch);
            zmqRcv.Subscribe(bypass);
            for (int i = 0; i < NUM_PIPES; i++)
            {
                DocumentFilterComponent rcv = new DocumentFilterComponent();
                rcv.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
                    Console.WriteLine("RCV " + doc.Name);
                    return(true);
                });
                DocumentCategorizerComponent cc = new DocumentCategorizerComponent();
                cc.BlockSelector = "TextBlock/Content";
                DocumentFilterComponent dfc = new DocumentFilterComponent();
                dfc.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(Filter);
                EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER);
                erc.BlockSelector = "TextBlock/Content";
                DocumentFilterComponent snd = new DocumentFilterComponent();
                snd.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
                    Console.WriteLine("SND " + doc.Name + " [" + doc.Features.GetFeatureValue("fullId") + "]");
                    return(true);
                });
                GenericStreamDataProcessor mkId = new GenericStreamDataProcessor();
                mkId.OnProcessData += new GenericStreamDataProcessor.ProcessDataHandler(delegate(IDataProducer sender, object data) {
                    DocumentCorpus c = (DocumentCorpus)data;
                    string corpusId  = c.Features.GetFeatureValue("guid").Replace("-", "");
                    DateTime timeEnd = DateTime.Parse(c.Features.GetFeatureValue("timeEnd"));
                    foreach (Document d in c.Documents)
                    {
                        string docId  = d.Features.GetFeatureValue("guid").Replace("-", "");
                        string fullId = timeEnd.ToString("HH_mm_ss_") + corpusId + "_" + docId;
                        d.Features.SetFeatureValue("fullId", fullId);
                    }
                    return(data);
                });
                oldBranch.Subscribe(rcv);
                rcv.Subscribe(cc);
                cc.Subscribe(dfc);
                dfc.Subscribe(erc);
                erc.Subscribe(mkId);
                mkId.Subscribe(snd);
                snd.Subscribe(zmqEmt);
            }
            // Petra's code
            Ontology o = new Ontology(ONTOLOGY_FOLDER_BYPASS);

            o.ToDb(CONNECTION_STRING_OCCURRENCE);    // fill DB tables entity and class
            // end of Petra's code
            for (int i = 0; i < NUM_PIPES_BYPASS; i++)
            {
                // create components
                EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER_BYPASS);
                erc.BlockSelector = "TextBlock/Content";
                OntologyCategorizerComponent occ = new OntologyCategorizerComponent();
                PumpIndexComponent           pic = new PumpIndexComponent();
                OccurrenceWriterComponent    owc = new OccurrenceWriterComponent(CONNECTION_STRING_OCCURRENCE);
                DocumentWriterComponent      dwc = new DocumentWriterComponent(null, /*cmdTimeout=*/ 0, XML_DATA_ROOT, null);
                // build branch
                bypass.Subscribe(erc);
                erc.Subscribe(occ);
                occ.Subscribe(pic);
                pic.Subscribe(owc);
                pic.Subscribe(dwc);
            }
            zmqRcv.Start();
            logger.Info("Main", "The pipeline is running.");
        }
示例#2
0
 static void Main(string[] args)
 {
     Logger logger = Logger.GetRootLogger();
     ZeroMqReceiverComponent zmqRcv = new ZeroMqReceiverComponent(delegate(string key) {
         if (key == "MessageSendAddress" || key == "ReceiveLoadBalancingAdress" || key == "FinishPublish") { return null; } // ignore these settings
         return ConfigurationManager.AppSettings.Get(key);
     });
     ZeroMqEmitterComponent zmqEmt = new ZeroMqEmitterComponent(delegate(string key) {
         if (key == "MessageReceiveAddress" || key == "SendLoadBalancingAddress" || key == "FinishReceive") { return null; } // ignore these settings
         return ConfigurationManager.AppSettings.Get(key);
     });
     PassOnComponent oldBranch = new PassOnComponent(); // first branch (goes to WP4)
     oldBranch.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
     PassOnComponent bypass = new PassOnComponent(); // second branch ("bypass", writes to DB)
     bypass.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
     zmqRcv.Subscribe(oldBranch);
     zmqRcv.Subscribe(bypass);
     for (int i = 0; i < NUM_PIPES; i++)
     {
         DocumentFilterComponent rcv = new DocumentFilterComponent();
         rcv.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
             Console.WriteLine("RCV " + doc.Name);
             return true;
         });
         DocumentCategorizerComponent cc = new DocumentCategorizerComponent();
         cc.BlockSelector = "TextBlock/Content";
         DocumentFilterComponent dfc = new DocumentFilterComponent();
         dfc.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(Filter);
         EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER);
         erc.BlockSelector = "TextBlock/Content";
         DocumentFilterComponent snd = new DocumentFilterComponent();
         snd.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
             Console.WriteLine("SND " + doc.Name + " [" + doc.Features.GetFeatureValue("fullId") + "]");
             return true;
         });
         GenericStreamDataProcessor mkId = new GenericStreamDataProcessor();
         mkId.OnProcessData += new GenericStreamDataProcessor.ProcessDataHandler(delegate(IDataProducer sender, object data) {
             DocumentCorpus c = (DocumentCorpus)data;
             string corpusId = c.Features.GetFeatureValue("guid").Replace("-", "");
             DateTime timeEnd = DateTime.Parse(c.Features.GetFeatureValue("timeEnd"));
             foreach (Document d in c.Documents)
             {
                 string docId = d.Features.GetFeatureValue("guid").Replace("-", "");
                 string fullId = timeEnd.ToString("HH_mm_ss_") + corpusId + "_" + docId;
                 d.Features.SetFeatureValue("fullId", fullId);
             }
             return data;
         });
         oldBranch.Subscribe(rcv);
         rcv.Subscribe(cc);
         cc.Subscribe(dfc);
         dfc.Subscribe(erc);
         erc.Subscribe(mkId);
         mkId.Subscribe(snd);
         snd.Subscribe(zmqEmt);
     }
     // Petra's code
     Ontology o = new Ontology(ONTOLOGY_FOLDER_BYPASS);
     o.ToDb(CONNECTION_STRING_OCCURRENCE);    // fill DB tables entity and class
     // end of Petra's code
     for (int i = 0; i < NUM_PIPES_BYPASS; i++)
     {
         // create components
         EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER_BYPASS);
         erc.BlockSelector = "TextBlock/Content";
         OntologyCategorizerComponent occ = new OntologyCategorizerComponent();
         PumpIndexComponent pic = new PumpIndexComponent();
         OccurrenceWriterComponent owc = new OccurrenceWriterComponent(CONNECTION_STRING_OCCURRENCE);
         DocumentWriterComponent dwc = new DocumentWriterComponent(null, /*cmdTimeout=*/0, XML_DATA_ROOT, null);
         // build branch
         bypass.Subscribe(erc);
         erc.Subscribe(occ);
         occ.Subscribe(pic);
         pic.Subscribe(owc);
         pic.Subscribe(dwc);
     }
     zmqRcv.Start();
     logger.Info("Main", "The pipeline is running.");
 }