Beispiel #1
0
        static void Main(string[] args)
        {
            WebUtils.DefaultTimeout     = 10000;
            WebUtils.NumRetries         = 3;
            WebUtils.WaitBetweenRetries = 5000;
            WebUtils.ReadWriteTimeout   = 30000;
            ServicePointManager.DefaultConnectionLimit = 8;

            DateTime startTime  = DateTime.MinValue;
            Logger   rootLogger = Logger.GetRootLogger();

            rootLogger.LocalLevel      = Logger.Level.Debug;
            rootLogger.LocalOutputType = Logger.OutputType.Console;
            if (Config.LogFileName != null)
            {
                rootLogger.LocalOutputWriter = new StreamWriter(Config.LogFileName, /*append=*/ true);
                rootLogger.LocalOutputType  |= Logger.OutputType.Writer;
            }
            ArrayList <StreamDataProducerPoll>    dataReaders   = new ArrayList <StreamDataProducerPoll>();
            ArrayList <StreamDataConsumer>        dataConsumers = new ArrayList <StreamDataConsumer>();
            Dictionary <IWorkflowComponent, Guid> components    = new Dictionary <IWorkflowComponent, Guid>();
            // init logging
            Logger logger = Logger.GetLogger("Latino.Workflows.Dacq");
            // start HTTP server
            bool exit = false;
            bool httpServerRunning = false;

            if (Config.ClientIp != null)
            {
                new Thread(new ThreadStart(delegate()
                {
                    HttpListener listener          = new HttpListener();
                    listener.AuthenticationSchemes = AuthenticationSchemes.Anonymous;
                    listener.Prefixes.Add(string.Format("http://localhost/{0}/", Config.WebSiteId));
                    listener.Prefixes.Add(string.Format("http://first.ijs.si/{0}/", Config.WebSiteId));
                    listener.Start();
                    logger.Info("Main.HttpServer", "HTTP server started.");
                    httpServerRunning        = true;
                    DateTime prevRequestTime = DateTime.MinValue;
                    while (!exit)
                    {
                        try
                        {
                            HttpListenerContext ctx = null;
                            listener.BeginGetContext(new AsyncCallback(delegate(IAsyncResult ar)
                            {
                                try { ctx = listener.EndGetContext(ar); }
                                catch { }
                            }), /*state=*/ null);
                            while (!exit && ctx == null)
                            {
                                Thread.Sleep(500);
                            }
                            if (!exit)
                            {
                                // process requests one by one
                                ctx.Response.AppendHeader("Content-Type", "application/xml");
                                XmlWriterSettings settings = new XmlWriterSettings();
                                settings.Encoding          = Encoding.UTF8;
                                settings.CheckCharacters   = false;
                                settings.Indent            = true;
                                XmlWriter w = XmlTextWriter.Create(ctx.Response.OutputStream, settings);
                                w.WriteStartElement("DacqResponse");
                                w.WriteElementString("DacqStartTime", startTime.ToString(TIME_FORMAT));
                                if (prevRequestTime == DateTime.MinValue)
                                {
                                    prevRequestTime = startTime;
                                }
                                DateTime thisRequestTime = DateTime.Now;
                                string command           = GetHttpRequestCommand(ctx.Request.Url.ToString());
                                if (command == "components")
                                {
                                    w.WriteElementString("PreviousRequestTime", prevRequestTime.ToString(TIME_FORMAT));
                                    w.WriteElementString("ThisRequestTime", thisRequestTime.ToString(TIME_FORMAT));
                                }
                                w.WriteElementString("Request", ctx.Request.Url.ToString());
                                w.WriteElementString("Command", command);
                                w.WriteStartElement("ResponseBody");
                                if (command == "help")
                                {
                                    WriteSupportedCommands(w);
                                }
                                else if (command == "components")
                                {
                                    WriteComponentInfo(w, components, thisRequestTime, prevRequestTime);
                                    prevRequestTime = thisRequestTime;
                                }
                                else if (command == "sources")
                                {
                                    WriteRssInfo(w, components);
                                }
                                w.WriteEndElement(); // ResponseBody
                                w.WriteEndElement(); // DacqResponse
                                w.Close();
                                ctx.Response.Close();
                            }
                        }
                        catch (Exception e)
                        {
                            logger.Warn("Main.HttpServer", e);
                        }
                    }
                    listener.Stop();
                    logger.Info("Main.HttpServer", "HTTP server stopped.");
                    httpServerRunning = false;
                })).Start();
            }
            Console.CancelKeyPress += delegate(object sender, ConsoleCancelEventArgs e)
            {
                logger.Info("Main", "*** Ctrl-C command received. ***");
                e.Cancel = true;
                exit     = true;
                string componentsStr = "";
                foreach (StreamDataProducerPoll c in dataReaders)
                {
                    if (c.IsRunning)
                    {
                        componentsStr += "\r\n" + c.GetType() + " : " + c.Name;
                    }
                }
                foreach (StreamDataConsumer dataConsumer in dataConsumers)
                {
                    if (dataConsumer.IsRunning)
                    {
                        componentsStr += "\r\n" + dataConsumer.GetType() + " : " + dataConsumer.Load;
                    }
                }
                logger.Info("Main", "Active components:" + componentsStr);
            };
            logger.Info("Main", "Starting Dacq ...");
            // initialize database writers
            PassOnComponent lb = new PassOnComponent(); // data load balancer

            lb.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
            dataConsumers.Add(lb);
            if (Config.DbConnectionString != null)
            {
                if (!Config.SkipBoilerplateHistoryInit)
                {
                    UrlTreeBoilerplateRemoverComponent.InitializeHistory(Config.DbConnectionString);
                    RssFeedComponent.DatabaseConnectionString = Config.DbConnectionString;
                }
            }
            for (int i = 0; i < Config.NumPipes; i++)
            {
                DocumentWriterComponent            dwc = new DocumentWriterComponent(Config.DbConnectionStringDump, /*cmdTimeout=*/ 0, Config.XmlDataDumpRoot, Config.HtmlDataDumpRoot, Config.HtmlDumpViewRoot);
                UrlTreeBoilerplateRemoverComponent bpr = new UrlTreeBoilerplateRemoverComponent();
                DocumentWriterComponent            dw  = new DocumentWriterComponent(Config.DbConnectionString, /*cmdTimeout=*/ 0, Config.XmlDataRoot, Config.HtmlDataRoot, Config.HtmlViewRoot);
                HtmlTokenizerComponent             htc = new HtmlTokenizerComponent();
                SentenceSplitterComponent          ssc = null;
                EnglishTokenizerComponent          tok = null;
                EnglishLemmatizerComponent         lem = null;
                EnglishPosTaggerComponent          pt  = null;
                if (Config.Language == "English")
                {
                    ssc = new SentenceSplitterComponent();
                    tok = new EnglishTokenizerComponent();
                    lem = new EnglishLemmatizerComponent(EnglishLemmatizerComponent.Type.Both);
                    pt  = new EnglishPosTaggerComponent();
                }
                LanguageDetectorComponent ld = new LanguageDetectorComponent();
                DocumentFilterComponent   df = new DocumentFilterComponent();
                df.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document document, Logger dfLogger) {
                    string docId = document.Features.GetFeatureValue("guid");
                    // remove items without link in RSS document
                    if (document.Features.GetFeatureValue("contentType") != "Text")
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: contentType not Text (id={0}).", docId);
                        return(false);
                    }
                    // remove items with blacklisted URL
                    if (document.Features.GetFeatureValue("blacklisted") == "True")
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: responseUrl blacklisted (id={0}).", docId);
                        return(false);
                    }
                    // remove items with not enough content
                    if (Convert.ToInt32(document.Features.GetFeatureValue("bprContentCharCount")) < 100)
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: bprContentCharCount < 100 (id={0}).", docId);
                        return(false);
                    }
                    // remove unsupported languages
                    if (document.Features.GetFeatureValue("detectedCharRange") != "Basic Latin")
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: detectedCharRange not Basic Latin (id={0}).", docId);
                        return(false);
                    }
                    if (document.Features.GetFeatureValue("detectedLanguage") != Config.Language)
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: detectedLanguage not {1} but {2} (id={0}).", docId, Config.Language, document.Features.GetFeatureValue("detectedLanguage"));
                        return(false);
                    }
                    // remove exact duplicates
                    if (document.Features.GetFeatureValue("unseenContent") == "No")
                    {
                        dfLogger.Info("OnFilterDocument", "Document rejected: no unseen content (id={0}).", docId);
                        return(false);
                    }
                    return(true);
                });
                ld.BlockSelector = "TextBlock/Content"; // due to problems with itar-tass.com
                df.SubscribeDumpConsumer(dwc);

                lb.Subscribe(htc);
                htc.Subscribe(bpr);
                bpr.Subscribe(ld);
                ld.Subscribe(df);

                if (Config.Language == "English")
                {
                    df.Subscribe(ssc);
                    ssc.Subscribe(tok);
                    tok.Subscribe(lem);
                    lem.Subscribe(pt);
                    pt.Subscribe(dw);
                    dataConsumers.AddRange(new StreamDataConsumer[] { dwc, df, ld, htc, ssc, tok, pt, dw, lem, bpr });
                }
                else
                {
                    df.Subscribe(dw);
                    dataConsumers.AddRange(new StreamDataConsumer[] { dwc, df, ld, htc, dw, bpr });
                }
            }
            // initialize stream simulator
            if (Config.OfflineSource != null)
            {
                DocumentStreamReaderComponent dsr = new DocumentStreamReaderComponent(Config.OfflineSource);
                dsr.Name = Config.OfflineSource;
                dataReaders.Add(dsr);
                dsr.Subscribe(lb);
            }
            // initialize RSS feed components
            int j = 0;
            RssFeedComponent rssComp = null;
            Set <string>     sites   = new Set <string>();

            if (Config.DataSourcesFileName != null)
            {
                string[] sources = File.ReadAllLines(Config.DataSourcesFileName);
                foreach (string _url in sources)
                {
                    string url = _url.Trim();
                    if (url != "" && !url.StartsWith("#"))
                    {
                        Match m;
                        if ((m = Regex.Match(url, @"^site\s*:(?<siteId>.*)$", RegexOptions.IgnoreCase)).Success)
                        {
                            string siteId = m.Result("${siteId}").Trim().ToLower();
                            if (sites.Contains(siteId))
                            {
                                throw new Exception(string.Format("Duplicated site identifier ({0}).", siteId));
                            }
                            sites.Add(siteId);
                            rssComp = new RssFeedComponent(siteId);
                            if (Config.Language != "")
                            {
                                rssComp.RssXmlCodePageDetectorLanguage = (Language)Enum.Parse(typeof(Language), Config.Language);
                            }
                            rssComp.MaxDocsPerCorpus   = Config.MaxDocsPerCorpus;
                            rssComp.RandomDelayAtStart = Config.RandomDelayAtStart;
                            rssComp.Name             = siteId;
                            rssComp.TimeBetweenPolls = Config.SleepBetweenPolls;
                            rssComp.IncludeRssXml    = true;
                            rssComp.Initialize();
                            rssComp.IncludeRawData = true;
                            rssComp.Subscribe(lb);
                            dataReaders.Add(rssComp);
                            j++;
                        }
                        else if (rssComp != null)
                        {
                            rssComp.AddSource(url);
                        }
                    }
                }
            }
            foreach (StreamDataProducerPoll c in dataReaders)
            {
                c.Start();
            }
            foreach (IWorkflowComponent obj in dataReaders)
            {
                components.Add(obj, Guid.NewGuid());
            }
            foreach (IWorkflowComponent obj in dataConsumers)
            {
                components.Add(obj, Guid.NewGuid());
            }
            startTime = DateTime.Now;
            while (!exit)
            {
                Thread.Sleep(500);
            }
            // shut down gracefully
            logger.Info("Main", "Please wait while shutting down ...");
            // wait for HTTP server shutdown
            while (httpServerRunning)
            {
                Thread.Sleep(500);
            }
            // stop RSS components
            foreach (StreamDataProducerPoll c in dataReaders)
            {
                c.Stop();
            }
            foreach (StreamDataProducerPoll c in dataReaders)
            {
                while (c.IsRunning)
                {
                    Thread.Sleep(500);
                }
            }
            // wait for all data consumers to finish
            foreach (StreamDataConsumer dataConsumer in dataConsumers)
            {
                if (dataConsumer.IsRunning)
                {
                    while (!dataConsumer.IsSuspended)
                    {
                        Thread.Sleep(500);
                    }
                }
                dataConsumer.Dispose();
            }
            logger.Info("Main", "Dacq successfully stopped.");
        }
Beispiel #2
0
        static void Main(string[] args)
        {
            Logger logger = Logger.GetRootLogger();
            ZeroMqReceiverComponent zmqRcv = new ZeroMqReceiverComponent(delegate(string key) {
                if (key == "MessageSendAddress" || key == "ReceiveLoadBalancingAdress" || key == "FinishPublish")
                {
                    return(null);
                }                                                                                                                  // ignore these settings
                return(ConfigurationManager.AppSettings.Get(key));
            });
            ZeroMqEmitterComponent zmqEmt = new ZeroMqEmitterComponent(delegate(string key) {
                if (key == "MessageReceiveAddress" || key == "SendLoadBalancingAddress" || key == "FinishReceive")
                {
                    return(null);
                }                                                                                                                   // ignore these settings
                return(ConfigurationManager.AppSettings.Get(key));
            });
            PassOnComponent oldBranch = new PassOnComponent(); // first branch (goes to WP4)

            oldBranch.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
            PassOnComponent bypass = new PassOnComponent(); // second branch ("bypass", writes to DB)

            bypass.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
            zmqRcv.Subscribe(oldBranch);
            zmqRcv.Subscribe(bypass);
            for (int i = 0; i < NUM_PIPES; i++)
            {
                DocumentFilterComponent rcv = new DocumentFilterComponent();
                rcv.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
                    Console.WriteLine("RCV " + doc.Name);
                    return(true);
                });
                DocumentCategorizerComponent cc = new DocumentCategorizerComponent();
                cc.BlockSelector = "TextBlock/Content";
                DocumentFilterComponent dfc = new DocumentFilterComponent();
                dfc.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(Filter);
                EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER);
                erc.BlockSelector = "TextBlock/Content";
                DocumentFilterComponent snd = new DocumentFilterComponent();
                snd.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
                    Console.WriteLine("SND " + doc.Name + " [" + doc.Features.GetFeatureValue("fullId") + "]");
                    return(true);
                });
                GenericStreamDataProcessor mkId = new GenericStreamDataProcessor();
                mkId.OnProcessData += new GenericStreamDataProcessor.ProcessDataHandler(delegate(IDataProducer sender, object data) {
                    DocumentCorpus c = (DocumentCorpus)data;
                    string corpusId  = c.Features.GetFeatureValue("guid").Replace("-", "");
                    DateTime timeEnd = DateTime.Parse(c.Features.GetFeatureValue("timeEnd"));
                    foreach (Document d in c.Documents)
                    {
                        string docId  = d.Features.GetFeatureValue("guid").Replace("-", "");
                        string fullId = timeEnd.ToString("HH_mm_ss_") + corpusId + "_" + docId;
                        d.Features.SetFeatureValue("fullId", fullId);
                    }
                    return(data);
                });
                oldBranch.Subscribe(rcv);
                rcv.Subscribe(cc);
                cc.Subscribe(dfc);
                dfc.Subscribe(erc);
                erc.Subscribe(mkId);
                mkId.Subscribe(snd);
                snd.Subscribe(zmqEmt);
            }
            // Petra's code
            Ontology o = new Ontology(ONTOLOGY_FOLDER_BYPASS);

            o.ToDb(CONNECTION_STRING_OCCURRENCE);    // fill DB tables entity and class
            // end of Petra's code
            for (int i = 0; i < NUM_PIPES_BYPASS; i++)
            {
                // create components
                EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER_BYPASS);
                erc.BlockSelector = "TextBlock/Content";
                OntologyCategorizerComponent occ = new OntologyCategorizerComponent();
                PumpIndexComponent           pic = new PumpIndexComponent();
                OccurrenceWriterComponent    owc = new OccurrenceWriterComponent(CONNECTION_STRING_OCCURRENCE);
                DocumentWriterComponent      dwc = new DocumentWriterComponent(null, /*cmdTimeout=*/ 0, XML_DATA_ROOT, null);
                // build branch
                bypass.Subscribe(erc);
                erc.Subscribe(occ);
                occ.Subscribe(pic);
                pic.Subscribe(owc);
                pic.Subscribe(dwc);
            }
            zmqRcv.Start();
            logger.Info("Main", "The pipeline is running.");
        }