static void Main(string[] args)
{
WebUtils.DefaultTimeout = 10000;
WebUtils.NumRetries = 3;
WebUtils.WaitBetweenRetries = 5000;
WebUtils.ReadWriteTimeout = 30000;
ServicePointManager.DefaultConnectionLimit = 8;
DateTime startTime = DateTime.MinValue;
Logger rootLogger = Logger.GetRootLogger();
rootLogger.LocalLevel = Logger.Level.Debug;
string LOG_FILE_NAME = ConfigurationManager.AppSettings["logFileName"];
rootLogger.LocalOutputType = Logger.OutputType.Console;
if (LOG_FILE_NAME != null)
{
rootLogger.LocalOutputWriter = new StreamWriter(LOG_FILE_NAME, /*append=*/true);
rootLogger.LocalOutputType |= Logger.OutputType.Writer;
}
string SOURCES_FILE_NAME = ConfigurationManager.AppSettings["dataSourcesFileName"];
string WEB_SITE_ID = Utils.GetConfigValue("webSiteId", "dacq");
string DB_CONNECTION_STRING = ConfigurationManager.AppSettings["dbConnectionString"];
string SQL_DB_CONNECTION_STRING = ConfigurationManager.AppSettings["SqlDbConnectionString"];
string DB_CONNECTION_STRING_DUMP = ConfigurationManager.AppSettings["dbConnectionStringDump"];
string CLIENT_IP = ConfigurationManager.AppSettings["clientIp"];
string XML_DATA_ROOT = ConfigurationManager.AppSettings["xmlDataRoot"];
string XML_DATA_ROOT_DUMP = ConfigurationManager.AppSettings["xmlDataRootDump"];
string HTML_DATA_ROOT = ConfigurationManager.AppSettings["htmlDataRoot"];
string HTML_DATA_ROOT_DUMP = ConfigurationManager.AppSettings["htmlDataRootDump"];
string XML_DATA_ROOT_NEW = XML_DATA_ROOT == null ? null : (XML_DATA_ROOT.TrimEnd('\\') + "\\" + "New");
string HTML_DATA_ROOT_NEW = HTML_DATA_ROOT == null ? null : (HTML_DATA_ROOT.TrimEnd('\\') + "\\" + "New");
string XML_DATA_ROOT_DUMP_NEW = XML_DATA_ROOT_DUMP == null ? null : (XML_DATA_ROOT_DUMP.TrimEnd('\\') + "\\" + "New");
string HTML_DATA_ROOT_DUMP_NEW = HTML_DATA_ROOT_DUMP == null ? null : (HTML_DATA_ROOT_DUMP.TrimEnd('\\') + "\\" + "New");
string DB_CONNECTION_STRING_NEW = ConfigurationManager.AppSettings["SqlDbConnectionStringNew"];
string tmp = ConfigurationManager.AppSettings["enableZeroMQ"];
bool ENABLE_ZEROMQ = tmp != null && new List<string>(new string[] { "true", "1", "yes", "on" }).Contains(tmp.ToLower());
const int NUM_WRITERS = 8;
const int SLEEP_BETWEEN_POLLS = 15 * 60000; // 15 minutes
ArrayList<StreamDataProducerPoll> dataReaders = new ArrayList<StreamDataProducerPoll>();
ArrayList<StreamDataConsumer> dataConsumers = new ArrayList<StreamDataConsumer>();
Dictionary<IWorkflowComponent, Guid> components = new Dictionary<IWorkflowComponent, Guid>();
// init logging
Logger logger = Logger.GetLogger("Latino.Workflows.Dacq");
// start HTTP server
bool exit = false;
bool httpServerRunning = false;
if (CLIENT_IP != null)
{
new Thread(new ThreadStart(delegate()
{
HttpListener listener = new HttpListener();
listener.AuthenticationSchemes = AuthenticationSchemes.Anonymous;
listener.Prefixes.Add(string.Format("http://localhost/{0}/", WEB_SITE_ID));
listener.Prefixes.Add(string.Format("http://first.ijs.si/{0}/", WEB_SITE_ID));
listener.Start();
logger.Info("Main.HttpServer", "HTTP server started.");
httpServerRunning = true;
DateTime prevRequestTime = DateTime.MinValue;
while (!exit)
{
try
{
HttpListenerContext ctx = null;
listener.BeginGetContext(new AsyncCallback(delegate(IAsyncResult ar)
{
try { ctx = listener.EndGetContext(ar); }
catch { }
}), /*state=*/null);
while (!exit && ctx == null) { Thread.Sleep(500); }
if (!exit)
{
// process requests one by one
ctx.Response.AppendHeader("Content-Type", "application/xml");
XmlWriterSettings settings = new XmlWriterSettings();
settings.Encoding = Encoding.UTF8;
settings.CheckCharacters = false;
settings.Indent = true;
XmlWriter w = XmlTextWriter.Create(ctx.Response.OutputStream, settings);
w.WriteStartElement("DacqResponse");
w.WriteElementString("DacqStartTime", startTime.ToString(TIME_FORMAT));
if (prevRequestTime == DateTime.MinValue) { prevRequestTime = startTime; }
DateTime thisRequestTime = DateTime.Now;
string command = GetHttpRequestCommand(ctx.Request.Url.ToString());
if (command == "components")
{
w.WriteElementString("PreviousRequestTime", prevRequestTime.ToString(TIME_FORMAT));
w.WriteElementString("ThisRequestTime", thisRequestTime.ToString(TIME_FORMAT));
}
w.WriteElementString("Request", ctx.Request.Url.ToString());
w.WriteElementString("Command", command);
w.WriteStartElement("ResponseBody");
if (command == "help")
{
WriteSupportedCommands(w);
}
else if (command == "components")
{
WriteComponentInfo(w, components, thisRequestTime, prevRequestTime);
prevRequestTime = thisRequestTime;
}
else if (command == "sources")
{
WriteRssInfo(w, components);
}
w.WriteEndElement(); // ResponseBody
w.WriteEndElement(); // DacqResponse
w.Close();
ctx.Response.Close();
}
}
catch (Exception e)
{
logger.Warn("Main.HttpServer", e);
}
}
listener.Stop();
logger.Info("Main.HttpServer", "HTTP server stopped.");
httpServerRunning = false;
})).Start();
}
Console.CancelKeyPress += delegate(object sender, ConsoleCancelEventArgs e)
{
logger.Info("Main", "*** Ctrl-C command received. ***");
e.Cancel = true;
exit = true;
string componentsStr = "";
foreach (StreamDataProducerPoll c in dataReaders)
{
if (c.IsRunning) { componentsStr += "\r\n" + c.GetType() + " : " + c.Name; }
}
foreach (StreamDataConsumer dataConsumer in dataConsumers)
{
if (dataConsumer.IsRunning) { componentsStr += "\r\n" + dataConsumer.GetType() + " : " + dataConsumer.Load; }
}
logger.Info("Main", "Active components:" + componentsStr);
};
logger.Info("Main", "Starting Dacq ...");
// initialize database writers
PassOnComponent lb = new PassOnComponent(); // data load balancer
lb.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
dataConsumers.Add(lb);
ZeroMqEmitterComponent zmq = null;
if (ENABLE_ZEROMQ)
{
zmq = new ZeroMqEmitterComponent();
dataConsumers.Add(zmq);
}
DatabaseConnection dbConnection = new DatabaseConnection();
if (DB_CONNECTION_STRING != null)
{
dbConnection.ConnectionString = DB_CONNECTION_STRING;
dbConnection.Connect();
UrlTreeBoilerplateRemoverComponent.InitializeHistory(dbConnection);
dbConnection.Disconnect();
RssFeedComponent.DatabaseConnectionString = SQL_DB_CONNECTION_STRING;
}
for (int i = 0; i < NUM_WRITERS; i++)
{
DocumentCorpusWriterComponent dcw = new DocumentCorpusWriterComponent(DB_CONNECTION_STRING_DUMP, /*xmlDataRoot=*/null);
DocumentWriterComponent dwc = new DocumentWriterComponent(/*connectionString=*/null, XML_DATA_ROOT_DUMP_NEW, HTML_DATA_ROOT_DUMP_NEW);
dcw.IsDumpWriter = true;
UrlTreeBoilerplateRemoverComponent bpr = new UrlTreeBoilerplateRemoverComponent(DB_CONNECTION_STRING);
DocumentCorpusWriterComponent cw = new DocumentCorpusWriterComponent(DB_CONNECTION_STRING, XML_DATA_ROOT);
DocumentWriterComponent dw = new DocumentWriterComponent(DB_CONNECTION_STRING_NEW, XML_DATA_ROOT_NEW, HTML_DATA_ROOT_NEW);
HtmlTokenizerComponent htc = new HtmlTokenizerComponent();
SentenceSplitterComponent ssc = new SentenceSplitterComponent();
EnglishTokenizerComponent tok = new EnglishTokenizerComponent();
EnglishLemmatizerComponent lem = new EnglishLemmatizerComponent(EnglishLemmatizerComponent.Type.Both);
EnglishPosTaggerComponent pt = new EnglishPosTaggerComponent();
LanguageDetectorComponent ld = new LanguageDetectorComponent();
DocumentFilterComponent df = new DocumentFilterComponent();
df.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document document, Logger dfLogger) {
string docId = document.Features.GetFeatureValue("guid");
// remove items without link in RSS document
if (document.Features.GetFeatureValue("contentType") != "Text")
{
dfLogger.Info("OnFilterDocument", "Document rejected: contentType not Text (id={0}).", docId);
return false;
}
// remove items with blacklisted URL
if (document.Features.GetFeatureValue("blacklisted") == "True")
{
dfLogger.Info("OnFilterDocument", "Document rejected: responseUrl blacklisted (id={0}).", docId);
return false;
}
// remove items with not enough content
if (Convert.ToInt32(document.Features.GetFeatureValue("bprContentCharCount")) < 100)
{
dfLogger.Info("OnFilterDocument", "Document rejected: bprContentCharCount < 100 (id={0}).", docId);
return false;
}
// remove non-English items
if (document.Features.GetFeatureValue("detectedCharRange") != "Basic Latin")
{
dfLogger.Info("OnFilterDocument", "Document rejected: detectedCharRange not Basic Latin (id={0}).", docId);
return false;
}
if (document.Features.GetFeatureValue("detectedLanguage") != "English")
{
dfLogger.Info("OnFilterDocument", "Document rejected: detectedLanguage not English (id={0}).", docId);
return false;
}
// remove exact duplicates
if (document.Features.GetFeatureValue("unseenContent") == "No")
{
dfLogger.Info("OnFilterDocument", "Document rejected: no unseen content (id={0}).", docId);
return false;
}
return true;
});
ld.BlockSelector = "TextBlock/Content"; // due to problems with itar-tass.com
df.SubscribeDumpConsumer(dcw);
df.SubscribeDumpConsumer(dwc);
lb.Subscribe(htc);
htc.Subscribe(bpr);
bpr.Subscribe(ld);
ld.Subscribe(df);
df.Subscribe(ssc);
ssc.Subscribe(tok);
tok.Subscribe(lem);
lem.Subscribe(pt);
pt.Subscribe(cw);
pt.Subscribe(dw);
if (ENABLE_ZEROMQ) { pt.Subscribe(zmq); }
dataConsumers.AddRange(new StreamDataConsumer[] { dcw, dwc, df, ld, htc, ssc, tok, pt, cw, dw, lem, bpr });
}
// initialize stream simulator
string offlineSource = ConfigurationManager.AppSettings["offlineSource"];
if (offlineSource != null)
{
DocumentStreamReaderComponent dsr = new DocumentStreamReaderComponent(offlineSource);
dsr.Name = offlineSource;
dataReaders.Add(dsr);
dsr.Subscribe(lb);
}
// initialize RSS feed components
int j = 0;
RssFeedComponent rssComp = null;
Set<string> sites = new Set<string>();
if (SOURCES_FILE_NAME != null)
{
string[] sources = File.ReadAllLines(SOURCES_FILE_NAME);
foreach (string _url in sources)
{
string url = _url.Trim();
if (url != "" && !url.StartsWith("#"))
{
Match m;
if ((m = Regex.Match(url, @"^site\s*:(?<siteId>.*)$", RegexOptions.IgnoreCase)).Success)
{
string siteId = m.Result("${siteId}").Trim().ToLower();
if (sites.Contains(siteId)) { throw new Exception(string.Format("Duplicated site identifier ({0}).", siteId)); }
sites.Add(siteId);
rssComp = new RssFeedComponent(siteId);
rssComp.MaxDocsPerCorpus = Convert.ToInt32(Utils.GetConfigValue("MaxDocsPerCorpus", "50"));
rssComp.RandomDelayAtStart = new ArrayList<string>("yes,on,true,1,y".Split(','))
.Contains(Utils.GetConfigValue("RandomDelayAtStart", "true").ToLower());
rssComp.Name = siteId;
rssComp.TimeBetweenPolls = SLEEP_BETWEEN_POLLS;
rssComp.IncludeRssXml = true;
if (SQL_DB_CONNECTION_STRING != null)
{
rssComp.Initialize(SQL_DB_CONNECTION_STRING);
}
rssComp.IncludeRawData = true;
rssComp.Subscribe(lb);
dataReaders.Add(rssComp);
j++;
}
else if (rssComp != null)
{
rssComp.AddSource(url);
}
}
}
}
foreach (StreamDataProducerPoll c in dataReaders)
{
c.Start();
}
foreach (IWorkflowComponent obj in dataReaders) { components.Add(obj, Guid.NewGuid()); }
foreach (IWorkflowComponent obj in dataConsumers) { components.Add(obj, Guid.NewGuid()); }
startTime = DateTime.Now;
while (!exit) { Thread.Sleep(500); }
// shut down gracefully
logger.Info("Main", "Please wait while shutting down ...");
// wait for HTTP server shutdown
while (httpServerRunning) { Thread.Sleep(500); }
// stop RSS components
foreach (StreamDataProducerPoll c in dataReaders)
{
c.Stop();
}
foreach (StreamDataProducerPoll c in dataReaders)
{
while (c.IsRunning) { Thread.Sleep(500); }
}
// wait for all data consumers to finish
foreach (StreamDataConsumer dataConsumer in dataConsumers)
{
if (dataConsumer.IsRunning) { while (!dataConsumer.IsSuspended) { Thread.Sleep(500); } }
dataConsumer.Dispose();
}
logger.Info("Main", "Dacq successfully stopped.");
}
static void Main(string[] args)
{
Logger logger = Logger.GetRootLogger();
ZeroMqReceiverComponent zmqRcv = new ZeroMqReceiverComponent(delegate(string key) {
if (key == "MessageSendAddress" || key == "ReceiveLoadBalancingAdress" || key == "FinishPublish") { return null; } // ignore these settings
return ConfigurationManager.AppSettings.Get(key);
});
ZeroMqEmitterComponent zmqEmt = new ZeroMqEmitterComponent(delegate(string key) {
if (key == "MessageReceiveAddress" || key == "SendLoadBalancingAddress" || key == "FinishReceive") { return null; } // ignore these settings
return ConfigurationManager.AppSettings.Get(key);
});
PassOnComponent oldBranch = new PassOnComponent(); // first branch (goes to WP4)
oldBranch.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
PassOnComponent bypass = new PassOnComponent(); // second branch ("bypass", writes to DB)
bypass.DispatchPolicy = DispatchPolicy.BalanceLoadMax;
zmqRcv.Subscribe(oldBranch);
zmqRcv.Subscribe(bypass);
for (int i = 0; i < NUM_PIPES; i++)
{
DocumentFilterComponent rcv = new DocumentFilterComponent();
rcv.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
Console.WriteLine("RCV " + doc.Name);
return true;
});
DocumentCategorizerComponent cc = new DocumentCategorizerComponent();
cc.BlockSelector = "TextBlock/Content";
DocumentFilterComponent dfc = new DocumentFilterComponent();
dfc.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(Filter);
EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER);
erc.BlockSelector = "TextBlock/Content";
DocumentFilterComponent snd = new DocumentFilterComponent();
snd.OnFilterDocument += new DocumentFilterComponent.FilterDocumentHandler(delegate(Document doc, Logger log) {
Console.WriteLine("SND " + doc.Name + " [" + doc.Features.GetFeatureValue("fullId") + "]");
return true;
});
GenericStreamDataProcessor mkId = new GenericStreamDataProcessor();
mkId.OnProcessData += new GenericStreamDataProcessor.ProcessDataHandler(delegate(IDataProducer sender, object data) {
DocumentCorpus c = (DocumentCorpus)data;
string corpusId = c.Features.GetFeatureValue("guid").Replace("-", "");
DateTime timeEnd = DateTime.Parse(c.Features.GetFeatureValue("timeEnd"));
foreach (Document d in c.Documents)
{
string docId = d.Features.GetFeatureValue("guid").Replace("-", "");
string fullId = timeEnd.ToString("HH_mm_ss_") + corpusId + "_" + docId;
d.Features.SetFeatureValue("fullId", fullId);
}
return data;
});
oldBranch.Subscribe(rcv);
rcv.Subscribe(cc);
cc.Subscribe(dfc);
dfc.Subscribe(erc);
erc.Subscribe(mkId);
mkId.Subscribe(snd);
snd.Subscribe(zmqEmt);
}
// Petra's code
Ontology o = new Ontology(ONTOLOGY_FOLDER_BYPASS);
o.ToDb(CONNECTION_STRING_OCCURRENCE); // fill DB tables entity and class
// end of Petra's code
for (int i = 0; i < NUM_PIPES_BYPASS; i++)
{
// create components
EntityRecognitionComponent erc = new EntityRecognitionComponent(ONTOLOGY_FOLDER_BYPASS);
erc.BlockSelector = "TextBlock/Content";
OntologyCategorizerComponent occ = new OntologyCategorizerComponent();
PumpIndexComponent pic = new PumpIndexComponent();
OccurrenceWriterComponent owc = new OccurrenceWriterComponent(CONNECTION_STRING_OCCURRENCE);
DocumentWriterComponent dwc = new DocumentWriterComponent(null, /*cmdTimeout=*/0, XML_DATA_ROOT, null);
// build branch
bypass.Subscribe(erc);
erc.Subscribe(occ);
occ.Subscribe(pic);
pic.Subscribe(owc);
pic.Subscribe(dwc);
}
zmqRcv.Start();
logger.Info("Main", "The pipeline is running.");
}