private static void Main(string[] args)
{
using (Process p = Process.GetCurrentProcess())
p.PriorityClass = ProcessPriorityClass.Idle;
while (true)
{
try
{
using (var ctx = new Db())
{
var haveWork = false;
Article a;
if ((a = ctx.Articles.FirstOrDefault(x => x.Processed == ProcessState.Waiting)) != null)
{
ShingleLogic.ProcessArticle(a.ID);
haveWork = true;
}
if (haveWork)
{
continue;
}
Console.WriteLine("................");
System.Threading.Tasks.Task.Delay(60000);
}
}
catch (Exception e)
{
DataLayer.LogException(e);
}
}
}
private void PerformShingleProcessing(object state)
{
//DataLayer.LogMessage(LogLevel.Service, "Processing Invoked");
try
{
//DataLayer.LogMessage(LogLevel.Info, "New processing run scheduled.");
using (var ctx = new Db())
{
ctx.Database.CommandTimeout = 120;
ShinglesToProcess = ShingleLogic.GetNextShingleList(ctx);
if (ShinglesToProcess == null || ShinglesToProcess.Count == 0)
{
System.Threading.Tasks.Task.Delay(20000);
}
// Thread.Sleep(20000);
else
{
foreach (var s in ShinglesToProcess)
{
ShingleLogic.AnalyzeShingle(s);
}
}
}
}
catch (Exception e)
{
DataLayer.LogException(e);
}
finally
{
thProcessShingles.timer.Start();
}
}
private void PerformScheduledWork(object state)
{
// Perform your work here, but be mindful of the _shutdownEvent in case
// the service is shutting down.
//
// Reschedule the work to be performed.
try
{
using (var ctx = new Db())
{
ctx.Database.CommandTimeout = 120;
ShinglesToProcess = ShingleLogic.GetNextShingleList(ctx);
if (ShinglesToProcess == null || ShinglesToProcess.Count == 0)
{
System.Threading.Tasks.Task.Delay(20000);
}
else
{
foreach (var s in ShinglesToProcess)
{
ShingleLogic.AnalyzeShingle(s);
}
}
}
}
catch (Exception e)
{
DataLayer.LogException(e);
}
finally
{
_scheduleTimer.Start();
}
}
static void Main(string[] args)
{
using (Process p = Process.GetCurrentProcess())
p.PriorityClass = ProcessPriorityClass.Idle;
var ctx = new Db();
ctx.Database.CommandTimeout = 120;
var shingleArray = ShingleLogic.GetNextShingles(ctx);
while (shingleArray.Length > 0)
{
foreach (int ShingleID in shingleArray)
{
ShingleLogic.AnalyzeShingle(ShingleID);
}
shingleArray = ShingleLogic.GetNextShingles(ctx);
}
}
private static void ProcessArticles()
{
var ctx = new Db();
Article ea;
while ((ea = ctx.Articles.FirstOrDefault(x => x.Processed == ProcessState.Waiting)) != null)
{
var sw = Stopwatch.StartNew();
ShingleLogic.ProcessArticle(ea.ID);
sw.Stop();
DataLayer.LogMessage(LogLevel.AnalyzedArticle, $"{sw.ElapsedMilliseconds}ms {ea.ID} {ea.Title}");
//RssLogic.ScoreArticle(ea, ctx);
//ea.ProcessedScore = ProcessState.Done;
ctx.SaveChanges();
ctx.Dispose();
ctx = new Db();
}
}
private static void DeleteShinglesContainingCompany()
{
var ctx = new Db();
var arr = ctx.Shingles.Where(x => x.kind == ShingleKind.newShingle).Take(1000).ToList();
while (arr.Count > 0)
{
foreach (var sh in arr)
{
if (ShingleLogic.ContainsCompanyName(sh, ctx))
{
ShingleLogic.SetShingleContainCompany(sh, ctx);
Console.ForegroundColor = ConsoleColor.Gray;
Console.Write(sh.ID);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine(" " + sh.text);
}
else
{
var sh2 = ctx.Shingles.Find(sh.ID);
if (sh2 == null)
{
continue;
}
if (sh2.ID < 2667191)
{
sh2.kind = ShingleKind.interesting;
}
else
{
ShingleLogic.SetShingleKind(sh2, ctx);
}
}
ctx.SaveChanges();
ctx.Dispose();
ctx = new Db();
}
arr = ctx.Shingles.Where(x => x.kind == ShingleKind.newShingle).Take(1000).ToList();
}
}
private static void ProcessShingles()
{
var ctx = new Db();
Shingle s;
while ((s = ctx.Shingles.FirstOrDefault(x => x.kind == ShingleKind.newShingle)) != null)
{
ShingleLogic.SetShingleKind(s, ctx);
//Console.ForegroundColor = (ConsoleColor)s.kind;
switch (s.kind)
{
case ShingleKind.common: Console.ForegroundColor = ConsoleColor.White; break;
case ShingleKind.ticker: Console.ForegroundColor = ConsoleColor.Yellow; break;
case ShingleKind.CEO: Console.ForegroundColor = ConsoleColor.Cyan; break;
case ShingleKind.companyName: Console.ForegroundColor = ConsoleColor.Green; break;
case ShingleKind.currencyPair: Console.ForegroundColor = ConsoleColor.Red; break;
case ShingleKind.containCommon: Console.ForegroundColor = ConsoleColor.DarkYellow; break;
case ShingleKind.containTicker: Console.ForegroundColor = ConsoleColor.DarkYellow; break;
case ShingleKind.upperCase: Console.ForegroundColor = ConsoleColor.Magenta; break;
}
Console.Write(s.kind + " " + s.ID);
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(" " + s.text);
ctx.SaveChanges();
ctx.Dispose();
ctx = new Db();
}
}
private void PerformArticleProcessing(object state)
{
try
{
using (var ctx = new Db())
{
ctx.Database.CommandTimeout = 120;
ArticlesToProcess = ShingleLogic.GetNextArticles(ctx);
if (ArticlesToProcess == null || ArticlesToProcess.Count == 0)
{
System.Threading.Tasks.Task.Delay(20000);
}
// Thread.Sleep(20000);
else
{
foreach (var a in ArticlesToProcess)
{
var sw = Stopwatch.StartNew();
ShingleLogic.ProcessArticle(a);
RssLogic.ScoreArticle(a);
var ea = ctx.Articles.Include("Feed").Single(x => x.ID == a);
string ticker = string.IsNullOrEmpty(ea.Ticker) ? "" : "Ticker:" + ea.Ticker + " ";
DataLayer.LogMessage(LogLevel.Article, $"A {sw.ElapsedMilliseconds}ms ID:{a} Score:{100 * (ea.ScoreMin + ea.ScoreMax)} {ticker}{ea.Title}");
}
}
}
}
catch (Exception e)
{
DataLayer.LogException(e);
}
finally
{
thProcessArticles.timer.Start();
}
}
public ArticleDetail GetDetail(int ArticleID)
{
var res = new ArticleDetail
{
ArticleEntity = _ctx.Articles.Find(ArticleID),
ScoredPhrases = new List <ScoredPhrase>()
};
if (res.ArticleEntity == null)
{
return(null);
}
var phrases = ShingleLogic.FindPhrases(res.ArticleEntity);
string text = res.ArticleEntity.Text();
var scores = new double[text.Length];
foreach (string phrase in phrases)
{
var sh = _ctx.Shingles.FirstOrDefault(s => s.text == phrase);
if (sh == null)
{
continue;
}
var sa = _ctx.ShingleActions.FirstOrDefault(x => x.shingleID == sh.ID && x.interval == 30);
if (sa == null)
{
continue;
}
if (sa.down == null || sa.up == null)
{
continue;
}
float score = (sa.up.Value + sa.down.Value - 2) * 100;
if (!res.ScoredPhrases.Any(p => p.Phrase == phrase))
{
res.ScoredPhrases.Add(new ScoredPhrase {
Score = score, Phrase = phrase, Color = GetColor(score), ShingleID = sh.ID
});
}
int wordIndex = text.IndexOf(phrase, StringComparison.OrdinalIgnoreCase);
if (wordIndex >= 0)
{
for (int i = wordIndex; i < wordIndex + sh.text.Length; i++)
{
scores[i] += score;
}
}
}
res.Colored = new List <ColoredText>();
for (int i = 0; i < text.Length; i++)
{
string newColor = GetColor(scores[i]);
if (res.Colored.Count == 0 || res.Colored.Last().Color != newColor)
{
res.Colored.Add(new ColoredText {
Color = newColor, Text = text.Substring(i, 1)
});
}
else
{
res.Colored[res.Colored.Count - 1] = new ColoredText
{
Color = newColor,
Text = res.Colored[res.Colored.Count - 1].Text + text.Substring(i, 1)
};
}
}
return(res);
}
static void Main(string[] args)
{
using (Process p = Process.GetCurrentProcess())
p.PriorityClass = ProcessPriorityClass.Idle;
var ctx = new Db();
//Shingle s;
//while ((s = ctx.Shingles.FirstOrDefault(x => x.kind == ShingleKind.newShingle)) != null)
//{
// ShingleLogic.SetShingleKind(s, ctx);
// //Console.ForegroundColor = (ConsoleColor)s.kind;
// switch (s.kind)
// {
// case ShingleKind.common: Console.ForegroundColor = ConsoleColor.White; break;
// case ShingleKind.ticker: Console.ForegroundColor = ConsoleColor.Yellow; break;
// case ShingleKind.CEO: Console.ForegroundColor = ConsoleColor.Cyan; break;
// case ShingleKind.companyName: Console.ForegroundColor = ConsoleColor.Green; break;
// case ShingleKind.currencyPair: Console.ForegroundColor = ConsoleColor.Red; break;
// case ShingleKind.containCommon: Console.ForegroundColor = ConsoleColor.DarkYellow; break;
// case ShingleKind.containTicker: Console.ForegroundColor = ConsoleColor.DarkYellow; break;
// case ShingleKind.upperCase: Console.ForegroundColor = ConsoleColor.Magenta; break;
// }
// Console.Write(s.kind + " " + s.ID);
// Console.ForegroundColor = ConsoleColor.Gray;
// Console.WriteLine(" " + s.text);
// ctx.SaveChanges();
// ctx.Dispose();
// ctx = new Db();
//}
//Article ea;
//while ((ea = ctx.Articles.FirstOrDefault(x => x.Processed5 == ProcessState.Waiting
// && x.Summary.Contains("ago"))) != null)
//{
// string newSummary = RssLogic.ClearText(ea.Summary);
// if (newSummary != ea.Summary)
// {
// Console.WriteLine($"Updating article {ea.ID}");
// ea.Summary = newSummary;
// ea.Hash32 = ea.ComputeHash();
// if (ctx.Articles.Any(x => x.ID != ea.ID && x.Hash32 == ea.Hash32 && x.Summary == ea.Summary))
// {
// Console.WriteLine($"Deleting duplicate article {ea.ID}");
// ctx.Entry(ea).State = EntityState.Deleted;
// }
// }
// ea.Processed5 = ProcessState.Done;
// //RssLogic.FindInstruments(ea.ID);
// //Console.ReadKey();
// //Console.WriteLine(ea.ID);
// ctx.SaveChanges();
// ctx.Dispose();
// ctx = new Db();
//}
//Article ea;
//while ((ea = ctx.Articles.FirstOrDefault(x => x.Processed == ProcessState.Waiting && x.Ticker != null )) != null)
//{
// //var sw = Stopwatch.StartNew();
// ShingleLogic.ProcessArticle(ea.ID);
// //sw.Stop();
// //DataLayer.LogMessage(LogLevel.NewArticle, $"{sw.ElapsedMilliseconds}ms {ea.ID} {ea.Title}");
// //RssLogic.ScoreArticle(ea, ctx);
// //ea.ProcessedScore = ProcessState.Done;
// ctx.SaveChanges();
// ctx.Dispose();
// ctx = new Db();
//}
var names = ctx.CompanyNames.Select(c => c.name).ToArray();
foreach (var name in names)
{
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(name);
var badShingles = ctx.Shingles.Where(s =>
s.text.StartsWith(name + " ") ||
s.text.StartsWith(name + "'") ||
s.text.Contains(" " + name + " ") ||
s.text.Contains("'" + name + " ") ||
s.text.Contains(" " + name + "'") ||
s.text.Contains("'" + name + "'") ||
s.text.EndsWith(" " + name));
foreach (var bad in badShingles)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write(bad.ID);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine(" " + bad.text);
ShingleLogic.SetShingleContainCompany(bad, ctx);
}
}
}
