public void SimulateClicksTest()
{
var context = new MobilniPortalNovicContext12();
FillDatabase f = new FillDatabase(context);
var d = DateTime.Now;
var categoryId = context.Categories.Where(x => x.Name == "Sportal").Select(x => x.CategoryId).First();
var userId = 1;
var count = 10;
var query = f.SimulateClicks(userId, categoryId, count, () => d, ()=>null);
var dict = CategoryHelpers.CategoryGetChildrensFromParent(context.Categories.ToList());
foreach (var r in query)
{
context.Clicks.Remove(r);
}
context.SaveChanges();
foreach (var r in query)
{
Assert.IsTrue(r.UserId == 1);
Assert.IsTrue(dict[categoryId].Select(x => x.CategoryId).Contains(r.CategoryId));
Assert.AreEqual(d, r.ClickDate);
}
}
public int UpdateFeedsForSites()
{
var count = 0;
using (var repo = new MobilniPortalNovicContext12())
{
#region InitializeRequiredFields
if (Titles == null)
{
///Select only last 30 from each list
var list = repo.NewsFiles.Select(x => x.Title);
/// Select all
/// var list = context.NewsFiles.Select(y => y.Title));
Titles = new HashSet<String>(list);
}
if (Categories == null)
{
var cat = repo.Categories.ToList();
cat.ForEach(x =>
{
if (x.ParentCategoryId.HasValue == false)
{
x.ParentCategoryId = null;
}
});
Categories = new HashSet<Category>(cat);
}
#endregion InitializeRequiredFields
#region ParseFeed
List<NewsFileExt> newsList = new List<NewsFileExt>();
var time = DateTime.Now;
//Get items from feed
var feeds = repo.Feeds.ToList();
feeds.ForEach(x => x.LastUpdated = time);
var newsFiles = feeds.AsParallel().Select(x => FeedParser.parseFeed(x)).SelectMany(x => x).ToList();
//remove duplicates
newsFiles = newsFiles.GroupBy(x => x.Title).Select(x => x.First()).ToList();
#endregion ParseFeed
#region ParseItems
//Remove already parsed ones
var newsFilesList = newsFiles.Where(x => FailedTitles.Contains(x.Title) == false && Titles.Contains(x.Title) == false).ToList();
//Process items
var items = NewsParser.parseItem(newsFilesList);
items.Where(x => !IsElementok(x)).ToList().ForEach(x =>
{
LogWriter.Instance.Log("Error parsing: " + x.Link);
FailedTitles.Add(x.Title);
});
#endregion ParseItems
#region SaveToDatabase
foreach (var item in items.Where(x => IsElementok(x)).ToList())
{
#region GetCategoryIdOrCreateNew
//Save categories into database
int? parentId = null;
foreach (var c in item.Categories.Take(2))
{
//Skip if already exists or parsed
if (Titles.Contains(item.Title) || FailedTitles.Contains(item.Title))
{
continue;
}
var s = Categories.Where(x => x.Name.Equals(c) && x.ParentCategoryId == parentId).FirstOrDefault();
if (s == null)
{
var category = repo.Categories.Add(new Category { Name = c, ParentCategoryId = parentId });
Console.WriteLine("Adding category {0}", c);
LogWriter.Instance.WriteToLog("Adding category " + c);
repo.SaveChanges();
parentId = category.CategoryId;
Categories.Add(category);
}
else
{
parentId = s.CategoryId;
}
}
item.CategoryId = parentId.Value;
#endregion GetCategoryIdOrCreateNew
Titles.Add(item.Title);
var i = AutoMapper.Mapper.Map<NewsFileExt, NewsFile>(item);
repo.NewsFiles.Add(i);
count += 1;
}
repo.SaveChanges();
Console.WriteLine("{0} new sites added.", count);
#endregion SaveToDatabase
}
return count;
}
private static void Main(string[] args)
{
ParsingService service = ParsingService.getParsingService();
Scheduler sched = new Scheduler(60 * 10, service);
inputDictionary = new Dictionary<String, CommandOption>();
inputDictionary.Add("start", new CommandOption { Description = "Start updating", Action = new Action(() => sched.StartUpdating()) });
inputDictionary.Add("stop", new CommandOption
{
Description = "Stop automatic updating.",
Action = new Action(
() =>
{
sched.Stop();
Console.WriteLine("Parser stopped");
})
});
inputDictionary.Add("simulate", new CommandOption { Description = "Simulate clicks", Action = new Action(() => FillDatabaseCmd.SimulateClicks()) });
inputDictionary.Add("check", new CommandOption
{
Description = "Check for duplicates",
Action = new Action(() =>
{
var dateToCheck = DateTime.Now.AddDays(-1);
var repo = new MobilniPortalNovicContext12();
var i = repo.NewsFiles.Where(x => x.PubDate > dateToCheck).GroupBy(x => x.Title).Where(x => x.Count() > 1).ToList();
var count = 0;
i.ForEach(x =>
{
count += x.Count() - 1;
x.Skip(1).ToList().ForEach(y=>repo.NewsFiles.Remove(y));
});
repo.SaveChanges();
Console.WriteLine("Removed {0} duplicates", count);
})
});
inputDictionary.Add("run",
new CommandOption
{
Description = "Run single update.",
Action = new Action(() =>
{
service.startParse();
})
});
inputDictionary.Add("exit", new CommandOption
{
Description = "Exit the program",
Action = new Action(() =>
{
sched.Stop();
Console.WriteLine("Stoping...");
while (ParsingService.getParsingService().State != State.WaitingToNextInterval)
{
Thread.Sleep(1000);
}
Environment.Exit(0);
})
});
inputDictionary.Add("stats", new CommandOption
{
Description = "Show parsers statistics",
Action = new Action(() =>
{
Console.WriteLine("Scheduler is {0}, with interval {1}.", sched.State, sched.RepeatInterval);
Console.WriteLine("Parsing service in currently: {0}", service.State);
Console.WriteLine("Total updated {0}.", service.TotalCount);
Console.WriteLine("Last run {0}.", service.LastRun);
})
});
inputDictionary.Add("AddNews", new CommandOption
{
Description="Manuly add news file",
Action = new Action(() =>
{
try
{
Console.WriteLine("CategoryName");
var i = Console.ReadLine();
var context = new MobilniPortalNovicContext12();
var cat = context.Categories.Where(x => x.Name == i).FirstOrDefault();
var catId = cat.CategoryId;
NewsFile f = new NewsFile {
CategoryId = catId,
FeedId = context.Feeds.First().FeedId,
Content = "Poljubna vsebina.",
Title = "Filler news",
ShortContent = "Ta novica je samo za testiranje",
PubDate=DateTime.Now,
Link="http//www.fake.si"
};
context.NewsFiles.Add(f);
context.SaveChanges();
Console.WriteLine("Added new news file");
}
catch (Exception e)
{
Console.WriteLine("Bad info");
}
})
});
inputDictionary.Add("clear", new CommandOption
{
Description = "Clear filler news",
Action = new Action(() =>
{
var c = new MobilniPortalNovicContext12();
c.NewsFiles.Where(x => x.Title == "Filler news").ToList().ForEach(x => c.NewsFiles.Remove(x));
c.SaveChanges();
})
});
while (true)
{
var input = Console.ReadLine();
if (inputDictionary.ContainsKey(input))
{
inputDictionary[input].Action();
}
else
{
DisplayChoices();
}
Console.WriteLine();
}
}
