public void SimulateClicksTest() { var context = new MobilniPortalNovicContext12(); FillDatabase f = new FillDatabase(context); var d = DateTime.Now; var categoryId = context.Categories.Where(x => x.Name == "Sportal").Select(x => x.CategoryId).First(); var userId = 1; var count = 10; var query = f.SimulateClicks(userId, categoryId, count, () => d, ()=>null); var dict = CategoryHelpers.CategoryGetChildrensFromParent(context.Categories.ToList()); foreach (var r in query) { context.Clicks.Remove(r); } context.SaveChanges(); foreach (var r in query) { Assert.IsTrue(r.UserId == 1); Assert.IsTrue(dict[categoryId].Select(x => x.CategoryId).Contains(r.CategoryId)); Assert.AreEqual(d, r.ClickDate); } }
public int UpdateFeedsForSites() { var count = 0; using (var repo = new MobilniPortalNovicContext12()) { #region InitializeRequiredFields if (Titles == null) { ///Select only last 30 from each list var list = repo.NewsFiles.Select(x => x.Title); /// Select all /// var list = context.NewsFiles.Select(y => y.Title)); Titles = new HashSet<String>(list); } if (Categories == null) { var cat = repo.Categories.ToList(); cat.ForEach(x => { if (x.ParentCategoryId.HasValue == false) { x.ParentCategoryId = null; } }); Categories = new HashSet<Category>(cat); } #endregion InitializeRequiredFields #region ParseFeed List<NewsFileExt> newsList = new List<NewsFileExt>(); var time = DateTime.Now; //Get items from feed var feeds = repo.Feeds.ToList(); feeds.ForEach(x => x.LastUpdated = time); var newsFiles = feeds.AsParallel().Select(x => FeedParser.parseFeed(x)).SelectMany(x => x).ToList(); //remove duplicates newsFiles = newsFiles.GroupBy(x => x.Title).Select(x => x.First()).ToList(); #endregion ParseFeed #region ParseItems //Remove already parsed ones var newsFilesList = newsFiles.Where(x => FailedTitles.Contains(x.Title) == false && Titles.Contains(x.Title) == false).ToList(); //Process items var items = NewsParser.parseItem(newsFilesList); items.Where(x => !IsElementok(x)).ToList().ForEach(x => { LogWriter.Instance.Log("Error parsing: " + x.Link); FailedTitles.Add(x.Title); }); #endregion ParseItems #region SaveToDatabase foreach (var item in items.Where(x => IsElementok(x)).ToList()) { #region GetCategoryIdOrCreateNew //Save categories into database int? parentId = null; foreach (var c in item.Categories.Take(2)) { //Skip if already exists or parsed if (Titles.Contains(item.Title) || FailedTitles.Contains(item.Title)) { continue; } var s = Categories.Where(x => x.Name.Equals(c) && x.ParentCategoryId == parentId).FirstOrDefault(); if (s == null) { var category = repo.Categories.Add(new Category { Name = c, ParentCategoryId = parentId }); Console.WriteLine("Adding category {0}", c); LogWriter.Instance.WriteToLog("Adding category " + c); repo.SaveChanges(); parentId = category.CategoryId; Categories.Add(category); } else { parentId = s.CategoryId; } } item.CategoryId = parentId.Value; #endregion GetCategoryIdOrCreateNew Titles.Add(item.Title); var i = AutoMapper.Mapper.Map<NewsFileExt, NewsFile>(item); repo.NewsFiles.Add(i); count += 1; } repo.SaveChanges(); Console.WriteLine("{0} new sites added.", count); #endregion SaveToDatabase } return count; }
private static void Main(string[] args) { ParsingService service = ParsingService.getParsingService(); Scheduler sched = new Scheduler(60 * 10, service); inputDictionary = new Dictionary<String, CommandOption>(); inputDictionary.Add("start", new CommandOption { Description = "Start updating", Action = new Action(() => sched.StartUpdating()) }); inputDictionary.Add("stop", new CommandOption { Description = "Stop automatic updating.", Action = new Action( () => { sched.Stop(); Console.WriteLine("Parser stopped"); }) }); inputDictionary.Add("simulate", new CommandOption { Description = "Simulate clicks", Action = new Action(() => FillDatabaseCmd.SimulateClicks()) }); inputDictionary.Add("check", new CommandOption { Description = "Check for duplicates", Action = new Action(() => { var dateToCheck = DateTime.Now.AddDays(-1); var repo = new MobilniPortalNovicContext12(); var i = repo.NewsFiles.Where(x => x.PubDate > dateToCheck).GroupBy(x => x.Title).Where(x => x.Count() > 1).ToList(); var count = 0; i.ForEach(x => { count += x.Count() - 1; x.Skip(1).ToList().ForEach(y=>repo.NewsFiles.Remove(y)); }); repo.SaveChanges(); Console.WriteLine("Removed {0} duplicates", count); }) }); inputDictionary.Add("run", new CommandOption { Description = "Run single update.", Action = new Action(() => { service.startParse(); }) }); inputDictionary.Add("exit", new CommandOption { Description = "Exit the program", Action = new Action(() => { sched.Stop(); Console.WriteLine("Stoping..."); while (ParsingService.getParsingService().State != State.WaitingToNextInterval) { Thread.Sleep(1000); } Environment.Exit(0); }) }); inputDictionary.Add("stats", new CommandOption { Description = "Show parsers statistics", Action = new Action(() => { Console.WriteLine("Scheduler is {0}, with interval {1}.", sched.State, sched.RepeatInterval); Console.WriteLine("Parsing service in currently: {0}", service.State); Console.WriteLine("Total updated {0}.", service.TotalCount); Console.WriteLine("Last run {0}.", service.LastRun); }) }); inputDictionary.Add("AddNews", new CommandOption { Description="Manuly add news file", Action = new Action(() => { try { Console.WriteLine("CategoryName"); var i = Console.ReadLine(); var context = new MobilniPortalNovicContext12(); var cat = context.Categories.Where(x => x.Name == i).FirstOrDefault(); var catId = cat.CategoryId; NewsFile f = new NewsFile { CategoryId = catId, FeedId = context.Feeds.First().FeedId, Content = "Poljubna vsebina.", Title = "Filler news", ShortContent = "Ta novica je samo za testiranje", PubDate=DateTime.Now, Link="http//www.fake.si" }; context.NewsFiles.Add(f); context.SaveChanges(); Console.WriteLine("Added new news file"); } catch (Exception e) { Console.WriteLine("Bad info"); } }) }); inputDictionary.Add("clear", new CommandOption { Description = "Clear filler news", Action = new Action(() => { var c = new MobilniPortalNovicContext12(); c.NewsFiles.Where(x => x.Title == "Filler news").ToList().ForEach(x => c.NewsFiles.Remove(x)); c.SaveChanges(); }) }); while (true) { var input = Console.ReadLine(); if (inputDictionary.ContainsKey(input)) { inputDictionary[input].Action(); } else { DisplayChoices(); } Console.WriteLine(); } }