Exemple #1
0
        public Form1()
        {
            InitializeComponent();
            MobilniPortalNovicLib.Models.MobilniPortalNovicContext12 context = new MobilniPortalNovicLib.Models.MobilniPortalNovicContext12();
            var counts1 = context.NewsFiles.GroupBy(x => x.Category.CategoryId).Select(x=> new { ID = x.Key, Count= x.Sum(p=>1)}).ToList();
            var parents = context.Categories.Where(x => x.ParentCategory == null).Select(x => new {ID = x.CategoryId, Name = x.Name}).ToList();
            var dict = new Dictionary<String, int>();
            foreach (var i in parents)
            {
                dict[i.Name] = context.NewsFiles.Where(x => x.CategoryId == i.ID || x.Category.ParentCategoryId == i.ID).Sum(x => 1);
            }

            var s = new Dictionary<String,int>();

            chart1.Series.Clear();
            foreach (var i in counts1.Where(x=>x.Count>50).OrderBy(x=>x.Count))
            {
                var cat = context.Categories.Find(i.ID);
                var name = cat.Name;
                if (cat.ParentCategory != null)
                {
                    name = cat.ParentCategory.Name + " -> " + name;
                }
                s[name] = i.Count;
                chart1.Series.Add(name);
                chart1.Series[name].Points.AddY(i.Count);
            }

             File.WriteAllLines(@"C:/stats.csv", s.Select(x => x.Key + ";" + x.Value + ";"));

            chart2.Series.Clear();
            foreach (var i in dict.OrderBy(x=>x.Value))
            {
                chart2.Series.Add(i.Key);
                chart2.Series[i.Key].Points.AddY(i.Value);
            }

            File.WriteAllLines(@"C:/rootCategorije.csv", dict.Select(x => x.Key + ";" + x.Value + ";"));

            var buckets = new List<int>();
            var k = 20;
            var size = counts1.Max(x=>x.Count);
            size = (size + (k - size % k))/k;

            for (int i = 0; i < k; i++)
            {
                buckets.Add(counts1.Where(x => x.Count < (i + 1) * size && x.Count > i * size).Sum(x => 1));
            }
            chart3.Series.Clear();
            int z = 0;
            foreach (var i in buckets)
            {
                var name = (z*size).ToString()+" - "+((z+1)*(size)).ToString();
                chart3.Series.Add(name);
                chart3.Series[name].Points.AddY(i);
                z++;
            }
        }
 public CategoryPersonalizer(MobilniPortalNovicContext12 context)
 {
     this.Context = context;
     CategoryTreshold = 70;
     Messages = new List<String>();
     GoodCategories = new List<Category>();
     MinimalClicks = 10;
     Filters = new List<Filter>();
 }
 //
 // GET: /Public/
 public ActionResult Login(User u)
 {
     ModelState.Clear();
     if (u != null)
     {
         var context = new MobilniPortalNovicContext12();
         if (context.Users.Where(x => x.Password == u.Password && x.Username == u.Username).FirstOrDefault() != null)
         {
             Session["username"] = u.Username;
             return RedirectToAction("Index");
         }
         ModelState.AddModelError("Error", "Username of parssword is invalid");
         return View(u);
     }
     return View();
 }
        public void SimulateClicksTest()
        {
            var context = new MobilniPortalNovicContext12();
            FillDatabase f = new FillDatabase(context);
            var d = DateTime.Now;
            var categoryId = context.Categories.Where(x => x.Name == "Sportal").Select(x => x.CategoryId).First();
            var userId = 1;
            var count = 10;
            var query = f.SimulateClicks(userId, categoryId, count, () => d, ()=>null);
            var dict = CategoryHelpers.CategoryGetChildrensFromParent(context.Categories.ToList());

            foreach (var r in query)
            {
                context.Clicks.Remove(r);
            }
            context.SaveChanges();

            foreach (var r in query)
            {
                Assert.IsTrue(r.UserId == 1);
                Assert.IsTrue(dict[categoryId].Select(x => x.CategoryId).Contains(r.CategoryId));
                Assert.AreEqual(d, r.ClickDate);
            }
        }
 public FillDatabase(MobilniPortalNovicContext12 context)
 {
     this.context = context;
 }
        public static void SimulateClicks()
        {
            try
            {
                using (var context = new MobilniPortalNovicContext12())
                {
                    //UserId
                    Console.WriteLine("UserId:");
                    int userId;
                    var i1 = Console.ReadLine();
                    if (Int32.TryParse(i1, out userId) == false)
                    {
                        userId = context.Users.Where(x => x.Username == i1).First().UserId;
                    }

                    //Number clicks
                    Console.WriteLine("Number of clicks");
                    var count = Int32.Parse(Console.ReadLine());

                    //CategoryId
                    Console.WriteLine("Category number");
                    int category;
                    var i2 = Console.ReadLine();
                    if (Int32.TryParse(i2, out category) == false)
                    {
                        category = context.Categories.Where(x => x.Name == i2).First().CategoryId;
                    }

                    //Time function
                    Console.WriteLine("Hour offset from now: (+-random minutes) dayOffset (+-randomDays)");
                    List<int> offset = Console.ReadLine().Split(' ').ToList().Select(x => Int32.Parse(x)).ToList();
                    List<int> defaults = new List<int> { 0, 60, 0, 0 };
                    //Adds default values to not entered values
                    offset.AddRange(defaults.Skip(offset.Count));

                    Func<Coordinates> coordinatesRandom;
                    Console.WriteLine("Location query String");
                    var city = Console.ReadLine();
                    if (city.Length != 0)
                    {
                        Console.WriteLine("Location distance offset:");
                        var distanceOffset = Console.ReadLine();

                        Random rnd = new Random();
                        coordinatesRandom = CreateCoordinatesRandomFunc(city, Int32.Parse(distanceOffset));
                    }
                    else
                    {

                        coordinatesRandom = () => null;
                    }

                    Func<DateTime> dateTimeRandom = CreateDateTimeRandomFunc(offset[0], offset[1], offset[2], offset[3]);

                    var clicks = new FillDatabase(new MobilniPortalNovicContext12()).SimulateClicks(userId,
                        category,
                        count,
                        dateTimeRandom,
                        coordinatesRandom
                    );

                    Console.WriteLine("{0} clicks added.", clicks.Count());
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("Bad input");
            }
        }
        public int UpdateFeedsForSites()
        {
            var count = 0;
            using (var repo = new MobilniPortalNovicContext12())
            {
                #region InitializeRequiredFields

                if (Titles == null)
                {
                    ///Select only last 30 from each list
                    var list = repo.NewsFiles.Select(x => x.Title);
                    /// Select all
                    /// var list = context.NewsFiles.Select(y => y.Title));
                    Titles = new HashSet<String>(list);
                }
                if (Categories == null)
                {
                    var cat = repo.Categories.ToList();
                    cat.ForEach(x =>
                    {
                        if (x.ParentCategoryId.HasValue == false)
                        {
                            x.ParentCategoryId = null;
                        }
                    });
                    Categories = new HashSet<Category>(cat);
                }

                #endregion InitializeRequiredFields

                #region ParseFeed

                List<NewsFileExt> newsList = new List<NewsFileExt>();
                var time = DateTime.Now;

                //Get items from feed
                var feeds = repo.Feeds.ToList();
                feeds.ForEach(x => x.LastUpdated = time);
                var newsFiles = feeds.AsParallel().Select(x => FeedParser.parseFeed(x)).SelectMany(x => x).ToList();

                //remove duplicates
                newsFiles = newsFiles.GroupBy(x => x.Title).Select(x => x.First()).ToList();

                #endregion ParseFeed

                #region ParseItems

                //Remove already parsed ones
                var newsFilesList = newsFiles.Where(x => FailedTitles.Contains(x.Title) == false && Titles.Contains(x.Title) == false).ToList();
                //Process items
                var items = NewsParser.parseItem(newsFilesList);

                items.Where(x => !IsElementok(x)).ToList().ForEach(x =>
                {
                    LogWriter.Instance.Log("Error parsing: " + x.Link);
                    FailedTitles.Add(x.Title);
                });

                #endregion ParseItems

                #region SaveToDatabase

                foreach (var item in items.Where(x => IsElementok(x)).ToList())
                {
                    #region GetCategoryIdOrCreateNew

                    //Save categories into database
                    int? parentId = null;
                    foreach (var c in item.Categories.Take(2))
                    {
                        //Skip if already exists or parsed
                        if (Titles.Contains(item.Title) || FailedTitles.Contains(item.Title))
                        {
                            continue;
                        }

                        var s = Categories.Where(x => x.Name.Equals(c) && x.ParentCategoryId == parentId).FirstOrDefault();
                        if (s == null)
                        {
                            var category = repo.Categories.Add(new Category { Name = c, ParentCategoryId = parentId });
                            Console.WriteLine("Adding category {0}", c);
                            LogWriter.Instance.WriteToLog("Adding category " + c);
                            repo.SaveChanges();
                            parentId = category.CategoryId;
                            Categories.Add(category);
                        }
                        else
                        {
                            parentId = s.CategoryId;
                        }
                    }
                    item.CategoryId = parentId.Value;

                    #endregion GetCategoryIdOrCreateNew

                    Titles.Add(item.Title);
                    var i = AutoMapper.Mapper.Map<NewsFileExt, NewsFile>(item);
                    repo.NewsFiles.Add(i);
                    count += 1;

                }
                repo.SaveChanges();
                Console.WriteLine("{0} new sites added.", count);

                #endregion SaveToDatabase
            }
            return count;
        }
        private static void Main(string[] args)
        {
            ParsingService service = ParsingService.getParsingService();
            Scheduler sched = new Scheduler(60 * 10, service);
            inputDictionary = new Dictionary<String, CommandOption>();
            inputDictionary.Add("start", new CommandOption { Description = "Start updating", Action = new Action(() => sched.StartUpdating()) });
            inputDictionary.Add("stop", new CommandOption
            {
                Description = "Stop automatic updating.",
                Action = new Action(
                    () =>
                    {
                        sched.Stop();
                        Console.WriteLine("Parser stopped");
                    })
            });
            inputDictionary.Add("simulate", new CommandOption { Description = "Simulate clicks", Action = new Action(() => FillDatabaseCmd.SimulateClicks()) });
            inputDictionary.Add("check", new CommandOption
            {
                Description = "Check for duplicates",
                Action = new Action(() =>
                {
                    var dateToCheck = DateTime.Now.AddDays(-1);
                    var repo = new MobilniPortalNovicContext12();
                    var i = repo.NewsFiles.Where(x => x.PubDate > dateToCheck).GroupBy(x => x.Title).Where(x => x.Count() > 1).ToList();
                    var count = 0;
                    i.ForEach(x =>
                        {
                            count += x.Count() - 1;
                            x.Skip(1).ToList().ForEach(y=>repo.NewsFiles.Remove(y));
                        });
                    repo.SaveChanges();
                    Console.WriteLine("Removed {0} duplicates", count);
                })
            });
            inputDictionary.Add("run",
                new CommandOption
                {
                    Description = "Run single update.",
                    Action = new Action(() =>
                {
                    service.startParse();
                })
                });
            inputDictionary.Add("exit", new CommandOption
            {
                Description = "Exit the program",
                Action = new Action(() =>
                {
                    sched.Stop();
                    Console.WriteLine("Stoping...");
                    while (ParsingService.getParsingService().State != State.WaitingToNextInterval)
                    {
                        Thread.Sleep(1000);
                    }
                    Environment.Exit(0);
                })
            });
            inputDictionary.Add("stats", new CommandOption
            {
                Description = "Show parsers statistics",
                Action = new Action(() =>
                {
                    Console.WriteLine("Scheduler is {0}, with interval {1}.", sched.State, sched.RepeatInterval);
                    Console.WriteLine("Parsing service in currently: {0}", service.State);
                    Console.WriteLine("Total updated {0}.", service.TotalCount);
                    Console.WriteLine("Last run {0}.", service.LastRun);
                })
            });
            inputDictionary.Add("AddNews", new CommandOption
            {
                Description="Manuly add news file",
                Action = new Action(() =>
                    {
                        try
                        {
                            Console.WriteLine("CategoryName");
                            var i = Console.ReadLine();
                            var context = new MobilniPortalNovicContext12();
                            var cat = context.Categories.Where(x => x.Name == i).FirstOrDefault();
                            var catId = cat.CategoryId;
                            NewsFile f = new NewsFile {
                                CategoryId = catId,
                                FeedId = context.Feeds.First().FeedId,
                                Content = "Poljubna vsebina.",
                                Title = "Filler news",
                                ShortContent = "Ta novica je samo za testiranje",
                                PubDate=DateTime.Now,
                                Link="http//www.fake.si"
                            };

                            context.NewsFiles.Add(f);
                            context.SaveChanges();
                            Console.WriteLine("Added new news file");
                        }
                        catch (Exception e)
                        {

                            Console.WriteLine("Bad info");
                        }
                    })
            });
            inputDictionary.Add("clear", new CommandOption
            {
                Description = "Clear filler news",
                Action = new Action(() =>
                {
                    var c = new MobilniPortalNovicContext12();
                    c.NewsFiles.Where(x => x.Title == "Filler news").ToList().ForEach(x => c.NewsFiles.Remove(x));
                    c.SaveChanges();
                })
            });

            while (true)
            {
                var input = Console.ReadLine();
                if (inputDictionary.ContainsKey(input))
                {
                    inputDictionary[input].Action();
                }
                else
                {
                    DisplayChoices();
                }
                Console.WriteLine();
            }
        }
 public static NewsRequest Construct(String username, MobilniPortalNovicContext12 context, Coordinates l = null)
 {
     var user = context.Users.Where(x => x.Username == username).First();
     return new NewsRequest(user, l);
 }
 public static NewsRequest Construct(Guid token, MobilniPortalNovicContext12 context, Coordinates l = null)
 {
     var user = context.Users.Where(x => x.AccessToken == token).First();
     return new NewsRequest(user, l);
 }
 public static NewsRequest Construct(int id, MobilniPortalNovicContext12 context, Coordinates l = null)
 {
     var user = context.Users.Find(id);
     return new NewsRequest(user, l);
 }