public Form1()
{
InitializeComponent();
MobilniPortalNovicLib.Models.MobilniPortalNovicContext12 context = new MobilniPortalNovicLib.Models.MobilniPortalNovicContext12();
var counts1 = context.NewsFiles.GroupBy(x => x.Category.CategoryId).Select(x=> new { ID = x.Key, Count= x.Sum(p=>1)}).ToList();
var parents = context.Categories.Where(x => x.ParentCategory == null).Select(x => new {ID = x.CategoryId, Name = x.Name}).ToList();
var dict = new Dictionary<String, int>();
foreach (var i in parents)
{
dict[i.Name] = context.NewsFiles.Where(x => x.CategoryId == i.ID || x.Category.ParentCategoryId == i.ID).Sum(x => 1);
}
var s = new Dictionary<String,int>();
chart1.Series.Clear();
foreach (var i in counts1.Where(x=>x.Count>50).OrderBy(x=>x.Count))
{
var cat = context.Categories.Find(i.ID);
var name = cat.Name;
if (cat.ParentCategory != null)
{
name = cat.ParentCategory.Name + " -> " + name;
}
s[name] = i.Count;
chart1.Series.Add(name);
chart1.Series[name].Points.AddY(i.Count);
}
File.WriteAllLines(@"C:/stats.csv", s.Select(x => x.Key + ";" + x.Value + ";"));
chart2.Series.Clear();
foreach (var i in dict.OrderBy(x=>x.Value))
{
chart2.Series.Add(i.Key);
chart2.Series[i.Key].Points.AddY(i.Value);
}
File.WriteAllLines(@"C:/rootCategorije.csv", dict.Select(x => x.Key + ";" + x.Value + ";"));
var buckets = new List<int>();
var k = 20;
var size = counts1.Max(x=>x.Count);
size = (size + (k - size % k))/k;
for (int i = 0; i < k; i++)
{
buckets.Add(counts1.Where(x => x.Count < (i + 1) * size && x.Count > i * size).Sum(x => 1));
}
chart3.Series.Clear();
int z = 0;
foreach (var i in buckets)
{
var name = (z*size).ToString()+" - "+((z+1)*(size)).ToString();
chart3.Series.Add(name);
chart3.Series[name].Points.AddY(i);
z++;
}
}
public CategoryPersonalizer(MobilniPortalNovicContext12 context)
{
this.Context = context;
CategoryTreshold = 70;
Messages = new List<String>();
GoodCategories = new List<Category>();
MinimalClicks = 10;
Filters = new List<Filter>();
}
//
// GET: /Public/
public ActionResult Login(User u)
{
ModelState.Clear();
if (u != null)
{
var context = new MobilniPortalNovicContext12();
if (context.Users.Where(x => x.Password == u.Password && x.Username == u.Username).FirstOrDefault() != null)
{
Session["username"] = u.Username;
return RedirectToAction("Index");
}
ModelState.AddModelError("Error", "Username of parssword is invalid");
return View(u);
}
return View();
}
public void SimulateClicksTest()
{
var context = new MobilniPortalNovicContext12();
FillDatabase f = new FillDatabase(context);
var d = DateTime.Now;
var categoryId = context.Categories.Where(x => x.Name == "Sportal").Select(x => x.CategoryId).First();
var userId = 1;
var count = 10;
var query = f.SimulateClicks(userId, categoryId, count, () => d, ()=>null);
var dict = CategoryHelpers.CategoryGetChildrensFromParent(context.Categories.ToList());
foreach (var r in query)
{
context.Clicks.Remove(r);
}
context.SaveChanges();
foreach (var r in query)
{
Assert.IsTrue(r.UserId == 1);
Assert.IsTrue(dict[categoryId].Select(x => x.CategoryId).Contains(r.CategoryId));
Assert.AreEqual(d, r.ClickDate);
}
}
public FillDatabase(MobilniPortalNovicContext12 context)
{
this.context = context;
}
public static void SimulateClicks()
{
try
{
using (var context = new MobilniPortalNovicContext12())
{
//UserId
Console.WriteLine("UserId:");
int userId;
var i1 = Console.ReadLine();
if (Int32.TryParse(i1, out userId) == false)
{
userId = context.Users.Where(x => x.Username == i1).First().UserId;
}
//Number clicks
Console.WriteLine("Number of clicks");
var count = Int32.Parse(Console.ReadLine());
//CategoryId
Console.WriteLine("Category number");
int category;
var i2 = Console.ReadLine();
if (Int32.TryParse(i2, out category) == false)
{
category = context.Categories.Where(x => x.Name == i2).First().CategoryId;
}
//Time function
Console.WriteLine("Hour offset from now: (+-random minutes) dayOffset (+-randomDays)");
List<int> offset = Console.ReadLine().Split(' ').ToList().Select(x => Int32.Parse(x)).ToList();
List<int> defaults = new List<int> { 0, 60, 0, 0 };
//Adds default values to not entered values
offset.AddRange(defaults.Skip(offset.Count));
Func<Coordinates> coordinatesRandom;
Console.WriteLine("Location query String");
var city = Console.ReadLine();
if (city.Length != 0)
{
Console.WriteLine("Location distance offset:");
var distanceOffset = Console.ReadLine();
Random rnd = new Random();
coordinatesRandom = CreateCoordinatesRandomFunc(city, Int32.Parse(distanceOffset));
}
else
{
coordinatesRandom = () => null;
}
Func<DateTime> dateTimeRandom = CreateDateTimeRandomFunc(offset[0], offset[1], offset[2], offset[3]);
var clicks = new FillDatabase(new MobilniPortalNovicContext12()).SimulateClicks(userId,
category,
count,
dateTimeRandom,
coordinatesRandom
);
Console.WriteLine("{0} clicks added.", clicks.Count());
}
}
catch (Exception e)
{
Console.WriteLine("Bad input");
}
}
public int UpdateFeedsForSites()
{
var count = 0;
using (var repo = new MobilniPortalNovicContext12())
{
#region InitializeRequiredFields
if (Titles == null)
{
///Select only last 30 from each list
var list = repo.NewsFiles.Select(x => x.Title);
/// Select all
/// var list = context.NewsFiles.Select(y => y.Title));
Titles = new HashSet<String>(list);
}
if (Categories == null)
{
var cat = repo.Categories.ToList();
cat.ForEach(x =>
{
if (x.ParentCategoryId.HasValue == false)
{
x.ParentCategoryId = null;
}
});
Categories = new HashSet<Category>(cat);
}
#endregion InitializeRequiredFields
#region ParseFeed
List<NewsFileExt> newsList = new List<NewsFileExt>();
var time = DateTime.Now;
//Get items from feed
var feeds = repo.Feeds.ToList();
feeds.ForEach(x => x.LastUpdated = time);
var newsFiles = feeds.AsParallel().Select(x => FeedParser.parseFeed(x)).SelectMany(x => x).ToList();
//remove duplicates
newsFiles = newsFiles.GroupBy(x => x.Title).Select(x => x.First()).ToList();
#endregion ParseFeed
#region ParseItems
//Remove already parsed ones
var newsFilesList = newsFiles.Where(x => FailedTitles.Contains(x.Title) == false && Titles.Contains(x.Title) == false).ToList();
//Process items
var items = NewsParser.parseItem(newsFilesList);
items.Where(x => !IsElementok(x)).ToList().ForEach(x =>
{
LogWriter.Instance.Log("Error parsing: " + x.Link);
FailedTitles.Add(x.Title);
});
#endregion ParseItems
#region SaveToDatabase
foreach (var item in items.Where(x => IsElementok(x)).ToList())
{
#region GetCategoryIdOrCreateNew
//Save categories into database
int? parentId = null;
foreach (var c in item.Categories.Take(2))
{
//Skip if already exists or parsed
if (Titles.Contains(item.Title) || FailedTitles.Contains(item.Title))
{
continue;
}
var s = Categories.Where(x => x.Name.Equals(c) && x.ParentCategoryId == parentId).FirstOrDefault();
if (s == null)
{
var category = repo.Categories.Add(new Category { Name = c, ParentCategoryId = parentId });
Console.WriteLine("Adding category {0}", c);
LogWriter.Instance.WriteToLog("Adding category " + c);
repo.SaveChanges();
parentId = category.CategoryId;
Categories.Add(category);
}
else
{
parentId = s.CategoryId;
}
}
item.CategoryId = parentId.Value;
#endregion GetCategoryIdOrCreateNew
Titles.Add(item.Title);
var i = AutoMapper.Mapper.Map<NewsFileExt, NewsFile>(item);
repo.NewsFiles.Add(i);
count += 1;
}
repo.SaveChanges();
Console.WriteLine("{0} new sites added.", count);
#endregion SaveToDatabase
}
return count;
}
private static void Main(string[] args)
{
ParsingService service = ParsingService.getParsingService();
Scheduler sched = new Scheduler(60 * 10, service);
inputDictionary = new Dictionary<String, CommandOption>();
inputDictionary.Add("start", new CommandOption { Description = "Start updating", Action = new Action(() => sched.StartUpdating()) });
inputDictionary.Add("stop", new CommandOption
{
Description = "Stop automatic updating.",
Action = new Action(
() =>
{
sched.Stop();
Console.WriteLine("Parser stopped");
})
});
inputDictionary.Add("simulate", new CommandOption { Description = "Simulate clicks", Action = new Action(() => FillDatabaseCmd.SimulateClicks()) });
inputDictionary.Add("check", new CommandOption
{
Description = "Check for duplicates",
Action = new Action(() =>
{
var dateToCheck = DateTime.Now.AddDays(-1);
var repo = new MobilniPortalNovicContext12();
var i = repo.NewsFiles.Where(x => x.PubDate > dateToCheck).GroupBy(x => x.Title).Where(x => x.Count() > 1).ToList();
var count = 0;
i.ForEach(x =>
{
count += x.Count() - 1;
x.Skip(1).ToList().ForEach(y=>repo.NewsFiles.Remove(y));
});
repo.SaveChanges();
Console.WriteLine("Removed {0} duplicates", count);
})
});
inputDictionary.Add("run",
new CommandOption
{
Description = "Run single update.",
Action = new Action(() =>
{
service.startParse();
})
});
inputDictionary.Add("exit", new CommandOption
{
Description = "Exit the program",
Action = new Action(() =>
{
sched.Stop();
Console.WriteLine("Stoping...");
while (ParsingService.getParsingService().State != State.WaitingToNextInterval)
{
Thread.Sleep(1000);
}
Environment.Exit(0);
})
});
inputDictionary.Add("stats", new CommandOption
{
Description = "Show parsers statistics",
Action = new Action(() =>
{
Console.WriteLine("Scheduler is {0}, with interval {1}.", sched.State, sched.RepeatInterval);
Console.WriteLine("Parsing service in currently: {0}", service.State);
Console.WriteLine("Total updated {0}.", service.TotalCount);
Console.WriteLine("Last run {0}.", service.LastRun);
})
});
inputDictionary.Add("AddNews", new CommandOption
{
Description="Manuly add news file",
Action = new Action(() =>
{
try
{
Console.WriteLine("CategoryName");
var i = Console.ReadLine();
var context = new MobilniPortalNovicContext12();
var cat = context.Categories.Where(x => x.Name == i).FirstOrDefault();
var catId = cat.CategoryId;
NewsFile f = new NewsFile {
CategoryId = catId,
FeedId = context.Feeds.First().FeedId,
Content = "Poljubna vsebina.",
Title = "Filler news",
ShortContent = "Ta novica je samo za testiranje",
PubDate=DateTime.Now,
Link="http//www.fake.si"
};
context.NewsFiles.Add(f);
context.SaveChanges();
Console.WriteLine("Added new news file");
}
catch (Exception e)
{
Console.WriteLine("Bad info");
}
})
});
inputDictionary.Add("clear", new CommandOption
{
Description = "Clear filler news",
Action = new Action(() =>
{
var c = new MobilniPortalNovicContext12();
c.NewsFiles.Where(x => x.Title == "Filler news").ToList().ForEach(x => c.NewsFiles.Remove(x));
c.SaveChanges();
})
});
while (true)
{
var input = Console.ReadLine();
if (inputDictionary.ContainsKey(input))
{
inputDictionary[input].Action();
}
else
{
DisplayChoices();
}
Console.WriteLine();
}
}
public static NewsRequest Construct(String username, MobilniPortalNovicContext12 context, Coordinates l = null)
{
var user = context.Users.Where(x => x.Username == username).First();
return new NewsRequest(user, l);
}
public static NewsRequest Construct(Guid token, MobilniPortalNovicContext12 context, Coordinates l = null)
{
var user = context.Users.Where(x => x.AccessToken == token).First();
return new NewsRequest(user, l);
}
public static NewsRequest Construct(int id, MobilniPortalNovicContext12 context, Coordinates l = null)
{
var user = context.Users.Find(id);
return new NewsRequest(user, l);
}