public void getCasts() { try { ScrapperContext _db = new ScrapperContext(); var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "CAST_ENDPOINT"); //var casts = new List<Cast>(); //var showcasts = new List<ShowCast>(); var shows = _db.Shows.ToList(); foreach (var show in shows) { var url = String.Format(endpoint.ParamValue, show.id); var people = CastAsync(url); foreach (var p in people.Result) { if (!_db.Casts.Any(x => x.id == p.person.id)) { _db.Casts.Add(new Cast { id = p.person.id, name = p.person.name, birthday = p.person.birthday }); } _db.ShowCasts.Add(new ShowCast { CastId = p.person.id, ShowId = show.id }); _db.SaveChanges(); } } } catch (Exception ex) { //Handle exceptions } }
public void getShows() { try { ScrapperContext _db = new ScrapperContext(); var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "SHOWS_ENDPOINT"); var shows = new List <Show>(); var url = endpoint.ParamValue; var episodes = ShowsAsync(url); foreach (var episode in episodes.Result) { if (!shows.Any(p => p.id == episode._embedded.show.id)) { shows.Add(new Show { id = episode._embedded.show.id, name = episode._embedded.show.name }); } } _db.Shows.AddRange(shows); _db.SaveChanges(); } catch (Exception ex) { //Handle exceptions } }
public void updateLastRun(DateTime lastRun) { ScrapperContext _db = new ScrapperContext(); var _update = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "LAST_RUN"); _update.ParamValue = lastRun.ToShortDateString(); _db.SaveChanges(); }
public void AddUrlToLibrary(UrlLibraryDTO newElement) { var dbElement = new UrlLibrary() { Url = newElement.Url }; var dbElementConfiguration = new ScrappingConfiguration() { Interval = newElement.ScrappingInterval, UrlLibrary = dbElement, ScrappingResult = new ScrappingResult() }; _context.Add(dbElementConfiguration); _context.SaveChanges(); //start the new url scrapping _scrapperFactory.AddAndStartOperation(dbElementConfiguration.ScrappingConfigurationId); }
public IActionResult Index() { ConsumerKey = Key.GetConsumerAPIKey(); ConsumerSecret = Key.GetConsumerSecretAPIKey(); AccessToken = Key.GetAccessTokenKey(); AccessTokenSecret = Key.GetAccessTokenSecretKey(); service = new TwitterService(ConsumerKey, ConsumerSecret); service.AuthenticateWith(AccessToken, AccessTokenSecret); Dictionary <string, List <string> > keyValuePairs = new Dictionary <string, List <string> >(); if (scrapperContext.backlogs.Count() == 0 && scrapperContext.users.Count() == 0) { keyValuePairs = new Utils().country_top_users; foreach (List <string> vs in keyValuePairs.Values) { vs.ForEach(x => scrapperContext.users.Add(ScrapData(x))); } scrapperContext.SaveChanges(); List <User> c = scrapperContext.users.ToList(); return(View(c)); } if (scrapperContext.backlogs.Count() != 0) { List <Backlog> values = scrapperContext.backlogs.ToList(); List <User> users = scrapperContext.users.ToList(); List <string> usernames = new List <string>(); foreach (Backlog backlog in values) { usernames.Add(backlog.ScreenName); scrapperContext.backlogs.Remove(backlog); scrapperContext.SaveChanges(); } keyValuePairs.Add("Random", usernames); List <string> usernames2 = new List <string>(); foreach (User u in users) { usernames2.Add(u.ScreenName); scrapperContext.users.Remove(u); scrapperContext.SaveChanges(); } keyValuePairs.Add("Old", usernames2); } if (scrapperContext.users.Count() == 0) { foreach (List <string> vs in keyValuePairs.Values) { vs.ForEach(x => scrapperContext.users.Add(ScrapData(x))); scrapperContext.SaveChanges(); } } scrapperContext.users.ToList().ForEach(x => { if (x.Name == null) { scrapperContext.users.Remove(x); scrapperContext.SaveChanges(); } }); List <User> collection = scrapperContext.users.ToList(); return(View(collection)); }