public void CanCreateRelatedDataFromScrappedData() { using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); repository.Add(new ScrappedData { Identifier = "Red Alert 2 ID", String1 = "Red Alert 2", RelatedData = new List <RelatedData> { new RelatedData { Description = "Image", String1 = "image1" }, new RelatedData { Description = "Image", String1 = "image2" } } }); } }
public bool refreshData() { ScrapperContext _db = new ScrapperContext(); var _update = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "LAST_RUN"); return(DateTime.Now.Subtract(Convert.ToDateTime(_update.ParamValue)).Days > 0); }
public void getCasts() { try { ScrapperContext _db = new ScrapperContext(); var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "CAST_ENDPOINT"); //var casts = new List<Cast>(); //var showcasts = new List<ShowCast>(); var shows = _db.Shows.ToList(); foreach (var show in shows) { var url = String.Format(endpoint.ParamValue, show.id); var people = CastAsync(url); foreach (var p in people.Result) { if (!_db.Casts.Any(x => x.id == p.person.id)) { _db.Casts.Add(new Cast { id = p.person.id, name = p.person.name, birthday = p.person.birthday }); } _db.ShowCasts.Add(new ShowCast { CastId = p.person.id, ShowId = show.id }); _db.SaveChanges(); } } } catch (Exception ex) { //Handle exceptions } }
public void getShows() { try { ScrapperContext _db = new ScrapperContext(); var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "SHOWS_ENDPOINT"); var shows = new List <Show>(); var url = endpoint.ParamValue; var episodes = ShowsAsync(url); foreach (var episode in episodes.Result) { if (!shows.Any(p => p.id == episode._embedded.show.id)) { shows.Add(new Show { id = episode._embedded.show.id, name = episode._embedded.show.name }); } } _db.Shows.AddRange(shows); _db.SaveChanges(); } catch (Exception ex) { //Handle exceptions } }
public List <TvShow> showCast(int page, int pagesize = 25) { try { ScrapperContext _db = new ScrapperContext(); IQueryable <Show> shows = _db.Shows.Skip((page - 1) * pagesize).Take(pagesize); List <TvShow> tvShows = (from show in shows select new TvShow() { id = show.id, name = show.name, cast = (from cast in show.ShowCasts select new TvCast { birthday = cast.Cast.birthday, id = cast.Cast.id, name = cast.Cast.name }).ToList() }).ToList(); return(tvShows); } catch (Exception ex) { //Handle exceptions return(new List <TvShow>()); } }
private static async Task HandleMenuAsync() { var configuration = GetConfiguration(settingsFileName); var options = GetDbContextOptions(configuration); var dbContext = new ScrapperContext(options); IDataHandler <Person> handler = new PersonDataHandler <Person>(dbContext); InputDataProvider inputData = await InputDataProviderFactory.GetInputDataProviderForPerson(); switch (Console.ReadLine()) { case "1": break; case "2": string PhoneTemplate = configuration[nameof(PhoneTemplate)]; inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(PhoneTemplate); break; case "3": string SiteTemplate = configuration[nameof(SiteTemplate)]; handler = new TelegramDataHandler <Person>(dbContext); inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(SiteTemplate); break; case "0": return; } Console.WriteLine(); Console.WriteLine("Start scrapping process...\n"); await RunScrappingAsync(handler, inputData); }
public void updateLastRun(DateTime lastRun) { ScrapperContext _db = new ScrapperContext(); var _update = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "LAST_RUN"); _update.ParamValue = lastRun.ToShortDateString(); _db.SaveChanges(); }
public void CanConnectToAccess() { var count = 0; using (var context = new ScrapperContext()) { count = context.ScrappedData.Count(); } }
public BaseScrapperClient(ScrapperContext context, BodyComparer comparer, string url, int resultId, int interval, int configurationId) { _comparer = comparer; _url = url; _resultId = resultId; _context = context; _interval = interval; ConfigurationId = configurationId; Initialize(); }
public virtual ICollection <ScrapeSetting> GetAllSettings() { if (Settings != null) { return(Settings); } using (var context = new ScrapperContext()) { var repository = new Repository <ScrapeSetting>(context); Settings = repository.GetAll().Include(x => x.ScrapeItemSettings).ToList(); } return(Settings); }
public virtual void SaveScrappedData(List <ScrappedData> data) { using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); foreach (var scrappedData in data) { repository.Add(scrappedData); } repository.SaveChanges(); } }
public void CanSearchScrapperData() { ScrappedData data; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); data = repository.FirstOrDefault(x => x.Identifier == "12345"); } Assert.That(data, Is.Not.Null); Assert.That(data.Identifier, Is.Not.Empty); }
public virtual ICollection <HeaderRequest> GetAll() { if (_headerRequests != null) { return(_headerRequests); } using (var context = new ScrapperContext()) { var repository = new Repository <HeaderRequest>(context); _headerRequests = repository.GetAll().ToList(); } return(_headerRequests); }
public void CanAddNewScrapperData() { using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); repository.Add(new ScrappedData { Identifier = "12345", String1 = "Witcher 3" }); } }
public void CanGetRelatedDataFromScrappedData() { //JetEntityFrameworkProvider.JetConnection.ShowSqlStatements = true; ScrappedData data; List <RelatedData> relatedData; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); data = repository.FirstOrDefault(x => x.Identifier == "Red Alert 2 ID"); relatedData = data.RelatedData.ToList(); } Assert.That(data, Is.Not.Null); Assert.That(data.Identifier, Is.Not.Empty); Assert.That(relatedData, Is.Not.Empty); Assert.That(relatedData.Count, Is.EqualTo(2)); }
public void CanUpdateScrapperData() { ScrappedData data; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); data = repository.FirstOrDefault(x => x.Identifier == "12345"); } data.String1 = "Witcher 4"; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); repository.Update(data); } }
public void CanRemoveScrapperData() { ScrappedData data; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); data = repository.FirstOrDefault(x => x.Identifier == "12345"); repository.Delete(data); } data = null; using (var context = new ScrapperContext()) { var repository = new Repository <ScrappedData>(context); data = repository.FirstOrDefault(x => x.Identifier == "12345"); } Assert.That(data, Is.Null); }
/// <summary> /// (tablename, columnname), definition /// example. ("ScrappedData", "String1"), "Image" /// </summary> /// <returns></returns> public virtual Dictionary <(string, string), string> GetAllRelatedDataColumnDefinitions() { var result = new Dictionary <(string, string), string>(); using (var context = new ScrapperContext()) { var repository = new Repository <ColumnDefinition>(context); var columns = repository.Find(x => x.TableName == nameof(RelatedData)).ToList(); if (!columns.Any()) { return(result); } foreach (var column in columns) { result.Add((column.TableName, column.ColumnName), column.Definition); } } return(result); }
public ScrappingManagerService(ScrapperContext context, ScrapperFactory scrapperFactory) { _context = context; _scrapperFactory = scrapperFactory; }
public UrlLibraryService(ScrapperContext context, ScrapperFactory scrapperFactory) { _context = context; _scrapperFactory = scrapperFactory; }
public BodyComparer(ScrapperContext context) { Context = context; // Configuration = configuration; _messageSender = new MessageSender(); }
public HomeController(ILogger <HomeController> logger, ScrapperContext context) { _logger = logger; scrapperContext = context; }
public ScrapperFactory(ScrapperContext context, IServiceScopeFactory scopeFactory) { _scopeFactory = scopeFactory; _context = context; }