public void CanCreateRelatedDataFromScrappedData()
{
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
repository.Add(new ScrappedData
{
Identifier = "Red Alert 2 ID",
String1 = "Red Alert 2",
RelatedData = new List <RelatedData>
{
new RelatedData
{
Description = "Image",
String1 = "image1"
},
new RelatedData
{
Description = "Image",
String1 = "image2"
}
}
});
}
}
public bool refreshData()
{
ScrapperContext _db = new ScrapperContext();
var _update = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "LAST_RUN");
return(DateTime.Now.Subtract(Convert.ToDateTime(_update.ParamValue)).Days > 0);
}
public void getCasts()
{
try
{
ScrapperContext _db = new ScrapperContext();
var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "CAST_ENDPOINT");
//var casts = new List<Cast>();
//var showcasts = new List<ShowCast>();
var shows = _db.Shows.ToList();
foreach (var show in shows)
{
var url = String.Format(endpoint.ParamValue, show.id);
var people = CastAsync(url);
foreach (var p in people.Result)
{
if (!_db.Casts.Any(x => x.id == p.person.id))
{
_db.Casts.Add(new Cast {
id = p.person.id, name = p.person.name, birthday = p.person.birthday
});
}
_db.ShowCasts.Add(new ShowCast {
CastId = p.person.id, ShowId = show.id
});
_db.SaveChanges();
}
}
}
catch (Exception ex)
{
//Handle exceptions
}
}
public void getShows()
{
try
{
ScrapperContext _db = new ScrapperContext();
var endpoint = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "SHOWS_ENDPOINT");
var shows = new List <Show>();
var url = endpoint.ParamValue;
var episodes = ShowsAsync(url);
foreach (var episode in episodes.Result)
{
if (!shows.Any(p => p.id == episode._embedded.show.id))
{
shows.Add(new Show {
id = episode._embedded.show.id, name = episode._embedded.show.name
});
}
}
_db.Shows.AddRange(shows);
_db.SaveChanges();
}
catch (Exception ex)
{
//Handle exceptions
}
}
public List <TvShow> showCast(int page, int pagesize = 25)
{
try
{
ScrapperContext _db = new ScrapperContext();
IQueryable <Show> shows = _db.Shows.Skip((page - 1) * pagesize).Take(pagesize);
List <TvShow> tvShows = (from show in shows
select new TvShow()
{
id = show.id,
name = show.name,
cast = (from cast in show.ShowCasts
select new TvCast
{
birthday = cast.Cast.birthday,
id = cast.Cast.id,
name = cast.Cast.name
}).ToList()
}).ToList();
return(tvShows);
}
catch (Exception ex)
{
//Handle exceptions
return(new List <TvShow>());
}
}
private static async Task HandleMenuAsync()
{
var configuration = GetConfiguration(settingsFileName);
var options = GetDbContextOptions(configuration);
var dbContext = new ScrapperContext(options);
IDataHandler <Person> handler = new PersonDataHandler <Person>(dbContext);
InputDataProvider inputData = await InputDataProviderFactory.GetInputDataProviderForPerson();
switch (Console.ReadLine())
{
case "1":
break;
case "2":
string PhoneTemplate = configuration[nameof(PhoneTemplate)];
inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(PhoneTemplate);
break;
case "3":
string SiteTemplate = configuration[nameof(SiteTemplate)];
handler = new TelegramDataHandler <Person>(dbContext);
inputData = await InputDataProviderFactory.GetInputDataProviderForPersonWithSpecialDomen(SiteTemplate);
break;
case "0":
return;
}
Console.WriteLine();
Console.WriteLine("Start scrapping process...\n");
await RunScrappingAsync(handler, inputData);
}
public void updateLastRun(DateTime lastRun)
{
ScrapperContext _db = new ScrapperContext();
var _update = _db.Parameters.AsQueryable().FirstOrDefault(p => p.ParamName == "LAST_RUN");
_update.ParamValue = lastRun.ToShortDateString();
_db.SaveChanges();
}
public void CanConnectToAccess()
{
var count = 0;
using (var context = new ScrapperContext())
{
count = context.ScrappedData.Count();
}
}
public BaseScrapperClient(ScrapperContext context, BodyComparer
comparer, string url, int resultId, int interval, int configurationId)
{
_comparer = comparer;
_url = url;
_resultId = resultId;
_context = context;
_interval = interval;
ConfigurationId = configurationId;
Initialize();
}
public virtual ICollection <ScrapeSetting> GetAllSettings()
{
if (Settings != null)
{
return(Settings);
}
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrapeSetting>(context);
Settings = repository.GetAll().Include(x => x.ScrapeItemSettings).ToList();
}
return(Settings);
}
public virtual void SaveScrappedData(List <ScrappedData> data)
{
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
foreach (var scrappedData in data)
{
repository.Add(scrappedData);
}
repository.SaveChanges();
}
}
public void CanSearchScrapperData()
{
ScrappedData data;
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
data = repository.FirstOrDefault(x => x.Identifier == "12345");
}
Assert.That(data, Is.Not.Null);
Assert.That(data.Identifier, Is.Not.Empty);
}
public virtual ICollection <HeaderRequest> GetAll()
{
if (_headerRequests != null)
{
return(_headerRequests);
}
using (var context = new ScrapperContext())
{
var repository = new Repository <HeaderRequest>(context);
_headerRequests = repository.GetAll().ToList();
}
return(_headerRequests);
}
public void CanAddNewScrapperData()
{
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
repository.Add(new ScrappedData
{
Identifier = "12345",
String1 = "Witcher 3"
});
}
}
public void CanGetRelatedDataFromScrappedData()
{
//JetEntityFrameworkProvider.JetConnection.ShowSqlStatements = true;
ScrappedData data;
List <RelatedData> relatedData;
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
data = repository.FirstOrDefault(x => x.Identifier == "Red Alert 2 ID");
relatedData = data.RelatedData.ToList();
}
Assert.That(data, Is.Not.Null);
Assert.That(data.Identifier, Is.Not.Empty);
Assert.That(relatedData, Is.Not.Empty);
Assert.That(relatedData.Count, Is.EqualTo(2));
}
public void CanUpdateScrapperData()
{
ScrappedData data;
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
data = repository.FirstOrDefault(x => x.Identifier == "12345");
}
data.String1 = "Witcher 4";
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
repository.Update(data);
}
}
public void CanRemoveScrapperData()
{
ScrappedData data;
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
data = repository.FirstOrDefault(x => x.Identifier == "12345");
repository.Delete(data);
}
data = null;
using (var context = new ScrapperContext())
{
var repository = new Repository <ScrappedData>(context);
data = repository.FirstOrDefault(x => x.Identifier == "12345");
}
Assert.That(data, Is.Null);
}
/// <summary>
/// (tablename, columnname), definition
/// example. ("ScrappedData", "String1"), "Image"
/// </summary>
/// <returns></returns>
public virtual Dictionary <(string, string), string> GetAllRelatedDataColumnDefinitions()
{
var result = new Dictionary <(string, string), string>();
using (var context = new ScrapperContext())
{
var repository = new Repository <ColumnDefinition>(context);
var columns = repository.Find(x => x.TableName == nameof(RelatedData)).ToList();
if (!columns.Any())
{
return(result);
}
foreach (var column in columns)
{
result.Add((column.TableName, column.ColumnName), column.Definition);
}
}
return(result);
}
public ScrappingManagerService(ScrapperContext context, ScrapperFactory scrapperFactory)
{
_context = context;
_scrapperFactory = scrapperFactory;
}
public UrlLibraryService(ScrapperContext context, ScrapperFactory scrapperFactory)
{
_context = context;
_scrapperFactory = scrapperFactory;
}
public BodyComparer(ScrapperContext context)
{
Context = context;
// Configuration = configuration;
_messageSender = new MessageSender();
}
public HomeController(ILogger <HomeController> logger, ScrapperContext context)
{
_logger = logger;
scrapperContext = context;
}
public ScrapperFactory(ScrapperContext context, IServiceScopeFactory scopeFactory)
{
_scopeFactory = scopeFactory;
_context = context;
}
