public async Task <IEnumerable <Event> > Scrap(EventCategories category)
{
var scrappedEvents = new List <Event>();
// https://concert.ua/ru/catalog/kharkiv/all-categories/date-from=2019-02-16&date-till=2019-02-17
var uri = $"{_koncertUaUri}/ru/catalog/kyiv/{category.ToString().ToLower()}/date-from={todayFilter}&date-till={todayFilter}";
var catalogPage = await _browser.NavigateToPageAsync(new Uri(uri));
var performances = catalogPage.Html.CssSelect(".event").ToList();
var multiplePerformances = catalogPage.Html.CssSelect(".selection");
performances.AddRange(multiplePerformances);
foreach (var performance in performances)
{
try
{
var newEvent = await scrapPerformance(performance, category);
scrappedEvents.Add(newEvent);
}
catch (Exception)
{
System.Console.WriteLine("Failed to scrap sessions, probably html markup has been changed");
}
}
return(scrappedEvents);
}
public async Task ScrapAsync_EventRepositoryAddRangeCalledOnce(EventCategories category)
{
var serviceMock = new ScrapperServiceMock(_scrapperMock.Object,
_eventRepositoryMock.Object,
_eventDateRepositoryMock.Object);
await serviceMock.ScrapAsync(category);
_eventRepositoryMock.Verify(sr => sr.AddRangeAsync(It.IsAny <IEnumerable <Event> >()), Times.Once);
}
public async Task ScrapAsync_KinoAfishaScrapperScrapCalledOnce(EventCategories category)
{
var serviceMock = new ScrapperServiceMock(_scrapperMock.Object,
_eventRepositoryMock.Object,
_eventDateRepositoryMock.Object);
await serviceMock.ScrapAsync(category);
_scrapperMock.Verify(sr => sr.Scrap(category), Times.Once);
}
public async Task <bool> ScrapAsync(EventCategories category)
{
var todaysDay = DateTime.UtcNow.Day;
var scrappedEvents = await _scrapper.Scrap(category);
foreach (var scrappedEvent in scrappedEvents)
{
var existingEvent = await checkEventExists(scrappedEvent);
await removeExistingEvent(existingEvent, todaysDay);
}
return(await this._eventRepository.AddRangeAsync(scrappedEvents));
}
public EventViewModel ToViewModel()
{
return(new EventViewModel
{
Id = Id,
Title = Title,
ShortTitle = ShortTitle,
Description = Description,
Dates = new List <EventDate>(Dates),
Images = new List <ImageModel>(Images),
Categories = EventCategories.Select(eventCat => eventCat.Category.Name).ToList(),
Tags = EventTags.Select(eventTag => eventTag.Tag.Name).ToList(),
Place = Place.ToViewModel()
});
}
private async Task <Event> scrapFilm(HtmlNode filmHtml, EventCategories category)
{
var filmUrl = scrapFilmUrl(filmHtml);
var detailsPage = await _browser.NavigateToPageAsync(new Uri(filmUrl));
var rating = scrapFilmRating(detailsPage);
var description = scrapFilmDescription(detailsPage);
var imageUrl = await scrapImageUrl(detailsPage, filmHtml.InnerText);
var sessions = scrapSessions(detailsPage);
var film = new Event(filmHtml.InnerText, description, filmUrl,
imageUrl, (int)category, rating, sessions);
return(film);
}
private async Task <Event> scrapPerformance(HtmlNode performanceCard, EventCategories category)
{
var performanceUrl = scrapUrl(performanceCard);
var detailsPage = await _browser.NavigateToPageAsync(new Uri(performanceUrl));
var performanceTitle = scrapTitle(performanceCard);
var description = scrapDescription(detailsPage);
var imageUrl = await scrapImageUrl(performanceCard, performanceTitle);
var performanceSessions = scrapSessions(performanceCard, detailsPage.Html).ToList();
var performance = new Event(performanceTitle, description, performanceUrl,
imageUrl, (int)category, 0.0m, performanceSessions);
return(performance);
}
public async Task <IEnumerable <Event> > Scrap(EventCategories category)
{
var scrappedEvents = new List <Event>();
var kinoafishaHomePage = await _browser.NavigateToPageAsync(new Uri($"{_kinoAfishaUri}/kinoafisha"));
var films = kinoafishaHomePage.Html.CssSelect(".list-films").First()?.CssSelect(".item > .text > h3");
foreach (var film in films)
{
var newEvent = await scrapFilm(film, category);
scrappedEvents.Add(newEvent);
}
return(scrappedEvents);
}
/// <summary>
/// Search Events - ALL options
/// </summary>
/// <param name="searchText">string text to search for e.g.: live music</param>
/// <param name="showMature">Include mature events in results</param>
/// <param name="eventDay">"u" for now and upcoming events, -or- number of days since/until event is scheduled
/// For example "0" = Today, "1" = tomorrow, "2" = following day, "-1" = yesterday, etc.</param>
/// <param name="queryStart">Page # to show, 0 for First Page</param>
/// <param name="category">EventCategory event is listed under.</param>
/// <param name="queryID">a LLUUID that can be used to track queries with results.</param>
/// <returns>UUID of query to correlate results in callback.</returns>
public LLUUID StartEventsSearch(string searchText, bool showMature, string eventDay, uint queryStart, EventCategories category, LLUUID queryID)
{
DirFindQueryPacket find = new DirFindQueryPacket();
find.AgentData.AgentID = Client.Self.AgentID;
find.AgentData.SessionID = Client.Self.SessionID;
find.QueryData.QueryID = queryID;
find.QueryData.QueryText = Helpers.StringToField(eventDay + "|" + (int)category + "|" + searchText);
find.QueryData.QueryFlags = showMature ? (uint)32 : (uint)8224;
find.QueryData.QueryStart = (int)queryStart;
Client.Network.SendPacket(find);
return(queryID);
}
/// <summary>
/// Search Events with Options to specify category and Mature events.
/// </summary>
/// <param name="searchText">Text to search for</param>
/// <param name="showMature">true to include Mature events</param>
/// <param name="category">category to search</param>
/// <returns>UUID of query to correlate results in callback.</returns>
public LLUUID StartEventsSearch(string searchText, bool showMature, EventCategories category)
{
return(StartEventsSearch(searchText, showMature, "u", 0, category, LLUUID.Random()));
}
public async Task ScrapAsync_EventsAlreadyExistInRepository_RemoveAllOldEventsFromRepositoryAddNewOnes(int eventsCount, EventCategories category)
{
var scrappedEvents = generateEvents(eventsCount);
_scrapperMock.Setup(scr => scr.Scrap(category)).ReturnsAsync(scrappedEvents);
var eventsMock = scrappedEvents.AsQueryable().BuildMock();
_eventRepositoryMock.Setup(rp => rp.Get()).Returns(eventsMock.Object);
var todaysEvents = new List <EventDate>();
var todaysEventsMock = todaysEvents.AsQueryable().BuildMock();
_eventDateRepositoryMock.Setup(rp => rp.Get()).Returns(todaysEventsMock.Object);
var serviceMock = new ScrapperServiceMock(_scrapperMock.Object,
_eventRepositoryMock.Object,
_eventDateRepositoryMock.Object);
await serviceMock.ScrapAsync(category);
foreach (var scrappedEvent in scrappedEvents)
{
_eventRepositoryMock.Verify(sr => sr.RemoveAsync(scrappedEvent.Id), Times.Once);
}
}
public async Task ScrapAsync_ScrapperReturnedEvents_AddedToRepository(int eventsCount, EventCategories category)
{
var scrappedEvents = generateEvents(eventsCount);
_scrapperMock.Setup(scr => scr.Scrap(category)).ReturnsAsync(scrappedEvents);
var eventsEmpty = new List <Event>();
var eventsMock = eventsEmpty.AsQueryable().BuildMock();
_eventRepositoryMock.Setup(rp => rp.Get()).Returns(eventsMock.Object);
var serviceMock = new ScrapperServiceMock(_scrapperMock.Object,
_eventRepositoryMock.Object,
_eventDateRepositoryMock.Object);
await serviceMock.ScrapAsync(category);
_eventRepositoryMock.Verify(sr => sr.AddRangeAsync(scrappedEvents), Times.Once);
}
/// <summary>
/// Search Events
/// </summary>
/// <param name="searchText">A string containing a list of keywords to search for separated by a space character</param>
/// <param name="queryFlags">One or more of the following flags: DateEvents, IncludePG, IncludeMature, IncludeAdult
/// from the <see cref="DirFindFlags"/> Enum
///
/// Multiple flags can be combined by separating the flags with the | (pipe) character</param>
/// <param name="eventDay">"u" for in-progress and upcoming events, -or- number of days since/until event is scheduled
/// For example "0" = Today, "1" = tomorrow, "2" = following day, "-1" = yesterday, etc.</param>
/// <param name="queryStart">Each request is limited to 100 entries
/// being returned. To get the first group of entries of a request use 0,
/// from 100-199 use 100, 200-299 use 200, etc.</param>
/// <param name="category">EventCategory event is listed under.</param>
/// <returns>UUID of query to correlate results in callback.</returns>
public UUID StartEventsSearch(string searchText, DirFindFlags queryFlags, string eventDay, uint queryStart, EventCategories category)
{
DirFindQueryPacket find = new DirFindQueryPacket();
find.AgentData.AgentID = Client.Self.AgentID;
find.AgentData.SessionID = Client.Self.SessionID;
UUID queryID = UUID.Random();
find.QueryData.QueryID = queryID;
find.QueryData.QueryText = Utils.StringToBytes(eventDay + "|" + (int)category + "|" + searchText);
find.QueryData.QueryFlags = (uint)queryFlags;
find.QueryData.QueryStart = (int)queryStart;
Client.Network.SendPacket(find);
return queryID;
}
/// <summary>
/// Search Events - ALL options
/// </summary>
/// <param name="searchText">string text to search for e.g.: live music</param>
/// <param name="showMature">Include mature events in results</param>
/// <param name="eventDay">"u" for now and upcoming events, -or- number of days since/until event is scheduled
/// For example "0" = Today, "1" = tomorrow, "2" = following day, "-1" = yesterday, etc.</param>
/// <param name="queryStart">Page # to show, 0 for First Page</param>
/// <param name="category">EventCategory event is listed under.</param>
/// <param name="queryID">a Guid that can be used to track queries with results.</param>
/// <returns>Guid of query to correlate results in callback.</returns>
public Guid StartEventsSearch(string searchText, bool showMature, string eventDay, uint queryStart, EventCategories category, Guid queryID)
{
DirFindQueryPacket find = new DirFindQueryPacket();
find.AgentData.AgentID = Client.Self.AgentID;
find.AgentData.SessionID = Client.Self.SessionID;
find.QueryData.QueryID = queryID;
find.QueryData.QueryText = Utils.StringToBytes(eventDay + "|" + (int)category + "|" + searchText);
find.QueryData.QueryFlags = showMature ? (uint)32 : (uint)8224;
find.QueryData.QueryStart = (int)queryStart;
Client.Network.SendPacket(find);
return queryID;
}
/// <summary>
/// Search Events with Options to specify category and Mature events.
/// </summary>
/// <param name="searchText">Text to search for</param>
/// <param name="showMature">true to include Mature events</param>
/// <param name="category">category to search</param>
/// <returns>Guid of query to correlate results in callback.</returns>
public Guid StartEventsSearch(string searchText, bool showMature, EventCategories category)
{
return StartEventsSearch(searchText, showMature, "u", 0, category, Guid.NewGuid());
}
/// <summary>
/// Search Events with Options to specify category and Mature events.
/// </summary>
/// <param name="searchText">Text to search for</param>
/// <param name="showMature">true to include Mature events</param>
/// <param name="category">category to search</param>
/// <returns>UUID of query to correlate results in callback.</returns>
public UUID StartEventsSearch(string searchText, bool showMature, EventCategories category)
{
return StartEventsSearch(searchText, showMature, "u", 0, category, UUID.Random());
}
