public override Context Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var uriConverter = options.GetConverter <Uri>();
Uri uri = null !;
Uri href = null !;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(uri, href, externalUrls));
}
public FeaturedPlaylistItem(
bool collaborative,
string href,
string name,
string snapshot_id,
string type,
string uri,
ExternalUrls external_urls,
List <Image> images,
Tracks tracks,
Owner owner)
{
Collaborative = collaborative;
ExternalUrls = external_urls;
Href = href;
Images = images;
Name = name;
Owner = owner;
SnapshotId = snapshot_id;
Tracks = tracks;
Type = type;
Uri = uri;
}
public override PrivateUser Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var followersConverter = options.GetConverter <Followers>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string? email = null;
string? displayName = null;
CountryCode? country = null;
ImageArray images = Array.Empty <Image>();
Product? product = null;
Followers followers = null !;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
case "email":
email = reader.GetString();
break;
case "display_name":
displayName = reader.GetString();
break;
case "country":
var countryCode = reader.GetString();
country = countryCode is not null
? EnumConverters.CountryCodeConverter.FromSpotifyString(countryCode)
: null;
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "product":
product = ProductConverter.FromSpotifyString(reader.GetString() !);
break;
case "followers":
followers = followersConverter.Read(ref reader, typeof(Followers), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(id, uri, href, email, displayName, country, images, product, followers, externalUrls));
}
protected bool Equals(ArgoApplicationStatusSummary other)
{
return(ExternalUrls.IsCollectionEqualTo(other.ExternalUrls));
}
public override int GetHashCode()
{
return(ExternalUrls != null ? ExternalUrls.GetCollectionHashCode() : 0);
}
public override Show Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var copyrightArrayConverter = options.GetConverter <CopyrightArray>();
var countryCodeArrayConverter = options.GetConverter <CountryCodeArray>();
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var simplifiedEpisodePagingConverter = options.GetConverter <Paging <SimplifiedEpisode> >();
var stringArrayConverter = options.GetConverter <StringArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
string description = string.Empty;
ImageArray images = Array.Empty <Image>();
Paging <SimplifiedEpisode> episodes = null !;
bool isExplicit = default;
StringArray languages = Array.Empty <string>();
CountryCodeArray availableMarkets = Array.Empty <CountryCode>();
string mediaType = string.Empty;
string publisher = string.Empty;
CopyrightArray copyrights = Array.Empty <Copyright>();
bool? isExternallyHosted = null;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "description":
description = reader.GetString() !;
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "episodes":
episodes = simplifiedEpisodePagingConverter.Read(ref reader, typeof(Paging <SimplifiedEpisode>), options) !;
break;
case "explicit":
isExplicit = reader.GetBoolean();
break;
case "is_externally_hosted":
isExternallyHosted = reader.TokenType switch
{
JsonTokenType.Null => null,
JsonTokenType.True => true,
JsonTokenType.False => false,
_ => throw new JsonException()
};
break;
case "languages":
languages = stringArrayConverter.Read(ref reader, typeof(StringArray), options) !;
break;
case "available_markets":
availableMarkets = countryCodeArrayConverter.Read(ref reader, typeof(CountryCodeArray), options) !;
break;
case "media_type":
mediaType = reader.GetString() !;
break;
case "publisher":
publisher = reader.GetString() !;
break;
case "copyrights":
copyrights = copyrightArrayConverter.Read(ref reader, typeof(CopyrightArray), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
description,
images,
episodes,
isExplicit,
isExternallyHosted,
languages,
availableMarkets,
mediaType,
publisher,
copyrights,
externalUrls));
}
public override Track Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var countryCodeArrayConverter = options.GetConverter <CountryCodeArray>();
var externalIdsConverter = options.GetConverter <ExternalIds>();
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var simplifiedAlbumConverter = options.GetConverter <SimplifiedAlbum>();
var simplifiedArtistArrayConverter = options.GetConverter <SimplifiedArtistArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
SimplifiedAlbum album = null !;
SimplifiedArtistArray artists = Array.Empty <SimplifiedArtist>();
int duration = default;
int trackNumber = default;
int discNumber = default;
bool isExplicit = default;
bool isLocal = default;
CountryCodeArray availableMarkets = Array.Empty <CountryCode>();
int popularity = default;
string previewUrl = string.Empty;
ExternalIds externalIds = null !;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "album":
album = simplifiedAlbumConverter.Read(ref reader, typeof(SimplifiedAlbum), options) !;
break;
case "artists":
artists = simplifiedArtistArrayConverter.Read(ref reader, typeof(SimplifiedArtistArray), options) !;
break;
case "duration_ms":
duration = reader.GetInt32();
break;
case "track_number":
trackNumber = reader.GetInt32();
break;
case "disc_number":
discNumber = reader.GetInt32();
break;
case "explicit":
isExplicit = reader.GetBoolean();
break;
case "is_local":
isLocal = reader.GetBoolean();
break;
case "available_markets":
availableMarkets = countryCodeArrayConverter.Read(ref reader, typeof(CountryCodeArray), options) !;
break;
case "popularity":
popularity = reader.GetInt32();
break;
case "preview_url":
previewUrl = reader.GetString() !;
break;
case "external_ids":
externalIds = externalIdsConverter.Read(ref reader, typeof(ExternalIds), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
album,
artists,
duration,
discNumber,
trackNumber,
isExplicit,
isLocal,
availableMarkets,
popularity,
previewUrl,
externalIds,
externalUrls));
}
public override SimplifiedEpisode Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var resumePointConverter = options.GetConverter <ResumePoint>();
var stringArrayConverter = options.GetConverter <StringArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
string description = string.Empty;
ImageArray images = Array.Empty <Image>();
int duration = default;
DateTime releaseDate = default;
ReleaseDatePrecision releaseDatePrecision = default;
bool isExplicit = default;
bool isPlayable = default;
bool isExternallyHosted = default;
StringArray languages = Array.Empty <string>();
Uri? audioPreviewUrl = null;
ExternalUrls externalUrls = null !;
ResumePoint? resumePoint = null;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "description":
description = reader.GetString() !;
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "duration_ms":
duration = reader.GetInt32();
break;
case "release_date":
releaseDate = reader.GetReleaseDate();
break;
case "release_date_precision":
releaseDatePrecision = ReleaseDatePrecisionConverter.FromSpotifyString(reader.GetString() !);
break;
case "explicit":
isExplicit = reader.GetBoolean();
break;
case "is_playable":
isPlayable = reader.GetBoolean();
break;
case "is_externally_hosted":
isExternallyHosted = reader.GetBoolean();
break;
case "languages":
languages = stringArrayConverter.Read(ref reader, typeof(StringArray), options) !;
break;
case "audio_preview_url":
audioPreviewUrl = (reader.TokenType is JsonTokenType.Null) ? null : uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
case "resume_point":
resumePoint = resumePointConverter.Read(ref reader, typeof(ResumePoint), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
description,
images,
duration,
releaseDate,
releaseDatePrecision,
isExplicit,
isPlayable,
isExternallyHosted,
languages,
audioPreviewUrl,
externalUrls,
resumePoint));
}
public override Album Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var copyrightArrayConverter = options.GetConverter <CopyrightArray>();
var countryCodeArrayConverter = options.GetConverter <CountryCodeArray>();
var externalIdsConverter = options.GetConverter <ExternalIds>();
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var simplifiedArtistArrayConverter = options.GetConverter <SimplifiedArtistArray>();
var simplifiedTrackPagingConverter = options.GetConverter <Paging <SimplifiedTrack> >();
var stringArrayConverter = options.GetConverter <StringArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
AlbumType type = default;
ImageArray images = Array.Empty <Image>();
SimplifiedArtistArray artists = Array.Empty <SimplifiedArtist>();
DateTime releaseDate = default;
ReleaseDatePrecision releaseDatePrecision = default;
CountryCodeArray availableMarkets = Array.Empty <CountryCode>();
CopyrightArray copyrights = Array.Empty <Copyright>();
ExternalIds externalIds = null !;
ExternalUrls externalUrls = null !;
StringArray genres = Array.Empty <string>();
string label = string.Empty;
int popularity = default;
Paging <SimplifiedTrack> tracks = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "album_type":
type = AlbumTypeConverter.FromSpotifyString(reader.GetString() !);
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "artists":
artists = simplifiedArtistArrayConverter.Read(ref reader, typeof(SimplifiedArtistArray), options) !;
break;
case "release_date":
releaseDate = reader.GetReleaseDate();
break;
case "release_date_precision":
releaseDatePrecision = ReleaseDatePrecisionConverter.FromSpotifyString(reader.GetString() !);
break;
case "tracks":
tracks = simplifiedTrackPagingConverter.Read(ref reader, typeof(Paging <SimplifiedTrack>), options) !;
break;
case "genres":
genres = stringArrayConverter.Read(ref reader, typeof(StringArray), options) !;
break;
case "popularity":
popularity = reader.GetInt32();
break;
case "available_markets":
availableMarkets = countryCodeArrayConverter.Read(ref reader, typeof(CountryCodeArray), options) !;
break;
case "label":
label = reader.GetString() !;
break;
case "copyrights":
copyrights = copyrightArrayConverter.Read(ref reader, typeof(CopyrightArray), options) !;
break;
case "external_ids":
externalIds = externalIdsConverter.Read(ref reader, typeof(ExternalIds), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
type,
images,
artists,
releaseDate,
releaseDatePrecision,
tracks,
genres,
popularity,
availableMarkets,
label,
copyrights,
externalIds,
externalUrls));
}
public override SimplifiedAlbum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var countryCodeArrayConverter = options.GetConverter <CountryCodeArray>();
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var simplifiedArtistArrayConverter = options.GetConverter <SimplifiedArtistArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
AlbumType type = default;
AlbumGroups? group = null;
ImageArray images = null !;
SimplifiedArtistArray artists = null !;
DateTime releaseDate = default;
ReleaseDatePrecision releaseDatePrecision = default;
CountryCodeArray availableMarkets = null !;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "album_type":
type = AlbumTypeConverter.FromSpotifyString(reader.GetString() !.ToLower());
break;
case "album_group":
group = AlbumGroupConverter.FromSpotifyString(reader.GetString() !.ToLower());
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "artists":
artists = simplifiedArtistArrayConverter.Read(ref reader, typeof(SimplifiedArtistArray), options) !;
break;
case "release_date":
var date = reader.GetString() !.Split('-');
releaseDate = new(
date.Length > 0 ? int.Parse(date[0]) : 1,
date.Length > 1 ? int.Parse(date[1]) : 1,
date.Length > 2 ? int.Parse(date[2]) : 1);
break;
case "release_date_precision":
releaseDatePrecision = ReleaseDatePrecisionConverter.FromSpotifyString(reader.GetString() !);
break;
case "available_markets":
availableMarkets = countryCodeArrayConverter.Read(ref reader, typeof(CountryCodeArray), options) !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
type,
group,
images,
artists,
releaseDate,
releaseDatePrecision,
availableMarkets,
externalUrls));
}
public override SimplifiedPlaylist Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var playlistTrackPagingConverter = options.GetConverter <Paging <PlaylistTrack> >();
var publicUserConverter = options.GetConverter <PublicUser>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
string? description = null;
ImageArray images = Array.Empty <Image>();
PublicUser owner = null !;
Paging <PlaylistTrack> tracks = null !;
bool? isPublic = null;
bool isCollaborative = default;
string snapshotId = string.Empty;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "description":
description = reader.GetString();
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "owner":
owner = publicUserConverter.Read(ref reader, typeof(PublicUser), options) !;
break;
case "tracks":
tracks = playlistTrackPagingConverter.Read(ref reader, typeof(Paging <PlaylistTrack>), options) !;
break;
case "public":
isPublic = reader.TokenType switch
{
JsonTokenType.Null => null,
JsonTokenType.True => true,
JsonTokenType.False => false,
_ => throw new JsonException()
};
break;
case "collaborative":
isCollaborative = reader.GetBoolean();
break;
case "snapshot_id":
snapshotId = reader.GetString() !;
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(
id,
uri,
href,
name,
description,
images,
owner,
tracks,
isPublic,
isCollaborative,
snapshotId,
externalUrls));
}
public override Artist Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType is not JsonTokenType.StartObject)
{
throw new JsonException();
}
var externalUrlsConverter = options.GetConverter <ExternalUrls>();
var followersConverter = options.GetConverter <Followers>();
var imageArrayConverter = options.GetConverter <ImageArray>();
var stringArrayConverter = options.GetConverter <StringArray>();
var uriConverter = options.GetConverter <Uri>();
string id = string.Empty;
Uri uri = null !;
Uri href = null !;
string name = string.Empty;
ImageArray images = Array.Empty <Image>();
Followers followers = null !;
StringArray genres = Array.Empty <string>();
int popularity = default;
ExternalUrls externalUrls = null !;
while (reader.Read())
{
if (reader.TokenType is JsonTokenType.EndObject)
{
break;
}
if (reader.TokenType is not JsonTokenType.PropertyName)
{
throw new JsonException();
}
var propertyName = reader.GetString();
reader.Read(); // Read to next token.
switch (propertyName)
{
case "id":
id = reader.GetString() !;
break;
case "uri":
uri = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "href":
href = uriConverter.Read(ref reader, typeof(Uri), options) !;
break;
case "name":
name = reader.GetString() !;
break;
case "images":
images = imageArrayConverter.Read(ref reader, typeof(ImageArray), options) !;
break;
case "followers":
followers = followersConverter.Read(ref reader, typeof(Followers), options) !;
break;
case "genres":
genres = stringArrayConverter.Read(ref reader, typeof(StringArray), options) !;
break;
case "popularity":
popularity = reader.GetInt32();
break;
case "external_urls":
externalUrls = externalUrlsConverter.Read(ref reader, typeof(ExternalUrls), options) !;
break;
default:
reader.Skip();
break;
}
}
return(new(id, uri, href, name, images, followers, genres, popularity, externalUrls));
}
public void crawlUrl(string urlToCrawl, List <string> parentWillCrawl = null)
{
Url aUrl = new Url()
{
Uri = urlToCrawl
};
CrawledDomainUrlsAsUrlObjects.Add(aUrl);
if (CrawledAllowedDomainUrls.Contains(urlToCrawl) || CrawledAllowedDomainUrls.Contains(urlToCrawl.Split('#').First()))
{
return;
}
CrawledAllowedDomainUrls.Add(urlToCrawl);
//Load Page And Capture Time To Load
DateTime preLoad = DateTime.Now;
WebDriver.Navigate().GoToUrl(urlToCrawl);
aUrl.MsToLoad = (DateTime.Now - preLoad).Milliseconds;
//Take Screenshot of Page
String basePath = SavePicturesToPath + aUrl.FileNameBase;
int height = WebDriver.FindElement(By.TagName("body")).Size.Height;
WebDriver.Manage().Window.Size = new Size(WebDriver.Manage().Window.Size.Width, height + 250);
((ITakesScreenshot)WebDriver).GetScreenshot().SaveAsFile(basePath + ".png",
ScreenshotImageFormat.Png);
aUrl.PicturePath = basePath + ".png";
aUrl.HtmlSourcePath = basePath + ".html";
File.WriteAllText(aUrl.HtmlSourcePath, WebDriver.PageSource);
//Save Source Code of Page
ReadOnlyCollection <IWebElement> links = WebDriver.FindElements(By.TagName("a"));
List <String> urlsFoundOnThisPage = new List <string>();
foreach (var link in links)
{
if (link.GetAttribute("href") != null)
{
urlsFoundOnThisPage.Add(link.GetAttribute("href").Split('#').First());
}
}
UpdateProgressListeners(urlToCrawl, urlsFoundOnThisPage);
//Generate list of known urls to pass to this method so that we don't crawl forever.
List <String> parentsWillCrawl = new List <string>();
if (parentWillCrawl != null)
{
parentsWillCrawl.AddRange(parentWillCrawl);
}
parentsWillCrawl.AddRange(urlsFoundOnThisPage);
foreach (var url in urlsFoundOnThisPage)
{
if (InAllowedDomain(url) && !CrawledAllowedDomainUrls.Contains(url) && (parentWillCrawl == null || !parentWillCrawl.Contains(url)))
{
crawlUrl(url, parentsWillCrawl);
}
else
{
ExternalUrls.Add(url);
//If it is a url that is in the list of external urls to capture, go ahead.
foreach (var captureExternalUrl in Config.CapturePagesLinkedInTheseDomains)
{
if (UrlsMatch(captureExternalUrl, url))
{
crawlUrl(url, parentsWillCrawl);
break;
}
}
}
}
}
internal void AddExternalUrl(string href)
{
ExternalUrls.Add(href);
}
/// <summary>
/// Parses a page looking for links.
/// </summary>
/// <param name="page">The page whose text is to be parsed.</param>
/// <param name="sourceUrl">The source url of the page.</param>
public void ParseLinks(InputSite inputSite, Page page, string sourceUrl)
{
if (sourceUrl.EndsWith(".xml"))
{
var matches = Regex.Matches(page.Text, _SITEMAP_REGEX);
for (var i = 0; i <= matches.Count - 1; i++)
{
var anchorMatch = matches[i];
var foundHref = BddJson.NormalizeUrl(anchorMatch.Value);
// TODO faire un Regex Match
foundHref = foundHref.Replace("<loc>", "");
foundHref = foundHref.Replace("</loc>", "");
if (!IsBad(foundHref) && !GoodUrls.Contains(foundHref))
{
GoodUrls.Add(foundHref);
}
}
}
else
{
var matches = Regex.Matches(page.Text, _LINK_REGEX);
for (var i = 0; i <= matches.Count - 1; i++)
{
var anchorMatch = matches[i];
if (anchorMatch.Value == string.Empty)
{
BadUrls.Add("Blank url value on page " + sourceUrl);
continue;
}
string foundHref = null;
try
{
foundHref = anchorMatch.Value.Replace("href=\"", "");
foundHref = foundHref.Substring(0, foundHref.IndexOf("\""));
}
catch (Exception exc)
{
Exceptions.Add("Error parsing matched href: " + exc.Message);
}
foundHref = BddJson.NormalizeUrl(foundHref);
if (!IsBad(foundHref) && !GoodUrls.Contains(foundHref))
{
if (IsExternalUrl(inputSite, foundHref))
{
ExternalUrls.Add(foundHref);
}
else if (!IsAWebPage(foundHref))
{
foundHref = Crawler.FixPath(inputSite, sourceUrl);
OtherUrls.Add(foundHref);
}
else
{
GoodUrls.Add(foundHref);
}
}
}
}
}
