public bool Do()
{
var result = _crawler.Crawl(new Uri(_startUrl));
//Console.WriteLine("Crawling Error Occurred ? -> {0}", result.ErrorOccurred);
if (!result.ErrorOccurred)
{
Console.WriteLine("Successfully Collected ArtistName/Page Data");
return(true);
}
else
{
Console.WriteLine("Error : {0}", result.ErrorException.Message);
return(false);
}
}
public void Scrap(string artist, string url, string outputDirPrefix)
{
_artistNameHint = artist;
var uri = new Uri(url);
var result = _crawler.Crawl(uri);
if (result.ErrorOccurred)
{
Console.WriteLine("Error : {0}", result.ErrorException.Message);
}
else
{
Console.WriteLine("done --> {0} ea", _songs.Count);
var normedArtist = _pathCorrector.NormalizedFileName(artist);
var directory = $"{outputDirPrefix}/{normedArtist}";
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
var serializer = new JsonSerializer();
foreach (var song in _songs)
{
var filePath = _pathCorrector.GetNormalizedPath(directory, song.Title);
filePath = filePath + ".json";
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read))
using (var textWriter = new StreamWriter(stream, Encoding.UTF8))
using (var jsonWriter = new JsonTextWriter(textWriter)
{
Formatting = Formatting.Indented
})
{
serializer.Serialize(jsonWriter, song);
}
}
}
}