public bool Do() { var result = _crawler.Crawl(new Uri(_startUrl)); //Console.WriteLine("Crawling Error Occurred ? -> {0}", result.ErrorOccurred); if (!result.ErrorOccurred) { Console.WriteLine("Successfully Collected ArtistName/Page Data"); return(true); } else { Console.WriteLine("Error : {0}", result.ErrorException.Message); return(false); } }
public void Scrap(string artist, string url, string outputDirPrefix) { _artistNameHint = artist; var uri = new Uri(url); var result = _crawler.Crawl(uri); if (result.ErrorOccurred) { Console.WriteLine("Error : {0}", result.ErrorException.Message); } else { Console.WriteLine("done --> {0} ea", _songs.Count); var normedArtist = _pathCorrector.NormalizedFileName(artist); var directory = $"{outputDirPrefix}/{normedArtist}"; if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var serializer = new JsonSerializer(); foreach (var song in _songs) { var filePath = _pathCorrector.GetNormalizedPath(directory, song.Title); filePath = filePath + ".json"; using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read)) using (var textWriter = new StreamWriter(stream, Encoding.UTF8)) using (var jsonWriter = new JsonTextWriter(textWriter) { Formatting = Formatting.Indented }) { serializer.Serialize(jsonWriter, song); } } } }