public void Reset(string BaseDirrectory) { RecordsDataAccessLayer recordsDataAccessLayer = new RecordsDataAccessLayer(); recordsDataAccessLayer.ResetDatabase(); try { System.IO.Directory.Delete(BaseDirrectory, true); //Deletes directory } catch { } }
public void GetRecords(int maxCtr, string BaseDirectory) { //Scrapes website and gets record information and album cover string url = @"https://www.discogs.com"; //Base url string urlPath = @"/search/?type=release"; //Fath url, first page int ctr = 0; int pageCtr = 0; Stopwatch stopwatch = new Stopwatch(); RecordsDataAccessLayer RecordsDataAccessLayer = new RecordsDataAccessLayer(); stopwatch.Start(); while (ctr < maxCtr) { HtmlWeb web = new HtmlWeb(); HtmlDocument htmlDoc = new HtmlDocument(); //Handles failed loading of pages //Infinite loop until loading of page successful for (; ;) { try { htmlDoc = web.Load(url + urlPath); break; } catch { Console.WriteLine("Failed to load web page"); urlPath = "/search/?type=release&page=" + pageCtr; pageCtr++; //Iterates to next page } } //Lambda functions used together with LINQ query, taking the Html node as input and returning a list of records var recordList = htmlDoc.DocumentNode.Descendants("div") .Where(node => node.GetAttributeValue("id", "") .Equals("search_results")).ToList(); var listItems = recordList[0].Descendants("a") //List of record links .Where(node => node.GetAttributeValue("class", "") .Equals("search_result_title")).ToList(); //Loops through every item in the list, corresponding with the records on a single web page foreach (var listItem in listItems) { //Checks that only the specified amount of records will be scraped if (ctr >= maxCtr) { break; } string recordUrlPath = listItem.GetAttributeValue("href", "").ToString(); //url to record page using (Record Record = new Record()) { //Call SetRecord to fill the record object with required information Record.SetRecord(url + recordUrlPath, recordUrlPath, Record); //If the record genre is empty the record will not be entered into the database if (Record.Genre == null) { Console.WriteLine("Error: Genre = Null"); } else { //Checks weather or not the record has been added and weather to download album cover if (RecordsDataAccessLayer.InsertRecord(Record)) { string id = RecordsDataAccessLayer.GetLatestId(); //If there is no album cover the path will be "-" string path; if (Record.ImgUrl == "thumbnail_border") { path = "-"; } else { //If the record does not use a stock image the image will be downloaded to the specified folder path = DownloadImage(Record.ImgUrl, id, BaseDirectory); } RecordsDataAccessLayer.UpdateRecordPtah(id, path); //Update file path to album cover RecordsDataAccessLayer.InsertTrackList(id, Record); //All tracks of the record will be inserted into the database ctr++; //Notifies the user that the record has been successfully added to the database Console.WriteLine($"Success: {Record.Name} - {Record.Artist}"); } } } } urlPath = GetNextPage(htmlDoc); pageCtr++; //Used if unable to get page (to iterate to next page) } stopwatch.Stop(); Console.WriteLine($"Done, records scraped: {ctr}, Time elapsed: {stopwatch.Elapsed}"); }