public void Reset(string BaseDirrectory)
{
RecordsDataAccessLayer recordsDataAccessLayer = new RecordsDataAccessLayer();
recordsDataAccessLayer.ResetDatabase();
try
{
System.IO.Directory.Delete(BaseDirrectory, true); //Deletes directory
}
catch
{
}
}
public void GetRecords(int maxCtr, string BaseDirectory)
{
//Scrapes website and gets record information and album cover
string url = @"https://www.discogs.com"; //Base url
string urlPath = @"/search/?type=release"; //Fath url, first page
int ctr = 0;
int pageCtr = 0;
Stopwatch stopwatch = new Stopwatch();
RecordsDataAccessLayer RecordsDataAccessLayer = new RecordsDataAccessLayer();
stopwatch.Start();
while (ctr < maxCtr)
{
HtmlWeb web = new HtmlWeb();
HtmlDocument htmlDoc = new HtmlDocument();
//Handles failed loading of pages
//Infinite loop until loading of page successful
for (; ;)
{
try
{
htmlDoc = web.Load(url + urlPath);
break;
}
catch
{
Console.WriteLine("Failed to load web page");
urlPath = "/search/?type=release&page=" + pageCtr;
pageCtr++; //Iterates to next page
}
}
//Lambda functions used together with LINQ query, taking the Html node as input and returning a list of records
var recordList = htmlDoc.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("id", "")
.Equals("search_results")).ToList();
var listItems = recordList[0].Descendants("a") //List of record links
.Where(node => node.GetAttributeValue("class", "")
.Equals("search_result_title")).ToList();
//Loops through every item in the list, corresponding with the records on a single web page
foreach (var listItem in listItems)
{
//Checks that only the specified amount of records will be scraped
if (ctr >= maxCtr)
{
break;
}
string recordUrlPath = listItem.GetAttributeValue("href", "").ToString(); //url to record page
using (Record Record = new Record())
{
//Call SetRecord to fill the record object with required information
Record.SetRecord(url + recordUrlPath, recordUrlPath, Record);
//If the record genre is empty the record will not be entered into the database
if (Record.Genre == null)
{
Console.WriteLine("Error: Genre = Null");
}
else
{
//Checks weather or not the record has been added and weather to download album cover
if (RecordsDataAccessLayer.InsertRecord(Record))
{
string id = RecordsDataAccessLayer.GetLatestId();
//If there is no album cover the path will be "-"
string path;
if (Record.ImgUrl == "thumbnail_border")
{
path = "-";
}
else
{
//If the record does not use a stock image the image will be downloaded to the specified folder
path = DownloadImage(Record.ImgUrl, id, BaseDirectory);
}
RecordsDataAccessLayer.UpdateRecordPtah(id, path); //Update file path to album cover
RecordsDataAccessLayer.InsertTrackList(id, Record); //All tracks of the record will be inserted into the database
ctr++;
//Notifies the user that the record has been successfully added to the database
Console.WriteLine($"Success: {Record.Name} - {Record.Artist}");
}
}
}
}
urlPath = GetNextPage(htmlDoc);
pageCtr++; //Used if unable to get page (to iterate to next page)
}
stopwatch.Stop();
Console.WriteLine($"Done, records scraped: {ctr}, Time elapsed: {stopwatch.Elapsed}");
}
