public static List <CrawlItem> LoadWithHelper(string path, bool fixPrerender)
{
int count = 0;
List <CrawlItem> ret = new List <CrawlItem>();
using (TextReader fileReader = File.OpenText(path))
{
var csv = new CsvReader(fileReader);
csv.Configuration.HasHeaderRecord = false;
while (csv.Read())
{
CrawlItem crawlItem = null;
if (count != 1 && count != 0)
{
if (fixPrerender)
{
if (csv.GetField <string>(1) != "image/jpeg")
{
crawlItem = new CrawlItem
{
URL = csv.GetField <string>(0),
PageTitle = csv.GetField <string>(4),
MetaDescription1 = csv.GetField <string>(7),
MetaDescription2 = csv.GetField <string>(10),
MetaKeyword1 = csv.GetField <string>(13),
H1 = csv.GetField <string>(17),
H2 = csv.GetField <string>(19),
StatusCode = csv.GetField <string>(3),
ResponseTime = csv.GetField <string>(40)
};
crawlItem.RealURL = crawlItem.FixURL();
}
}
else
{
if (csv.GetField <string>(1) != "image/jpeg")
{
crawlItem = new CrawlItem
{
URL = csv.GetField <string>(0),
PageTitle = csv.GetField <string>(4),
MetaDescription1 = csv.GetField <string>(7),
MetaDescription2 = csv.GetField <string>(10),
MetaKeyword1 = csv.GetField <string>(16),
H1 = csv.GetField <string>(22),
H2 = csv.GetField <string>(26),
StatusCode = csv.GetField <string>(3),
ResponseTime = csv.GetField <string>(45)
};
crawlItem.RealURL = crawlItem.URL;
}
}
//if(control && fixPrerender){
// crawlItem.RealURL = crawlItem.FixURL();
//}else{
// crawlItem.RealURL = crawlItem.URL;
//}
if (crawlItem != null)
{
ret.Add(crawlItem);
}
}
count++;
}
}
return(ret);
}
