public static List <CrawlItem> LoadCSV(string csvDir, bool checkPrerender)
{
int count = 0;
string[] header = new string[4];
List <CrawlItem> ret = new List <CrawlItem>();
using (var reader = new StreamReader(csvDir))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(',');
if (count == 0)
{
}
else if (count == 1)
{
header = values;
}
else
{
CrawlItem crawlItem = new CrawlItem
{
URL = values[0],
PageTitle = values[4],
MetaDescription1 = values[7],
MetaDescription2 = values[10],
MetaKeyword1 = values[16],
H1 = values[22],
H2 = values[26],
StatusCode = values[3],
ResponseTime = values[45]
};
ret.Add(crawlItem);
}
count++;
}
}
return(null);
}
public static List <CrawlItem> LoadWithHelper(string path, bool fixPrerender)
{
int count = 0;
List <CrawlItem> ret = new List <CrawlItem>();
using (TextReader fileReader = File.OpenText(path))
{
var csv = new CsvReader(fileReader);
csv.Configuration.HasHeaderRecord = false;
while (csv.Read())
{
CrawlItem crawlItem = null;
if (count != 1 && count != 0)
{
if (fixPrerender)
{
if (csv.GetField <string>(1) != "image/jpeg")
{
crawlItem = new CrawlItem
{
URL = csv.GetField <string>(0),
PageTitle = csv.GetField <string>(4),
MetaDescription1 = csv.GetField <string>(7),
MetaDescription2 = csv.GetField <string>(10),
MetaKeyword1 = csv.GetField <string>(13),
H1 = csv.GetField <string>(17),
H2 = csv.GetField <string>(19),
StatusCode = csv.GetField <string>(3),
ResponseTime = csv.GetField <string>(40)
};
crawlItem.RealURL = crawlItem.FixURL();
}
}
else
{
if (csv.GetField <string>(1) != "image/jpeg")
{
crawlItem = new CrawlItem
{
URL = csv.GetField <string>(0),
PageTitle = csv.GetField <string>(4),
MetaDescription1 = csv.GetField <string>(7),
MetaDescription2 = csv.GetField <string>(10),
MetaKeyword1 = csv.GetField <string>(16),
H1 = csv.GetField <string>(22),
H2 = csv.GetField <string>(26),
StatusCode = csv.GetField <string>(3),
ResponseTime = csv.GetField <string>(45)
};
crawlItem.RealURL = crawlItem.URL;
}
}
//if(control && fixPrerender){
// crawlItem.RealURL = crawlItem.FixURL();
//}else{
// crawlItem.RealURL = crawlItem.URL;
//}
if (crawlItem != null)
{
ret.Add(crawlItem);
}
}
count++;
}
}
return(ret);
}
