public List <CategoryStructure> GetCategories()
{
List <CategoryStructure> lcateogry = new List <CategoryStructure>();
WebClient webClient = new WebClient();
String jsonString = webClient.DownloadString(GetCategoriesUrl());
Dictionary <String, dynamic> apiGroups = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(jsonString);
if (apiGroups.ContainsKey("apiGroups"))
{
Dictionary <String, dynamic> affiliate = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(Convert.ToString(apiGroups["apiGroups"]));
if (affiliate.ContainsKey("affiliate"))
{
Dictionary <String, dynamic> apiListings = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(Convert.ToString(affiliate["affiliate"]));
if (apiListings.ContainsKey("apiListings"))
{
Dictionary <String, dynamic> dictCateogry = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(Convert.ToString(apiListings["apiListings"]));
foreach (var key in dictCateogry.Keys)
{
Dictionary <string, dynamic> dictCat = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(Convert.ToString(dictCateogry[key]));
if (dictCat.ContainsKey("availableVariants"))
{
Dictionary <string, dynamic> availableVariants = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(Convert.ToString(dictCat["availableVariants"]));
if (availableVariants.ContainsKey("v0.1.0"))
{
CategoryStructure objCat = JsonConvert.DeserializeObject <CategoryStructure>(Convert.ToString(availableVariants["v0.1.0"]));
lcateogry.Add(objCat);
}
}
}
}
else
{
throw new Exception("Json does not contains key apiListings");
}
}
else
{
throw new Exception("Json does not contains key affiliate");
}
}
else
{
throw new Exception("Json does not contains key apiGroups");
}
return(lcateogry);
}
public void GetProductListing(CategoryStructure objcat)
{
WebClient client = new WebClient();
client.Headers.Add("Fk-Affiliate-Id", m_affiliateId);
client.Headers.Add("Fk-Affiliate-Token", m_affiliteToken);
string nextUrl = objcat.get;
string lastProduct = String.Empty;
bool bcontinue = true;
string fileName = Constants.LINK_EXTRACTED_FOLDER + "//" + objcat.resourceName + ".csv";
if (!System.IO.File.Exists(fileName))
{
string line = "Model_Name" + Constants.CSV_DELIMITER +
"productBrand" + Constants.CSV_DELIMITER +
"productUrl" + Constants.CSV_DELIMITER +
"productId";
Logger.WriteToCSVFile(line, objcat.resourceName, Constants.LINK_EXTRACTED_FOLDER);
}
//populate the csv first in memory
List <List <string> > matcsv = Utilities.GetCSVSheet(fileName);
Dictionary <string, string> dictProduct = new Dictionary <string, string>();
string[] colNames = matcsv.First().ToArray();
matcsv.RemoveAt(0);
foreach (List <string> row in matcsv)
{
dictProduct.Add(row.ElementAt(3), null);
}
while (bcontinue)
{
string jsonString = client.DownloadString(nextUrl);
Dictionary <String, dynamic> productObject = JsonConvert.DeserializeObject <Dictionary <String, dynamic> >(jsonString);
string newLastProduct = productObject["lastProductId"];
if (lastProduct.Equals(newLastProduct, StringComparison.InvariantCultureIgnoreCase))
{
throw new Exception("Last product id is either null or same as current last product id");
}
else
{
lastProduct = newLastProduct;
}
if (productObject.ContainsKey("nextUrl"))
{
nextUrl = productObject["nextUrl"];
if (String.IsNullOrEmpty(nextUrl))
{
bcontinue = false;
}
}
else
{
bcontinue = false;
}
if (productObject.ContainsKey("productInfoList"))
{
string strpodcut = Convert.ToString(productObject["productInfoList"]);
List <ProductInfoList> lproductInfoList = JsonConvert.DeserializeObject <List <ProductInfoList> >(Convert.ToString(productObject["productInfoList"]));
List <ProductStructure> lproducts = new List <ProductStructure>();
foreach (ProductInfoList objpr in lproductInfoList)
{
objpr.productBaseInfo.productAttributes.productId = objpr.productBaseInfo.productIdentifier.productId;
lproducts.Add(objpr.productBaseInfo.productAttributes);
}
Logger.WriteToLogFile("Total Count : " + lproducts.Count);
WorkerThread objworkerThread = new WorkerThread(lproducts, Constants.LINK_EXTRACTED_FOLDER + "//" + objcat.resourceName, dictProduct);
System.Threading.Thread t = new System.Threading.Thread(objworkerThread.ScrapModelNumberAndWriteToCSV);
t.Start();
m_threadCount++;
while (m_threadCount == 3)
{
Thread.Sleep(5000);
}
}
else
{
throw new Exception("Json does not contains key productInfoList");
}
}
}
