private string GetPageTypeByUrl(Dictionary <string, string> debug, string domain, string url)
{
var pageTypeMappingDt = CseToMongoEtlProvider.GetDataTable("Get_Domain_PageTypes",
new Dictionary <string, object> {
{ "@domain", domain }
},
"SQLCacheConnectionString", 60);
if (pageTypeMappingDt == null || pageTypeMappingDt.Rows.Count == 0)
{
debug.Add("could not find page type mappings for", domain);
return(null);
}
for (int i = 0; i < pageTypeMappingDt.Rows.Count; i++)
{
if (Regex.IsMatch(url, (string)pageTypeMappingDt.Rows[i]["Regex"]))
{
string pageType = (string)pageTypeMappingDt.Rows[i]["pagetype"];
debug.Add("pagetype", pageType.ToLower());
return(pageType);
}
}
debug.Add("could determin page type for", url);
return(null);
}
private string GetSearchKeyword(Dictionary <string, string> debug, string domain, string url)
{
var searchParamsDt = CseToMongoEtlProvider.GetDataTable("Get_Domain_SearchParam",
new Dictionary <string, object> {
{ "@domain", domain }
},
"SQLCacheConnectionString", 60);
if (searchParamsDt == null || searchParamsDt.Rows.Count == 0)
{
debug.Add("could not find search params for", domain);
return(null);
}
var uri = new Uri(url);
var query = HttpUtility.ParseQueryString(uri.Query);
for (int i = 0; i < searchParamsDt.Rows.Count; i++)
{
var param = (string)searchParamsDt.Rows[i]["SearchParam"];
var searchKeyword = query[param];
if (!string.IsNullOrEmpty(searchKeyword))
{
debug.Add("search keyword", searchKeyword);
return(searchKeyword);
}
}
debug.Add("could not find search params in", url);
return(null);
}
public dynamic GetRtSettings()
{
var dataTable = CseToMongoEtlProvider.GetDataTable("sp_GET_DealFinder_RT_Settings",
new Dictionary <string, object>
{
{ "@CampaignId", _context.CampaignId }
},
"SQLCacheConnectionString", 60);
if (dataTable == null || dataTable.Rows.Count == 0)
{
return(new
{
rt_campaign_id = "",
is_active = false,
freq = 0,
non_e_com_rec = 0,
e_com_rec = 0
});
}
return(new
{
rt_campaign_id = (string)dataTable.Rows[0]["CampaignID_RT"],
is_active = ((byte)dataTable.Rows[0]["active"]) == 1 ? true : false,
freq = (short)dataTable.Rows[0]["Frequency_Not30K"],
non_e_com_rec = (short)dataTable.Rows[0]["Reccency_Not30K"],
e_com_rec = (short)dataTable.Rows[0]["Reccency_30K"]
});
}
private List <string> GetDomainTriggersAndCountry(DealFinderContext context)
{
var timer = new TimerProvider(context, "get-domain-triggers");
var dataTable = CseToMongoEtlProvider.GetDataTable("sp_Get_DomainTrigger",
new Dictionary <string, object>
{
{ "Domain", context.DomainName }
},
"SQLCacheConnectionString", 60);
timer.StopTimer();
timer = new TimerProvider(context, "domain-triggers-match-time");
var domainTriggers = new List <string>();
for (int i = 0; i < dataTable.Rows.Count; i++)
{
domainTriggers.Add((string)dataTable.Rows[i][0]);
context.Country = dataTable.Rows[i][1] == DBNull.Value ? "US" : (string)dataTable.Rows[i][1];
}
timer.StopTimer();
return(domainTriggers);
}
private DataRow[] GetDomainCache(string strDomain)
{
string exp = "DomainName = '" + strDomain + "'";
string sortOrder = "Priority DESC,isPrefixCacheRegex DESC";
DataTable dtCacheDomainsCache = CseToMongoEtlProvider.GetDataTable("dtCacheDomains", "", "SQLCacheConnectionString", 60);
// DataTable dtCacheDomainsCache = CU.GetCacheTable("dtCacheDomains", "SQLCacheConnectionString", 11);
DataRow[] drw = new DataRow[0];
try
{
//drw = dtCacheDomainsCache.Select(exp, sortOrder);
DataView dv;
lock (dtCacheDomainsCache)
{
dv = new DataView(dtCacheDomainsCache, exp, sortOrder, DataViewRowState.Added);
if (dv.Count == 0)
{
dv = new DataView(dtCacheDomainsCache, exp, sortOrder, DataViewRowState.OriginalRows);
}
drw = dv.ToTable().Select();
}
}
catch
{
}
return(drw);
}
private string GetDomainToMidMapping(int cse)
{
var dt = CseToMongoEtlProvider.GetDataTable("Get_Domain_Mids",
new Dictionary <string, object>
{
{ "@domain", _context.DomainName }
},
"SQLCacheConnectionString", 24);
if (dt == null || dt.Rows.Count == 0)
{
return("undefined");
}
var rows = dt.Select("api = " + cse.ToString());
if (rows.Length == 0)
{
return("undefined");
}
string mid = rows[0][0] as string;
return(mid);
}
private dynamic GetDomainPriceInterval()
{
var dt = CseToMongoEtlProvider.GetDataTable("Get_DomainPriceRange",
new Dictionary <string, object>
{
{ "@domain", _context.DomainName }
},
"SQLCacheConnectionString", 24);
if (dt == null || dt.Rows.Count == 0)
{
return(new
{
min = 0,
max = 0
});
}
else
{
return(new
{
min = (double)((int)dt.Rows[0]["min"]) / 100,
max = (double)((int)dt.Rows[0]["max"]) / 100
});
}
}
private List <short> GetCseApisAndCurrencyForCurrentCampaignId()
{
var dt = CseToMongoEtlProvider.GetDataTable("Get_Campaign_Apis",
new Dictionary <string, object>
{
{ "CampaignID", string.IsNullOrEmpty(_context.Country) ? _context.CampaignId.ToString() : _context.Country }
},
"SQLCacheConnectionString", 24);
if (dt == null || dt.Rows.Count == 0)
{
return(new List <short>());
}
_context.Currency = dt.Rows[0][1].ToString();
var apis = new List <short>();
for (int i = 0; i < dt.Rows.Count; i++)
{
apis.Add((short)dt.Rows[i][0]);
}
return(apis);
}
public static string GenerateOuterClickUrl(string offerUrl, string pageId, int shownProducts, int shownStores, int storePos,
string prodPrice, string storePrice, string prodSku, string minStorePrice,
string maxStorePrice, DealFinderContext dealFinderContext, string api,
bool encode)
{
var queryParams = new StringBuilder();
queryParams.Append(string.Format("productURL={0}&", HttpUtility.UrlEncode(offerUrl)));
queryParams.Append(string.Format("domainName={0}&", dealFinderContext.DomainName));
queryParams.Append(string.Format("pageId={0}&", pageId));
queryParams.Append(string.Format("currency={0}&", dealFinderContext.Currency));
queryParams.Append(string.Format("category={0}&", dealFinderContext.Category));
queryParams.Append(string.Format("country={0}&", dealFinderContext.Country));
queryParams.Append(string.Format("impressionId={0}&", dealFinderContext.ImpressionId));
queryParams.Append(string.Format("userId={0}&", dealFinderContext.UserId));
queryParams.Append(string.Format("campaignId={0}&", dealFinderContext.CampaignId));
queryParams.Append(string.Format("shownProducts={0}&", shownProducts));
queryParams.Append(string.Format("shownStores={0}&", shownStores));
queryParams.Append(string.Format("storeApi={0}&", api));
queryParams.Append(string.Format("storePos={0}&", storePos));
queryParams.Append(string.Format("storePrice={0}&", storePrice));
queryParams.Append(string.Format("productPrice={0}&", prodPrice));
queryParams.Append(string.Format("minStorePrice={0}&", minStorePrice));
queryParams.Append(string.Format("maxStorePrice={0}&", maxStorePrice));
queryParams.Append(string.Format("clickType={0}&", 1));
queryParams.Append(string.Format("prodSku={0}", prodSku));
string url = queryParams.ToString();
string clickUrl = null;
if (!encode)
{
clickUrl = string.Format("{0}/?{1}", ConfigurationManager.AppSettings["ClickUrl"], url);
}
else
{
var query = new StringBuilder();
var mongoProvider = new CseToMongoEtlProvider();
var offerUrlId = mongoProvider.GetUrlId(url);
query.Append(string.Format("data={0}", offerUrlId));
query.Append(string.Format("&nid={0}", dealFinderContext.Nid));
query.Append(string.Format("&sid={0}", dealFinderContext.Sid == null ? "0" : dealFinderContext.Sid));
clickUrl = string.Format("{0}/?{1}", ConfigurationManager.AppSettings["ClickUrl"], query.ToString());
}
return(clickUrl);
}
private string GetOP(string campaignId, string sid, string nid)
{
Dictionary <string, object> parameters = new Dictionary <string, object>();
parameters.Add("AID", campaignId);
parameters.Add("PID", nid == null ? "" : nid);
parameters.Add("SE", 0);
parameters.Add("SID", sid == null ? "" : sid);
DataTable aidMapping = CseToMongoEtlProvider.GetDataTable("SP_DF_GET_MappedCampaign", parameters, "SQLCacheConnectionString", 24);
string op = aidMapping.Rows[0]["OP"].ToString();
return(op);
}
public bool IsEcommerceDomain()
{
var dataTable = CseToMongoEtlProvider.GetDataTable("sp_Domain_in30K",
new Dictionary <string, object>
{
{ "@Domain", _context.DomainName }
},
"SQLCacheConnectionString", 60);
if (dataTable == null || dataTable.Rows.Count == 0)
{
return(false);
}
return(((int)dataTable.Rows[0][0]) == 0 ? false : true);
}
private string GetProductName(Dictionary <string, string> debug, string domain, string url)
{
var urlKeywordMappingsDt = CseToMongoEtlProvider.GetDataTable("Get_Domain_UrlProduct",
new Dictionary <string, object> {
{ "@domain", domain }
},
"SQLCacheConnectionString", 60);
if (urlKeywordMappingsDt == null || urlKeywordMappingsDt.Rows.Count == 0)
{
return(null);
//debug.Add("could not find url mapping for domain, executing get site data", domain);
//try
//{
// var siteData = MarketProvider.GetSiteData(debug, "6000666", url, false);
// return siteData.productName;
//}
//catch
//{
//}
}
else
{
debug.Add("url mapping found", domain);
short level = (short)urlKeywordMappingsDt.Rows[0]["Level"];
string stopString = (string)urlKeywordMappingsDt.Rows[0]["StopString"];
string divider = (string)urlKeywordMappingsDt.Rows[0]["Divider"];
var uri = new Uri(url);
string keyword = uri.Segments[level - 1];
keyword = keyword.Replace(stopString, "");
keyword = keyword.Replace(divider, " ");
return(keyword);
}
return(null);
}
private List <CseProductInfo> GetProductsFromPcmAction(CsePcmInfo pcmInfo, string op)
{
var matches = pcmInfo.Matches;
var cseDataProvider = new CseToMongoEtlProvider();
var products = new List <CseProductInfo>();
foreach (var match in matches)
{
foreach (var pid in match.Pids)
{
CseProductInfo product = null;
if (match.Cse == 9 || match.Cse == 10)
{
product = cseDataProvider.GetProduct(match.Cse, op + "_" + pid);
}
else
{
product = cseDataProvider.GetProduct(match.Cse, pid);
}
if (product == null)
{
return(new List <CseProductInfo>());
}
foreach (var offer in product.Offers)
{
offer.MerchantInfo = cseDataProvider.GetMerchant(match.Cse, offer.MerchantId);
}
products.Add(product);
}
_context.AddCseOptimizationMapping(match.Cse.ToString(), match.ModifiedBy);
}
return(products);
}
private List <CseProductInfo> GetProductsFromCseAction(int minNumberOfOffers)
{
var timer = new TimerProvider(_context, "get-campaign-apis-sql");
var apis = GetCseApisAndCurrencyForCurrentCampaignId();
timer.StopTimer();
if (apis.Count == 0)
{
return(new List <CseProductInfo>());
}
var cseDataProvider = new CseToMongoEtlProvider();
var productsList = new List <CseProductInfo>();
var rawProductsList = new List <CseProductInfo>();
var merchantsList = new List <CseMerchantInfo>();
int totalNumberOfOffers = 0;
while (true)
{
for (int i = 0; i < apis.Count; i++)
{
MatchingData matchingData = null;
try
{
matchingData = GetProtuctsFromCseByKeyword(apis[i]);
if (matchingData == null)
{
matchingData = new MatchingData();
}
}
catch (Exception ex)
{
_logger.ErrorFormat("failed get data from cse [{0}], ex: [{1}]", apis[i], ex.Message);
_context.AddCseExecutionFailure(apis[i].ToString(), "Failed");
}
_context.AddCseNumberOfRawResults(apis[i].ToString(), matchingData.Matches.Count.ToString());
if (matchingData.Matches.Count == 0)
{
continue;
}
_context.AddCseExecutionFailure(apis[i].ToString(), "OK");
List <CseProductInfo> products = null;
List <CseMerchantInfo> merchants = null;
cseDataProvider.ParseProductAndMerchants(apis[i], "widget", matchingData.ds, out products, out merchants);
if (products.Count == 0)
{
continue;
}
merchantsList.AddRange(merchants.Select(x => (CseMerchantInfo)x.Clone()));
rawProductsList.AddRange(products.Select(x => (CseProductInfo)x.Clone()));
var filterProvider = new FilterProvider(_context);
products = filterProvider.FilterSameStore(products);
products = filterProvider.FilterPriceRange(products);
if (products.Count == 0)
{
continue;
}
productsList.AddRange(products);
productsList = filterProvider.FilterDuplicateStores(productsList);
productsList = filterProvider.FilterDuplicateProductId(productsList);
if (productsList.Count == 0)
{
continue;
}
totalNumberOfOffers = CountProdcutOffers(productsList);
if (totalNumberOfOffers >= minNumberOfOffers)
{
break;
}
}
totalNumberOfOffers = CountProdcutOffers(productsList);
if (totalNumberOfOffers >= minNumberOfOffers)
{
break;
}
if (!_context.IsDefaultPrice)
{
var filterProvider = new FilterProvider(_context);
rawProductsList = filterProvider.FilterDuplicateStores(rawProductsList);
rawProductsList = filterProvider.FilterDuplicateProductId(rawProductsList);
totalNumberOfOffers = CountProdcutOffers(rawProductsList);
if (totalNumberOfOffers >= minNumberOfOffers)
{
productsList = rawProductsList;
break;
}
}
if (!ShrinkKeywordStart())
{
break;
}
}
ThreadPool.QueueUserWorkItem(state =>
{
dynamic param = state as dynamic;
cseDataProvider.AddCseResultToMongoDb("widget", _context.Op, param.products, param.merchants, false);
}, new
{
products = new List <CseProductInfo>(rawProductsList.Select(x => (CseProductInfo)x.Clone())),
merchants = new List <CseMerchantInfo>(merchantsList.Select(x => (CseMerchantInfo)x.Clone()))
});
return(productsList);
}
