/// <summary>
///
/// </summary>
/// <param name="actionContext"></param>
/// <param name="httpContext"></param>
/// <returns></returns>
public void GetGridData(ActionContext actionContext, HttpContext httpContext)
{
string otherUnid = httpContext.Request.Params["otherUnid"];
int pageNo = int.Parse(httpContext.Request.Params["pageNo"].Trim());
int pageSize = int.Parse(httpContext.Request.Params["pageSize"].Trim());
Newtonsoft.Json.JavaScriptObject jsonObject = new Newtonsoft.Json.JavaScriptObject();
jsonObject.Add("rows", new Newtonsoft.Json.JavaScriptArray());
jsonObject.Add("totalCount", 0);
jsonObject.Add("id", "ID");
PageInfo pageInfo = this.userService.FindAllByRelationShipParentUnid(otherUnid, pageNo, pageSize);
if (!string.IsNullOrEmpty(otherUnid))
{
if (pageInfo != null)
{
IList list = pageInfo.Objs;
if (!(list == null || list.Count == 0))
{
Newtonsoft.Json.JavaScriptArray jsonArray = this.GetArray <User>(list);
jsonObject["rows"] = jsonArray;
jsonObject["totalCount"] = pageInfo.TotalCount;
}
}
}
httpContext.Response.ContentType = ("text/plain;charset=UTF-8");
httpContext.Response.AppendHeader("Cache-Control", "no-cache");
httpContext.Response.Write(Newtonsoft.Json.JavaScriptConvert.SerializeObject(jsonObject));
}
public List <T> ToList <T>(string json)
{
Newtonsoft.Json.JavaScriptArray jsonArray = (Newtonsoft.Json.JavaScriptArray)Newtonsoft.Json.JavaScriptConvert.DeserializeObject(json);
List <T> list = new List <T>();
T model = NewObject <T>();
System.Reflection.PropertyInfo[] propertys = model.GetType().GetProperties();
for (int i = 0; i < jsonArray.Count; i++)
{
T _model = (T)((ICloneable)model).Clone();
Newtonsoft.Json.JavaScriptObject Jobject = ((Newtonsoft.Json.JavaScriptObject)((Newtonsoft.Json.JavaScriptArray)jsonArray)[i]);
for (int n = 0; n < Jobject.Count; n++)
{
for (int j = 0; j < propertys.Length; j++)
{
if (propertys[j].Name == "WorkId")
{
break;
}
if (Jobject.ToList()[n].Key.Trim().ToUpper() == propertys[j].Name.ToUpper())
{
if (propertys[j].PropertyType.Equals(typeof(Int32)))
{
propertys[j].SetValue(_model, Convert.ToInt32(Jobject.ToList()[n].Value.ToString().Trim() == "" ? 0 : Jobject.ToList()[n].Value), null);
}
else if (propertys[j].PropertyType.Equals(typeof(Int64)))
{
propertys[j].SetValue(_model, Convert.ToInt64(Jobject.ToList()[n].Value.ToString().Trim() == "" ? 0 : Jobject.ToList()[n].Value), null);
}
else if (propertys[j].PropertyType.Equals(typeof(decimal)))
{
propertys[j].SetValue(_model, Convert.ToDecimal(Jobject.ToList()[n].Value.ToString().Trim() == "" ? 0 : Jobject.ToList()[n].Value), null);
}
else if (propertys[j].PropertyType.Equals(typeof(DateTime)))
{
propertys[j].SetValue(_model, Convert.ToDateTime(Jobject.ToList()[n].Value.ToString().Trim() == "" ? DateTime.Now.ToString() : Jobject.ToList()[n].Value), null);
}
else
{
propertys[j].SetValue(_model, Jobject.ToList()[n].Value.ToString(), null);
}
break;
}
}
}
list.Add(_model);
}
return(list);
}
/// <summary>
/// IList 转换成 JavascriptArray
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="list"></param>
/// <returns></returns>
private Newtonsoft.Json.JavaScriptArray GetArray <T>(IList list) where T : Entry
{
Newtonsoft.Json.JavaScriptArray jsonArray = new Newtonsoft.Json.JavaScriptArray();
if (!(null == list || list.Count == 0))
{
foreach (object obj in list)
{
if (obj != null)
{
T t = obj as T;
jsonArray.Add(t);
}
}
}
return(jsonArray);
}
private JavaScriptArray PopulateJavaScriptArray(JsonReader reader)
{
JavaScriptArray jsArray = new JavaScriptArray();
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonToken.EndArray:
return(jsArray);
case JsonToken.Comment:
break;
default:
object value = GetObject(reader, null);
jsArray.Add(value);
break;
}
}
throw new JsonSerializationException("Unexpected end while deserializing array.");
}
static void Main(string[] args)
{
string errr = "";
Console.WriteLine("开始分析数据:");
List <string> uids = new List <string>();
for (int d = int.Parse(dfrom); d < 0; d++)
{
string writeFile1 = string.Format("{0}\\result{1}.txt", System.Threading.Thread.GetDomain().BaseDirectory, DateTime.Now.ToFileTimeUtc().ToString());
List <UserVisitInfo> userList = new List <UserVisitInfo>();
string dt = DateTime.Now.AddDays(d).ToString("yyyy-MM-dd");
int pageType1 = 31;
if (!DataFarm.ExistGaData(pageType1, DateTime.Parse(dt), DateTime.Parse(dt)))
{
//数据读取完毕退出
bool exitFlag = false;
Int64 maxVisitID = 0;
try
{
#region 循环拉取
for (int mi = 0; mi < int.MaxValue; mi++)
{
if (exitFlag)
{
break;
}
#region 循环读取数据
int filter_limit = intent; // intent * (mi + 1);
int filter_offset = intent * mi;
string url = string.Format("http://10.0.0.131:922/index.php?module=API&filter_limit={0}&method=Live.getLastVisitsDetails&format=json&idSite=1&period=day&date={1}&expanded=1&token_auth={2}", filter_limit, dt, token);
if (maxVisitID > 0)
{
url = url + "&maxIdVisit=" + maxVisitID;
}
string xml = Boodoll.PageBL.ProductSearch.ProductSearchBLL.GetHtml(url, Encoding.GetEncoding("GB2312"));
xml = xml.Replace("\\u", "\\\\u");
if (xml == null || xml.Length < 10)
{
exitFlag = true;
continue;
}
Newtonsoft.Json.JavaScriptArray jsonObject = (Newtonsoft.Json.JavaScriptArray)Newtonsoft.Json.JavaScriptConvert.DeserializeObject(xml);
int count = 0;
if (jsonObject != null && xml.Length > 5)
{
count = jsonObject.Count();
}
else
{
continue;
}
for (int i = 0; i < count; i++)
{
JavaScriptObject qcount = (JavaScriptObject)jsonObject[i];
JavaScriptArray actionDetails = (JavaScriptArray)qcount["actionDetails"];
string lastActionDateTime = qcount["serverDate"].ToString() + " " + qcount["serverTimePretty"].ToString();
string visitIp = qcount["visitIp"].ToString();
string location = ConvertUnicodeStringToChinese(qcount["location"].ToString());
string locationsina = updateLocation(visitIp, location);
if (Convert.ToDateTime(lastActionDateTime) < Convert.ToDateTime(dt))
{
exitFlag = true;
continue;
}
//Console.WriteLine(visitIp+",time,"+lastActionDateTime);
if (i == 0)
{
maxVisitID = Convert.ToInt64(qcount["idVisit"]);
}
else
{
maxVisitID = Convert.ToInt64(qcount["idVisit"]) > maxVisitID ? maxVisitID : Convert.ToInt64(qcount["idVisit"]);
}
if (actionDetails.Count > 0)
{
string userid = "-1";
string guid = "";
string referurl = "";
JavaScriptObject customVariables = null;
try
{
if (qcount.Keys.Contains("referrerUrl"))
{
referurl = Converter.ParseString(qcount["referrerUrl"], "");
if (referurl.Length > 0)
{
if (referurl.Length > 2000)
{
referurl = referurl.Substring(0, 2000);
}
}
}
customVariables = (JavaScriptObject)qcount["customVariables"];
userid = ((new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customVariables)).ElementAt(0).Value)))).ElementAt(1).Value.ToString());
guid = (new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customVariables)).ElementAt(1).Value)))).ElementAt(1).Value.ToString();
if (!uids.Any(u => u.ToString() == guid))
{
uids.Add(guid);
}
}
catch (Exception ex)
{
userid = "-1";
guid = "";
continue;
}
actionDetails.ForEach(item =>
{
JavaScriptObject itemobject = (JavaScriptObject)item;
string tmpUrl = "";
if (itemobject.Keys.Contains("url") && itemobject["url"] != null)
{
tmpUrl = itemobject["url"].ToString();
}
UserVisitInfo vinfo = new UserVisitInfo();
if (vinfo != null)
{
vinfo.Guid = guid;
vinfo.Userid = userid;
if (itemobject.Keys.Contains("timeSpent"))
{
vinfo.Spent = Converter.ParseString(itemobject["timeSpent"], "");
}
vinfo.Url = tmpUrl;
vinfo.LastVisitTime = lastActionDateTime;
if (itemobject.Keys.Contains("pageTitle"))
{
vinfo.PageTitle = UnicodeToString(Converter.ParseString(itemobject["pageTitle"], ""));
}
else
{
vinfo.PageTitle = "";
}
if (itemobject.Keys.Contains("type"))
{
vinfo.Action = Converter.ParseString(itemobject["type"], "");
}
else
{
vinfo.Action = "";
}
vinfo.VisitIp = visitIp;
vinfo.Location = location;
vinfo.Locationsina = locationsina;
if (itemobject.Keys.Contains("customVariables"))
{
JavaScriptObject customerDetail = (JavaScriptObject)itemobject["customVariables"];
// var q = customerDetail.Values.First();
string q1 = (new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customerDetail)).ElementAt(0).Value)))).ElementAt(1).Value.ToString();
string q2 = (new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customerDetail)).ElementAt(0).Value)))).ElementAt(0).Value.ToString();
vinfo.Event_action = UnicodeToString(q2);
if (vinfo.Event_action.Length >= 1000)
{
vinfo.Event_action = vinfo.Event_action.Substring(0, 1000);
}
}
vinfo.Referurl = referurl;
userList.Add(vinfo);
string tmpmsg = "";
try
{
tmpmsg = "1";
if (DataFarm.insert_piwiklog(userList, out tmpmsg))
{
tmpmsg = "2";
}
else
{
tmpmsg = "3";
}
userList = new List <UserVisitInfo>();
}
catch (Exception ex)
{
string sss = ex.Message;
}
}
});
}
}
Console.WriteLine(dt + "," + mi + "," + "finish." + uids.Distinct().Count() + "," + maxVisitID + "," + DateTime.Now);
#endregion
}
#endregion 结束循环
}
catch (Exception ex)
{
errr += ex.Message;
Console.WriteLine(ex.ToString());
}
userList = userList.Distinct().ToList();
if (userList.Count > 0 || true)
{
string msg = "";
Console.WriteLine(DateTime.Now + "," + pageType1 + "," + dt + "," + "update piwik_log DB.");
if (DataFarm.insert_piwiklog(userList, out msg))
{
Console.WriteLine(DateTime.Now + "," + pageType1 + "," + dt + "," + "update piwik_log_reffer DB.");
DataFarm.UpdateGaBaseData(pageType1, DateTime.Parse(dt), DateTime.Parse(dt));
Console.WriteLine(DateTime.Now + "," + pageType1 + "," + dt + "," + "finish.");
}
if (!string.IsNullOrEmpty(msg))
{
errr += msg;
}
}
writeLog(writeFile1, userList, errr);
}
int pageType2 = 32;
if (!DataFarm.ExistGaData(pageType2, DateTime.Parse(dt), DateTime.Parse(dt)))
{
List <Piwik_CustomerAction> customerActionList = new List <Piwik_CustomerAction>();
string url = "http://10.0.0.131:920/index.php?module=API&method=CustomVariables.getCustomVariables&format=JSON&idSite=1&period=day&date=" + dt + "&expanded=1&filter_limit=500&token_auth=453170c79e8f0ad5dcd1f0b2ce1ecf23";
string xml = Boodoll.PageBL.ProductSearch.ProductSearchBLL.GetHtml(url, System.Text.Encoding.UTF8);
try
{
xml = xml.Replace("\\u", "\\\\u");
Newtonsoft.Json.JavaScriptArray jsonObject = (Newtonsoft.Json.JavaScriptArray)Newtonsoft.Json.JavaScriptConvert.DeserializeObject(xml);
int count = jsonObject.Count;
for (int i = 0; i < count; i++)
{
JavaScriptObject qcount = (JavaScriptObject)jsonObject[i];
// qcount["actionDetails"];
int nb_actions = 0;
if (qcount.Keys.Contains("nb_uniq_visitors"))
{
nb_actions = Converter.ParseInt(qcount["nb_uniq_visitors"], 0);
}
else
{
nb_actions = Converter.ParseInt(qcount["nb_actions"], 0);
}
customerActionList.Add(
new Piwik_CustomerAction()
{
dt = DateTime.Parse(dt),
idsubdatatable = Converter.ParseInt(qcount["idsubdatatable"], 1),
label = UnicodeToString(qcount["label"].ToString()),
nb_actions = nb_actions
}
);
}
string msg = "";
if (DataFarm.insert_piwikCsAction(customerActionList, out msg))
{
DataFarm.UpdateGaBaseData(pageType2, DateTime.Parse(dt), DateTime.Parse(dt));
Console.WriteLine(DateTime.Now + "," + pageType2 + "," + dt + "," + "finish.");
}
}
catch (Exception ex)
{
string exx = ex.Message;
}
}
}
Console.WriteLine("数据生成完毕");
}
static void Main(string[] args)
{
// DMSService.DMSServiceClient dmsc=new RunPiwikData.DMSService.DMSServiceClient();
//string message=dmsc.FinishFsMain("66978", "1408", "0", "85038.9000", "0", "0", "0", "-100", "", "0", "85038.90");
//Console.WriteLine(message);
//string pp = "http://www.sogou.com/sogou?pid=Af11228-1464&query=\\u6bcd\\u5a74\\u4e4b\\u5bb6&p=50040111&sourceid=sugg&w=01015004&oq=muying&ri=1";
//string pp = "http://www.soso.com/q?unc=i400044&sc=web&bs=%C0%D6%D3%D1&ch=w.uf&num=10&w=%C4%B8%D3%A4%D6%AE%BC%D2";
//string pp = "http://www.baidu.com/s?wd=%E5%9F%BA%E8%AF%BA%E6%B5%A6%20%E4%BA%A7%E5%9C%B0&rsp=9&f=1&oq=%E6%83%A0%E6%AD%A5%E8%88%92%20%E5%9F%BA%E8%AF%BA%E6%B5%A6%20%E5%8E%82%E5%AE%B6&tn=srcindex_hao&ie=utf-8&rs_src=0";
//string pp = "http://www.sogou.com/sogou?query=%C4%B8%D3%A4%D6%AE%BC%D2&pid=5F0EF-5842&iv=6.2.0.7270";
//Uri u11 = new Uri(pp);
//foreach (string uss in u11.Segments)
//{
// Console.WriteLine(UrlDecode(uss));
//}
//Console.WriteLine(System.Web.HttpUtility.UrlDecode(pp, System.Text.Encoding.GetEncoding("GB2312")));
//Console.WriteLine(System.Web.HttpUtility.UrlDecode(pp, System.Text.Encoding.UTF8));
//Console.WriteLine(Microsoft.JScript.GlobalObject.decodeURI(pp));
//byte[] bt = Encoding.UTF8.GetBytes(pp);
//MemoryStream stm = new MemoryStream(bt);
//StreamReader stmr = new StreamReader(stm);
//string sssurl = stmr.ReadToEnd();
// Console.WriteLine(MyUrlDeCode(pp,null));
////Console.ReadLine();
////string pageURL = "http://www.google.com.hk/search?hl=zh-CN&source=hp&q=%E5%8D%9A%E6%B1%87%E6%95%B0%E7%A0%81&aq=f&aqi=g2&aql=&oq=&gs_rfai=";
//Console.WriteLine(GetNewString(pp, null, true));
//Console.ReadLine();
Dictionary <string, string> dicStrs = new Dictionary <string, string>();
using (OffBBhomeDataContext ctx = new OffBBhomeDataContext())
{
// ctx.ObjectTrackingEnabled = false;
// var log = ctx.piwik_log_reffers.Select(c => c.refferurl).Distinct().ToList();
var log = ctx.piwik_log_reffers.Where(c => c.refergbk == null).Distinct();
int count = 0;
int total = log.Count();
Console.WriteLine(total);
foreach (piwik_log_reffer l in log)
{
string tstr = l.refferurl.ToString();
string durl = tstr;
if (!string.IsNullOrEmpty(tstr) && tstr != "http://about:blank")
{
durl = GetNewString(tstr, null, true);
}
if (tstr.Length > 0)
{
dicStrs.Add(tstr, durl);
Console.WriteLine(l.Id + "," + durl);
l.refergbk = durl;
count++;
//if (tstr != durl)
//{
// Console.WriteLine(l.Id + "," + durl);
// l.refergbk = durl;
// count++;
//}
//else
// Console.WriteLine(l.Id);
if (count % 10000 == 0)
{
ctx.SubmitChanges();
string writeFile1 = string.Format("{0}\\result{1}.txt", System.Threading.Thread.GetDomain().BaseDirectory, DateTime.Now.ToFileTimeUtc().ToString());
writeLog(writeFile1, dicStrs);
dicStrs = new Dictionary <string, string>();
}
if (count == total)
{
ctx.SubmitChanges();
}
}
}
;
ctx.SubmitChanges();
writeLog(string.Format("{0}\\r{1}.txt", System.Threading.Thread.GetDomain().BaseDirectory, DateTime.Now.ToFileTimeUtc().ToString()), dicStrs);
dicStrs = new Dictionary <string, string>();
}
Console.ReadLine();
return;
string dt = "2013-5-18";
string token = "453170c79e8f0ad5dcd1f0b2ce1ecf23";
//数据读取完毕退出
Int64 maxVisitID = 0;
try
{
for (int mi = 0; mi < int.MaxValue; mi++)
{
#region 循环读取数据
// string url = "http://click.muyingzhijia.com/index.php?module=API&filter_limit=100&method=Live.getLastVisitsDetails&format=json&idSite=1&period=day&date=" + dtStr + "&expanded=1&token_auth=453170c79e8f0ad5dcd1f0b2ce1ecf23";
string url = "http://10.0.0.131:920/index.php?module=API&filter_limit=50&method=Live.getLastVisitsDetails&format=json&idSite=1&period=day&date=" + dt + "&expanded=1&token_auth=" + token;
if (maxVisitID > 0)
{
url = url + "&maxIdVisit=" + maxVisitID;
}
string xml = Boodoll.PageBL.ProductSearch.ProductSearchBLL.GetHtml(url, Encoding.GetEncoding("GB2312"));
xml = xml.Replace("\\u", "\\\\u");
Newtonsoft.Json.JavaScriptArray jsonObject = (Newtonsoft.Json.JavaScriptArray)Newtonsoft.Json.JavaScriptConvert.DeserializeObject(xml);
int count = jsonObject.Count();
for (int i = 0; i < count; i++)
{
JavaScriptObject qcount = (JavaScriptObject)jsonObject[i];
JavaScriptArray actionDetails = (JavaScriptArray)qcount["actionDetails"];
if (i == 0)
{
maxVisitID = Convert.ToInt64(qcount["idVisit"]);
}
else
{
maxVisitID = Convert.ToInt64(qcount["idVisit"]) > maxVisitID ? maxVisitID : Convert.ToInt64(qcount["idVisit"]);
}
if (actionDetails.Count > 0)
{
string userid = "";
string guid = "";
string referurl = "";
JavaScriptObject customVariables = null;
try
{
customVariables = (JavaScriptObject)qcount["customVariables"];
userid = ((new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customVariables)).ElementAt(0).Value)))).ElementAt(1).Value.ToString());
guid = (new Dictionary <string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary <string, object>(customVariables)).ElementAt(1).Value)))).ElementAt(1).Value.ToString();
actionDetails.ForEach(item =>
{
JavaScriptObject itemobject = (JavaScriptObject)item;
if (itemobject["type"].Equals("ecommerceAbandonedCart"))
{
//
Newtonsoft.Json.JavaScriptArray ItemDetails = (Newtonsoft.Json.JavaScriptArray)itemobject["itemDetails"];
ItemDetails.ForEach(tmp_item =>
{
Console.WriteLine(((JavaScriptObject)(tmp_item))["price"]);
Console.WriteLine(((JavaScriptObject)(tmp_item))["itemSKU"]);
Console.WriteLine(((JavaScriptObject)(tmp_item))["quantity"]);
// + [4] {[quantity, 1]} System.Collections.Generic.KeyValuePair<string,object>
// (new System.Collections.Generic.Mscorlib_DictionaryDebugView<string,object>(((Newtonsoft.Json.JavaScriptObject)((tmp_item))))).Items[0].Value "33202" object {string}
// Console.WriteLine((Newtonsoft.Json.JavaScriptArray)tmp_item["sku"]);
});
Console.WriteLine(ItemDetails.Count);
// var q = customerDetail.Values.First();
//string q1 = (new Dictionary<string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary<string, object>(ItemDetail)).ElementAt(0).Value)))).ElementAt(1).Value.ToString();
//string q2 = (new Dictionary<string, object>(((Newtonsoft.Json.JavaScriptObject)((new Dictionary<string, object>(ItemDetail)).ElementAt(0).Value)))).ElementAt(0).Value.ToString();
// (new Dictionary<string, object>(customerDetail));
//(new System.Collections.Generic.Mscorlib_DictionaryDebugView<string,object>(((Newtonsoft.Json.JavaScriptObject)((new System.Collections.Generic.Mscorlib_DictionaryDebugView<string,object>(customerDetail)).Items[0].Value)))).Items[1].Value
// (new System.Collections.Generic.Mscorlib_DictionaryDebugView<string,object>(((Newtonsoft.Json.JavaScriptObject)((new System.Collections.Generic.Mscorlib_DictionaryDebugView<string,object>(q)).Items[0].Value)))).Items[1].Value
}
});
Console.WriteLine(mi);
}
catch (Exception ex)
{
continue;
}
}
}
#endregion
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}