public void Test_KeyValue() { //STRING_KEY / STRING_VALUE == DATA CACHE //STRING_KEY / STRING_VALUE == TTL ABS|TTL SLI. // 239.8MB x 38 Sec. // 128MB (compression active) x 52 Sec. for (int i = 0; i < 100001; i++) { string k = Guid.NewGuid().ToString(); string v = k + " :: " + Properties.Settings.Default.Value_Text; //byte[] v = Utility.Deflate(Utility.Serialize(k + " :: " + Properties.Settings.Default.Value_Text), System.IO.Compression.CompressionMode.Compress); DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddItem("DATA_" + k, v); // add data dal.AddItem("TTL_" + k, dt_TTL_ABS_str + "|" + dt_TTL_SLI_str); // add ttl } }
public void Test_KeyValue() { //STRING_KEY / STRING_VALUE == DATA CACHE //STRING_KEY / STRING_VALUE == TTL ABS|TTL SLI. // 239.8MB x 38 Sec. // 128MB (compression active) x 52 Sec. for (int i = 0; i < 100001; i++) { string k = Guid.NewGuid().ToString(); string v = k + " :: " + Properties.Settings.Default.Value_Text; //byte[] v = Utility.Deflate(Utility.Serialize(k + " :: " + Properties.Settings.Default.Value_Text), System.IO.Compression.CompressionMode.Compress); DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0,0,10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddItem("DATA_" + k, v); // add data dal.AddItem("TTL_" + k, dt_TTL_ABS_str + "|" + dt_TTL_SLI_str); // add ttl } }
public void Test_HashBalanced() { //HASH / K / DATA == DATA CACHE // K / TTL == TTL ABS|TTL SLI // 245.5MB int k_hash = 0; for (int i = 0; i < 100001; i++) { if ((i % 50) == 0) { k_hash++; } string k = Guid.NewGuid().ToString(); string v = k + " :: " + Properties.Settings.Default.Value_Text; DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddHashItem(k_hash.ToString(), "DATA_" + k, v); dal.AddHashItem(k_hash.ToString(), "TTL_" + k, dt_TTL_ABS_str + "|" + dt_TTL_SLI_str); } }
public void Test_KeyValue_2() { //STRING_KEY / STRING_VALUE == TTL ABS|TTL SLI|DATA. // 228.8MB for (int i = 0; i < 100001; i++) { string k = Guid.NewGuid().ToString(); string v = k + " :: " + Properties.Settings.Default.Value_Text; DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddItem(k, dt_TTL_ABS_str + "|" + dt_TTL_SLI_str + "|" + v); // add data with TTL } }
/// <summary> /// 按手机号统计一段时间内短讯发送接口的调用此时 /// 时间单位:分钟 /// </summary> /// <param name="list">待处理数据</param> public void PhoneSendCounts(List <LogEntity> list) { lock (this) { RedisDal dal = new RedisDal(); //每个手机号每分钟调用指定接口的次数 List <Count> listPhoneCount = new List <Count>(); foreach (LogEntity l in list) { if (string.Compare(l.url, "send") == 0) { string minuteTime = l.time.ToString("yyyy/MM/dd HH:mm:00"); bool thisMinute = false; foreach (Count c in listPhoneCount) { string time = c.time.ToString("yyyy/MM/dd HH:mm:00"); if (string.Compare(time, minuteTime) == 0 && string.Compare(c.phone, l.phone) == 0) { c.count += 1; thisMinute = true; break; } } if (!thisMinute) { if (!string.IsNullOrEmpty(l.phone)) { listPhoneCount.Add(new Count(l.phone, Convert.ToDateTime(minuteTime), 1)); } } } } //将数据添加到redis中 dal.PhoneMinuteCount(listPhoneCount); dal.PhoneMinuteTopCount(listPhoneCount); dal.Dispose(); } }
public void Test_List() { //LIST / DATA == DATA CACHE // TTL == TTL ABS|TTL SLI // 243.4MB X 22 Sec. // 132.3MB X 45 Sec. (with compression) for (int i = 0; i < 100001; i++) { string k = Guid.NewGuid().ToString(); //string v = k + " :: " + Properties.Settings.Default.Value_Text; byte[] v = Utility.Deflate(Utility.Serialize(k + " :: " + Properties.Settings.Default.Value_Text), System.IO.Compression.CompressionMode.Compress); DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddListItem(k, v, dt_TTL_ABS_str + "|" + dt_TTL_SLI_str); } }
public void Test_Hash_2() { //HASH / K / DATA == DATA CACHE // K / TTL == TTL ABS|TTL SLI // ... // 259.3MB string k_hash = "123456"; for (int i = 0; i < 100001; i++) { string k = Guid.NewGuid().ToString(); string v = k + " :: " + Properties.Settings.Default.Value_Text; DateTime dt_TTL_ABS = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 10)); DateTime dt_TTL_SLI = DateTime.Now.ToUniversalTime().Add(new TimeSpan(0, 0, 5)); string dt_TTL_ABS_str = dt_TTL_ABS.ToString("yyMMddTHHmmss"); string dt_TTL_SLI_str = dt_TTL_SLI.ToString("yyMMddTHHmmss"); Redis.Cache.RedisDal dal = new RedisDal(); dal.AddHashItem(k, "DATA", v); dal.AddHashItem(k, "TTL", dt_TTL_ABS_str + "|" + dt_TTL_SLI_str); } }
public static void ListLine() { //开始本次服务时间计算 Stopwatch st = new Stopwatch(); st.Start(); logger.Info("=====开始读取数据====="); try { if (File.Exists(path)) { using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { //获取上次文件流结束位置(若旧流位置大于新流长度则视为新文件,流位置初始化) RedisDal dal = new RedisDal(); long oldPosition = dal.GetFilePosition(); if (oldPosition > fs.Length) { fs.Position = 0L; } else { fs.Position = oldPosition; } //每100行文本创建一个线程,最后余量创建一个线程 using (StreamReader sr = new StreamReader(fs, Encoding.UTF8)) { string line = string.Empty; List <string> listStr = new List <string>(); List <Task> listTask = new List <Task>(); long length = fs.Length; while (true) { line = sr.ReadLine(); if (line != null) { if (listStr.Count == 100) { dal.SetFilePosition(fs.Position.ToString()); AnalyticalArithmetic anal = new AnalyticalArithmetic(); anal.listStr = listStr.ToList(); Task t = new Task(() => anal.AllFun(123)); listTask.Add(t); t.Start(); listStr.Clear(); } listStr.Add(line); if (fs.Position == length) { break; } } else { break; } } dal.SetFilePosition(fs.Position.ToString()); dal.Dispose(); if (listStr.Count > 0) { AnalyticalArithmetic analytickal = new AnalyticalArithmetic(); analytickal = new AnalyticalArithmetic(); analytickal.listStr = listStr.ToList(); Task tt = new Task(() => analytickal.AllFun(1)); listTask.Add(tt); tt.Start(); listStr.Clear(); Console.WriteLine(listTask.Count); Task.WaitAll(listTask.ToArray()); } } logger.Info("=====本次服务结束====="); } } //打印本次服务所用时间 st.Stop(); TimeSpan ts = st.Elapsed; logger.Info("本地服务使用时间:" + ts); } catch (Exception ex) { Console.WriteLine(ex); logger.Error("服务发生异常:" + ex); } }
/// <summary> /// 从指定位置 /// 获取一百行数据 /// </summary> /// <param name="path">文件路径</param> /// <returns></returns> public static void ListLine() { //开始本次服务时间计算 Stopwatch st = new Stopwatch(); st.Start(); logger.Info("=====开始读取数据====="); try { if (File.Exists(path)) { using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { //设置线程池中最多有20个线程 ThreadPool.SetMaxThreads(20, 20); //获取上次文件流结束位置(若旧流位置大于新流长度则视为新文件,流位置初始化) RedisDal dal = new RedisDal(); long oldPosition = dal.GetFilePosition(); if (oldPosition > fs.Length) { fs.Position = 0L; } else { fs.Position = oldPosition; } //每100行文本创建一个线程,最后余量创建一个线程 using (StreamReader sr = new StreamReader(fs, Encoding.UTF8)) { string line = string.Empty; List <string> listStr = new List <string>(); long length = fs.Length; while (true) { line = sr.ReadLine(); if (line != null) { if (listStr.Count == 100) { dal.SetFilePosition(fs.Position.ToString()); AnalyticalArithmetic anal = new AnalyticalArithmetic(); anal.listStr = listStr.ToList(); // ThreadPool.QueueUserWorkItem(new WaitCallback(anal.AllFun)); listStr.Clear(); } listStr.Add(line); if (fs.Position == length) { break; } } else { break; } } dal.SetFilePosition(fs.Position.ToString()); dal.Dispose(); AnalyticalArithmetic analytickal = new AnalyticalArithmetic(); analytickal = new AnalyticalArithmetic(); analytickal.listStr = listStr.ToList(); // ThreadPool.QueueUserWorkItem(new WaitCallback(analytickal.AllFun)); } logger.Info("=====本次服务结束====="); } } //打印本次服务所用时间 st.Stop(); TimeSpan ts = st.Elapsed; logger.Info("本地服务使用时间:" + ts); } catch (Exception ex) { logger.Error("服务发生异常:" + ex); } }
/// <summary> /// 统计每家医院每个接口每秒的调用次数 /// 统计每家医院每个接口每天的调用次数 /// </summary> /// <param name="list">待处理数据</param> public void HospPhoneStatistics(List <LogEntity> list) { lock (this) { RedisDal dal = new RedisDal(); #region 用linq将数据分组处理 var result = list.GroupBy( e => e.url, (url, urlGroup) => new { url, hospGroups = urlGroup .GroupBy( e2 => e2.hospid, (hospid, hospGroup) => new { hospid, timeGroups = hospGroup .OrderBy(e3 => e3.time) .GroupBy(e3 => e3.time) .Select(g => new { time = g.Key, count = g.Count() }) } ).Select(e4 => new { hospid = e4.hospid, timeGroups = e4.timeGroups, timeGroupsCount = e4.timeGroups.Count() } ) } ).Select(s => new { url = s.url, hospGroups = s.hospGroups } ); #endregion //每家医院每个接口没秒的调用次数 List <Count> listSecondCount = new List <Count>(); //每家医院每个接口每天的调用次数 List <Count> listDayCount = new List <Count>(); //将linq结果添加到对应的集合中 foreach (var e in result) { foreach (var e2 in e.hospGroups) { foreach (var e3 in e2.timeGroups) { //将按秒分组的数据取出 判断集合中是否已经有当前类型数据,有则追加,无则添加 listSecondCount.Add(new Count(e.url, e2.hospid, e3.count, e3.time)); bool thisSecond = false; foreach (var c in listSecondCount) { if (c.time == e3.time && c.hospid == e2.hospid && c.url == e.url) { thisSecond = true; c.count += e3.count; break; } } if (!thisSecond) { listSecondCount.Add(new Count(e.url, e2.hospid, e3.count, e3.time)); } string daytime = e3.time.Year + "-" + e3.time.Month + "-" + e3.time.Day; //获取每个医院每个接口当天的调用次数 bool thisday = false; foreach (Count c in listDayCount) { if (c.daytime == daytime && c.hospid == e2.hospid && c.url == e.url) { thisday = true; c.count += e3.count; break; } } if (!thisday) { listDayCount.Add(new Count(e.url, e2.hospid, e3.count, daytime)); } } } } //将数据添加到redis中 dal.DayCount(listDayCount); dal.SecondCount(listSecondCount); dal.SecondTopCount(listSecondCount); dal.Dispose(); } }