public void TestMillionData_group_in_memory_for_market30_in_redis()
{
SingletonBase<ConfigurableSet>.Instance["redis_read_write_servers"] = "127.0.0.1:6379";
SingletonBase<ConfigurableSet>.Instance["redis_readonly_servers"] = "127.0.0.1:6379";
_redisService = new RedisService();
PrepareDataForRedis();
var dt = DateTime.Now;
const int MAX_RETRIEVE = 500000;
// get from redis
var allKeys = _redisService.GetAllKeysFromHash(hash_key);
var finalValues = new List<MarketImeiItem>();
this.Items = new Dictionary<string, MarketImeiItem>();
if (allKeys.Count <= MAX_RETRIEVE)
{
finalValues = _redisService.GetValuesFromHash<MarketImeiItem>(hash_key, allKeys);
}
else
{
var times = (allKeys.Count / MAX_RETRIEVE) + 1;
for (int t = 0; t < times; t++)
{
var value = _redisService.GetValuesFromHash<MarketImeiItem>(hash_key, allKeys.Skip(t * MAX_RETRIEVE).Take(MAX_RETRIEVE).ToList());
if (value != null) finalValues.AddRange(value);
}
}
for (int i = 0; i < allKeys.Count; i++)
{
this.Items[allKeys[i]] = finalValues[i];
}
this.NewIMEI = _redisService.DiffSets(imei_key, "333333333333333");
var news = from i in NewIMEI
where this.Items.ContainsKey(i)
select new { i, FirmWareMode = Items[i].MD, Manufacturer = Items[i].CN };
var group = (from c in news
group c by new { c.FirmWareMode, c.Manufacturer } into grp
select new { grp.Key.FirmWareMode, grp.Key.Manufacturer, Count = grp.Count() }).ToList();
var total = 0;
foreach (var g in group)
{
total += g.Count;
//Console.WriteLine(g.FirmWareMode + "|" + g.Manufacturer + "|" + g.Count);
}
Console.WriteLine(total);
Console.WriteLine("Cost:" + (DateTime.Now - dt).TotalSeconds);
}