/// <summary>
/// 基于Linq的查询
/// </summary>
/// <param name="filter">过滤器</param>
/// <returns></returns>
public List <Dictionary <string, object> > Find2(string dbName, string collectionName, string query, string protection = "{}", int pageIndex = 0, int pageSize = int.MaxValue, Dictionary <string, object> updateData = null)
{
List <Dictionary <string, object> > res = new List <Dictionary <string, object> >();
//var filterDict = Convertor.FromJsonToDict2(jsonString);
//var filterBuilder = Builders<BsonDocument>.Filter;
//var filter = this.FilterConvertor(jsonString);
FilterDefinition <BsonDocument> filter = query;
ProjectionDefinition <BsonDocument> protectionD = protection;
var db = this.client.GetDatabase(dbName);
var collection = db.GetCollection <BsonDocument>(collectionName);
var cursor = collection.Find(filter).Project(protectionD).Skip(pageIndex * pageSize).Limit(pageSize).ToCursor();
foreach (var document in cursor.ToEnumerable())
{
if (null == this.EventTraverse)
{
res.Add(document.ToDictionary());
}
else
{
EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary()));
}
}
return(res);
}
/// <summary>
/// 注冊事件委托回調方法
/// </summary>
/// <param name="callback"></param>
/// <returns></returns>
public bool RegisterDelegate(EventProc callback)
{
if (eventProc != null)
{
return(false);
}
eventProc = callback;
return(true);
}
/// <summary>
/// 以GZip的格式进行下载
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public string GetGzip2(string url, Encoding encoding, Dictionary <HttpRequestHeader, string> headers = null)
{
try
{
this.SetHeaders(headers);
byte[] byteArray = this.http.DownloadData(url);
// 处理 gzip
string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"];
if (sContentEncoding == "gzip")
{
var sourceStream = new MemoryStream(byteArray);
var targetStream = new MemoryStream();
int count = 0;
// 解压
GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress);
byte[] buf = new byte[512];
while ((count = gzip.Read(buf, 0, buf.Length)) > 0)
{
targetStream.Write(buf, 0, count);
}
var res = encoding.GetString(targetStream.GetBuffer());
sourceStream.Close();
targetStream.Close();
return(res);
}
else if (string.IsNullOrWhiteSpace(sContentEncoding))
{
var res = encoding.GetString(byteArray);
return(res);
}
else
{
var p = 0;
}
}
catch (Exception e)
{
var dict = new Dictionary <string, object>();
dict["Url"] = url;
dict["Exception"] = e.Message;
dict["CreateTime"] = DateTime.Now;
EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict));
Console.WriteLine("异常：{0}", e.Message);
}
return(string.Empty);
}
public string PostGZip(string url, string postData)
{
string res = string.Empty;
try
{
this.http.Headers.Clear();
this.http.Headers.Add("Accept-Encoding", "gzip,deflate");
this.http.Headers.Add("Accept", "application/json, text/javascript, */*; q=0.01");
this.http.Headers.Add("X-Requested-With", "XMLHttpRequest");
this.http.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
this.http.Headers.Add("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6");
this.http.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36");
byte[] byteArray = this.http.UploadData(url, Encoding.UTF8.GetBytes(postData));
// 处理 gzip
string sContentEncoding = this.http.ResponseHeaders["Content-Encoding"];
if (sContentEncoding == "gzip")
{
var sourceStream = new MemoryStream(byteArray);
var targetStream = new MemoryStream();
int count = 0;
// 解压
GZipStream gzip = new GZipStream(sourceStream, CompressionMode.Decompress);
byte[] buf = new byte[512];
while ((count = gzip.Read(buf, 0, buf.Length)) > 0)
{
targetStream.Write(buf, 0, count);
}
res = Encoding.UTF8.GetString(targetStream.GetBuffer());
sourceStream.Close();
targetStream.Close();
return(res);
}
}
catch (Exception e)
{
var dict = new Dictionary <string, object>();
dict["Url"] = url;
dict["Exception"] = e.Message;
dict["CreateTime"] = DateTime.Now;
EventProc.TriggerEvent(this.EventException, this, EventProcEventArgs.Create(dict));
}
return(string.Empty);
}
/// <summary>
/// 遍历处理
/// </summary>
/// <param name="dbName"></param>
/// <param name="tableName"></param>
/// <param name="jsonString"></param>
public void Traverse(string dbName, string tableName, string query)
{
var pageSize = 1000;
var index = 0;
var startTime = DateTime.Now;
var prevTime = startTime;
var list = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize);
var hasData = (0 < list.Count) ? true : false;
var queue = new Queue <List <Dictionary <string, object> > >();
queue.Enqueue(list);
while (0 < queue.Count)
{
var cost = DateTime.Now - prevTime;
list = queue.Dequeue();
foreach (var item in list)
{
EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(item));
}
var task = Task.Factory.StartNew <object>(() => {
prevTime = DateTime.Now;
Console.WriteLine(" 准备查找第{0}页数据 页面大小:{1} 上次耗时:{2}", index, pageSize, cost);
var resList = mongo.Find3(dbName, tableName, query, "{}", "{}", index++, pageSize);
return(resList);
});
var nextList = task.Result as List <Dictionary <string, object> >;
if (0 < nextList.Count)
{
queue.Enqueue(nextList);
Console.WriteLine(" 队列深度 {0} ", queue.Count);
}
else
{
Console.WriteLine(" 数据已经全部遍历完毕 ");
}
}
}
/// <summary>
/// 添加聚合计算
/// </summary>
/// <param name="dbName"></param>
/// <param name="collectionName"></param>
/// <param name="pipeline"></param>
/// <returns></returns>
public List <Dictionary <string, object> > Aggregate(string dbName, string collectionName, string[] pipeline)
{
List <Dictionary <string, object> > res = new List <Dictionary <string, object> >();
var pipelineDefinition = PipelineDefinition <BsonDocument, BsonDocument> .Create(pipeline);//"{$match:{}}", "{$group:{_id:'$CategoryName',total:{$sum:1}}}"
var db = this.client.GetDatabase(dbName);
var collection = db.GetCollection <BsonDocument>(collectionName);
var cursor = collection.Aggregate(pipelineDefinition);
foreach (var document in cursor.ToEnumerable())
{
if (null == this.EventTraverse)
{
res.Add(document.ToDictionary());
}
else
{
EventProc.TriggerEvent(this.EventTraverse, this, EventProcEventArgs.Create(document.ToDictionary()));
}
}
return(res);
}
/// <summary>
/// 准备数据
/// </summary>
protected void PrepareData()
{
EventProc.TriggerEvent(this.EventUrlEnqueue, this, null);
}
