static async Task BsonDocMethod()
{
var doc = new BsonDocument
{
{"name", "Sanya" }
};
doc.Add("age", 30);
doc["profession"] = "hacker";
var nestedArray = new BsonArray();
nestedArray.Add(new BsonDocument("color", "red"));
doc.Add("array", nestedArray);
Console.WriteLine(1);
//Console.WriteLine(doc.TryGetElement("name", out name));
Console.WriteLine(doc.Count());
Console.WriteLine();
Console.WriteLine(doc.ContainsValue("Sanya"));
Console.WriteLine(doc.Contains("name"));
Console.WriteLine(doc);
//var doc2 = new BsonDocument
//{
// { "Name", "Smith"},
// {"Something", "something else" },
// {"profession", "hacker" }
//};
//await collectionUsed.InsertManyAsync(new[] { doc, doc2 });
}
static async Task BsonDocMethod()
{
var doc = new BsonDocument
{
{ "name", "Sanya" }
};
doc.Add("age", 30);
doc["profession"] = "hacker";
var nestedArray = new BsonArray();
nestedArray.Add(new BsonDocument("color", "red"));
doc.Add("array", nestedArray);
Console.WriteLine(1);
//Console.WriteLine(doc.TryGetElement("name", out name));
Console.WriteLine(doc.Count());
Console.WriteLine();
Console.WriteLine(doc.ContainsValue("Sanya"));
Console.WriteLine(doc.Contains("name"));
Console.WriteLine(doc);
//var doc2 = new BsonDocument
//{
// { "Name", "Smith"},
// {"Something", "something else" },
// {"profession", "hacker" }
//};
//await collectionUsed.InsertManyAsync(new[] { doc, doc2 });
}
/// <summary>
/// Aggregation用的$project和$order
/// </summary>
/// <returns></returns>
public List<BsonDocument> GetAggregation()
{
//如果有改名的话，则其他没有改名的，也需要设定输出
var projectAggr = new BsonDocument();
var suppressAggr = new BsonDocument();
var project = new BsonDocument();
var suppress = new BsonDocument();
bool HasRename = false;
foreach (var item in _mQueryFieldList)
{
var ctl = ((CtlFieldInfo)Controls.Find(item.ColName, true)[0]).QueryFieldItem;
if (ctl.ColName == ConstMgr.KeyId)
{
//id
if (!ctl.IsShow)
{
//id抑制
suppress.Add(new BsonElement(ConstMgr.KeyId, 0));
}
}
else
{
//其他字段
if (ctl.IsShow)
{
if (string.IsNullOrEmpty(ctl.ProjectName))
{
project.Add(new BsonElement(ctl.ColName, 1));
}
else
{
project.Add(new BsonElement(ctl.ProjectName, "$" + ctl.ColName));
HasRename = true;
}
}
else
{
suppress.Add(new BsonElement(ctl.ColName, 0));
}
}
}
var aggrDocList = new List<BsonDocument>();
//如果有抑制操作和改名操作，则需要分开执行
suppressAggr.Add(new BsonElement("$project", suppress));
aggrDocList.Add(suppressAggr);
if (!HasRename && suppress.Count() == 0)
{
//没有改名和抑制的时候，则project全部去除
project.Clear();
}
projectAggr.Add(new BsonElement("$project", project));
aggrDocList.Add(projectAggr);
return aggrDocList;
}
public IQueryable <T> FilterByJSON(int ps, int cp, string sc, string sd, string fo)
{
IQueryable <T> items;
BsonDocument filters = new BsonDocument();
BsonDocument firstFilter = new BsonDocument();
BsonDocument finalFilter = new BsonDocument();
BsonDocument sort = new BsonDocument();
int sortOrder = (sd == "ASC") ? 1 : -1;
if (fo != null && fo != "{}")
{
var objFilter = (JObject)JsonConvert.DeserializeObject(fo);
var lstFilters = objFilter.Children().Cast <JProperty>().Select(j => new { Name = j.Name, Value = (string)j.Value }).Where(x => x.Value != null && x.Value != "").ToList();
if (lstFilters.Count > 0)
{
foreach (var f in lstFilters)
{
BsonDocument filter = new BsonDocument();
// Se for booleano testa true/false
Boolean boolValue;
int intValue;
if (Boolean.TryParse(f.Value, out boolValue))
{
filter.Add(f.Name, boolValue);
}
else if (Int32.TryParse(f.Value, out intValue))
{
filter.Add(f.Name, Convert.ToInt32(f.Value));
}
// Se for string testa com lowerCase e contains
else
{
if (f.Name.Substring(0, 2).ToLower() == "id")
{
filter.Add(f.Name, f.Value);
}
else
{
filter.Add(f.Name, new BsonRegularExpression(".*" + f.Value + ".*", "i"));
}
}
if (firstFilter.Count() == 0)
{
firstFilter.AddRange(filter);
}
else
{
filters.AddRange(filter);
}
}
finalFilter.AddRange(firstFilter);
finalFilter.AddRange(new BsonDocument("$and", new BsonArray().Add(filters)));
}
}
if (!String.IsNullOrEmpty(sc))
{
sort.Add(sc, sortOrder);
}
items = _collection.Find(finalFilter).Sort(sort).Skip(ps * (cp - 1)).Limit(ps).ToList().AsQueryable <T>();
return(items);
}
public BsonDocument foToFilter(string fo = null)
{
BsonDocument filters = new BsonDocument();
BsonDocument firstFilter = new BsonDocument();
BsonDocument finalFilter = new BsonDocument();
if (fo != null && fo != "{}")
{
var objFilter = (JObject)JsonConvert.DeserializeObject(fo);
var lstFilters = objFilter.Children().Cast <JProperty>().Select(j => new { Name = j.Name, Value = (string)j.Value }).Where(x => x.Value != null && x.Value != "").ToList();
if (lstFilters.Count > 0)
{
foreach (var f in lstFilters)
{
BsonDocument filter = new BsonDocument();
// Se for booleano testa true/false
Boolean boolValue;
int intValue;
if (Boolean.TryParse(f.Value, out boolValue))
{
filter.Add(f.Name, boolValue);
}
else if (f.Name.Substring(0, 4).ToLower() != "nome" && Int32.TryParse(f.Value, out intValue))
{
filter.Add(f.Name, Convert.ToInt32(f.Value));
}
else if (f.Name.Substring(0, 2).ToLower() == "dt")
{
if (f.Value != "")
{
//var dtValue = Convert.ToDateTime(f.Value).ToString("yyyy/MM/dd");
var dtIni = DateTime.Parse(f.Value.Substring(0, f.Value.Length - 9), new CultureInfo("en-US", true)).Date;
var dtEnd = dtIni.AddDays(1);
filter.AddRange(new BsonDocument(f.Name,
new BsonDocument {
{
"$gte", dtIni
}, {
"$lt", dtEnd
}
}));
}
}
// Se for string testa com lowerCase e contains
else
{
if (f.Name.Substring(0, 2).ToLower() == "id")
{
filter.Add(f.Name, f.Value);
}
else
{
filter.Add(f.Name, new BsonRegularExpression(".*" + f.Value + ".*", "i"));
}
}
if (firstFilter.Count() == 0)
{
firstFilter.AddRange(filter);
}
else
{
filters.AddRange(filter);
}
}
finalFilter.AddRange(firstFilter);
finalFilter.AddRange(new BsonDocument("$and", new BsonArray().Add(filters)));
}
}
return(finalFilter);
}
public int Count(string fo = null)
{
BsonDocument filters = new BsonDocument();
BsonDocument firstFilter = new BsonDocument();
BsonDocument finalFilter = new BsonDocument();
BsonDocument sort = new BsonDocument();
if (fo != null && fo != "{}")
{
var objFilter = (JObject)JsonConvert.DeserializeObject(fo);
var lstFilters = objFilter.Children().Cast <JProperty>().Select(j => new { Name = j.Name, Value = (string)j.Value }).Where(x => x.Value != null && x.Value != "").ToList();
if (lstFilters.Count > 0)
{
foreach (var f in lstFilters)
{
BsonDocument filter = new BsonDocument();
// Se for booleano testa true/false
Boolean boolValue;
int intValue;
if (Boolean.TryParse(f.Value, out boolValue))
{
filter.Add(f.Name, boolValue);
}
else if (Int32.TryParse(f.Value, out intValue))
{
filter.Add(f.Name, Convert.ToInt32(f.Value));
}
// Se for string testa com lowerCase e contains
else
{
if (f.Name.Substring(0, 2).ToLower() == "id")
{
filter.Add(f.Name, f.Value);
}
else
{
filter.Add(f.Name, new BsonRegularExpression(".*" + f.Value + ".*", "i"));
}
}
if (firstFilter.Count() == 0)
{
firstFilter.AddRange(filter);
}
else
{
filters.AddRange(filter);
}
}
finalFilter.AddRange(firstFilter);
finalFilter.AddRange(new BsonDocument("$and", new BsonArray().Add(filters)));
return(Convert.ToInt32(_collection.CountDocuments(finalFilter)));
}
else
{
return(Convert.ToInt32(_collection.EstimatedDocumentCount()));
}
}
else
{
return(Convert.ToInt32(_collection.EstimatedDocumentCount()));
}
}
public async Task <List <BsonDocument> > ConvertMongoToBsonAsync(List <BsonDocument> lstBson, BsonDocument parentSession, BsonDocument quetionIntent, BsonDocument customerInfo)
{
try
{
List <Task> tasks = new List <Task>();
var newList = new List <BsonDocument>();
BsonDocument newBson = new BsonDocument();
var template = new BsonDocument();
string business_id = "", template_id = "";
if (lstBson != null && lstBson.Count > 0 && lstBson[0].Contains("template_id"))
{
business_id = (string)lstBson[0]["business_id"];
template_id = (string)lstBson[0]["template_id"];
template = await _templatesRepository.GetById(business_id, template_id);
}
foreach (var bson in lstBson)
{
try
{
//tasks.Add(Task.Factory.StartNew(() =>
//{
var intent = "product";
var strData = bson.ToString().ToLower();
if (quetionIntent != null)
{
intent = quetionIntent["quetionintent"].ToString();
}
newBson = new BsonDocument();
if (template.GetValue(intent, null) != null)
{
#region get title
try
{
var str = "";
var title = template[intent]["reply"]["title"];
if (title != null && title != "")
{
var sp = title.ToString().Split(new string[] { "{{", "}}" }, StringSplitOptions.None);
var i = 0;
foreach (var r in sp)
{
if (i % 2 != 0)
{
var y = r.Split('.');
if (bson[y[0]][y[1]] != "")
{
str += bson[y[0]][y[1]];
}
}
else
{
if (sp[i] != "")
{
str += sp[i];
}
}
i++;
}
}
if (!newBson.Contains("messager"))
{
newBson.Add("messager", str);
}
else
{
newBson["messager"] = newBson["messager"].ToString() + " - 
 " + str;
}
}
catch (Exception ex) { }
#endregion
#region add image
try
{
var imageUrl = "";
var image = template[intent]["reply"]["image"].ToString();
if (image != "")
{
var sp = image.ToString().Split(new string[] { "{{", "}}" }, StringSplitOptions.None);
var i = 0;
foreach (var r in sp)
{
if (i % 2 != 0)
{
var y = r.Split('.');
if (bson[y[0]][y[1]] != "")
{
imageUrl = (bson[y[0]][y[1]].ToString());
}
}
i++;
}
}
if (!newBson.Contains("image"))
{
newBson.Add("image", imageUrl);
}
else
{
newBson["image"] = newBson["image"].ToString() + " - 
 " + imageUrl;
}
}
catch (Exception ex) { }
#endregion
#region add website link
try
{
if (!newBson.Contains("webdetailurl") && bson.Contains("websiteurl"))
{
var website = template[intent]["reply"]["website"].ToString();
if (website != "")
{
var sp = website.ToString().Split(new string[] { "{{", "}}" }, StringSplitOptions.None);
var i = 0;
foreach (var r in sp)
{
if (i % 2 != 0)
{
var y = r.Split('.');
if (bson[y[0]][y[1]] != "")
{
newBson.Add("webdetailurl", bson[y[0]][y[1]].ToString());
}
}
i++;
}
}
}
}
catch (Exception ex) { }
#endregion
if (!newBson.Contains("intent"))
{
newBson.Add("intent", parentSession != null ? parentSession["intent"] : "buy");
}
if (!newBson.Contains("last_message"))
{
newBson.Add("last_message", parentSession != null ? parentSession["last_message"] : "");
}
if (!newBson.Contains("using_full_search"))
{
newBson.Add("using_full_search", parentSession != null ? parentSession["defaultsearch"] : "1");
}
if (!newBson.Contains("currentsession"))
{
newBson.Add("currentsession", parentSession != null ? parentSession["product"] : "");
}
if (!newBson.Contains("reply_format"))
{
newBson.Add("reply_format", template[intent]["reply"]["format"]);
}
if (!newBson.Contains("product") && bson.Contains("product"))
{
newBson.Add("product", bson["product"]["value"]);
}
if (newBson.Count() > 0)
{
// thêm thay thông tin anh chị
try
{
var str = newBson["messager"].ToString();
if (str != "" && str.IndexOf(">>") > 0)
{
var sp = str.Split(new string[] { "<<", ">>" }, StringSplitOptions.None);
var i = 0;
var text = "";
foreach (var r in sp)
{
if (i % 2 != 0)
{
if (r != "")
{
text += customerInfo != null ? customerInfo[r] == "male" ? "anh" : "chị" : customerInfo == null && r == "sex" ? "anh/chị" : "";
}
}
else
{
if (r != "")
{
text += r;
}
}
i++;
}
newBson["messager"] = text;
}
newList.Add(newBson);
}
catch (Exception ex) { }
}
}
else
{
#region them truong hop check ton kho
if (intent == "inventory")
{
try
{
var url = _appSettings.BaseUrls.StockBalance;
var dic = new Dictionary <string, string>();
dic.Add("Id", bson["productid"]["value"].ToString());
dic.Add("accesstoken", "bazavietnam2017");
var para = JsonConvert.SerializeObject(dic);
var client = new HttpClient();
var response = client.PostAsync(url, new StringContent(para, Encoding.UTF8, "application/json")).Result;
var contents = response.Content.ReadAsStringAsync().Result;
var obj = JsonConvert.DeserializeObject <dynamic>(contents);
newBson.Add("messager", bson["attributes"]["desc"].ToString() + " - " +
(string)obj.StockInfo);
newBson.Add("image", bson["imageurl"]["value"].ToString());
if (!newBson.Contains("intent"))
{
newBson.Add("intent", parentSession != null ? parentSession["intent"] : "buy");
}
if (!newBson.Contains("last_message"))
{
newBson.Add("last_message", parentSession != null ? parentSession["last_message"] : "");
}
if (!newBson.Contains("using_full_search"))
{
newBson.Add("using_full_search", parentSession != null ? parentSession["defaultsearch"] : "1");
}
if (!newBson.Contains("currentsession"))
{
newBson.Add("currentsession", parentSession != null ? parentSession["product"] : "");
}
if (!newBson.Contains("reply_format"))
{
newBson.Add("reply_format", "text");
}
if (!newBson.Contains("product") && bson.Contains("product"))
{
newBson.Add("product", bson["product"]["value"]);
}
if (!newBson.Contains("webdetailurl"))
{
newBson.Add("webdetailurl", "");
}
newList.Add(newBson);
}
catch { }
}
#endregion
}
//}));
}
catch (Exception ex)
{
}
//Task.WaitAll(tasks.ToArray());
}
var t = newList.GroupBy(i => i["messager"], (key, group) => group.First()).ToArray().Distinct().ToList().GroupBy(i => i["messager"], (key, group) => group.First()).Distinct().ToList();
return(t);
}
catch (Exception ex) { return(null); }
}
public static void Input(string Endpoint, string[] Metadata)
{
// Initialize variables
FilterDefinition <BsonDocument> Filter;
UpdateDefinition <BsonDocument> Update;
DateTime Event;
// Connect database
var Client = new MongoClient("mongodb://" + ConfigurationManager.AppSettings["MongoDB.Username"] + ":" + ConfigurationManager.AppSettings["MongoDB.Password"] + "@" + ConfigurationManager.AppSettings["MongoDB.Server"] + ":" + ConfigurationManager.AppSettings["MongoDB.Port"]);
var Database = Client.GetDatabase(ConfigurationManager.AppSettings["MongoDB.Database"]);
// Select collection
var Collection = Database.GetCollection <BsonDocument>(Endpoint);
// Handle document
var Document = new BsonDocument();
// Register datapoint
if (Endpoint == "Datapoint")
{
// Validate input
Filter = Builders <BsonDocument> .Filter.Eq("IdNumber", Metadata[0]);
var Datapoints = Database.GetCollection <BsonDocument>("Datapoint").Find(Filter).ToList();
// Avoid duplicates
if (Datapoints.Count > 0)
{
throw new System.ArgumentException("Duplicate record");
}
// Count parameters
if (Metadata.Length < 2)
{
throw new System.ArgumentException("Missing parameters");
}
// Build location
var Location = new BsonDocument();
// Record event
Event = DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Utc);
Location.Add("Event", Event);
// Append coordinates
Location.Add("Coordinates", new BsonArray {
Metadata[1], Metadata[2]
});
// Append metadata
Document.Add("IdNumber", Metadata[0]);
Document.Add("Location", new BsonArray {
Location
});
if (Metadata.Length > 2)
{
Document.Add("Range", Metadata[3]);
}
// Insert document
Collection.InsertOne(Document);
}
// Register people
else if (Endpoint == "People")
{
// Validate input
Filter = Builders <BsonDocument> .Filter.Eq("IdNumber", Metadata[0]);
var People = Database.GetCollection <BsonDocument>("People").Find(Filter).ToList();
// Avoid duplicates
if (People.Count != 0)
{
throw new System.ArgumentException("Duplicate record");
}
// Count parameters
if (Metadata.Length < 4)
{
throw new System.ArgumentException("Missing parameters");
}
// Append common metadata
Document.Add("IdNumber", Metadata[0]);
Document.Add("Nationality", Metadata[1]);
Document.Add("FirstName", Metadata[2]);
// Append surname/s metadata
if (Metadata.Length == 5)
{
// Middle and last name
Document.Add("MiddleName", Metadata[3]);
Document.Add("LastName", Metadata[4]);
}
// Just last name
else
{
Document.Add("LastName", Metadata[3]);
}
// Insert document
Collection.InsertOne(Document);
}
// Register object
else if (Endpoint == "Object")
{
// Validate input
if (Metadata.Length < 2)
{
throw new System.ArgumentException("Missing parameters");
}
// Build ownership
var Ownership = new BsonDocument();
// Retrieve owners
Filter = Builders <BsonDocument> .Filter.Eq("IdNumber", Metadata[1]);
var Owners = Database.GetCollection <BsonDocument>("People").Find(Filter).ToList();
// Append metadata
if (Owners.Count != 0)
{
// Record event
Event = DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Utc);
Ownership.Add("Event", Event);
// Append first owner
Ownership.Add("ObjectId", Owners.First()["_id"]);
}
// Handle exceptions
else if (Metadata[1].Length == 15)
{
throw new System.ArgumentException("Invalid ID");
}
else if (Owners.Count == 0)
{
Ownership = null;
}
// Build attributes
var Attributes = new BsonDocument();
// Build tagcloud
var Tagcloud = new BsonDocument();
// Skip parameters
if (Ownership != null)
{
foreach (string Attribute in Metadata.Skip(2).ToArray())
{
Tagcloud.Add(Attribute.Split(":")[0], Attribute.Split(":")[1]);
}
}
// Append directly
else if (Ownership == null)
{
foreach (string Attribute in Metadata.Skip(1).ToArray())
{
Tagcloud.Add(Attribute.Split(":")[0], Attribute.Split(":")[1]);
}
}
// Append metadata
if (Tagcloud.Count() != 0)
{
// Record event
Event = DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Utc);
Attributes.Add("Event", Event);
// Append metadata
Attributes.Add("Tags", Tagcloud);
}
// Handle exceptions
else if (Tagcloud.Count() == 0)
{
Attributes = null;
}
// Handle object
Filter = Builders <BsonDocument> .Filter.Eq("IdNumber", Metadata[0]);
var Objects = Collection.Find(Filter).ToList();
// Create object
if (Objects.Count == 0)
{
// Append metadata
Document.Add("IdNumber", Metadata[0]);
if (Ownership != null)
{
Document.Add("Ownership", new BsonArray {
Ownership
});
}
if (Attributes != null)
{
Document.Add("Attributes", new BsonArray {
Attributes
});
}
// Insert document
Collection.InsertOne(Document);
}
// Update object
else if (Objects.Count > 0)
{
// Transfer ownership
if (Ownership != null)
{
Update = Builders <BsonDocument> .Update.Push("Ownership", Ownership);
Collection.UpdateOne(Filter, Update);
}
// Include attributes
if (Attributes != null)
{
Update = Builders <BsonDocument> .Update.Push("Attributes", Attributes);
Collection.UpdateOne(Filter, Update);
}
}
}
// Return signal
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine();
Console.WriteLine("Completed");
Console.WriteLine();
Console.ResetColor();
// Return menu
Program.Main(new string[0]);
}
BsonDocument WhereExpress(string key, object obj)
{
var rtn = new BsonDocument(true);
var filter = Builders <BsonDocument> .Filter;
if (key.StartsWith("$"))
{
if (key.ToLower() == "$or")
{
BsonDocument q = new BsonDocument(true);
if (obj is object[])
{
var aobj = (object[])obj;
foreach (var item in aobj)
{
if (item is FrameDLRObject)
{
var ditem = (FrameDLRObject)item;
foreach (var itemkey in ditem.Keys)
{
q.AddRange(filter.And(WhereExpress(itemkey, ditem.GetValue(itemkey))).ToBsonDocument());
}
}
}
}
//else if (obj is FrameDLRObject)
//{
// var ditem = (FrameDLRObject)obj;
// foreach (var itemkey in ditem.Keys)
// {
// l.AddRange(WhereExpress(itemkey, ditem.GetValue(itemkey)));
// }
//}
if (q.Count() > 0)
{
rtn.AddRange(filter.Or(q).ToBsonDocument());
}
}
else if (key.ToLower() == "$and")
{
BsonDocument q = new BsonDocument(true);
if (obj is object[])
{
var aobj = (object[])obj;
foreach (var item in aobj)
{
if (item is FrameDLRObject)
{
var ditem = (FrameDLRObject)item;
foreach (var itemkey in ditem.Keys)
{
q.AddRange(filter.And(WhereExpress(itemkey, ditem.GetValue(itemkey))).ToBsonDocument());
}
}
}
}
if (q.Count() > 0)
{
rtn.AddRange(filter.And(q).ToBsonDocument());
}
}
}
else
{
if (obj is FrameDLRObject)
{
var dobj = (FrameDLRObject)obj;
foreach (var k in dobj.Keys)
{
var vobj = dobj.GetValue(k);
switch (k.ToLower())
{
case "$eq":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime || obj is bool)
{
rtn.AddRange(filter.Eq(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$neq":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime || obj is bool)
{
rtn.AddRange(filter.Ne(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$lt":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime)
{
rtn.AddRange(filter.Lt(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$gt":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime)
{
rtn.AddRange(filter.Gt(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$lte":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime)
{
rtn.AddRange(filter.Lte(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$gte":
if (vobj is string || vobj is double || vobj is int || vobj is DateTime)
{
rtn.AddRange(filter.Gte(key, BsonValue.Create(dobj.GetValue(k))).ToBsonDocument());
}
break;
case "$like":
if (vobj is string)
{
rtn.AddRange(filter.Regex(key, "/" + vobj + "/").ToBsonDocument());
}
break;
case "$likel":
if (vobj is string)
{
rtn.AddRange(filter.Regex(key, "/^" + vobj + "/").ToBsonDocument());
}
break;
case "$liker":
if (vobj is string)
{
rtn.AddRange(filter.Regex(key, "/" + vobj + "$/").ToBsonDocument());
}
break;
case "$in":
List <BsonValue> inl = new List <BsonValue>();
if (vobj is object[])
{
var array = (object[])vobj;
foreach (var item in array)
{
if (item is string || item is int || item is double)
{
inl.Add(BsonValue.Create(item));
}
}
}
rtn.AddRange(filter.In(key, inl.ToArray()).ToBsonDocument());
break;
case "$nin":
List <BsonValue> ninl = new List <BsonValue>();
if (vobj is object[])
{
var array = (object[])vobj;
foreach (var item in array)
{
if (item is string || item is int || item is double)
{
ninl.Add(BsonValue.Create(item));
}
}
}
rtn.AddRange(filter.Not(filter.In(key, ninl.ToArray())).ToBsonDocument());
break;
default:
break;
}
}
}
else if (obj is string || obj is int || obj is double || obj is DateTime || obj is bool)
{
rtn.AddRange(new BsonDocument(key, BsonValue.Create(obj)));
}
}
return(rtn);
}
private static BsonDocument CorrectDateField(BsonDocument bsonDocument)
{
BsonDocument result = new BsonDocument();
int cpt = bsonDocument.Count();
for (int i = 0; i < cpt; i++)
{
var element = bsonDocument.ElementAt(i);
var value = element.Value;
var name = element.Name;
if(value is BsonDocument)
{
value =CorrectDateField(value as BsonDocument);
element = new BsonElement(name, value);
}
else
{
DateTime Out;
if (DateTime.TryParse(value.ToString(), out Out))
{
value = new BsonDateTime(Out);
element = new BsonElement(name, value);
}
}
result.Add(element);
}
return result;
}
public static async Task <int> GetNewsList(Task <int> imageSaverTask, BufferBlock <string> imageTargetBlock)
{
BufferBlock <string> newsDetailTargetBlock = new BufferBlock <string>();
int errorTime = 0;
var newsDetailPageGenerate = GetNewsDetail.GenerateNewsDetail(newsDetailTargetBlock, imageTargetBlock);
int pageNumber = DebugHelperTools.IsDebugMode() ? 2 : 255;
for (int page = 1; page < pageNumber; page++)
{
if (errorTime > 10)
{
Console.WriteLine("reach the limitation of error time");
break;
}
Console.WriteLine("starting process {0} page", page);
string pageURL = String.Format("{0}?category_id=9&page={1}&limit=0", GetIhChina.NewsListUrl, page);
var result = WebpageHelper.GetRequest(pageURL);
var jsonObject = JsonConvert.DeserializeObject <NewsListResponse>(result);
if (jsonObject.more != 1 && String.IsNullOrEmpty(jsonObject.data))
{
Console.WriteLine("GetNewsList: There is no more here, break from the loop");
break;
}
var doc = new HtmlDocument();
doc.LoadHtml(jsonObject.data);
var newsListNodes = from links in doc.DocumentNode.Descendants()
where links.Name == "div" && links.Attributes["class"] != null && links.Attributes["class"].Value == "list-item"
select links;
var newslistBsons = new List <BsonDocument>();
foreach (var node in newsListNodes)
{
var newsBson = new BsonDocument();
var titleNode = node.SelectSingleNode(".//div[@class='h16']/a");
if (titleNode != null)
{
var link = titleNode.Attributes["href"].Value;
if (MongodbChecker.CheckNewsExist(link))
{
Console.WriteLine("duplicated url: page {0}", link);
errorTime++;
continue;
}
newsDetailTargetBlock.Post(link);
newsBson.Add("link", link);
newsBson.Add("title", titleNode.Attributes["title"].Value);
}
var imgNode = node.SelectSingleNode(".//img");
if (imgNode != null)
{
var imgUrl = WebpageHelper.GetSubUrl(imgNode.Attributes["src"].Value);
newsBson.Add("img", imgUrl);
newsBson.Add("compressImg", WebImageSaver.Instance.GetComressImageName(imgUrl));
imageTargetBlock.Post(GetIhChina.MainPage + imgNode.Attributes["src"].Value);
}
var dataNode = node.SelectSingleNode(".//div[@class='date']/div");
if (dataNode != null)
{
newsBson.Add("date", dataNode.InnerText);
}
var contentNode = node.SelectSingleNode(".//div[@class='p']");
if (contentNode != null)
{
newsBson.Add("content", contentNode.InnerText);
}
if (newsBson.Count() != 0)
{
newslistBsons.Add(newsBson);
}
//每10条进行一次数据库插入，减少内存负担
if (newslistBsons.Count == 10)
{
MongodbSaver.SaveNewsList(newslistBsons);
for (int i = 0; i < 10; i++)
{
newslistBsons[i] = null;
}
newslistBsons.Clear();
}
}
if (newslistBsons.Count != 0)
{
MongodbSaver.SaveNewsList(newslistBsons);
}
}
newsDetailTargetBlock.Complete();
return(await newsDetailPageGenerate);
}
