protected override PollResult OnPoll() { List <BsonDocument> resultList; DateTime limitDate = DateTime.Now.AddSeconds(-30); resultList = _collection .Find(Query.And( Query.GTE("ts", _checkpoint.LastCheckpoint), Query.LT("ts", limitDate))) .SetSortOrder(SortBy.Ascending("ts")) .Take(1000) .ToList(); if (resultList.Count == 0) { return(PollResult.Empty); } DateTime lastCheckpoint = resultList.Select(b => b["ts"].AsDateTime).Max(); var ready = resultList.Select(b => { try { b["_id"] = b["_id"].ToString(); //Fix string format to avoid problems with locales. DateTime timestamp = b["ts"].ToUniversalTime(); b["ts"] = timestamp.ToString(DateTimeFormatForNestQuery, CultureInfo.InvariantCulture); b["collection"] = Collection; b["mongo-server"] = Connection; b["source"] = Connection + "/" + Collection; var level = b["le"].AsString; String ttl = "30d"; if (level == "DEBUG") { ttl = "2d"; } else if (level == "INFO") { ttl = "5d"; } else if (level == "WARN") { ttl = "15d"; } var jsonString = b.ToJson(); var replaced = Regex.Replace(jsonString, "CSUUID\\(\"(?<csuid>.+?)\"\\)", "\"${csuid}\""); return((JObject)JsonConvert.DeserializeObject(replaced)); } catch (Exception ex) { _log.Error("Error converting mongo log: " + b.ToString(), ex); return(null); } }); var results = ready.Where(e => e != null).ToList(); //var request = new BulkRequest(); //request.Index = importer.Index; //request.Operations = new List<IBulkOperation>(); //foreach (var result in results) //{ // var obj = (JObject)JsonConvert.DeserializeObject(result); // var operation = new BulkIndexOperation<JObject>(obj); // operation.Id = obj["_id"].ToString(); // request.Operations.Add(new BulkIndexOperation<JObject>(obj)); //} //var retvalue = _client.Bulk(request); StringBuilder request = new StringBuilder(); foreach (var result in results) { var requestString = String.Format("{{ \"index\" : {{ \"_type\" : \"log\", \"_id\" : \"{0}\" }} }}", result["_id"].ToString()); request.AppendLine(requestString); request.AppendLine(result.ToString(Formatting.None)); } var response = new PollResult(); response.FullJsonForElasticBulkEndpoint = request.ToString(); response.HasMore = results.Count == 1000; response.Checkpoint = lastCheckpoint; response.Count = results.Count; return(response); }
static PollResult() { Empty = new PollResult(); }