protected override PollResult OnPoll()
{
List <BsonDocument> resultList;
DateTime limitDate = DateTime.Now.AddSeconds(-30);
resultList = _collection
.Find(Query.And(
Query.GTE("ts", _checkpoint.LastCheckpoint),
Query.LT("ts", limitDate)))
.SetSortOrder(SortBy.Ascending("ts"))
.Take(1000)
.ToList();
if (resultList.Count == 0)
{
return(PollResult.Empty);
}
DateTime lastCheckpoint = resultList.Select(b => b["ts"].AsDateTime).Max();
var ready = resultList.Select(b =>
{
try
{
b["_id"] = b["_id"].ToString();
//Fix string format to avoid problems with locales.
DateTime timestamp = b["ts"].ToUniversalTime();
b["ts"] = timestamp.ToString(DateTimeFormatForNestQuery, CultureInfo.InvariantCulture);
b["collection"] = Collection;
b["mongo-server"] = Connection;
b["source"] = Connection + "/" + Collection;
var level = b["le"].AsString;
String ttl = "30d";
if (level == "DEBUG")
{
ttl = "2d";
}
else if (level == "INFO")
{
ttl = "5d";
}
else if (level == "WARN")
{
ttl = "15d";
}
var jsonString = b.ToJson();
var replaced = Regex.Replace(jsonString, "CSUUID\\(\"(?<csuid>.+?)\"\\)", "\"${csuid}\"");
return((JObject)JsonConvert.DeserializeObject(replaced));
}
catch (Exception ex)
{
_log.Error("Error converting mongo log: " + b.ToString(), ex);
return(null);
}
});
var results = ready.Where(e => e != null).ToList();
//var request = new BulkRequest();
//request.Index = importer.Index;
//request.Operations = new List<IBulkOperation>();
//foreach (var result in results)
//{
// var obj = (JObject)JsonConvert.DeserializeObject(result);
// var operation = new BulkIndexOperation<JObject>(obj);
// operation.Id = obj["_id"].ToString();
// request.Operations.Add(new BulkIndexOperation<JObject>(obj));
//}
//var retvalue = _client.Bulk(request);
StringBuilder request = new StringBuilder();
foreach (var result in results)
{
var requestString = String.Format("{{ \"index\" : {{ \"_type\" : \"log\", \"_id\" : \"{0}\" }} }}", result["_id"].ToString());
request.AppendLine(requestString);
request.AppendLine(result.ToString(Formatting.None));
}
var response = new PollResult();
response.FullJsonForElasticBulkEndpoint = request.ToString();
response.HasMore = results.Count == 1000;
response.Checkpoint = lastCheckpoint;
response.Count = results.Count;
return(response);
}
static PollResult()
{
Empty = new PollResult();
}
