public static async Task CopyDocuments(
ElasticCopySettings source,
string targetServerurl, string targetUsername, string targetPassword, string targetIndex,
DateTime starttime, DateTime endtime, long diff_ms,
Dictionary <string, string> extraFields)
{
Log($"Copying: {targetServerurl}/{targetUsername}/{new string('*', targetPassword.Length)}/{targetIndex ?? "<null>"}, " +
$"starttime: {starttime:yyyy-MM-dd HH:mm:ss.fff}, endtime: {endtime:yyyy-MM-dd HH:mm:ss.fff}, diffms: {diff_ms}");
string timestampfieldname = source.TimestampField;
var newDocuments = new List <ElasticBulkDocument>();
for (DateTime spanStart = starttime; spanStart < endtime; spanStart = spanStart.AddMinutes(2))
{
DateTime spanEnd = spanStart.AddMinutes(2) > endtime ? endtime : spanStart.AddMinutes(2);
Log($"time span: {spanStart:yyyy-MM-dd HH:mm:ss.fff} - {spanEnd:yyyy-MM-dd HH:mm:ss.fff}");
dynamic sourceDocuments = await Elastic.GetRowsAsync(source.SourceServerurl, source.SourceUsername, source.SourcePassword, source.SourceIndex,
source.ElasticFilterField, source.ElasticFilterValue,
timestampfieldname, spanStart, spanEnd);
if (sourceDocuments == null || sourceDocuments.Count == 0)
{
Log("Got no documents.");
continue;
}
Log($"Source document count: {sourceDocuments.Count}");
foreach (var sourceDocument in sourceDocuments)
{
dynamic jobject = new JObject(sourceDocument._source);
DateTime timestamp = jobject[timestampfieldname];
jobject[$"Rebased{timestampfieldname}"] = timestamp.AddMilliseconds(diff_ms).ToString("o");
foreach (var field in extraFields)
{
jobject[field.Key] = field.Value;
}
var bulkDocument = new ElasticBulkDocument
{
Index = GetIndexWithDate(targetIndex, timestamp) ?? sourceDocument._index,
Id = sourceDocument._id,
Document = jobject
};
newDocuments.Add(bulkDocument);
}
}
if (newDocuments.Count == 0)
{
Log("No documents to copy.");
return;
}
MakeSureDoublesAreDoubles(newDocuments.Select(d => d.Document).ToArray());
Log($"New document count: {newDocuments.Count}");
await Elastic.PutIntoIndex(targetServerurl, targetUsername, targetPassword, newDocuments.ToArray());
}