protected override Guid FlushBatch(List <RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
var results = _database.Batch(batch.Select(x =>
{
var metadata = x.Value <RavenJObject>("@metadata");
var key = metadata.Value <string>("@id");
x.Remove("@metadata");
return(new PutCommandData
{
Document = x,
Etag = null,
Key = key,
Metadata = metadata
});
}).ToArray());
ShowProgress("Wrote {0:#,#} documents in {1:#,#;;0} ms", batch.Count, sw.ElapsedMilliseconds);
batch.Clear();
if (results.Length == 0)
{
return(Guid.Empty);
}
return(results.Last().Etag.Value);
}
protected override int SetupData()
{
DocumentDatabase.Put("Raven/Hilo/users", null, new RavenJObject(), new RavenJObject(), null);
DocumentDatabase.Put("Raven/Hilo/posts", null, new RavenJObject(), new RavenJObject(), null);
DocumentDatabase.Batch(new[]
{
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "users/1",
Metadata = new RavenJObject(),
TransactionInformation = null
},
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "posts/1",
Metadata = new RavenJObject(),
TransactionInformation = null
},
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "posts/2",
Metadata = new RavenJObject(),
TransactionInformation = null
},
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "posts/3",
Metadata = new RavenJObject(),
TransactionInformation = null
},
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "posts/4",
Metadata = new RavenJObject(),
TransactionInformation = null
},
new PutCommandData
{
Document = new RavenJObject(),
Etag = null,
Key = "posts/5",
Metadata = new RavenJObject(),
TransactionInformation = null
},
});
return(8);
}
public BatchResult[] Batch(ICommandData[] commandDatas)
{
foreach (var commandData in commandDatas)
{
commandData.TransactionInformation = GetTransactionInformation();
}
return(database.Batch(commandDatas));
}
/// <summary>
/// Executed the specified commands as a single batch
/// </summary>
/// <param name="commandDatas">The command data.</param>
public BatchResult[] Batch(IEnumerable <ICommandData> commandDatas)
{
foreach (var commandData in commandDatas)
{
commandData.TransactionInformation = TransactionInformation;
}
CurrentOperationContext.Headers.Value = OperationsHeaders;
return(database.Batch(commandDatas));
}
public BatchResult[] Batch(ICommandData[] commandDatas)
{
foreach (var commandData in commandDatas)
{
commandData.TransactionInformation = RavenTransactionAccessor.GetTransactionInformation();
}
CurrentRavenOperation.Headers.Value = OperationsHeaders;
return(database.Batch(commandDatas));
}
/// <summary>
/// Executed the specified commands as a single batch
/// </summary>
/// <param name="commandDatas">The command data.</param>
public BatchResult[] Batch(IEnumerable <ICommandData> commandDatas)
{
CurrentOperationContext.Headers.Value = OperationsHeaders;
var batchResults = database.Batch(commandDatas.Select(cmd =>
{
cmd.TransactionInformation = TransactionInformation;
return(cmd);
}));
if (batchResults != null)
{
foreach (var batchResult in batchResults.Where(batchResult => batchResult != null && batchResult.Metadata != null && batchResult.Metadata.IsSnapshot))
{
batchResult.Metadata = (RavenJObject)batchResult.Metadata.CreateSnapshot();
}
}
return(batchResults);
}
protected override void FlushBatch(List <RavenJObject> batch)
{
var sw = Stopwatch.StartNew();
_database.Batch(batch.Select(x =>
{
var metadata = x.Value <RavenJObject>("@metadata");
var key = metadata.Value <string>("@id");
x.Remove("@metadata");
return(new PutCommandData
{
Document = x,
Etag = null,
Key = key,
Metadata = metadata
});
}).ToArray());
ShowProgress("Wrote {0} documents in {1}", batch.Count, sw.ElapsedMilliseconds);
ShowProgress(" in {0:#,#;;0} ms", sw.ElapsedMilliseconds);
batch.Clear();
}
public void GetDocumentAfterAnEtagWhileAddingDocsFromMultipleThreadsEnumeratesAllDocs()
{
var numberOfDocsAdded = 0;
var threads = new List <Thread>();
try
{
for (var i = 0; i < 10; i++)
{
var thread = new Thread(() =>
{
var cmds = new List <ICommandData>();
for (var k = 0; k < 100; k++)
{
var newId = Interlocked.Increment(ref numberOfDocsAdded);
cmds.Add(new PutCommandData
{
Document = new RavenJObject(),
Metadata = new RavenJObject(),
Key = newId.ToString()
});
}
;
db.Batch(cmds);
});
threads.Add(thread);
thread.Start();
}
var docs = new List <string>();
var lastEtag = Guid.Empty;
var total = 0;
var stop = false;
do
{
var etag = lastEtag;
var jsonDocuments = new JsonDocument[0];
db.TransactionalStorage.Batch(actions =>
{
jsonDocuments = actions.Documents.GetDocumentsAfter(etag).Where(x => x != null).ToArray();
});
docs.AddRange(jsonDocuments.Select(x => x.Key));
total += jsonDocuments.Length;
if (jsonDocuments.Length > 0)
{
lastEtag = jsonDocuments.Last().Etag.Value;
}
if (stop)
{
break;
}
if (threads.All(x => !x.IsAlive))
{
stop = true;
}
} while (true);
Assert.Equal(numberOfDocsAdded, total);
}
finally
{
foreach (var thread in threads)
{
thread.Join();
}
}
}
public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
{
var stopwatch = Stopwatch.StartNew();
var items = new List <ICommandData>(deletionBatchSize);
try
{
var query = new IndexQuery
{
Start = 0,
DisableCaching = true,
Cutoff = SystemTime.UtcNow,
PageSize = deletionBatchSize,
Query = $"LastModified:[* TO {expiryThreshold.Ticks}]",
FieldsToFetch = new[]
{
"__document_id",
},
SortedFields = new[]
{
new SortedField("LastModified")
{
Field = "LastModified",
Descending = false
}
}
};
var indexName = new ExpirySagaAuditIndex().IndexName;
database.Query(indexName, query, database.WorkContext.CancellationToken,
doc =>
{
var id = doc.Value <string>("__document_id");
if (string.IsNullOrEmpty(id))
{
return;
}
items.Add(new DeleteCommandData
{
Key = id
});
});
}
catch (OperationCanceledException)
{
//Ignore
}
var deletionCount = 0;
Chunker.ExecuteInChunks(items.Count, (s, e) =>
{
logger.InfoFormat("Batching deletion of {0}-{1} sagahistory documents.", s, e);
var results = database.Batch(items.GetRange(s, e - s + 1), CancellationToken.None);
logger.InfoFormat("Batching deletion of {0}-{1} sagahistory documents completed.", s, e);
deletionCount += results.Count(x => x.Deleted == true);
});
if (deletionCount == 0)
{
logger.Info("No expired sagahistory documents found");
}
else
{
logger.InfoFormat("Deleted {0} expired sagahistory documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
}
}
public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
{
var stopwatch = Stopwatch.StartNew();
var items = new List <ICommandData>(deletionBatchSize);
var attachments = new List <string>(deletionBatchSize);
try
{
var query = new IndexQuery
{
Start = 0,
PageSize = deletionBatchSize,
Cutoff = SystemTime.UtcNow,
DisableCaching = true,
Query = $"Status:[2 TO 4] AND LastModified:[* TO {expiryThreshold.Ticks}]",
FieldsToFetch = new[]
{
"__document_id",
"ProcessingAttempts[0].MessageId"
},
SortedFields = new[]
{
new SortedField("LastModified")
{
Field = "LastModified",
Descending = false
}
}
};
var indexName = new ExpiryErrorMessageIndex().IndexName;
database.Query(indexName, query, database.WorkContext.CancellationToken,
null,
doc =>
{
var id = doc.Value <string>("__document_id");
if (string.IsNullOrEmpty(id))
{
return;
}
items.Add(new DeleteCommandData
{
Key = id
});
var bodyid = doc.Value <string>("ProcessingAttempts[0].MessageId");
attachments.Add(bodyid);
});
}
catch (OperationCanceledException)
{
//Ignore
}
var deletionCount = 0;
Chunker.ExecuteInChunks(items.Count, (s, e) =>
{
logger.InfoFormat("Batching deletion of {0}-{1} error documents.", s, e);
var results = database.Batch(items.GetRange(s, e - s + 1));
logger.InfoFormat("Batching deletion of {0}-{1} error documents completed.", s, e);
deletionCount += results.Count(x => x.Deleted == true);
});
Chunker.ExecuteInChunks(attachments.Count, (s, e) =>
{
database.TransactionalStorage.Batch(accessor =>
{
logger.InfoFormat("Batching deletion of {0}-{1} attachment error documents.", s, e);
for (var idx = s; idx <= e; idx++)
{
accessor.Attachments.DeleteAttachment("messagebodies/" + attachments[idx], null);
}
logger.InfoFormat("Batching deletion of {0}-{1} attachment error documents completed.", s, e);
});
});
if (deletionCount == 0)
{
logger.Info("No expired error documents found");
}
else
{
logger.InfoFormat("Deleted {0} expired error documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
}
}
public static void Clean(int deletionBatchSize, DocumentDatabase database, DateTime expiryThreshold)
{
var stopwatch = Stopwatch.StartNew();
var items = new List <ICommandData>(deletionBatchSize);
var attachments = new List <string>(deletionBatchSize);
try
{
var query = new IndexQuery
{
Start = 0,
PageSize = deletionBatchSize,
Cutoff = SystemTime.UtcNow,
DisableCaching = true,
Query = $"ProcessedAt:[* TO {expiryThreshold.Ticks}]",
FieldsToFetch = new[]
{
"__document_id",
"MessageMetadata"
},
SortedFields = new[]
{
new SortedField("ProcessedAt")
{
Field = "ProcessedAt",
Descending = false
}
}
};
var indexName = new ExpiryProcessedMessageIndex().IndexName;
database.Query(indexName, query, database.WorkContext.CancellationToken,
doc =>
{
var id = doc.Value <string>("__document_id");
if (string.IsNullOrEmpty(id))
{
return;
}
items.Add(new DeleteCommandData
{
Key = id
});
string bodyId;
if (TryGetBodyId(doc, out bodyId))
{
attachments.Add(bodyId);
}
});
}
catch (OperationCanceledException)
{
//Ignore
}
var deletionCount = 0;
Chunker.ExecuteInChunks(items.Count, (s, e) =>
{
logger.InfoFormat("Batching deletion of {0}-{1} audit documents.", s, e);
var results = database.Batch(items.GetRange(s, e - s + 1), CancellationToken.None);
logger.InfoFormat("Batching deletion of {0}-{1} audit documents completed.", s, e);
deletionCount += results.Count(x => x.Deleted == true);
});
Chunker.ExecuteInChunks(attachments.Count, (s, e) =>
{
database.TransactionalStorage.Batch(accessor =>
{
logger.InfoFormat("Batching deletion of {0}-{1} attachment audit documents.", s, e);
for (var idx = s; idx <= e; idx++)
{
//We want to continue using attachments for now
#pragma warning disable 618
accessor.Attachments.DeleteAttachment(attachments[idx], null);
#pragma warning restore 618
}
logger.InfoFormat("Batching deletion of {0}-{1} attachment audit documents completed.", s, e);
});
});
if (deletionCount == 0)
{
logger.Info("No expired audit documents found");
}
else
{
logger.InfoFormat("Deleted {0} expired audit documents. Batch execution took {1}ms", deletionCount, stopwatch.ElapsedMilliseconds);
}
}