// protected override void LogItemToConsole(JsonDocumentMergerQueueItem workItem)
// {
// // don't log here
// }
/// <summary>
/// The add to json object.
/// </summary>
/// <param name="queryId">
/// The query id.
/// </param>
/// <param name="id">
/// The id.
/// </param>
/// <param name="propertyName">
/// The property name.
/// </param>
/// <param name="newJObjects">
/// The new j objects.
/// </param>
/// <param name="batchNumber">
/// The batch number.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
private async Task AddToJsonObjectAsync(string queryId, string id, string propertyName, JObject[] newJObjects, int batchNumber)
{
// BlockIfMaxSizeReached(id);
// lock on id so multiple threads cannot update the same document at the same time
var semaphoreSlim = this.locks.GetOrAdd(id, s => new SemaphoreSlim(1, 1));
// Asynchronously wait to enter the Semaphore. If no-one has been granted access to the Semaphore, code execution will proceed, otherwise this thread waits here until the semaphore is released
await semaphoreSlim.WaitAsync();
try
{
JObject document;
if (!this.documentDictionary.ContainsKey(id))
{
document = new JObject {
{ this.Config.TopLevelKeyColumn, id }
};
var jsonObjectCacheItem = new JsonObjectQueueItem
{
BatchNumber = batchNumber,
Id = id,
Document = document
};
this.documentDictionary.Add(id, jsonObjectCacheItem);
Interlocked.Increment(ref this.numDocumentsModified);
this.MyLogger.Verbose($"AddToJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}");
await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document);
}
else
{
document = this.documentDictionary.GetById(id).Document;
this.MyLogger.Verbose($"UpdatedJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}");
await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document);
}
}
finally
{
// When the task is ready, release the semaphore. It is vital to ALWAYS release the semaphore when we are ready, or else we will end up with a Semaphore that is forever locked.
// This is why it is important to do the Release within a try...finally clause; program execution may crash or take a different path, this way you are guaranteed execution
semaphoreSlim.Release();
}
var minimum = SequenceBarrier.UpdateMinimumEntityIdProcessed(queryId, id);
// Console.Write($"\r{LoggerName} Id:{id} Remaining: {_inQueue.Count:N0} queryId:{queryId} Minimum id:{minimum}");
this.MyLogger.Verbose($"Processed id: {id} for queryId:{queryId} Minimum id:{minimum}");
// AddDocumentsToOutQueue(minimum);
// AddDocumentToOutputQueueByKey(id);
}
public async Task TestSuccess()
{
// Arrange
var job = new Job
{
Config = new QueryConfig
{
LocalSaveFolder = Path.GetTempPath(),
EntitiesPerUploadFile = 1
}
};
var queueManager = new QueueManager();
var logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateLogger();
using (var cancellationTokenSource = new CancellationTokenSource())
{
var createBatchItemsQueueProcessor = new CreateBatchItemsPipelineStep(
job.Config,
logger,
queueManager,
new MockProgressMonitor(),
cancellationTokenSource.Token);
var stepNumber = 1;
queueManager.CreateInputQueue <IJsonObjectQueueItem>(stepNumber);
createBatchItemsQueueProcessor.CreateOutQueue(stepNumber);
createBatchItemsQueueProcessor.InitializeWithStepNumber(stepNumber);
string queryId = "1";
var jsonObjectQueueItem1 = new JsonObjectQueueItem
{
Document = JObject.Parse(@"{test:'ff'}"),
PropertyName = "foo",
QueryId = queryId
};
await createBatchItemsQueueProcessor.InternalHandleAsync(jsonObjectQueueItem1);
var jsonObjectQueueItem2 = new JsonObjectQueueItem
{
Document = JObject.Parse(@"{test2:'ff'}"),
PropertyName = "foo2",
QueryId = queryId
};
// Act
await createBatchItemsQueueProcessor.InternalHandleAsync(jsonObjectQueueItem2);
// Assert
var queues = queueManager.Queues;
Assert.AreEqual(2, queues.Count);
var meteredBlockingCollection = queues.First(queue => queue.Key == "SaveBatchQueueItem2").Value;
var outputQueue = meteredBlockingCollection as IQueue <SaveBatchQueueItem>;
Assert.IsNotNull(outputQueue);
Assert.AreEqual(1, outputQueue.Count);
var saveBatchQueueItem = outputQueue.Take(cancellationTokenSource.Token);
Assert.AreEqual(0, outputQueue.Count);
Assert.AreEqual(1, saveBatchQueueItem.ItemsToSave.Count);
var jsonObjectQueueItem = saveBatchQueueItem.ItemsToSave.First();
Assert.AreEqual(jsonObjectQueueItem1, jsonObjectQueueItem);
Assert.AreEqual(jsonObjectQueueItem1.PropertyName, jsonObjectQueueItem.PropertyName);
// Act
// now complete the queue
createBatchItemsQueueProcessor.TestComplete(queryId, true);
// Assert
Assert.AreEqual(1, outputQueue.Count);
saveBatchQueueItem = outputQueue.Take(cancellationTokenSource.Token);
Assert.AreEqual(0, outputQueue.Count);
Assert.AreEqual(1, saveBatchQueueItem.ItemsToSave.Count);
jsonObjectQueueItem = saveBatchQueueItem.ItemsToSave.First();
Assert.AreEqual(jsonObjectQueueItem2, jsonObjectQueueItem);
Assert.AreEqual(jsonObjectQueueItem2.PropertyName, jsonObjectQueueItem.PropertyName);
}
}
