// protected override void LogItemToConsole(JsonDocumentMergerQueueItem workItem)
// {
// // don't log here
// }
/// <summary>
/// The add to json object.
/// </summary>
/// <param name="queryId">
/// The query id.
/// </param>
/// <param name="id">
/// The id.
/// </param>
/// <param name="propertyName">
/// The property name.
/// </param>
/// <param name="newJObjects">
/// The new j objects.
/// </param>
/// <param name="batchNumber">
/// The batch number.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
private async Task AddToJsonObjectAsync(string queryId, string id, string propertyName, JObject[] newJObjects, int batchNumber)
{
// BlockIfMaxSizeReached(id);
// lock on id so multiple threads cannot update the same document at the same time
var semaphoreSlim = this.locks.GetOrAdd(id, s => new SemaphoreSlim(1, 1));
// Asynchronously wait to enter the Semaphore. If no-one has been granted access to the Semaphore, code execution will proceed, otherwise this thread waits here until the semaphore is released
await semaphoreSlim.WaitAsync();
try
{
JObject document;
if (!this.documentDictionary.ContainsKey(id))
{
document = new JObject {
{ this.Config.TopLevelKeyColumn, id }
};
var jsonObjectCacheItem = new JsonObjectQueueItem
{
BatchNumber = batchNumber,
Id = id,
Document = document
};
this.documentDictionary.Add(id, jsonObjectCacheItem);
Interlocked.Increment(ref this.numDocumentsModified);
this.MyLogger.Verbose($"AddToJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}");
await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document);
}
else
{
document = this.documentDictionary.GetById(id).Document;
this.MyLogger.Verbose($"UpdatedJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}");
await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document);
}
}
finally
{
// When the task is ready, release the semaphore. It is vital to ALWAYS release the semaphore when we are ready, or else we will end up with a Semaphore that is forever locked.
// This is why it is important to do the Release within a try...finally clause; program execution may crash or take a different path, this way you are guaranteed execution
semaphoreSlim.Release();
}
var minimum = SequenceBarrier.UpdateMinimumEntityIdProcessed(queryId, id);
// Console.Write($"\r{LoggerName} Id:{id} Remaining: {_inQueue.Count:N0} queryId:{queryId} Minimum id:{minimum}");
this.MyLogger.Verbose($"Processed id: {id} for queryId:{queryId} Minimum id:{minimum}");
// AddDocumentsToOutQueue(minimum);
// AddDocumentToOutputQueueByKey(id);
}
/// <summary>
/// The write object to json.
/// </summary>
/// <param name="wt">
/// The wt.
/// </param>
/// <returns>
/// The <see cref="Task"/>.
/// </returns>
private async Task WriteObjectToJsonAsync(ConvertDatabaseToJsonQueueItem wt)
{
var id = this.GetId(wt);
var jsonForRows = await this.entityJsonWriter.GetJsonForRowForMergeAsync(
wt.Columns,
wt.Rows,
wt.PropertyName,
wt.PropertyTypes);
var path = Path.Combine(this.folder, wt.PropertyName ?? "main");
this.fileWriter.CreateDirectory(path);
var sb = new StringBuilder();
foreach (var jsonForRow in jsonForRows)
{
sb.AppendLine(jsonForRow.ToString());
}
await this.fileWriter.WriteToFileAsync(Path.Combine(path, $"{id}.json"), sb.ToString());
await this.AddToOutputQueueAsync(
new JsonDocumentMergerQueueItem
{
BatchNumber = wt.BatchNumber,
QueryId = wt.QueryId,
Id = id,
PropertyName = wt.PropertyName,
//JoinColumnValue = workItem.JoinColumnValue,
NewJObjects = jsonForRows
});
var minimumEntityIdProcessed = SequenceBarrier.UpdateMinimumEntityIdProcessed(wt.QueryId, id);
this.MyLogger.Verbose($"Add to queue: {id}");
this.CleanListIfNeeded(wt.QueryId, minimumEntityIdProcessed);
}
private void WriteObjectToJson(ConvertDatabaseToJsonQueueItem wt)
{
var id = GetId(wt);
var jsonForRows = GetJsonForRowForMerge(wt.Columns, wt.JsonValueWriter, wt.Rows, wt.PropertyName, wt.PropertyType);
if (Config.WriteDetailedTemporaryFilesToDisk)
{
var path = Path.Combine(_folder, wt.PropertyName ?? "main");
Directory.CreateDirectory(path);
var sb = new StringBuilder();
foreach (var jsonForRow in jsonForRows)
{
sb.AppendLine(jsonForRow.ToString());
}
File.WriteAllText(Path.Combine(path, id), sb.ToString());
}
AddToOutputQueue(new JsonDocumentMergerQueueItem
{
BatchNumber = wt.BatchNumber,
QueryId = wt.QueryId,
Id = id,
PropertyName = wt.PropertyName,
//JoinColumnValue = wt.JoinColumnValue,
NewJObjects = jsonForRows
});
var minimumEntityIdProcessed = SequenceBarrier.UpdateMinimumEntityIdProcessed(wt.QueryId, id);
MyLogger.Trace($"Add to queue: {id}");
CleanListIfNeeded(wt.QueryId, minimumEntityIdProcessed);
}
private void AddToJsonObject(string queryId, string id, string propertyName, JObject[] newJObjects, int batchNumber)
{
//BlockIfMaxSizeReached(id);
// lock on id so multiple threads cannot update the same document at the same time
lock (_locks.GetOrAdd(id, s => new object()))
{
JObject document;
if (!_documentDictionary.ContainsKey(id))
{
document = new JObject {
{ Config.TopLevelKeyColumn, id }
};
var jsonObjectCacheItem = new JsonObjectQueueItem
{
BatchNumber = batchNumber,
Id = id,
Document = document
};
_documentDictionary.Add(id, jsonObjectCacheItem);
Interlocked.Increment(ref _numDocumentsModified);
MyLogger.Trace($"AddToJsonObject: id:{id} _numDocumentsModified={_numDocumentsModified:N0} _documentDictionary.Count={_documentDictionary.Count:N0}");
SetPropertiesByMerge(propertyName, newJObjects, document);
}
else
{
document = _documentDictionary[id].Document;
MyLogger.Trace($"UpdatedJsonObject: id:{id} _numDocumentsModified={_numDocumentsModified:N0} _documentDictionary.Count={_documentDictionary.Count:N0}");
SetPropertiesByMerge(propertyName, newJObjects, document);
}
}
var minimum = SequenceBarrier.UpdateMinimumEntityIdProcessed(queryId, id);
//QueueContext.
// ProgressMonitor.SetProgressItem(new ProgressMonitorItem
// {
// LoggerName = LoggerName,
// Id = id,
// StepNumber = _stepNumber,
// //QueryId = queryId,
// InQueueCount = _inQueue.Count,
// Minimum = minimum,
// TimeElapsed = TimeSpan.Zero,
// LastCompletedEntityIdForEachQuery = SequenceBarrier.LastCompletedEntityIdForEachQuery.ToList(),
// DocumentDictionaryCount = _documentDictionary.Count,
// TotalItemsProcessed = _totalItemsProcessed,
// TotalItemsAddedToOutputQueue = _totalItemsAddedToOutputQueue,
// });
//Console.Write($"\r{LoggerName} Id:{id} Remaining: {_inQueue.Count:N0} queryId:{queryId} Minimum id:{minimum}");
MyLogger.Trace($"Processed id: {id} for queryId:{queryId} Minimum id:{minimum}");
// AddDocumentsToOutQueue(minimum);
//AddDocumentToOutputQueueByKey(id);
}