// protected override void LogItemToConsole(JsonDocumentMergerQueueItem workItem) // { // // don't log here // } /// <summary> /// The add to json object. /// </summary> /// <param name="queryId"> /// The query id. /// </param> /// <param name="id"> /// The id. /// </param> /// <param name="propertyName"> /// The property name. /// </param> /// <param name="newJObjects"> /// The new j objects. /// </param> /// <param name="batchNumber"> /// The batch number. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> private async Task AddToJsonObjectAsync(string queryId, string id, string propertyName, JObject[] newJObjects, int batchNumber) { // BlockIfMaxSizeReached(id); // lock on id so multiple threads cannot update the same document at the same time var semaphoreSlim = this.locks.GetOrAdd(id, s => new SemaphoreSlim(1, 1)); // Asynchronously wait to enter the Semaphore. If no-one has been granted access to the Semaphore, code execution will proceed, otherwise this thread waits here until the semaphore is released await semaphoreSlim.WaitAsync(); try { JObject document; if (!this.documentDictionary.ContainsKey(id)) { document = new JObject { { this.Config.TopLevelKeyColumn, id } }; var jsonObjectCacheItem = new JsonObjectQueueItem { BatchNumber = batchNumber, Id = id, Document = document }; this.documentDictionary.Add(id, jsonObjectCacheItem); Interlocked.Increment(ref this.numDocumentsModified); this.MyLogger.Verbose($"AddToJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}"); await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document); } else { document = this.documentDictionary.GetById(id).Document; this.MyLogger.Verbose($"UpdatedJsonObject: id:{id} _numDocumentsModified={this.numDocumentsModified:N0} _documentDictionary.Count={this.documentDictionary.Count:N0}"); await this.entityJsonWriter.SetPropertiesByMergeAsync(propertyName, newJObjects, document); } } finally { // When the task is ready, release the semaphore. It is vital to ALWAYS release the semaphore when we are ready, or else we will end up with a Semaphore that is forever locked. // This is why it is important to do the Release within a try...finally clause; program execution may crash or take a different path, this way you are guaranteed execution semaphoreSlim.Release(); } var minimum = SequenceBarrier.UpdateMinimumEntityIdProcessed(queryId, id); // Console.Write($"\r{LoggerName} Id:{id} Remaining: {_inQueue.Count:N0} queryId:{queryId} Minimum id:{minimum}"); this.MyLogger.Verbose($"Processed id: {id} for queryId:{queryId} Minimum id:{minimum}"); // AddDocumentsToOutQueue(minimum); // AddDocumentToOutputQueueByKey(id); }
/// <summary> /// The write object to json. /// </summary> /// <param name="wt"> /// The wt. /// </param> /// <returns> /// The <see cref="Task"/>. /// </returns> private async Task WriteObjectToJsonAsync(ConvertDatabaseToJsonQueueItem wt) { var id = this.GetId(wt); var jsonForRows = await this.entityJsonWriter.GetJsonForRowForMergeAsync( wt.Columns, wt.Rows, wt.PropertyName, wt.PropertyTypes); var path = Path.Combine(this.folder, wt.PropertyName ?? "main"); this.fileWriter.CreateDirectory(path); var sb = new StringBuilder(); foreach (var jsonForRow in jsonForRows) { sb.AppendLine(jsonForRow.ToString()); } await this.fileWriter.WriteToFileAsync(Path.Combine(path, $"{id}.json"), sb.ToString()); await this.AddToOutputQueueAsync( new JsonDocumentMergerQueueItem { BatchNumber = wt.BatchNumber, QueryId = wt.QueryId, Id = id, PropertyName = wt.PropertyName, //JoinColumnValue = workItem.JoinColumnValue, NewJObjects = jsonForRows }); var minimumEntityIdProcessed = SequenceBarrier.UpdateMinimumEntityIdProcessed(wt.QueryId, id); this.MyLogger.Verbose($"Add to queue: {id}"); this.CleanListIfNeeded(wt.QueryId, minimumEntityIdProcessed); }
private void WriteObjectToJson(ConvertDatabaseToJsonQueueItem wt) { var id = GetId(wt); var jsonForRows = GetJsonForRowForMerge(wt.Columns, wt.JsonValueWriter, wt.Rows, wt.PropertyName, wt.PropertyType); if (Config.WriteDetailedTemporaryFilesToDisk) { var path = Path.Combine(_folder, wt.PropertyName ?? "main"); Directory.CreateDirectory(path); var sb = new StringBuilder(); foreach (var jsonForRow in jsonForRows) { sb.AppendLine(jsonForRow.ToString()); } File.WriteAllText(Path.Combine(path, id), sb.ToString()); } AddToOutputQueue(new JsonDocumentMergerQueueItem { BatchNumber = wt.BatchNumber, QueryId = wt.QueryId, Id = id, PropertyName = wt.PropertyName, //JoinColumnValue = wt.JoinColumnValue, NewJObjects = jsonForRows }); var minimumEntityIdProcessed = SequenceBarrier.UpdateMinimumEntityIdProcessed(wt.QueryId, id); MyLogger.Trace($"Add to queue: {id}"); CleanListIfNeeded(wt.QueryId, minimumEntityIdProcessed); }
private void AddToJsonObject(string queryId, string id, string propertyName, JObject[] newJObjects, int batchNumber) { //BlockIfMaxSizeReached(id); // lock on id so multiple threads cannot update the same document at the same time lock (_locks.GetOrAdd(id, s => new object())) { JObject document; if (!_documentDictionary.ContainsKey(id)) { document = new JObject { { Config.TopLevelKeyColumn, id } }; var jsonObjectCacheItem = new JsonObjectQueueItem { BatchNumber = batchNumber, Id = id, Document = document }; _documentDictionary.Add(id, jsonObjectCacheItem); Interlocked.Increment(ref _numDocumentsModified); MyLogger.Trace($"AddToJsonObject: id:{id} _numDocumentsModified={_numDocumentsModified:N0} _documentDictionary.Count={_documentDictionary.Count:N0}"); SetPropertiesByMerge(propertyName, newJObjects, document); } else { document = _documentDictionary[id].Document; MyLogger.Trace($"UpdatedJsonObject: id:{id} _numDocumentsModified={_numDocumentsModified:N0} _documentDictionary.Count={_documentDictionary.Count:N0}"); SetPropertiesByMerge(propertyName, newJObjects, document); } } var minimum = SequenceBarrier.UpdateMinimumEntityIdProcessed(queryId, id); //QueueContext. // ProgressMonitor.SetProgressItem(new ProgressMonitorItem // { // LoggerName = LoggerName, // Id = id, // StepNumber = _stepNumber, // //QueryId = queryId, // InQueueCount = _inQueue.Count, // Minimum = minimum, // TimeElapsed = TimeSpan.Zero, // LastCompletedEntityIdForEachQuery = SequenceBarrier.LastCompletedEntityIdForEachQuery.ToList(), // DocumentDictionaryCount = _documentDictionary.Count, // TotalItemsProcessed = _totalItemsProcessed, // TotalItemsAddedToOutputQueue = _totalItemsAddedToOutputQueue, // }); //Console.Write($"\r{LoggerName} Id:{id} Remaining: {_inQueue.Count:N0} queryId:{queryId} Minimum id:{minimum}"); MyLogger.Trace($"Processed id: {id} for queryId:{queryId} Minimum id:{minimum}"); // AddDocumentsToOutQueue(minimum); //AddDocumentToOutputQueueByKey(id); }