/// <summary> /// Parse the log file associated with this processor. /// </summary> /// <returns>True if at least one document was successfully parsed.</returns> public bool ProcessFile() { bool processedSuccessfully = false; using (var fs = new FileStream(logFile.FilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var reader = new StreamReader(fs)) { while (!parser.FinishedParsing) { // Parse a document. JObject jObject = parser.ParseLogDocument(reader); if (jObject != null) { BsonDocument bson = MongoJsonHelper.GetBsonDocument(jObject); long bsonDocumentSizeBytes = bson.ToBson().LongLength; if (!IsPersistableLogEntry(bson)) { continue; } // Check if we need to flush prior to inserting this document. if (insertionQueue.Count > 0 && insertionQueueByteCount + bsonDocumentSizeBytes > LogsharkConstants.MONGO_INSERTION_BATCH_SIZE) { processedSuccessfully = true; FlushInsertionQueue(); } // Add document to insertion queue. insertionQueue.Add(bson); insertionQueueByteCount += bsonDocumentSizeBytes; } } } // Flush any outstanding documents. if (insertionQueue.Count > 0) { processedSuccessfully = true; FlushInsertionQueue(); } WaitForInsertionThreadsToFinish(LogsharkConstants.MONGO_INSERTION_THREAD_POLL_INTERVAL, LogsharkConstants.MONGO_INSERTION_THREAD_TIMEOUT); return(processedSuccessfully); }
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { JToken jToken = JToken.FromObject(value); if (jToken.Type != JTokenType.Object) { jToken.WriteTo(writer, dateTimeConverter); } else { JObject jObject = (JObject)jToken; // Find any properties with illegal field names and replace them with well-formed properties to avoid an insert failure down the road. var propertiesWithIllegalNames = MongoJsonHelper.FindPropertiesWithIllegalNames(jObject); foreach (var propertyWithIllegalName in propertiesWithIllegalNames) { JProperty propertyWithLegalName = MongoJsonHelper.CreateLegalCopy(propertyWithIllegalName); propertyWithIllegalName.Replace(propertyWithLegalName); } jObject.WriteTo(writer, dateTimeConverter); } }