private FileObjectCollection PopulateCollection <T>(FileObject fileObject) where T : IRecord { FileObjectCollection collection = new FileObjectCollection() { fileObject }; _instance.TotalFilesFormatted++; _instance.TotalRecords += fileObject.RecordCount; if (Config.IsKustoConfigured()) { // kusto native format is Csv // kusto json ingest is 2 to 3 times slower and does *not* use standard json format. uses json document per line no comma // using csv and compression for best performance collection = SerializeCsv <T>(fileObject); if (Config.KustoCompressed) { collection.ForEach(x => x.Stream.Compress()); } } else if (Config.IsLogAnalyticsConfigured()) { // la is kusto based but only accepts non compressed json format ingest collection = SerializeJson <T>(fileObject); } collection.ForEach(x => SaveToCache(x)); return(collection); }
public FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord { Log.Debug("enter"); string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.JsonExtension.ToCharArray()); fileObject.FileUri = $"{sourceFile}{Constants.JsonExtension}"; FileObjectCollection collection = new FileObjectCollection(); string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + Constants.JsonExtension; if (fileObject.Length > Constants.MaxJsonTransmitBytes) { FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri) { Status = FileStatus.formatting }; _instance.FileObjects.Add(newFileObject); int counter = 0; foreach (T record in fileObject.Stream.Read <T>()) { record.RelativeUri = relativeUri; counter++; if (newFileObject.Length < Constants.WarningJsonTransmitBytes) { newFileObject.Stream.Write <T>(new List <T>(1) { record }, true); } else { collection.Add(newFileObject); record.RelativeUri = relativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + $".{counter}{Constants.JsonExtension}"; newFileObject = new FileObject(record.RelativeUri, fileObject.BaseUri) { Status = FileStatus.formatting }; _instance.FileObjects.Add(newFileObject); } } _instance.FileObjects.Remove(fileObject); newFileObject.FileUri = $"{sourceFile}.{counter}{Constants.JsonExtension}"; collection.Add(newFileObject); } else { collection.Add(fileObject); } Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}"); return(collection); }
private FileObjectCollection SerializeJson <T>(FileObject fileObject, IList <T> records) { Log.Debug("enter"); FileObjectCollection collection = new FileObjectCollection() { fileObject }; int counter = 0; string sourceFile = fileObject.FileUri.ToLower().Replace(JsonExtension, ""); fileObject.FileUri = $"{sourceFile}{JsonExtension}"; List <byte> jsonSerializedBytes = new List <byte>(); byte[] leftBracket = Encoding.UTF8.GetBytes("["); byte[] rightBracket = Encoding.UTF8.GetBytes("]"); byte[] comma = Encoding.UTF8.GetBytes(","); jsonSerializedBytes.AddRange(leftBracket); foreach (T record in records) { byte[] recordBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(record, new JsonSerializerSettings() { })); if (jsonSerializedBytes.Count + recordBytes.Length + rightBracket.Length > MaxJsonTransmitBytes) { jsonSerializedBytes.AddRange(rightBracket); fileObject.Stream.Set(jsonSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; jsonSerializedBytes.Clear(); fileObject = new FileObject($"{sourceFile}.{counter}{JsonExtension}", fileObject.BaseUri); jsonSerializedBytes.AddRange(leftBracket); Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}"); collection.Add(fileObject); } jsonSerializedBytes.AddRange(recordBytes); jsonSerializedBytes.AddRange(comma); counter++; } jsonSerializedBytes.RemoveRange(jsonSerializedBytes.Count - comma.Length, comma.Length); jsonSerializedBytes.AddRange(rightBracket); fileObject.Stream.Set(jsonSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}"); return(collection); }
public FileObjectCollection SerializeCsv <T>(FileObject fileObject) where T : IRecord { Log.Debug("enter"); FileObjectCollection collection = new FileObjectCollection() { fileObject }; int counter = 0; string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.CsvExtension.ToCharArray()); fileObject.FileUri = $"{sourceFile}{Constants.CsvExtension}"; List <byte> csvSerializedBytes = new List <byte>(); string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + Constants.CsvExtension; foreach (T record in fileObject.Stream.Read <T>()) { record.RelativeUri = relativeUri; byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString()); if (csvSerializedBytes.Count + recordBytes.Length > Constants.MaxCsvTransmitBytes) { record.RelativeUri = relativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + $".{counter}{Constants.CsvExtension}"; recordBytes = Encoding.UTF8.GetBytes(record.ToString()); fileObject.Stream.Set(csvSerializedBytes.ToArray()); csvSerializedBytes.Clear(); fileObject = new FileObject(record.RelativeUri, fileObject.BaseUri) { Status = FileStatus.formatting }; _instance.FileObjects.Add(fileObject); Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); collection.Add(fileObject); } csvSerializedBytes.AddRange(recordBytes); counter++; } fileObject.Stream.Set(csvSerializedBytes.ToArray()); Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); return(collection); }
private FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord { Log.Debug("enter"); string sourceFile = fileObject.FileUri.ToLower().TrimEnd(JsonExtension.ToCharArray()); fileObject.FileUri = $"{sourceFile}{JsonExtension}"; FileObjectCollection collection = new FileObjectCollection(); string relativeUri = fileObject.RelativeUri.TrimEnd(JsonExtension.ToCharArray()) + JsonExtension; if (fileObject.Length > MaxJsonTransmitBytes) { FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri); int counter = 0; foreach (T record in fileObject.Stream.Read <T>()) { record.RelativeUri = relativeUri; counter++; if (newFileObject.Length < WarningJsonTransmitBytes) { newFileObject.Stream.Write <T>(new List <T> { record }, true); } else { collection.Add(newFileObject); record.RelativeUri = relativeUri.TrimEnd(JsonExtension.ToCharArray()) + $".{counter}{JsonExtension}"; newFileObject = new FileObject(record.RelativeUri, fileObject.BaseUri); } } newFileObject.FileUri = $"{sourceFile}.{counter}{JsonExtension}"; collection.Add(newFileObject); } else { collection.Add(fileObject); } Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}"); return(collection); }
private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records) { Log.Debug("enter"); FileObjectCollection collection = new FileObjectCollection() { fileObject }; int counter = 0; string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, ""); fileObject.FileUri = $"{sourceFile}{CsvExtension}"; List <byte> csvSerializedBytes = new List <byte>(); foreach (T record in records) { byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString()); if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes) { fileObject.Stream.Set(csvSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; csvSerializedBytes.Clear(); fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri); Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); collection.Add(fileObject); } csvSerializedBytes.AddRange(recordBytes); counter++; } fileObject.Stream.Set(csvSerializedBytes.ToArray()); fileObject.Length = fileObject.Stream.Get().Length; Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}"); return(collection); }