private FileObjectCollection PopulateCollection <T>(FileObject fileObject) where T : IRecord
{
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
_instance.TotalFilesFormatted++;
_instance.TotalRecords += fileObject.RecordCount;
if (Config.IsKustoConfigured())
{
// kusto native format is Csv
// kusto json ingest is 2 to 3 times slower and does *not* use standard json format. uses json document per line no comma
// using csv and compression for best performance
collection = SerializeCsv <T>(fileObject);
if (Config.KustoCompressed)
{
collection.ForEach(x => x.Stream.Compress());
}
}
else if (Config.IsLogAnalyticsConfigured())
{
// la is kusto based but only accepts non compressed json format ingest
collection = SerializeJson <T>(fileObject);
}
collection.ForEach(x => SaveToCache(x));
return(collection);
}
public FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord
{
Log.Debug("enter");
string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.JsonExtension.ToCharArray());
fileObject.FileUri = $"{sourceFile}{Constants.JsonExtension}";
FileObjectCollection collection = new FileObjectCollection();
string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + Constants.JsonExtension;
if (fileObject.Length > Constants.MaxJsonTransmitBytes)
{
FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri)
{
Status = FileStatus.formatting
};
_instance.FileObjects.Add(newFileObject);
int counter = 0;
foreach (T record in fileObject.Stream.Read <T>())
{
record.RelativeUri = relativeUri;
counter++;
if (newFileObject.Length < Constants.WarningJsonTransmitBytes)
{
newFileObject.Stream.Write <T>(new List <T>(1)
{
record
}, true);
}
else
{
collection.Add(newFileObject);
record.RelativeUri = relativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + $".{counter}{Constants.JsonExtension}";
newFileObject = new FileObject(record.RelativeUri, fileObject.BaseUri)
{
Status = FileStatus.formatting
};
_instance.FileObjects.Add(newFileObject);
}
}
_instance.FileObjects.Remove(fileObject);
newFileObject.FileUri = $"{sourceFile}.{counter}{Constants.JsonExtension}";
collection.Add(newFileObject);
}
else
{
collection.Add(fileObject);
}
Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}");
return(collection);
}
private FileObjectCollection SerializeJson <T>(FileObject fileObject, IList <T> records)
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().Replace(JsonExtension, "");
fileObject.FileUri = $"{sourceFile}{JsonExtension}";
List <byte> jsonSerializedBytes = new List <byte>();
byte[] leftBracket = Encoding.UTF8.GetBytes("[");
byte[] rightBracket = Encoding.UTF8.GetBytes("]");
byte[] comma = Encoding.UTF8.GetBytes(",");
jsonSerializedBytes.AddRange(leftBracket);
foreach (T record in records)
{
byte[] recordBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(record, new JsonSerializerSettings()
{
}));
if (jsonSerializedBytes.Count + recordBytes.Length + rightBracket.Length > MaxJsonTransmitBytes)
{
jsonSerializedBytes.AddRange(rightBracket);
fileObject.Stream.Set(jsonSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
jsonSerializedBytes.Clear();
fileObject = new FileObject($"{sourceFile}.{counter}{JsonExtension}", fileObject.BaseUri);
jsonSerializedBytes.AddRange(leftBracket);
Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
jsonSerializedBytes.AddRange(recordBytes);
jsonSerializedBytes.AddRange(comma);
counter++;
}
jsonSerializedBytes.RemoveRange(jsonSerializedBytes.Count - comma.Length, comma.Length);
jsonSerializedBytes.AddRange(rightBracket);
fileObject.Stream.Set(jsonSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}
public FileObjectCollection SerializeCsv <T>(FileObject fileObject) where T : IRecord
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.CsvExtension.ToCharArray());
fileObject.FileUri = $"{sourceFile}{Constants.CsvExtension}";
List <byte> csvSerializedBytes = new List <byte>();
string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + Constants.CsvExtension;
foreach (T record in fileObject.Stream.Read <T>())
{
record.RelativeUri = relativeUri;
byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());
if (csvSerializedBytes.Count + recordBytes.Length > Constants.MaxCsvTransmitBytes)
{
record.RelativeUri = relativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + $".{counter}{Constants.CsvExtension}";
recordBytes = Encoding.UTF8.GetBytes(record.ToString());
fileObject.Stream.Set(csvSerializedBytes.ToArray());
csvSerializedBytes.Clear();
fileObject = new FileObject(record.RelativeUri, fileObject.BaseUri)
{
Status = FileStatus.formatting
};
_instance.FileObjects.Add(fileObject);
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
csvSerializedBytes.AddRange(recordBytes);
counter++;
}
fileObject.Stream.Set(csvSerializedBytes.ToArray());
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}
private FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord
{
Log.Debug("enter");
string sourceFile = fileObject.FileUri.ToLower().TrimEnd(JsonExtension.ToCharArray());
fileObject.FileUri = $"{sourceFile}{JsonExtension}";
FileObjectCollection collection = new FileObjectCollection();
string relativeUri = fileObject.RelativeUri.TrimEnd(JsonExtension.ToCharArray()) + JsonExtension;
if (fileObject.Length > MaxJsonTransmitBytes)
{
FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri);
int counter = 0;
foreach (T record in fileObject.Stream.Read <T>())
{
record.RelativeUri = relativeUri;
counter++;
if (newFileObject.Length < WarningJsonTransmitBytes)
{
newFileObject.Stream.Write <T>(new List <T> {
record
}, true);
}
else
{
collection.Add(newFileObject);
record.RelativeUri = relativeUri.TrimEnd(JsonExtension.ToCharArray()) + $".{counter}{JsonExtension}";
newFileObject = new FileObject(record.RelativeUri, fileObject.BaseUri);
}
}
newFileObject.FileUri = $"{sourceFile}.{counter}{JsonExtension}";
collection.Add(newFileObject);
}
else
{
collection.Add(fileObject);
}
Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}");
return(collection);
}
private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records)
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, "");
fileObject.FileUri = $"{sourceFile}{CsvExtension}";
List <byte> csvSerializedBytes = new List <byte>();
foreach (T record in records)
{
byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());
if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes)
{
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
csvSerializedBytes.Clear();
fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri);
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
csvSerializedBytes.AddRange(recordBytes);
counter++;
}
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}