private IList <CsvCounterRecord> ExtractPerfCsvData(FileObject fileObject)
{
List <CsvCounterRecord> csvRecords = new List <CsvCounterRecord>();
try
{
IList <string> allLines = fileObject.Stream.Read();
MatchCollection matchList = Regex.Matches(allLines[0], "\"(.+?)\"");
string[] headers = matchList.Cast <Match>().Select(match => match.Value).ToArray();
for (int csvRecordIndex = 1; csvRecordIndex < allLines.Count; csvRecordIndex++)
{
string[] counterValues = allLines[csvRecordIndex].Split(',');
for (int headerIndex = 1; headerIndex < headers.Length; headerIndex++)
{
if (counterValues.Length > headerIndex)
{
string stringValue = counterValues[headerIndex].Trim('"').Trim(' ');
if (!string.IsNullOrEmpty(stringValue))
{
try
{
csvRecords.Add(new CsvCounterRecord()
{
Timestamp = Convert.ToDateTime(counterValues[0].Trim('"').Trim(' ')),
CounterName = headers[headerIndex],
CounterValue = Decimal.Parse(stringValue, NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint),
NodeName = fileObject.NodeName,
FileType = fileObject.FileDataType.ToString(),
RelativeUri = fileObject.RelativeUri
});
}
catch (Exception ex)
{
Log.Exception($"stringValue:{stringValue} exception:{ex}");
}
}
}
}
}
return(csvRecords);
}
catch (Exception e)
{
Log.Exception($"{e}", csvRecords);
return(csvRecords);
}
}
private void SaveToCache(FileObject fileObject, bool force = false)
{
try
{
if (force || (!Config.UseMemoryStream & !fileObject.Exists))
{
fileObject.Stream.SaveToFile();
}
}
catch (Exception e)
{
Log.Exception($"{e}", fileObject);
}
}
public void SaveToCache(FileObject fileObject, bool force = false)
{
try
{
if (force || (!_config.UseMemoryStream && fileObject.FileUriType == FileUriTypesEnum.fileUri && !fileObject.Exists))
{
fileObject.Stream.SaveToFile();
}
}
catch (Exception e)
{
Log.Exception($"{e}", fileObject);
}
}
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null)
{
Match match = Regex.Match(traceRecord, @"/(?<Type>\w+?)(\.\w+?)?\.(?<PID>\d+?)\.dmp");
Timestamp = fileObject.LastModified.UtcDateTime;
PID = Convert.ToInt32(match.Groups["PID"].Value);
Type = match.Groups["Type"].Value;
Text = traceRecord;
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
private FileObjectCollection FormatTraceFile <T>(FileObject fileObject) where T : ITraceRecord, new()
{
Log.Debug($"enter:{fileObject.FileUri}");
IList <IRecord> records = new List <IRecord>();
// handles dtr, setup, and deployer file timestamp formats
string newEventPattern = @"^[0-9]{2,4}(-|/)[0-9]{1,2}(-|/)[0-9]{1,2}(-| )[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}";
Regex regex = new Regex(newEventPattern, RegexOptions.Compiled);
string record = string.Empty;
try
{
foreach (string tempLine in fileObject.Stream.ReadLine())
{
if (regex.IsMatch(tempLine))
{
// new record, write old record
if (record.Length > 0)
{
records.Add(new T().Populate(fileObject, record, Config.ResourceUri));
}
record = string.Empty;
}
record += tempLine;
}
// last record
if (record.Length > 0)
{
records.Add(new T().Populate(fileObject, record, Config.ResourceUri));
}
Log.Debug($"finished format:{fileObject.FileUri}");
fileObject.Stream.ResetPosition();
fileObject.Stream.Write(records);
return(PopulateCollection <DtrTraceRecord>(fileObject));
}
catch (Exception e)
{
Log.Exception($"file:{fileObject.FileUri} exception:{e}");
return(new FileObjectCollection()
{
fileObject
});
}
}
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null)
{
string[] fields = ParseRecord(traceRecord);
Timestamp = Convert.ToDateTime(fields[0].Replace("-", " "));
Level = fields[1];
PID = Convert.ToInt32(fields[2]);
Type = fields[3];
Text = fields[4];
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
public FileObjectCollection SerializeCsv <T>(FileObject fileObject) where T : IRecord
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.CsvExtension.ToCharArray());
fileObject.FileUri = $"{sourceFile}{Constants.CsvExtension}";
List <byte> csvSerializedBytes = new List <byte>();
string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + Constants.CsvExtension;
foreach (T record in fileObject.Stream.Read <T>())
{
record.RelativeUri = relativeUri;
byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());
if (csvSerializedBytes.Count + recordBytes.Length > Constants.MaxCsvTransmitBytes)
{
record.RelativeUri = relativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + $".{counter}{Constants.CsvExtension}";
recordBytes = Encoding.UTF8.GetBytes(record.ToString());
fileObject.Stream.Set(csvSerializedBytes.ToArray());
csvSerializedBytes.Clear();
fileObject = new FileObject(record.RelativeUri, fileObject.BaseUri)
{
Status = FileStatus.formatting
};
_instance.FileObjects.Add(fileObject);
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
csvSerializedBytes.AddRange(recordBytes);
counter++;
}
fileObject.Stream.Set(csvSerializedBytes.ToArray());
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}
public IRecord Populate(FileObject fileObject, string dtrRecord, string resourceUri = null)
{
string[] fields = ParseRecord(dtrRecord);
Timestamp = Convert.ToDateTime(fields[EtlInputFields.TimeStamp]);
Level = fields[EtlInputFields.Level];
TID = Convert.ToInt32(fields[EtlInputFields.TID]);
PID = Convert.ToInt32(fields[EtlInputFields.PID]);
Type = fields[EtlInputFields.Type];
Text = fields[EtlInputFields.Text];
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
private FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord
{
Log.Debug("enter");
string sourceFile = fileObject.FileUri.ToLower().TrimEnd(JsonExtension.ToCharArray());
fileObject.FileUri = $"{sourceFile}{JsonExtension}";
FileObjectCollection collection = new FileObjectCollection();
string relativeUri = fileObject.RelativeUri.TrimEnd(JsonExtension.ToCharArray()) + JsonExtension;
if (fileObject.Length > MaxJsonTransmitBytes)
{
FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri);
int counter = 0;
foreach (T record in fileObject.Stream.Read <T>())
{
record.RelativeUri = relativeUri;
counter++;
if (newFileObject.Length < WarningJsonTransmitBytes)
{
newFileObject.Stream.Write <T>(new List <T> {
record
}, true);
}
else
{
collection.Add(newFileObject);
record.RelativeUri = relativeUri.TrimEnd(JsonExtension.ToCharArray()) + $".{counter}{JsonExtension}";
newFileObject = new FileObject(record.RelativeUri, fileObject.BaseUri);
}
}
newFileObject.FileUri = $"{sourceFile}.{counter}{JsonExtension}";
collection.Add(newFileObject);
}
else
{
collection.Add(fileObject);
}
Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}");
return(collection);
}
private FileObjectCollection FormatCounterFile(FileObject fileObject)
{
Log.Debug($"enter:{fileObject.FileUri}");
string outputFile = fileObject.FileUri + PerfCsvExtension;
bool result;
fileObject.Stream.SaveToFile();
DeleteFile(outputFile);
Log.Info($"Writing {outputFile}");
if (Config.UseTx)
{
result = TxBlg(fileObject, outputFile);
}
else
{
result = RelogBlg(fileObject, outputFile);
}
if (result)
{
_instance.TotalFilesConverted++;
if (!Config.UseTx)
{
fileObject.Stream.ReadFromFile(outputFile);
fileObject.Stream.Write <CsvCounterRecord>(ExtractPerfRelogCsvData(fileObject));
}
}
else
{
_instance.TotalErrors++;
}
DeleteFile(outputFile);
if (Config.UseMemoryStream | !Config.IsCacheLocationPreConfigured())
{
DeleteFile(fileObject.FileUri);
}
return(PopulateCollection <CsvCounterRecord>(fileObject));
}
private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records)
{
Log.Debug("enter");
FileObjectCollection collection = new FileObjectCollection()
{
fileObject
};
int counter = 0;
string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, "");
fileObject.FileUri = $"{sourceFile}{CsvExtension}";
List <byte> csvSerializedBytes = new List <byte>();
foreach (T record in records)
{
byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());
if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes)
{
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
csvSerializedBytes.Clear();
fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri);
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
collection.Add(fileObject);
}
csvSerializedBytes.AddRange(recordBytes);
counter++;
}
fileObject.Stream.Set(csvSerializedBytes.ToArray());
fileObject.Length = fileObject.Stream.Get().Length;
Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
return(collection);
}
public IRecord Populate(FileObject fileObject, string traceRecord, string resourceUri = null)
{
Match match = Regex.Match(traceRecord, @"(?:/|^)(?<Type>\w+?)(?:\.+?){0,2}(?<PID>\d+?){0,1}\.dmp", RegexOptions.IgnoreCase);
int pid = 0;
string pidMatch = match.Groups["PID"]?.Value;
if (!string.IsNullOrEmpty(pidMatch))
{
pid = Convert.ToInt32(pidMatch);
}
Timestamp = fileObject.LastModified.UtcDateTime;
PID = pid;
Type = match.Groups["Type"]?.Value;
Text = traceRecord;
NodeName = fileObject.NodeName;
FileType = fileObject.FileDataType.ToString();
RelativeUri = fileObject.RelativeUri;
ResourceUri = resourceUri;
return(this);
}
public MemoryStream Compress(FileObject fileObject = null)
{
fileObject = fileObject ?? _fileObject;
if (_fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip))
{
string error = $"{_fileObject.FileUri} is already zip file";
Log.Error(error);
throw new ArgumentException();
}
Open();
_memoryStream.Position = 0;
Log.Debug($"compressing memoryStream start. start size: {fileObject.Length} dest size: {_memoryStream.Length} position: {_memoryStream.Position}");
fileObject.Length = _memoryStream.Length;
MemoryStream compressedStream = new MemoryStream();
using (ZipArchive archive = new ZipArchive(compressedStream, ZipArchiveMode.Create, true))
{
ZipArchiveEntry entry = archive.CreateEntry(Path.GetFileName(_fileObject.FileUri));
using (Stream archiveStream = entry.Open())
{
_memoryStream.CopyTo(archiveStream);
}
}
compressedStream.Position = 0;
_fileObject.FileUri += ZipExtension;
_fileObject.Stream.Set(compressedStream);
Log.Debug($"compressing memoryStream complete. size: {compressedStream.Length} position: {compressedStream.Position}");
return(compressedStream);
}
public bool TxBlg(FileObject fileObject, string outputFile)
{
// this forces blg output timestamps to use local capture timezone which is utc for azure
// Tx module is not able to determine with PDH api blg source timezone
TimeUtil.DateTimeKind = DateTimeKind.Unspecified;
DateTime startTime = DateTime.Now;
IObservable <PerformanceSample> observable = default(IObservable <PerformanceSample>);
TraceObserver <PerformanceSample> counterSession = default(TraceObserver <PerformanceSample>);
List <PerformanceSample> records = new List <PerformanceSample>();
List <CsvCounterRecord> csvRecords = new List <CsvCounterRecord>();
// testing pdh found invalid data when using concurrently
lock (_lockObj)
{
Log.Debug($"observable creating: {fileObject.FileUri}");
observable = PerfCounterObservable.FromFile(fileObject.FileUri);
Log.Debug($"observable created: {fileObject.FileUri}");
counterSession = ReadTraceRecords(observable);
Log.Debug($"finished total ms: {DateTime.Now.Subtract(startTime).TotalMilliseconds} reading: {fileObject.FileUri}");
records = counterSession.Records;
}
foreach (PerformanceSample record in records)
{
if (!string.IsNullOrEmpty(record.Value.ToString()))
{
string counterValue = record.Value.ToString() == "NaN" ? "0" : record.Value.ToString();
try
{
csvRecords.Add(new CsvCounterRecord()
{
Timestamp = record.Timestamp,
CounterName = record.CounterPath.Replace("\"", "").Trim(),
CounterValue = Decimal.Parse(counterValue, NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint),
Object = record.CounterSet?.Replace("\"", "").Trim(),
Counter = record.CounterName.Replace("\"", "").Trim(),
Instance = record.Instance?.Replace("\"", "").Trim(),
NodeName = fileObject.NodeName,
FileType = fileObject.FileDataType.ToString(),
RelativeUri = fileObject.RelativeUri
});
}
catch (Exception ex)
{
Log.Exception($"stringValue:{counterValue} exception:{ex}", record);
}
}
else
{
Log.Warning($"empty counter value:", record);
}
}
fileObject.Stream.Write(csvRecords);
Log.Info($"records: {records.Count()} {csvRecords.Count}");
return(true);
}
public DtrTraceRecord(string traceRecord, FileObject fileObject, string resourceUri = null)
{
Populate(fileObject, traceRecord, resourceUri);
}
private FileObjectCollection FormatTableFile(FileObject fileObject)
{
Log.Debug($"enter:{fileObject.FileUri}");
return(PopulateCollection <CsvTableRecord>(fileObject));
}
private FileObjectCollection FormatSetupFile(FileObject fileObject)
{
return(FormatTraceFile <CsvSetupRecord>(fileObject));
}
private FileObjectCollection FormatDtrFile(FileObject fileObject)
{
return(FormatTraceFile <DtrTraceRecord>(fileObject));
}
public CsvExceptionRecord(string traceRecord, FileObject fileObject, string resourceUri = null)
{
Populate(fileObject, traceRecord, resourceUri);
}
public FileObjectCollection ProcessFile(FileObject fileObject)
{
Log.Debug($"enter:{fileObject.FileUri}");
if (fileObject.DownloadAction != null)
{
Log.Info($"downloading:{fileObject.FileUri}", ConsoleColor.Cyan, ConsoleColor.DarkBlue);
_fileTasks.TaskAction(fileObject.DownloadAction).Wait();
Log.Info($"downloaded:{fileObject.FileUri}", ConsoleColor.DarkCyan, ConsoleColor.DarkBlue);
}
if (fileObject.DownloadAction != null && fileObject.Length < 1 && !fileObject.Exists)
{
string error = $"memoryStream does not exist and file does not exist {fileObject.FileUri}";
Log.Error(error);
throw new ArgumentException(error);
}
if (!fileObject.FileType.Equals(FileTypesEnum.any))
{
if (fileObject.Length < 1 & fileObject.Exists)
{
// for cached directory uploads
fileObject.Stream.ReadFromFile();
}
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.zip))
{
fileObject.Stream.Decompress();
SaveToCache(fileObject);
}
}
switch (fileObject.FileDataType)
{
case FileDataTypesEnum.unknown:
{
if (fileObject.FileType.Equals(FileTypesEnum.any))
{
SaveToCache(fileObject);
return(new FileObjectCollection());
}
Log.Warning("unknown file", fileObject);
break;
}
case FileDataTypesEnum.counter:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.blg))
{
return(FormatCounterFile(fileObject));
}
break;
}
case FileDataTypesEnum.data:
case FileDataTypesEnum.fabriccrashdumps:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dmp))
{
return(FormatExceptionFile(fileObject));
}
SaveToCache(fileObject);
return(new FileObjectCollection());
}
case FileDataTypesEnum.fabric:
case FileDataTypesEnum.lease:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.dtr))
{
return(FormatDtrFile(fileObject));
}
break;
}
case FileDataTypesEnum.bootstrap:
case FileDataTypesEnum.fabricsetup:
case FileDataTypesEnum.fabricdeployer:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.trace))
{
return(FormatSetupFile(fileObject));
}
break;
}
case FileDataTypesEnum.table:
{
if (fileObject.FileExtensionType.Equals(FileExtensionTypesEnum.csv))
{
return(FormatTableFile(fileObject));
}
break;
}
default:
{
Log.Warning("unsupported file data type", fileObject.FileDataType);
break;
}
}
Log.Warning($"returning: empty fileObjectCollection for file: {fileObject.FileUri}");
return(new FileObjectCollection());
}
public StreamManager(FileObject fileObject = null)
{
_fileObject = fileObject;
}
public IRecord Populate(FileObject fileObject, string dtrRecord, string resourceUri = null)
{
return(this as ITraceRecord);
}
public bool TxEtl(FileObject fileObject, string outputFile)
{
// this forces blg output timestamps to use local capture timezone which is utc for azure
// Tx module is not able to determine with PDH api blg source timezone
// todo: verify if needed for etl...
//TimeUtil.DateTimeKind = DateTimeKind.Unspecified;
DateTime startTime = DateTime.Now;
IObservable <EtwNativeEvent> observable = default(IObservable <EtwNativeEvent>);
TraceObserver <EtwNativeEvent> traceSession = default(TraceObserver <EtwNativeEvent>);
//List<EtwNativeEvent> records = new List<EtwNativeEvent>();
List <DtrTraceRecord> csvRecords = new List <DtrTraceRecord>();
// todo: verify if needed for etl...testing pdh found invalid data when using concurrently
// lock (_lockObj)
// {
Log.Debug($"observable creating: {fileObject.FileUri}");
observable = EtwObservable.FromFiles(fileObject.FileUri);
Log.Debug($"observable created: {fileObject.FileUri}");
traceSession = ReadTraceRecords(observable);
Log.Debug($"finished total ms: {DateTime.Now.Subtract(startTime).TotalMilliseconds} reading: {fileObject.FileUri}");
// records = traceSession.Records;
// }
//foreach (EtwNativeEvent record in records)
foreach (EtwNativeEvent record in traceSession.Records)
{
//Log.Info("record", record);
//if (!string.IsNullOrEmpty(record.Value.ToString()))
//{
// string counterValue = record.Value.ToString() == "NaN" ? "0" : record.Value.ToString();
// try
// {
// csvRecords.Add(new CsvCounterRecord()
// {
// Timestamp = record.Timestamp,
// CounterName = record.CounterPath.Replace("\"", "").Trim(),
// CounterValue = Decimal.Parse(counterValue, NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint),
// Object = record.CounterSet?.Replace("\"", "").Trim(),
// Counter = record.CounterName.Replace("\"", "").Trim(),
// Instance = record.Instance?.Replace("\"", "").Trim(),
// NodeName = fileObject.NodeName,
// FileType = fileObject.FileDataType.ToString(),
// RelativeUri = fileObject.RelativeUri
// });
// }
// catch (Exception ex)
// {
// Log.Exception($"stringValue:{counterValue} exception:{ex}", record);
// }
//}
//else
//{
// Log.Warning($"empty counter value:", record);
//}
}
fileObject.Stream.Write(csvRecords);
Log.Info($"records: {traceSession.Records.Count()} {csvRecords.Count}");
traceSession.Dispose();
return(true);
}