private static void Exec(Options o)
{
Console.WriteLine($"XEL Console");
if (!File.Exists(o.InputFileName))
{
Console.WriteLine($"ERROR: Input file {o.InputFileName} not exists");
return;
}
Console.WriteLine($"Parsing {o.InputFileName} ...");
var outDir = Path.GetDirectoryName(o.OutputFileName);
if (!Directory.Exists(outDir))
{
Console.WriteLine($"ERROR: Output directory {outDir} not exists");
return;
}
using (var outputStream = File.CreateText(o.OutputFileName))
{
WriteFileStart(o, outputStream);
var xelStream = new XEFileEventStreamer(o.InputFileName);
xelStream.ReadEventStream(
xevent =>
{
switch (o.Format)
{
case ExportFormat.txt:
ExportTxt(xevent, outputStream);
break;
case ExportFormat.html:
ExportHtml(xevent, outputStream);
break;
case ExportFormat.json:
default:
ExportJson(xevent, outputStream);
break;
}
return(Task.CompletedTask);
},
CancellationToken.None).Wait();
WriteFileEnd(o, outputStream);
}
Console.WriteLine($"Exported as {o.Format} to {o.OutputFileName}");
}
public async Task <long> Count(CancellationToken cancellationToken)
{
long eventCount = 0;
foreach (var logFile in _logFilesWithData)
{
using (Stream logFileStream = new FileStream(logFile,
FileMode.Open, FileAccess.Read, FileShare.ReadWrite,
81920, false))
{
XEFileEventStreamer xeReader = new XEFileEventStreamer(logFileStream);
await xeReader.ReadEventStream((eventData) =>
{
eventCount++;
return(Task.CompletedTask);
}, CancellationToken.None);
}
}
return(eventCount);
}
public void Process()
{
if (!File.Exists(InputFilePath))
{
return;
}
string inputFileName = Path.GetFileName(InputFilePath);
Console.WriteLine($"{inputFileName}");
string extension = Path.GetExtension(InputFilePath);
var xeStream = new XEFileEventStreamer(InputFilePath);
xeStream.ReadEventStream(
xevent =>
{
Console.WriteLine(xevent);
Console.WriteLine("");
return(Task.CompletedTask);
},
CancellationToken.None).Wait();
}
static async Task Main(string[] args)
{
// Create a BlobServiceClient object which will be used to create a container client
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(containerName);
await foreach (BlobItem blobItem in containerClient.GetBlobsAsync())
{
var blobNameSplit = blobItem.Name.Split("/");
var blobDate = DateTime.Parse(blobNameSplit[3]);
// Structure based on servername/databasename/audit_name/date/filename.xel
if (
blobNameSplit[0] == serverName &&
blobNameSplit[1] == databaseName &&
blobItem.Name.EndsWith(".xel") &&
blobDate >= startDate &&
blobDate <= endDate
)
{
Console.WriteLine("\t" + blobItem.Name);
//Assuming we meet all the filtering criteria, parse and output to log analytics
BlobClient blobClient = containerClient.GetBlobClient(blobItem.Name);
BlobDownloadInfo download = await blobClient.DownloadAsync();
string downloadFilePath = "holdinglog.xel";
using (FileStream downloadFileStream = File.OpenWrite(downloadFilePath))
{
await download.Content.CopyToAsync(downloadFileStream);
downloadFileStream.Close();
}
List <IReadOnlyDictionary <string, object> > events = new List <IReadOnlyDictionary <string, object> >();
XEFileEventStreamer xelreader = new XEFileEventStreamer(downloadFilePath);
xelreader.ReadEventStream(
() =>
{
return(Task.CompletedTask);
},
xevent =>
{
events.Add(xevent.Fields);
return(Task.CompletedTask);
},
CancellationToken.None).Wait();
string json = JsonSerializer.Serialize(events);
var jsonBytesSize = Encoding.UTF8.GetBytes(json).Length;
if (jsonBytesSize > 30000000)
{
splitAndPost(events);
}
else
{
post(json);
}
File.Delete(downloadFilePath);
}
}
}
/// <summary>
/// Read a XEL file, consume all callstacks, hash them and return the equivalent XML
/// </summary>
/// <param name="xelFiles">List of paths to XEL files to read</param>
/// <returns>XML equivalent of the histogram corresponding to these events</returns>
public string ExtractFromXEL(string[] xelFiles, bool bucketize)
{
var callstackSlots = new Dictionary <string, long>();
var callstackRaw = new Dictionary <string, string>();
var xmlEquivalent = new StringBuilder();
// the below feels quite hacky. Unfortunately till such time that we have strong typing in XELite I believe this is necessary
var relevantKeyNames = new string[] { "callstack", "call_stack", "stack_frames" };
foreach (var xelFileName in xelFiles)
{
if (File.Exists(xelFileName))
{
var xeStream = new XEFileEventStreamer(xelFileName);
xeStream.ReadEventStream(
() =>
{
return(Task.CompletedTask);
},
evt =>
{
var allStacks = (from actTmp in evt.Actions
where relevantKeyNames.Contains(actTmp.Key.ToLower())
select actTmp.Value as string)
.Union(
from fldTmp in evt.Fields
where relevantKeyNames.Contains(fldTmp.Key.ToLower())
select fldTmp.Value as string);
foreach (var callStackString in allStacks)
{
if (string.IsNullOrEmpty(callStackString))
{
continue;
}
if (bucketize)
{
lock (callstackSlots)
{
if (!callstackSlots.ContainsKey(callStackString))
{
callstackSlots.Add(callStackString, 1);
}
else
{
callstackSlots[callStackString]++;
}
}
}
else
{
var evtId = string.Format("File: {0}, Timestamp: {1}, UUID: {2}:",
xelFileName,
evt.Timestamp.ToString("yyyy-MM-dd HH:mm:ss.mi"),
evt.UUID);
lock (callstackRaw)
{
callstackRaw.Add(evtId, callStackString);
}
}
}
return(Task.CompletedTask);
},
CancellationToken.None).Wait();
}
}
if (bucketize)
{
xmlEquivalent.AppendLine("<HistogramTarget truncated=\"0\" buckets=\"256\">");
foreach (var item in callstackSlots.OrderByDescending(key => key.Value))
{
xmlEquivalent.AppendFormat("<Slot count=\"{0}\"><value>{1}</value></Slot>", item.Value, item.Key);
xmlEquivalent.AppendLine();
}
xmlEquivalent.AppendLine("</HistogramTarget>");
}
else
{
foreach (var item in callstackRaw)
{
xmlEquivalent.AppendLine(item.Key);
xmlEquivalent.AppendLine(item.Value);
}
}
return(xmlEquivalent.ToString());
}
public async Task StartReadEvents(CancellationToken cancellationToken)
{
while (CurrentFile != null)
{
if (!InitializeReadFileStream())
{
return;
}
RaiseBeforeReadFileEvent(out bool cancelBeforeReadFile);
if (cancelBeforeReadFile)
{
NextFile();
await StartReadEvents(cancellationToken);
return;
}
_position = null;
_currentRow = null;
_currentFileEventNumber = 1;
var currentLogFile = _logFilesWithData[_indexCurrentFile];
FileInfo currentLogFileInfo = new FileInfo(currentLogFile);
try
{
XEFileEventStreamer xeReader = new XEFileEventStreamer(_stream);
await xeReader.ReadEventStream(() =>
{
_position = new ExtendedEventsPosition(
_currentFileEventNumber,
CurrentFile,
null,
null,
false,
currentLogFileInfo.CreationTimeUtc,
currentLogFileInfo.LastWriteTimeUtc);
RaiseOnReadMetadata(new OnReadMetadataArgs(_position));
return(Task.CompletedTask);
},
(eventData) =>
{
_position = new ExtendedEventsPosition(
_currentFileEventNumber,
CurrentFile,
eventData.UUID.ToString(),
eventData.Timestamp,
false,
currentLogFileInfo.CreationTimeUtc,
currentLogFileInfo.LastWriteTimeUtc);
_currentRow = new ExtendedEvent(_currentFileEventNumber, eventData);
RaiseOnRead(new OnReadEventArgs(_currentRow, _position, _currentFileEventNumber));
_currentFileEventNumber++;
return(Task.CompletedTask);
}, cancellationToken);
_currentRow = null;
if (_position != null)
{
_position = new ExtendedEventsPosition(
_position.EventNumber,
_position.CurrentFileData,
_position.EventUUID,
_position.EventPeriod,
true,
currentLogFileInfo.CreationTimeUtc,
currentLogFileInfo.LastWriteTimeUtc
);
}
RaiseOnRead(new OnReadEventArgs(_currentRow, _position, _currentFileEventNumber));
NextFile();
}
catch (Exception ex)
{
RaiseOnError(new OnErrorEventArgs(ex));
break;
}
}
}
