/// <summary>
/// Starts the processing data of the technology log
/// </summary>
/// <returns></returns>
private async Task ProcessTlData()
{
// Options of the many-threads blocks
// Limit a max bounded capacity to improve a consumption of memory
var parallelBlockOptions = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount, BoundedCapacity = 1000
};
using (var contextWriter = Common.GetOutputStream(CONTEXT_TEMP_FILEPATH))
using (var dbmssqlWriter = Common.GetOutputStream(DBMSSQL_TEMP_FILEPATH))
{
HashSet <(string eventName, ITargetBlock <string> nextBlock)> events = new HashSet <(string, ITargetBlock <string>)>();
// Blocks with the output streams for processed data
var contextOutputBlock = new ActionBlock <string>(async(text) => await Common.WriteToOutputStream(text, contextWriter));
var dbmssqlOutputBlock = new ActionBlock <string>(async(text) => await Common.WriteToOutputStream(text, dbmssqlWriter));
// Blocks of the processing data
var contextBlock = new ActionBlock <string>((text) => ProcessContextEvent(text, contextOutputBlock), parallelBlockOptions);
events.Add(("Context", contextBlock));
var dbmssqlBlock = new ActionBlock <string>((text) => ProcessDbmssqlEvent(text, dbmssqlOutputBlock), parallelBlockOptions);
events.Add(("DBMSSQL", dbmssqlBlock));
// Reading tech log block
var readBlock = new ActionBlock <string>((filePath) => TlHelper.ReadFile(events, filePath), parallelBlockOptions);
foreach (var file in TlHelper.GetLogFiles(_logcfg))
{
await readBlock.SendAsync(file);
}
// Mark block as completed
readBlock.Complete();
// Create relations between blocks (signals "complete" to the next blocks)
await readBlock.Completion.ContinueWith(c => dbmssqlBlock.Complete());
await dbmssqlBlock.Completion.ContinueWith(c => dbmssqlOutputBlock.Complete());
await readBlock.Completion.ContinueWith(c => contextBlock.Complete());
await contextBlock.Completion.ContinueWith(c => contextOutputBlock.Complete());
// Wait writing processed data to the temp files
await contextOutputBlock.Completion;
await dbmssqlOutputBlock.Completion;
}
}
/// <summary>
/// Initializes the _logcfg variable
/// </summary>
private void InitializeLogCfg()
{
_logcfg = new Logcfg(_settings);
_logcfg.AddLog(TlHelper.GetCollectPeriod(_settings.CollectPeriod).ToString(), _settings.TlFolder);
_logcfg.AddEvent(Logcfg.DBMSSQL_EV);
_logcfg.AddProperty(Logcfg.USER_PR);
_logcfg.AddProperty(Logcfg.CONTEXT_PR);
_logcfg.AddProperty(Logcfg.CONNECT_ID_PR);
_logcfg.AddProperty(Logcfg.SQL_PR);
_logcfg.AddProperty(Logcfg.CLIENT_ID_PR);
if (_settings.FilterByDatabase)
{
_logcfg.AddFilter(Logcfg.EQ_CT, Logcfg.PROCESS_NAME_PR, _settings.Database1C);
}
if (_settings.FilterByDuration)
{
_logcfg.AddFilter(Logcfg.GE_CT, Logcfg.DURATION_PR, _settings.Duration.ToString());
}
}
/// <summary>
/// Processes a Context event and send the result to the next block
/// </summary>
/// <param name="text">Event data</param>
/// <param name="targetBlock">Next block</param>
/// <returns></returns>
private async Task ProcessContextEvent(string text, ITargetBlock <string> targetBlock)
{
var clientId = await TlHelper.GetPropertyValue(text, Logcfg.CLIENT_ID_PR);
// If "client id" property is empty then it`s a system call, such event skipping
if (clientId == string.Empty)
{
return;
}
var dateTime = await TlHelper.GetPropertyValue(text, Logcfg.DATETIME_PR);
var context = await TlHelper.GetPropertyValue(text, Logcfg.CONTEXT_PR);
var firstLineContext = await TlHelper.GetFirstLineContext(context);
var lastLineContext = await TlHelper.GetLastLineContext(context);
var user = await TlHelper.GetPropertyValue(text, Logcfg.USER_PR);
var connectId = await TlHelper.GetPropertyValue(text, Logcfg.CONNECT_ID_PR);
var data =
Common.FS +
dateTime +
Common.FS +
user +
Common.FS +
connectId +
Common.FS +
clientId +
Common.FS +
firstLineContext +
Common.FS +
lastLineContext +
Common.RS;
await targetBlock.SendAsync(data);
}
/// <summary>
/// Run this analyzer
/// </summary>
/// <returns></returns>
public async Task Run()
{
try
{
Logger.Log("Start the queries analyze");
Logger.Log("Check settings");
await _settings.Check();
Logger.Log("Create the database");
await SqlHelper.CreateDatabase(_settings);
Logger.Log("Clean the folders");
Common.CleanFolder(_settings.TlFolder);
Common.CleanFolder(_settings.SqlTraceFolder);
Common.CleanFolder(_settings.TempFolder);
Logger.Log("Start the data collection");
await _session.Create();
await _logcfg.Write();
Logger.Log("Wait while the data collection will be started...");
await TlHelper.WaitStartCollectData(_logcfg);
await _session.Start();
Logger.Log("Wait while the data collection is going...");
// Wait while the data collection is going
await Task.Delay(_settings.CollectPeriod * 60 * 1000);
Logger.Log("Stop the data collection");
_logcfg.Delete();
await _session.Stop();
Logger.Log("Process the tech log data");
await ProcessTlData();
Logger.Log("Process the extended events data");
await ProcessXESessionData();
await _session.Delete();
Logger.Log("Insert data into the database");
await SqlHelper.InsertFileIntoTable(_settings, SQL_QUERIES_TABLENAME, SQL_QUERIES_TEMP_FILEPATH);
await SqlHelper.InsertFileIntoTable(_settings, CONTEXT_TABLENAME, CONTEXT_TEMP_FILEPATH);
await SqlHelper.InsertFileIntoTable(_settings, DBMSSQL_TABLENAME, DBMSSQL_TEMP_FILEPATH);
Logger.Log("Join contexts and queries");
await FillContext();
Logger.Log("Fill AVG table");
await FillQueriesAvgTable();
Logger.Log("Analyze was successfully completed");
}
catch (Exception ex)
{
Logger.Log(ex);
Logger.Log("Analyze was completed with errors");
throw new Exception();
}
}
/// <summary>
/// Processes a DBMSSQL event and send the result to the next block
/// </summary>
/// <param name="text">Event data</param>
/// <param name="targetBlock">Next block</param>
/// <returns></returns>
private async Task ProcessDbmssqlEvent(string text, ITargetBlock <string> targetBlock)
{
var user = await TlHelper.GetPropertyValue(text, Logcfg.USER_PR);
// If "usr" property is empty then skip this event
if (user == string.Empty)
{
return;
}
var sql = await TlHelper.GetPropertyValue(text, Logcfg.SQL_PR);
// If "sql" property is empty then skip this event
if (sql == string.Empty)
{
return;
}
var clientId = await TlHelper.GetPropertyValue(text, Logcfg.CLIENT_ID_PR);
// If "client id" property is empty then it`s a system call, such event skipping
if (clientId == string.Empty)
{
return;
}
var clearedSql = await TlHelper.CleanSql(sql);
var normalizedSql = await Common.GetNormalizedSql(clearedSql);
var hash = await Common.GetMD5Hash(normalizedSql);
var dateTime = await TlHelper.GetPropertyValue(text, Logcfg.DATETIME_PR);
var context = await TlHelper.GetPropertyValue(text, Logcfg.CONTEXT_PR);
var firstLine = await TlHelper.GetFirstLineContext(context);
var lastLine = await TlHelper.GetLastLineContext(context);
var connectId = await TlHelper.GetPropertyValue(text, Logcfg.CONNECT_ID_PR);
var contextExists = firstLine.Length == 0 ? 0 : 1;
var data =
Common.FS +
dateTime +
Common.FS +
user +
Common.FS +
connectId +
Common.FS +
clientId +
Common.FS +
sql +
Common.FS +
normalizedSql +
Common.FS +
firstLine +
Common.FS +
lastLine +
Common.FS +
contextExists +
Common.FS +
hash +
Common.RS;
await targetBlock.SendAsync(data);
}
