/// <summary> /// Starts the processing data of the technology log /// </summary> /// <returns></returns> private async Task ProcessTlData() { // Options of the many-threads blocks // Limit a max bounded capacity to improve a consumption of memory var parallelBlockOptions = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, BoundedCapacity = 1000 }; using (var contextWriter = Common.GetOutputStream(CONTEXT_TEMP_FILEPATH)) using (var dbmssqlWriter = Common.GetOutputStream(DBMSSQL_TEMP_FILEPATH)) { HashSet <(string eventName, ITargetBlock <string> nextBlock)> events = new HashSet <(string, ITargetBlock <string>)>(); // Blocks with the output streams for processed data var contextOutputBlock = new ActionBlock <string>(async(text) => await Common.WriteToOutputStream(text, contextWriter)); var dbmssqlOutputBlock = new ActionBlock <string>(async(text) => await Common.WriteToOutputStream(text, dbmssqlWriter)); // Blocks of the processing data var contextBlock = new ActionBlock <string>((text) => ProcessContextEvent(text, contextOutputBlock), parallelBlockOptions); events.Add(("Context", contextBlock)); var dbmssqlBlock = new ActionBlock <string>((text) => ProcessDbmssqlEvent(text, dbmssqlOutputBlock), parallelBlockOptions); events.Add(("DBMSSQL", dbmssqlBlock)); // Reading tech log block var readBlock = new ActionBlock <string>((filePath) => TlHelper.ReadFile(events, filePath), parallelBlockOptions); foreach (var file in TlHelper.GetLogFiles(_logcfg)) { await readBlock.SendAsync(file); } // Mark block as completed readBlock.Complete(); // Create relations between blocks (signals "complete" to the next blocks) await readBlock.Completion.ContinueWith(c => dbmssqlBlock.Complete()); await dbmssqlBlock.Completion.ContinueWith(c => dbmssqlOutputBlock.Complete()); await readBlock.Completion.ContinueWith(c => contextBlock.Complete()); await contextBlock.Completion.ContinueWith(c => contextOutputBlock.Complete()); // Wait writing processed data to the temp files await contextOutputBlock.Completion; await dbmssqlOutputBlock.Completion; } }
/// <summary> /// Initializes the _logcfg variable /// </summary> private void InitializeLogCfg() { _logcfg = new Logcfg(_settings); _logcfg.AddLog(TlHelper.GetCollectPeriod(_settings.CollectPeriod).ToString(), _settings.TlFolder); _logcfg.AddEvent(Logcfg.DBMSSQL_EV); _logcfg.AddProperty(Logcfg.USER_PR); _logcfg.AddProperty(Logcfg.CONTEXT_PR); _logcfg.AddProperty(Logcfg.CONNECT_ID_PR); _logcfg.AddProperty(Logcfg.SQL_PR); _logcfg.AddProperty(Logcfg.CLIENT_ID_PR); if (_settings.FilterByDatabase) { _logcfg.AddFilter(Logcfg.EQ_CT, Logcfg.PROCESS_NAME_PR, _settings.Database1C); } if (_settings.FilterByDuration) { _logcfg.AddFilter(Logcfg.GE_CT, Logcfg.DURATION_PR, _settings.Duration.ToString()); } }
/// <summary> /// Processes a Context event and send the result to the next block /// </summary> /// <param name="text">Event data</param> /// <param name="targetBlock">Next block</param> /// <returns></returns> private async Task ProcessContextEvent(string text, ITargetBlock <string> targetBlock) { var clientId = await TlHelper.GetPropertyValue(text, Logcfg.CLIENT_ID_PR); // If "client id" property is empty then it`s a system call, such event skipping if (clientId == string.Empty) { return; } var dateTime = await TlHelper.GetPropertyValue(text, Logcfg.DATETIME_PR); var context = await TlHelper.GetPropertyValue(text, Logcfg.CONTEXT_PR); var firstLineContext = await TlHelper.GetFirstLineContext(context); var lastLineContext = await TlHelper.GetLastLineContext(context); var user = await TlHelper.GetPropertyValue(text, Logcfg.USER_PR); var connectId = await TlHelper.GetPropertyValue(text, Logcfg.CONNECT_ID_PR); var data = Common.FS + dateTime + Common.FS + user + Common.FS + connectId + Common.FS + clientId + Common.FS + firstLineContext + Common.FS + lastLineContext + Common.RS; await targetBlock.SendAsync(data); }
/// <summary> /// Run this analyzer /// </summary> /// <returns></returns> public async Task Run() { try { Logger.Log("Start the queries analyze"); Logger.Log("Check settings"); await _settings.Check(); Logger.Log("Create the database"); await SqlHelper.CreateDatabase(_settings); Logger.Log("Clean the folders"); Common.CleanFolder(_settings.TlFolder); Common.CleanFolder(_settings.SqlTraceFolder); Common.CleanFolder(_settings.TempFolder); Logger.Log("Start the data collection"); await _session.Create(); await _logcfg.Write(); Logger.Log("Wait while the data collection will be started..."); await TlHelper.WaitStartCollectData(_logcfg); await _session.Start(); Logger.Log("Wait while the data collection is going..."); // Wait while the data collection is going await Task.Delay(_settings.CollectPeriod * 60 * 1000); Logger.Log("Stop the data collection"); _logcfg.Delete(); await _session.Stop(); Logger.Log("Process the tech log data"); await ProcessTlData(); Logger.Log("Process the extended events data"); await ProcessXESessionData(); await _session.Delete(); Logger.Log("Insert data into the database"); await SqlHelper.InsertFileIntoTable(_settings, SQL_QUERIES_TABLENAME, SQL_QUERIES_TEMP_FILEPATH); await SqlHelper.InsertFileIntoTable(_settings, CONTEXT_TABLENAME, CONTEXT_TEMP_FILEPATH); await SqlHelper.InsertFileIntoTable(_settings, DBMSSQL_TABLENAME, DBMSSQL_TEMP_FILEPATH); Logger.Log("Join contexts and queries"); await FillContext(); Logger.Log("Fill AVG table"); await FillQueriesAvgTable(); Logger.Log("Analyze was successfully completed"); } catch (Exception ex) { Logger.Log(ex); Logger.Log("Analyze was completed with errors"); throw new Exception(); } }
/// <summary> /// Processes a DBMSSQL event and send the result to the next block /// </summary> /// <param name="text">Event data</param> /// <param name="targetBlock">Next block</param> /// <returns></returns> private async Task ProcessDbmssqlEvent(string text, ITargetBlock <string> targetBlock) { var user = await TlHelper.GetPropertyValue(text, Logcfg.USER_PR); // If "usr" property is empty then skip this event if (user == string.Empty) { return; } var sql = await TlHelper.GetPropertyValue(text, Logcfg.SQL_PR); // If "sql" property is empty then skip this event if (sql == string.Empty) { return; } var clientId = await TlHelper.GetPropertyValue(text, Logcfg.CLIENT_ID_PR); // If "client id" property is empty then it`s a system call, such event skipping if (clientId == string.Empty) { return; } var clearedSql = await TlHelper.CleanSql(sql); var normalizedSql = await Common.GetNormalizedSql(clearedSql); var hash = await Common.GetMD5Hash(normalizedSql); var dateTime = await TlHelper.GetPropertyValue(text, Logcfg.DATETIME_PR); var context = await TlHelper.GetPropertyValue(text, Logcfg.CONTEXT_PR); var firstLine = await TlHelper.GetFirstLineContext(context); var lastLine = await TlHelper.GetLastLineContext(context); var connectId = await TlHelper.GetPropertyValue(text, Logcfg.CONNECT_ID_PR); var contextExists = firstLine.Length == 0 ? 0 : 1; var data = Common.FS + dateTime + Common.FS + user + Common.FS + connectId + Common.FS + clientId + Common.FS + sql + Common.FS + normalizedSql + Common.FS + firstLine + Common.FS + lastLine + Common.FS + contextExists + Common.FS + hash + Common.RS; await targetBlock.SendAsync(data); }