public void SaveLogPosition(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position) { using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection) { DestinationTableName = "LogFiles", BatchSize = 100000 }) { long logFileNewId = GetLogFileInfoNewId(system); IEnumerable <object[]> values = new List <object[]>() { new object[] { system.Name, logFileNewId, logFileInfo.Name, logFileInfo.CreationTimeUtc, logFileInfo.LastWriteTimeUtc, position.EventNumber, position.CurrentFileReferences.Replace("\\", "\\\\"), position.CurrentFileData.Replace("\\", "\\\\"), position.StreamPosition ?? 0 } }.AsEnumerable(); var bulkResult = bulkCopyInterface.WriteToServerAsync(values); bulkResult.Wait(); } }
public static bool RowDataExistOnDatabase(this EventLogContext context, InformationSystemsBase system, YY.EventLogReaderAssistant.Models.RowData itemRow) { var checkExist = context.RowsData .FirstOrDefault(e => e.InformationSystemId == system.Id && e.Period == itemRow.Period && e.Id == itemRow.RowId); return(checkExist != null); }
public LogDataElement(InformationSystemsBase system, RowData item) { Id = $"[{system.Name}][{item.Period:yyyyMMddhhmmss}][{item.RowId}]"; Application = item.Application.Name; Comment = item.Comment; Computer = item.Computer?.Name; ConnectionId = item.ConnectId; Data = item.Data; DataPresentation = item.DataPresentation; DataUUID = item.DataUuid; Event = item.Event?.Name; RowId = item.RowId; InformationSystem = system.Name; Metadata = item.Metadata?.Name; MetadataUUID = item.Metadata?.Uuid.ToString(); Period = item.Period; PrimaryPort = item.PrimaryPort?.Name; SecondaryPort = item.SecondaryPort?.Name; Session = item.Session; Severity = item.Severity.ToString(); TransactionDate = item.TransactionDate; TransactionId = item.TransactionId; TransactionStatus = item.TransactionStatus.ToString(); User = item.User?.Name; UserUUID = item.User?.Uuid.ToString(); WorkServer = item.WorkServer?.Name; }
public static bool ReferenceExistInDB <T>(this EventLogContext context, InformationSystemsBase system, T item) where T : ReferenceObject { T foundItem = context.Set <T>() .FirstOrDefault(e => e.InformationSystemId == system.Id && e.Name == item.Name); return(foundItem != null); }
public DateTime GetRowsDataMaxPeriod(InformationSystemsBase system) { DateTime output = DateTime.MinValue; using (var command = _connection.CreateCommand()) { command.CommandText = @"SELECT MAX(Period) AS MaxPeriod FROM RowsData AS RD WHERE InformationSystem = {InformationSystem:String} "; command.Parameters.Add(new ClickHouseDbParameter { ParameterName = "InformationSystem", Value = system.Name }); using (var cmdReader = command.ExecuteReader()) { if (cmdReader.Read()) { output = cmdReader.GetDateTime(0); } } } return(output); }
public void BeforeSaveData(InformationSystemsBase system, List <RowData> rowsData, ref IEnumerable <object[]> dataToSave) { var dataToSaveNew = new List <object[]>(); foreach (var objectItem in dataToSave) { string metadataName = objectItem[15].ToString(); string dataPresentation = objectItem[19].ToString(); DateTime documentDate = _minDateTime; if (metadataName != null && !string.IsNullOrEmpty(dataPresentation)) { if (metadataName.Contains("Документ", StringComparison.InvariantCultureIgnoreCase) || metadataName.Contains("Document", StringComparison.InvariantCultureIgnoreCase)) { var matchResult = _parseDateFromDataPresentation.Match(dataPresentation); DateTime.TryParseExact( matchResult.Value, new[] { "dd.MM.yyyy HH:mm:ss", "dd.MM.yyyy H:mm:ss" }, CultureInfo.InvariantCulture, DateTimeStyles.None, out documentDate); } } dataToSaveNew.Add(new [] { objectItem[0], objectItem[1], objectItem[2], objectItem[3], objectItem[4], objectItem[5], objectItem[6], objectItem[7], objectItem[8], objectItem[9], objectItem[10], objectItem[11], objectItem[12], objectItem[13], objectItem[14], metadataName, objectItem[16], objectItem[17], objectItem[18], dataPresentation, objectItem[20], objectItem[21], objectItem[22], (documentDate < _minDateTime ? _minDateTime : documentDate) }); } dataToSave = dataToSaveNew; }
public LogFiles(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position) { InformationSystemId = system.Id; FileName = logFileInfo.Name; CreateDate = logFileInfo.CreationTimeUtc; ModificationDate = logFileInfo.LastWriteTimeUtc; LastCurrentFileData = position.CurrentFileData; LastCurrentFileReferences = position.CurrentFileReferences; LastEventNumber = position.EventNumber; LastStreamPosition = position.StreamPosition; }
public static void FillReferencesToSave(this EventLogContext context, InformationSystemsBase system, ReferencesData data) { FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Applications>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Computers>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Events>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Metadata>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <PrimaryPorts>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <SecondaryPorts>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Severities>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <TransactionStatuses>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Users>(system)); FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <WorkServers>(system)); }
public override void SetInformationSystem(InformationSystemsBase system) { _system = new InformationSystems() { Id = system.Id, Name = system.Name, Description = system.Description }; if (_indexName == null) { _indexName = _system.Name; } }
public EventLogPosition GetLogFilePosition(InformationSystemsBase system) { var cmdGetLastLogFileInfo = _connection.CreateCommand(); cmdGetLastLogFileInfo.CommandText = @"SELECT LastEventNumber, LastCurrentFileReferences, LastCurrentFileData, LastStreamPosition FROM LogFiles AS LF WHERE InformationSystem = {informationSystem:String} AND Id IN ( SELECT MAX(Id) LastId FROM LogFiles AS LF_LAST WHERE LF_LAST.InformationSystem = {informationSystem:String} )"; cmdGetLastLogFileInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "informationSystem", DbType = DbType.AnsiString, Value = system.Name }); EventLogPosition output = null; using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader()) { if (cmdReader.Read()) { string fileData = cmdReader.GetString(2) .Replace("\\\\", "\\") .FixNetworkPath(); string fileReferences = cmdReader.GetString(1) .Replace("\\\\", "\\") .FixNetworkPath(); output = new EventLogPosition( cmdReader.GetInt64(0), fileReferences, fileData, cmdReader.GetInt64(3)); } } return(output); }
private static void FillReferencesToSave <T>(EventLogContext context, InformationSystemsBase system, IReadOnlyList <T> sourceReferences) where T : ReferenceObject { if (sourceReferences == null) { return; } foreach (var itemReference in sourceReferences) { if (!context.ReferenceExistInDB(system, itemReference)) { context.AddReferenceToSaveInDB(itemReference); } } }
public void SaveRowsData(InformationSystemsBase system, List <RowData> rowsData) { if (rowsData == null || rowsData.Count == 0) { return; } using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection) { DestinationTableName = "RowsData", BatchSize = 100000 }) { var values = rowsData.Select(i => new object[] { system.Name, i.RowId, i.Period, Severities.GetPresentationByName(i.Severity.ToString()), i.ConnectId ?? 0, i.Session ?? 0, TransactionStatuses.GetPresentationByName(i.TransactionStatus.ToString()), (i.TransactionDate == null || i.TransactionDate < _minDateTime ? _minDateTime : i.TransactionDate), i.TransactionId ?? 0, i.User?.Name ?? string.Empty, i.User?.Uuid.ToString() ?? _emptyGuidAsString, i.Computer?.Name ?? string.Empty, Applications.GetPresentationByName(i.Application?.Name ?? string.Empty), Events.GetPresentationByName(i.Event?.Name ?? string.Empty), i.Comment ?? string.Empty, i.Metadata?.Name ?? string.Empty, i.Metadata?.Uuid.ToString() ?? _emptyGuidAsString, i.Data ?? string.Empty, (i.DataUuid ?? string.Empty).NormalizeShortUUID(), i.DataPresentation ?? string.Empty, i.WorkServer?.Name ?? string.Empty, i.PrimaryPort?.Name ?? string.Empty, i.SecondaryPort?.Name ?? string.Empty }).AsEnumerable(); _extendedActions?.BeforeSaveData(system, rowsData, ref values); var bulkResult = bulkCopyInterface.WriteToServerAsync(values); bulkResult.Wait(); } }
public bool RowDataExistOnDatabase(InformationSystemsBase system, RowData rowData) { bool output = false; using (var command = _connection.CreateCommand()) { command.CommandText = @"SELECT InformationSystem, Id, Period FROM RowsData AS RD WHERE InformationSystem = {existInfSysId:String} AND Id = {existId:Int64} AND Period = {existPeriod:DateTime}"; command.Parameters.Add(new ClickHouseDbParameter { ParameterName = "existInfSysId", DbType = DbType.AnsiString, Value = system.Name }); command.Parameters.Add(new ClickHouseDbParameter { ParameterName = "existId", DbType = DbType.Int64, Value = rowData.RowId }); command.Parameters.Add(new ClickHouseDbParameter { ParameterName = "existPeriod", DbType = DbType.DateTime, Value = rowData.Period.DateTime }); using (var cmdReader = command.ExecuteReader()) { if (cmdReader.Read()) { output = true; } } } return(output); }
public static DateTime GetRowsDataMaxPeriod(this EventLogContext context, InformationSystemsBase system) { DateTime maxPeriodRowData = DateTime.MinValue; RowData firstRow = context.RowsData.FirstOrDefault(); if (firstRow != null) { var _maxPeriodData = context.RowsData .Where(p => p.InformationSystemId == system.Id); if (_maxPeriodData.Any()) { DateTimeOffset _maxPeriodRowDataTimeOffset = _maxPeriodData.Max(m => m.Period); maxPeriodRowData = _maxPeriodRowDataTimeOffset.DateTime; } } return(maxPeriodRowData); }
public static EventLogPosition GetLastPosition(this EventLogContext context, InformationSystemsBase system) { var lastLogFile = context.LogFiles .SingleOrDefault(e => e.InformationSystemId == system.Id && e.Id == context.LogFiles.Where(i => i.InformationSystemId == system.Id).Max(m => m.Id)); if (lastLogFile == null) { return(null); } else { return(new EventLogPosition( lastLogFile.LastEventNumber, lastLogFile.LastCurrentFileReferences, lastLogFile.LastCurrentFileData, lastLogFile.LastStreamPosition)); } }
public void RemoveArchiveLogFileRecords(InformationSystemsBase system) { var commandRemoveArchiveLogInfo = _connection.CreateCommand(); commandRemoveArchiveLogInfo.CommandText = @"ALTER TABLE LogFiles DELETE WHERE InformationSystem = {InformationSystem:String} AND Id < ( SELECT MAX(Id) AS LastId FROM LogFiles lf WHERE InformationSystem = {InformationSystem:String} )"; commandRemoveArchiveLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "InformationSystem", DbType = DbType.AnsiString, Value = system.Name }); commandRemoveArchiveLogInfo.ExecuteNonQuery(); }
public static void SaveLogPosition(this EventLogContext context, InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position) { LogFiles foundLogFile = context.LogFiles .FirstOrDefault(l => l.InformationSystemId == system.Id && l.FileName == logFileInfo.Name && l.CreateDate == logFileInfo.CreationTimeUtc); if (foundLogFile == null) { context.LogFiles.Add(new LogFiles(system, logFileInfo, position)); } else { foundLogFile.ModificationDate = logFileInfo.LastWriteTimeUtc; foundLogFile.LastCurrentFileData = position.CurrentFileData; foundLogFile.LastCurrentFileReferences = position.CurrentFileReferences; foundLogFile.LastEventNumber = position.EventNumber; foundLogFile.LastStreamPosition = position.StreamPosition; context.Entry(foundLogFile).State = EntityState.Modified; } context.SaveChanges(); }
public long GetLogFileInfoNewId(InformationSystemsBase system) { long output = 0; if (logFileLastId < 0) { using (var command = _connection.CreateCommand()) { command.CommandText = @"SELECT MAX(Id) FROM LogFiles WHERE InformationSystem = {InformationSystem:String}"; command.Parameters.Add(new ClickHouseDbParameter { ParameterName = "InformationSystem", Value = system.Name }); using (var cmdReader = command.ExecuteReader()) { if (cmdReader.Read()) { output = cmdReader.GetInt64(0); } } } } else { output = logFileLastId; } output += 1; logFileLastId = output; return(output); }
public RowData(InformationSystemsBase system, EventLogReaderAssistant.Models.RowData sourceRow, ReferencesDataCache referencesCache) { ApplicationId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Applications>(sourceRow); Comment = sourceRow.Comment; ComputerId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Computers>(sourceRow); ConnectId = sourceRow.ConnectId; Data = sourceRow.Data; DataPresentation = sourceRow.DataPresentation; DataUUID = sourceRow.DataUuid; EventId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Events>(sourceRow); Id = sourceRow.RowId; InformationSystemId = system.Id; MetadataId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Metadata>(sourceRow); Period = sourceRow.Period; PrimaryPortId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.PrimaryPorts>(sourceRow); SecondaryPortId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.SecondaryPorts>(sourceRow); Session = sourceRow.Session; SeverityId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Severity>(sourceRow); TransactionDate = sourceRow.TransactionDate; TransactionId = sourceRow.TransactionId; TransactionStatusId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.TransactionStatus>(sourceRow); UserId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Users>(sourceRow); WorkServerId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.WorkServers>(sourceRow); }
public ReferencesDataCache(InformationSystemsBase system) { _system = system; }
public override void SetInformationSystem(InformationSystemsBase system) { using (EventLogContext _context = EventLogContext.Create(_databaseOptions, _databaseActions)) _system = _context.CreateOrUpdateInformationSystem(system); }
public override void SetInformationSystem(InformationSystemsBase system) { _system = system; }
public static InformationSystems CreateOrUpdateInformationSystem(this EventLogContext context, InformationSystemsBase system) { InformationSystems existSystem = context.InformationSystems.FirstOrDefault(e => e.Name == system.Name); if (existSystem == null) { context.InformationSystems.Add(new InformationSystems() { Name = system.Name, Description = system.Description }); context.SaveChanges(); existSystem = context.InformationSystems.FirstOrDefault(e => e.Name == system.Name); } else { if (existSystem.Description != system.Description) { existSystem.Description = system.Description; context.Update(system); context.SaveChanges(); } } return(existSystem); }
public static IReadOnlyList <T> PrepareItemsToSave <T>(InformationSystemsBase system, ReferencesData data) where T : ReferenceObject { IReadOnlyList <T> sourceReferenceList = data.GetReferencesListForDatabaseType <T>(system); return(sourceReferenceList); }
public void SaveLogPosition(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position) { var commandAddLogInfo = _connection.CreateCommand(); commandAddLogInfo.CommandText = @"INSERT INTO LogFiles ( InformationSystem, Id, FileName, CreateDate, ModificationDate, LastEventNumber, LastCurrentFileReferences, LastCurrentFileData, LastStreamPosition ) VALUES ( {isId:String}, {newId:Int64}, {FileName:String}, {CreateDate:DateTime}, {ModificationDate:DateTime}, {LastEventNumber:Int64}, {LastCurrentFileReferences:String}, {LastCurrentFileData:String}, {LastStreamPosition:Int64} )"; commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "isId", DbType = DbType.Int64, Value = system.Name }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "newId", DbType = DbType.Int64, Value = GetLogFileInfoNewId(system) }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "FileName", DbType = DbType.AnsiString, Value = logFileInfo.Name }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "CreateDate", DbType = DbType.DateTime, Value = logFileInfo.CreationTimeUtc }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "ModificationDate", DbType = DbType.DateTime, Value = logFileInfo.LastWriteTimeUtc }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "LastEventNumber", DbType = DbType.Int64, Value = position.EventNumber }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "LastCurrentFileReferences", DbType = DbType.AnsiString, Value = position.CurrentFileReferences.Replace("\\", "\\\\") }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "LastCurrentFileData", DbType = DbType.AnsiString, Value = position.CurrentFileData.Replace("\\", "\\\\") }); commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter { ParameterName = "LastStreamPosition", DbType = DbType.Int64, Value = position?.StreamPosition ?? 0 }); commandAddLogInfo.ExecuteNonQuery(); }