public void SaveLogPosition(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position)
{
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "LogFiles",
BatchSize = 100000
})
{
long logFileNewId = GetLogFileInfoNewId(system);
IEnumerable <object[]> values = new List <object[]>()
{
new object[]
{
system.Name,
logFileNewId,
logFileInfo.Name,
logFileInfo.CreationTimeUtc,
logFileInfo.LastWriteTimeUtc,
position.EventNumber,
position.CurrentFileReferences.Replace("\\", "\\\\"),
position.CurrentFileData.Replace("\\", "\\\\"),
position.StreamPosition ?? 0
}
}.AsEnumerable();
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
bulkResult.Wait();
}
}
public static bool RowDataExistOnDatabase(this EventLogContext context, InformationSystemsBase system, YY.EventLogReaderAssistant.Models.RowData itemRow)
{
var checkExist = context.RowsData
.FirstOrDefault(e => e.InformationSystemId == system.Id && e.Period == itemRow.Period && e.Id == itemRow.RowId);
return(checkExist != null);
}
public LogDataElement(InformationSystemsBase system, RowData item)
{
Id = $"[{system.Name}][{item.Period:yyyyMMddhhmmss}][{item.RowId}]";
Application = item.Application.Name;
Comment = item.Comment;
Computer = item.Computer?.Name;
ConnectionId = item.ConnectId;
Data = item.Data;
DataPresentation = item.DataPresentation;
DataUUID = item.DataUuid;
Event = item.Event?.Name;
RowId = item.RowId;
InformationSystem = system.Name;
Metadata = item.Metadata?.Name;
MetadataUUID = item.Metadata?.Uuid.ToString();
Period = item.Period;
PrimaryPort = item.PrimaryPort?.Name;
SecondaryPort = item.SecondaryPort?.Name;
Session = item.Session;
Severity = item.Severity.ToString();
TransactionDate = item.TransactionDate;
TransactionId = item.TransactionId;
TransactionStatus = item.TransactionStatus.ToString();
User = item.User?.Name;
UserUUID = item.User?.Uuid.ToString();
WorkServer = item.WorkServer?.Name;
}
public static bool ReferenceExistInDB <T>(this EventLogContext context, InformationSystemsBase system, T item) where T : ReferenceObject
{
T foundItem = context.Set <T>()
.FirstOrDefault(e => e.InformationSystemId == system.Id && e.Name == item.Name);
return(foundItem != null);
}
public DateTime GetRowsDataMaxPeriod(InformationSystemsBase system)
{
DateTime output = DateTime.MinValue;
using (var command = _connection.CreateCommand())
{
command.CommandText =
@"SELECT
MAX(Period) AS MaxPeriod
FROM RowsData AS RD
WHERE InformationSystem = {InformationSystem:String} ";
command.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "InformationSystem",
Value = system.Name
});
using (var cmdReader = command.ExecuteReader())
{
if (cmdReader.Read())
{
output = cmdReader.GetDateTime(0);
}
}
}
return(output);
}
public void BeforeSaveData(InformationSystemsBase system, List <RowData> rowsData, ref IEnumerable <object[]> dataToSave)
{
var dataToSaveNew = new List <object[]>();
foreach (var objectItem in dataToSave)
{
string metadataName = objectItem[15].ToString();
string dataPresentation = objectItem[19].ToString();
DateTime documentDate = _minDateTime;
if (metadataName != null && !string.IsNullOrEmpty(dataPresentation))
{
if (metadataName.Contains("Документ", StringComparison.InvariantCultureIgnoreCase) ||
metadataName.Contains("Document", StringComparison.InvariantCultureIgnoreCase))
{
var matchResult = _parseDateFromDataPresentation.Match(dataPresentation);
DateTime.TryParseExact(
matchResult.Value,
new[] { "dd.MM.yyyy HH:mm:ss", "dd.MM.yyyy H:mm:ss" },
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out documentDate);
}
}
dataToSaveNew.Add(new []
{
objectItem[0],
objectItem[1],
objectItem[2],
objectItem[3],
objectItem[4],
objectItem[5],
objectItem[6],
objectItem[7],
objectItem[8],
objectItem[9],
objectItem[10],
objectItem[11],
objectItem[12],
objectItem[13],
objectItem[14],
metadataName,
objectItem[16],
objectItem[17],
objectItem[18],
dataPresentation,
objectItem[20],
objectItem[21],
objectItem[22],
(documentDate < _minDateTime ? _minDateTime : documentDate)
});
}
dataToSave = dataToSaveNew;
}
public LogFiles(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position)
{
InformationSystemId = system.Id;
FileName = logFileInfo.Name;
CreateDate = logFileInfo.CreationTimeUtc;
ModificationDate = logFileInfo.LastWriteTimeUtc;
LastCurrentFileData = position.CurrentFileData;
LastCurrentFileReferences = position.CurrentFileReferences;
LastEventNumber = position.EventNumber;
LastStreamPosition = position.StreamPosition;
}
public static void FillReferencesToSave(this EventLogContext context, InformationSystemsBase system, ReferencesData data)
{
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Applications>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Computers>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Events>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Metadata>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <PrimaryPorts>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <SecondaryPorts>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Severities>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <TransactionStatuses>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <Users>(system));
FillReferencesToSave(context, system, data.GetReferencesListForDatabaseType <WorkServers>(system));
}
public override void SetInformationSystem(InformationSystemsBase system)
{
_system = new InformationSystems()
{
Id = system.Id,
Name = system.Name,
Description = system.Description
};
if (_indexName == null)
{
_indexName = _system.Name;
}
}
public EventLogPosition GetLogFilePosition(InformationSystemsBase system)
{
var cmdGetLastLogFileInfo = _connection.CreateCommand();
cmdGetLastLogFileInfo.CommandText =
@"SELECT
LastEventNumber,
LastCurrentFileReferences,
LastCurrentFileData,
LastStreamPosition
FROM LogFiles AS LF
WHERE InformationSystem = {informationSystem:String}
AND Id IN (
SELECT
MAX(Id) LastId
FROM LogFiles AS LF_LAST
WHERE LF_LAST.InformationSystem = {informationSystem:String}
)";
cmdGetLastLogFileInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "informationSystem",
DbType = DbType.AnsiString,
Value = system.Name
});
EventLogPosition output = null;
using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader())
{
if (cmdReader.Read())
{
string fileData = cmdReader.GetString(2)
.Replace("\\\\", "\\")
.FixNetworkPath();
string fileReferences = cmdReader.GetString(1)
.Replace("\\\\", "\\")
.FixNetworkPath();
output = new EventLogPosition(
cmdReader.GetInt64(0),
fileReferences,
fileData,
cmdReader.GetInt64(3));
}
}
return(output);
}
private static void FillReferencesToSave <T>(EventLogContext context, InformationSystemsBase system, IReadOnlyList <T> sourceReferences)
where T : ReferenceObject
{
if (sourceReferences == null)
{
return;
}
foreach (var itemReference in sourceReferences)
{
if (!context.ReferenceExistInDB(system, itemReference))
{
context.AddReferenceToSaveInDB(itemReference);
}
}
}
public void SaveRowsData(InformationSystemsBase system, List <RowData> rowsData)
{
if (rowsData == null || rowsData.Count == 0)
{
return;
}
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "RowsData",
BatchSize = 100000
})
{
var values = rowsData.Select(i => new object[]
{
system.Name,
i.RowId,
i.Period,
Severities.GetPresentationByName(i.Severity.ToString()),
i.ConnectId ?? 0,
i.Session ?? 0,
TransactionStatuses.GetPresentationByName(i.TransactionStatus.ToString()),
(i.TransactionDate == null || i.TransactionDate < _minDateTime ? _minDateTime : i.TransactionDate),
i.TransactionId ?? 0,
i.User?.Name ?? string.Empty,
i.User?.Uuid.ToString() ?? _emptyGuidAsString,
i.Computer?.Name ?? string.Empty,
Applications.GetPresentationByName(i.Application?.Name ?? string.Empty),
Events.GetPresentationByName(i.Event?.Name ?? string.Empty),
i.Comment ?? string.Empty,
i.Metadata?.Name ?? string.Empty,
i.Metadata?.Uuid.ToString() ?? _emptyGuidAsString,
i.Data ?? string.Empty,
(i.DataUuid ?? string.Empty).NormalizeShortUUID(),
i.DataPresentation ?? string.Empty,
i.WorkServer?.Name ?? string.Empty,
i.PrimaryPort?.Name ?? string.Empty,
i.SecondaryPort?.Name ?? string.Empty
}).AsEnumerable();
_extendedActions?.BeforeSaveData(system, rowsData, ref values);
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
bulkResult.Wait();
}
}
public bool RowDataExistOnDatabase(InformationSystemsBase system, RowData rowData)
{
bool output = false;
using (var command = _connection.CreateCommand())
{
command.CommandText =
@"SELECT
InformationSystem,
Id,
Period
FROM RowsData AS RD
WHERE InformationSystem = {existInfSysId:String}
AND Id = {existId:Int64}
AND Period = {existPeriod:DateTime}";
command.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "existInfSysId",
DbType = DbType.AnsiString,
Value = system.Name
});
command.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "existId",
DbType = DbType.Int64,
Value = rowData.RowId
});
command.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "existPeriod",
DbType = DbType.DateTime,
Value = rowData.Period.DateTime
});
using (var cmdReader = command.ExecuteReader())
{
if (cmdReader.Read())
{
output = true;
}
}
}
return(output);
}
public static DateTime GetRowsDataMaxPeriod(this EventLogContext context, InformationSystemsBase system)
{
DateTime maxPeriodRowData = DateTime.MinValue;
RowData firstRow = context.RowsData.FirstOrDefault();
if (firstRow != null)
{
var _maxPeriodData = context.RowsData
.Where(p => p.InformationSystemId == system.Id);
if (_maxPeriodData.Any())
{
DateTimeOffset _maxPeriodRowDataTimeOffset = _maxPeriodData.Max(m => m.Period);
maxPeriodRowData = _maxPeriodRowDataTimeOffset.DateTime;
}
}
return(maxPeriodRowData);
}
public static EventLogPosition GetLastPosition(this EventLogContext context, InformationSystemsBase system)
{
var lastLogFile = context.LogFiles
.SingleOrDefault(e => e.InformationSystemId == system.Id &&
e.Id == context.LogFiles.Where(i => i.InformationSystemId == system.Id).Max(m => m.Id));
if (lastLogFile == null)
{
return(null);
}
else
{
return(new EventLogPosition(
lastLogFile.LastEventNumber,
lastLogFile.LastCurrentFileReferences,
lastLogFile.LastCurrentFileData,
lastLogFile.LastStreamPosition));
}
}
public void RemoveArchiveLogFileRecords(InformationSystemsBase system)
{
var commandRemoveArchiveLogInfo = _connection.CreateCommand();
commandRemoveArchiveLogInfo.CommandText =
@"ALTER TABLE LogFiles DELETE
WHERE InformationSystem = {InformationSystem:String}
AND Id < (
SELECT MAX(Id) AS LastId
FROM LogFiles lf
WHERE InformationSystem = {InformationSystem:String}
)";
commandRemoveArchiveLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "InformationSystem",
DbType = DbType.AnsiString,
Value = system.Name
});
commandRemoveArchiveLogInfo.ExecuteNonQuery();
}
public static void SaveLogPosition(this EventLogContext context, InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position)
{
LogFiles foundLogFile = context.LogFiles
.FirstOrDefault(l => l.InformationSystemId == system.Id && l.FileName == logFileInfo.Name && l.CreateDate == logFileInfo.CreationTimeUtc);
if (foundLogFile == null)
{
context.LogFiles.Add(new LogFiles(system, logFileInfo, position));
}
else
{
foundLogFile.ModificationDate = logFileInfo.LastWriteTimeUtc;
foundLogFile.LastCurrentFileData = position.CurrentFileData;
foundLogFile.LastCurrentFileReferences = position.CurrentFileReferences;
foundLogFile.LastEventNumber = position.EventNumber;
foundLogFile.LastStreamPosition = position.StreamPosition;
context.Entry(foundLogFile).State = EntityState.Modified;
}
context.SaveChanges();
}
public long GetLogFileInfoNewId(InformationSystemsBase system)
{
long output = 0;
if (logFileLastId < 0)
{
using (var command = _connection.CreateCommand())
{
command.CommandText =
@"SELECT
MAX(Id)
FROM LogFiles
WHERE InformationSystem = {InformationSystem:String}";
command.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "InformationSystem",
Value = system.Name
});
using (var cmdReader = command.ExecuteReader())
{
if (cmdReader.Read())
{
output = cmdReader.GetInt64(0);
}
}
}
}
else
{
output = logFileLastId;
}
output += 1;
logFileLastId = output;
return(output);
}
public RowData(InformationSystemsBase system, EventLogReaderAssistant.Models.RowData sourceRow, ReferencesDataCache referencesCache)
{
ApplicationId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Applications>(sourceRow);
Comment = sourceRow.Comment;
ComputerId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Computers>(sourceRow);
ConnectId = sourceRow.ConnectId;
Data = sourceRow.Data;
DataPresentation = sourceRow.DataPresentation;
DataUUID = sourceRow.DataUuid;
EventId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Events>(sourceRow);
Id = sourceRow.RowId;
InformationSystemId = system.Id;
MetadataId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Metadata>(sourceRow);
Period = sourceRow.Period;
PrimaryPortId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.PrimaryPorts>(sourceRow);
SecondaryPortId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.SecondaryPorts>(sourceRow);
Session = sourceRow.Session;
SeverityId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Severity>(sourceRow);
TransactionDate = sourceRow.TransactionDate;
TransactionId = sourceRow.TransactionId;
TransactionStatusId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.TransactionStatus>(sourceRow);
UserId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.Users>(sourceRow);
WorkServerId = referencesCache.GetReferenceDatabaseId <EventLogReaderAssistant.Models.WorkServers>(sourceRow);
}
public ReferencesDataCache(InformationSystemsBase system)
{
_system = system;
}
public override void SetInformationSystem(InformationSystemsBase system)
{
using (EventLogContext _context = EventLogContext.Create(_databaseOptions, _databaseActions))
_system = _context.CreateOrUpdateInformationSystem(system);
}
public override void SetInformationSystem(InformationSystemsBase system)
{
_system = system;
}
public static InformationSystems CreateOrUpdateInformationSystem(this EventLogContext context, InformationSystemsBase system)
{
InformationSystems existSystem = context.InformationSystems.FirstOrDefault(e => e.Name == system.Name);
if (existSystem == null)
{
context.InformationSystems.Add(new InformationSystems()
{
Name = system.Name,
Description = system.Description
});
context.SaveChanges();
existSystem = context.InformationSystems.FirstOrDefault(e => e.Name == system.Name);
}
else
{
if (existSystem.Description != system.Description)
{
existSystem.Description = system.Description;
context.Update(system);
context.SaveChanges();
}
}
return(existSystem);
}
public static IReadOnlyList <T> PrepareItemsToSave <T>(InformationSystemsBase system, ReferencesData data) where T : ReferenceObject
{
IReadOnlyList <T> sourceReferenceList = data.GetReferencesListForDatabaseType <T>(system);
return(sourceReferenceList);
}
public void SaveLogPosition(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position)
{
var commandAddLogInfo = _connection.CreateCommand();
commandAddLogInfo.CommandText =
@"INSERT INTO LogFiles (
InformationSystem,
Id,
FileName,
CreateDate,
ModificationDate,
LastEventNumber,
LastCurrentFileReferences,
LastCurrentFileData,
LastStreamPosition
) VALUES (
{isId:String},
{newId:Int64},
{FileName:String},
{CreateDate:DateTime},
{ModificationDate:DateTime},
{LastEventNumber:Int64},
{LastCurrentFileReferences:String},
{LastCurrentFileData:String},
{LastStreamPosition:Int64}
)";
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "isId",
DbType = DbType.Int64,
Value = system.Name
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "newId",
DbType = DbType.Int64,
Value = GetLogFileInfoNewId(system)
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "FileName",
DbType = DbType.AnsiString,
Value = logFileInfo.Name
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "CreateDate",
DbType = DbType.DateTime,
Value = logFileInfo.CreationTimeUtc
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "ModificationDate",
DbType = DbType.DateTime,
Value = logFileInfo.LastWriteTimeUtc
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "LastEventNumber",
DbType = DbType.Int64,
Value = position.EventNumber
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "LastCurrentFileReferences",
DbType = DbType.AnsiString,
Value = position.CurrentFileReferences.Replace("\\", "\\\\")
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "LastCurrentFileData",
DbType = DbType.AnsiString,
Value = position.CurrentFileData.Replace("\\", "\\\\")
});
commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter
{
ParameterName = "LastStreamPosition",
DbType = DbType.Int64,
Value = position?.StreamPosition ?? 0
});
commandAddLogInfo.ExecuteNonQuery();
}
