public bool TryReadNext(out ILogRecord record)
{
var res = TryReadNext();
record = res.LogRecord;
return(res.Success);
}
public JObject ReadToJson(ILogRecord row)
{
var json = new JObject();
foreach (var field in this.fields)
{
if (this.columnMap.ContainsKey(field.Name))
{
object v = row.getValue(field.Name);
if (field.DataType == typeof(DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
{
json.Add(new JProperty(field.Name, v));
}
}
}
AddTimestamp(json);
return(json);
}
//TODO: add methods for other types
// W3C - COMW3CInputContextClassClass
// NCSA - COMIISNCSAInputContextClassClass
private static DataTable Execute <T>(string query) where T : new()
{
LogQueryClassClass log = new LogQueryClassClass();
ILogRecordset recordset = log.Execute(query, new T());
ILogRecord record = null;
DataTable dt = new DataTable();
Int32 columnCount = recordset.getColumnCount();
for (int i = 0; i < columnCount; i++)
{
dt.Columns.Add(recordset.getColumnName(i), types[recordset.getColumnType(i) - 1]);
}
for (; !recordset.atEnd(); recordset.moveNext())
{
DataRow dr = dt.NewRow();
record = recordset.getRecord();
for (int i = 0; i < columnCount; i++)
{
dr[i] = record.getValue(i);
}
dt.Rows.Add(dr);
}
return(dt);
}
public void dumpCache()
{
// If there are any records in the queue, dump it
if (this.cache.Count > 0)
{
Debug.Log("[CompetitiveLogger] Dumping cache of " + this.cache.Count + " log records...");
// Log each queued record
while (this.cache.Count > 0)
{
ILogRecord theRecord = this.cache.Dequeue();
// Handle record type accordingly
if (theRecord is RegularRecord)
{
RegularRecord record = (RegularRecord)theRecord;
this.LogFormat(record.logType, record.context, record.format, record.args);
}
else if (theRecord is ExceptionRecord)
{
ExceptionRecord record = (ExceptionRecord)theRecord;
this.LogException(record.exception, record.context);
}
}
Debug.Log("[CompetitiveLogger] Cache dump complete.");
}
}
public RecordReadResult(bool success, long nextPosition, ILogRecord logRecord, int recordLength)
{
Success = success;
LogRecord = logRecord;
NextPosition = nextPosition;
RecordLength = recordLength;
}
public void Log(ILogRecord record)
{
var log = (LogRecord)record;
string query = @"INSERT INTO [dbo].[RequestLog]
([ClientIp]
,[ClientBrowser]
,[PostData]
,[CreatedDate]
,[IsBotRequest])
VALUES
(@ClientIp
,@ClientBrowser
,@PostData
,@CreatedDate
,@IsBotRequest)";
SqlCommand insertCmd = new SqlCommand(query, MyConnection);
insertCmd.Parameters.AddWithValue("ClientIp", log.ClientIP);
insertCmd.Parameters.AddWithValue("ClientBrowser", log.ClientBrowser);
insertCmd.Parameters.AddWithValue("PostData", log.PostData);
insertCmd.Parameters.AddWithValue("CreatedDate", log.RequestDate);
insertCmd.Parameters.AddWithValue("IsBotRequest", log.IsBotRequest);
insertCmd.ExecuteNonQuery();
}
private static bool IsCommitAlike(ILogRecord rec)
{
return(rec.RecordType == LogRecordType.Commit ||
((rec.RecordType == LogRecordType.Prepare ||
rec.RecordType == LogRecordType.EventType ||
rec.RecordType == LogRecordType.Stream) &&
((IPrepareLogRecord)rec).Flags.HasAnyOf(PrepareFlags.IsCommitted)));
}
protected bool TryReadForwardInternal(ReaderWorkItem workItem, long actualPosition, out int length,
out ILogRecord record)
{
length = -1;
record = null;
workItem.Stream.Position = GetRawPosition(actualPosition);
// no space even for length prefix and suffix
if (actualPosition + 2 * sizeof(int) > Chunk.PhysicalDataSize)
{
_log.Warning(
"Tried to read actual position {actualPosition}, but there isn't enough space for a record left in the chunk. Chunk's data size: {chunkPhysicalDataSize}",
actualPosition, Chunk.PhysicalDataSize);
return(false);
}
length = workItem.Reader.ReadInt32();
if (length <= 0)
{
throw new InvalidReadException(
string.Format("Log record at actual pos {0} has non-positive length: {1}. "
+ " in chunk {2}.", actualPosition, length, Chunk));
}
if (length > TFConsts.MaxLogRecordSize)
{
throw new InvalidReadException(
string.Format("Log record at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes. In chunk {3}.",
actualPosition, length, TFConsts.MaxLogRecordSize, Chunk));
}
if (actualPosition + length + 2 * sizeof(int) > Chunk.PhysicalDataSize)
{
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (length prefix: {0}). "
+ "Actual pre-position: {1}. Something is seriously wrong in chunk {2}.",
length, actualPosition, Chunk));
}
record = LogRecord.ReadFrom(workItem.Reader, length);
// verify suffix length == prefix length
int suffixLength = workItem.Reader.ReadInt32();
if (suffixLength != length)
{
throw new Exception(
string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1}).\n"
+ "Actual pre-position: {2}. Something is seriously wrong in chunk {3}.",
length, suffixLength, actualPosition, Chunk));
}
return(true);
}
public static void WriteWithLengthPrefixAndSuffixTo(this ILogRecord record, BinaryWriter writer)
{
using (var memoryStream = new MemoryStream()) {
record.WriteTo(new BinaryWriter(memoryStream));
var length = (int)memoryStream.Length;
writer.Write(length);
writer.Write(memoryStream.GetBuffer(), 0, (int)memoryStream.Length);
writer.Write(length);
}
}
public static void Log(ILogRecord record)
{
string persisterNameSpace = HoneypotSettings.Settings.RequestPersister;
Type requestPersister = TypeHelper.GetTypeFromAllAssemblies(persisterNameSpace);
using (var logWriter = Activator.CreateInstance(requestPersister) as IRequestPersister)
{
logWriter.Log(record);
}
}
protected bool TryReadBackwardInternal(ReaderWorkItem workItem, long actualPosition, out int length,
out ILogRecord record) {
length = -1;
record = null;
// no space even for length prefix and suffix
if (actualPosition < 2 * sizeof(int)) {
_log.Warning(
"Tried to read actual position {actualPosition}, but the position isn't large enough to contain a record",
actualPosition);
return false;
}
var realPos = GetRawPosition(actualPosition);
workItem.Stream.Position = realPos - sizeof(int);
length = workItem.Reader.ReadInt32();
if (length <= 0) {
throw new InvalidReadException(
string.Format("Log record that ends at actual pos {0} has non-positive length: {1}. "
+ "In chunk {2}.",
actualPosition, length, Chunk));
}
if (length > TFConsts.MaxLogRecordSize) {
throw new ArgumentException(
string.Format("Log record that ends at actual pos {0} has too large length: {1} bytes, "
+ "while limit is {2} bytes. In chunk {3}.",
actualPosition, length, TFConsts.MaxLogRecordSize, Chunk));
}
if (actualPosition < length + 2 * sizeof(int)) // no space for record + length prefix and suffix
{
throw new UnableToReadPastEndOfStreamException(
string.Format("There is not enough space to read full record (length suffix: {0}). "
+ "Actual post-position: {1}. Something is seriously wrong in chunk {2}.",
length, actualPosition, Chunk));
}
workItem.Stream.Position = realPos - length - 2 * sizeof(int);
// verify suffix length == prefix length
int prefixLength = workItem.Reader.ReadInt32();
if (prefixLength != length) {
throw new Exception(
string.Format("Prefix/suffix length inconsistency: prefix length({0}) != suffix length ({1})"
+ "Actual post-position: {2}. Something is seriously wrong in chunk {3}.",
prefixLength, length, actualPosition, Chunk));
}
record = LogRecord.ReadFrom(workItem.Reader, length);
return true;
}
void Write(int chunkNum, TFChunk chunk, ILogRecord record, out long newPos)
{
var writerRes = chunk.TryAppend(record);
if (!writerRes.Success)
{
throw new Exception(string.Format("Could not write log record: {0}", record));
}
_db.Config.WriterCheckpoint.Write(chunkNum * (long)_db.Config.ChunkSize + writerRes.NewPosition);
newPos = _db.Config.WriterCheckpoint.ReadNonFlushed();
}
public RequestResponseLoggingMiddleware(RequestDelegate next,
ILoggerFactory loggerFactory,
ILogRecord record,
IConfiguration configuration)
{
_next = next;
_logger = loggerFactory.CreateLogger <RequestResponseLoggingMiddleware>();
_configuration = configuration;
_record = record;
var connectionString = _configuration.GetConnectionString(AppConst.DefaultDb);
_writer = new SqlServerLogWriter(connectionString);
}
public SeqReadResult(bool success,
bool eof,
ILogRecord logRecord,
int recordLength,
long recordPrePosition,
long recordPostPosition)
{
Success = success;
Eof = eof;
LogRecord = logRecord;
RecordLength = recordLength;
RecordPrePosition = recordPrePosition;
RecordPostPosition = recordPostPosition;
}
private async void LogRequest(HttpRequest request)
{
if (HoneypotSettings.Settings.LogingEnabled)
{
string recordModelNameSpace = HoneypotSettings.Settings.RecordModel;
Type requestPersister = TypeHelper.GetTypeFromAllAssemblies(recordModelNameSpace);
ILogRecord record = Activator.CreateInstance(requestPersister) as ILogRecord;
if (record != null)
{
record.MapModelToRequest(request, IsTrapped);
Logger.Log(record);
}
}
}
public int GerErrorsCount()
{
int result = 0;
ILogRecordset logRecordset = _logQuery.Execute($"SELECT COUNT(*) AS rowCount FROM {_logFileName} WHERE Field2 = 'ERROR'", _input);
while (!logRecordset.atEnd())
{
ILogRecord record = logRecordset.getRecord();
result = record.getValue("rowCount");
logRecordset.moveNext();
}
return(result);
}
public bool Write(ILogRecord record, out long newPos)
{
var result = _currentChunk.TryAppend(record);
if (result.Success)
{
_writerCheckpoint.Write(result.NewPosition + _currentChunk.ChunkHeader.ChunkStartPosition);
}
else
{
CompleteChunk(); // complete updates checkpoint internally
}
newPos = _writerCheckpoint.ReadNonFlushed();
return(result.Success);
}
public long Write(ILogRecord record)
{
lock (_lockObj)
{
if (_isClosed)
{
return(-1L);
}
//如果当前文件已经写完，则需要新建文件
if (_currentChunk.IsCompleted)
{
_currentChunk = _chunkManager.AddNewChunk();
}
//先尝试写文件
var result = _currentChunk.TryAppend(record);
//如果当前文件已满
if (!result.Success)
{
//结束当前文件
_currentChunk.Complete();
//新建新的文件
_currentChunk = _chunkManager.AddNewChunk();
//再尝试写入新的文件
result = _currentChunk.TryAppend(record);
//如果还是不成功，则报错
if (!result.Success)
{
throw new ChunkWriteException(_currentChunk.ToString(), "Write record to chunk failed.");
}
}
//如果需要同步刷盘，则立即同步刷盘
if (!_chunkManager.IsMemoryMode && _chunkManager.Config.SyncFlush)
{
_currentChunk.Flush();
}
//返回数据写入位置
return(result.Position);
}
}
public long Write(ILogRecord record)
{
lock (_lockObj)
{
if (_isClosed)
{
throw new ChunkWriteException(_currentChunk.ToString(), "Chunk writer is closed.");
}
//如果当前文件已经写完，则需要新建文件
if (_currentChunk.IsCompleted)
{
_currentChunk = _chunkManager.AddNewChunk();
}
//先尝试写文件
var result = _currentChunk.TryAppend(record);
//如果当前文件已满
if (!result.Success)
{
//结束当前文件
_currentChunk.Complete();
//新建新的文件
_currentChunk = _chunkManager.AddNewChunk();
//再尝试写入新的文件
result = _currentChunk.TryAppend(record);
//如果还是不成功，则报错
if (!result.Success)
{
throw new ChunkWriteException(_currentChunk.ToString(), "Write record to chunk failed.");
}
}
//如果需要同步刷盘，则立即同步刷盘
if (_chunkManager.Config.SyncFlush)
{
_currentChunk.Flush();
}
//返回数据写入位置
return result.Position;
}
}
public string GetLogMetadata()
{
StringBuilder result = new StringBuilder();
ILogRecordset logRecordset = _logQuery.Execute($"SELECT Field2 as logLevel, COUNT(*) AS rowCount FROM {_logFileName} GROUP BY Field2", _input);
while (!logRecordset.atEnd())
{
ILogRecord record = logRecordset.getRecord();
if (!string.IsNullOrEmpty(record.getValue("logLevel").ToString()))
{
result.Append($"Level: {record.getValue("logLevel")}; Count: {record.getValue("rowCount")}\n");
}
logRecordset.moveNext();
}
return(result.ToString());
}
public RecordWriteResult TryAppend(ILogRecord record)
{
if (_isReadOnly)
{
throw new InvalidOperationException("Cannot write to a read-only block.");
}
var workItem = _writerWorkItem;
var buffer = workItem.Buffer;
var bufferWriter = workItem.BufferWriter;
buffer.SetLength(4);
buffer.Position = 4;
record.WriteTo(bufferWriter);
var length = (int)buffer.Length - 4;
bufferWriter.Write(length); // length suffix
buffer.Position = 0;
bufferWriter.Write(length); // length prefix
if (workItem.StreamPosition + length + 2 * sizeof(int) > ChunkHeader.Size + _chunkHeader.ChunkSize)
{
return(RecordWriteResult.Failed(GetDataPosition(workItem)));
}
var oldPosition = WriteRawData(workItem, buffer);
_physicalDataSize = (int)GetDataPosition(workItem); // should fit 32 bits
_logicalDataSize = ChunkHeader.GetLocalLogPosition(record.LogPosition + length + 2 * sizeof(int));
// for non-scavenged chunk _physicalDataSize should be the same as _logicalDataSize
// for scavenged chunk _logicalDataSize should be at least the same as _physicalDataSize
if ((!ChunkHeader.IsScavenged && _logicalDataSize != _physicalDataSize) ||
(ChunkHeader.IsScavenged && _logicalDataSize < _physicalDataSize))
{
throw new Exception(string.Format(
"Data sizes violation. Chunk: {0}, IsScavenged: {1}, LogicalDataSize: {2}, PhysicalDataSize: {3}.",
FileName, ChunkHeader.IsScavenged, _logicalDataSize, _physicalDataSize));
}
return(RecordWriteResult.Successful(oldPosition, _physicalDataSize));
}
/// <summary>
/// ログ・ファイルに記録します。失敗しても無視します。
/// </summary>
/// <param name="line"></param>
static void XLine(ILogRecord record, string level, string line, IResFile targetOrNull)
{
// ログ出力オフ
if (!record.Enabled)
{
return;
}
// ログ追記
try
{
StringBuilder sb = new StringBuilder();
// タイムスタンプ
if (record.TimeStampPrintable)
{
sb.Append($"[{DateTime.Now.ToString()}] ");
}
sb.Append($"{level} {line}");
sb.AppendLine();
string message = sb.ToString();
if (targetOrNull != null)
{
System.IO.File.AppendAllText(targetOrNull.Name, message);
}
else
{
System.IO.File.AppendAllText(record.LogFile.Name, message);
}
}
catch (Exception)
{
//>>>>> エラーが起こりました。
// どうにもできないので 無視します。
}
}
public ILogRecordset executeCommand()
{
// prepare LogParser Recordset & Record objects
ILogRecordset rsLP = null;
ILogRecord rowLP = null;
LogQueryClassClass LogParser = null;
COMCSVInputContextClass W3Clog = null;
LogParser = new LogQueryClassClass();
W3Clog = new COMCSVInputContextClass();
try
{
//W3C Logparsing SQL. Replace this SQL query with whatever
//you want to retrieve. The example below
//will sum up all the bandwidth
//Usage of a specific folder with name
//"userID". Download Log Parser 2.2
//from Microsoft and see sample queries.
foreach (string s in logList)
{
cmd = Utils.ReplaceFirst(cmd, s, "'" + currentDir + "\\Logs\\" + s + "'");
}
Debug.WriteLine(cmd);
// run the query against W3C log
rsLP = LogParser.Execute(cmd, W3Clog);
}
catch
{
throw;
}
return(rsLP);
}
public override void UndoLog(ILogRecord record, ITransaction tran)
{
lock (this.lockObject)
{
var undoContent = record.GetUndoContent();
RowHolder rs = new RowHolder(this.columnTypes, undoContent.DataToUndo);
if (record.GetRecordType() == LogRecordType.RowModify)
{
this.items.SetRow(undoContent.RowPosition, rs);
}
else if (record.GetRecordType() == LogRecordType.RowInsert)
{
this.items.DeleteRow(undoContent.RowPosition);
this.rowCount--;
}
else
{
throw new NotImplementedException();
}
}
}
public override void RedoLog(ILogRecord record, ITransaction tran)
{
lock (this.lockObject)
{
var redoContent = record.GetRedoContent();
RowHolder rs = new RowHolder(this.columnTypes, redoContent.DataToApply);
if (record.GetRecordType() == LogRecordType.RowModify)
{
this.items.SetRow(redoContent.RowPosition, rs);
}
else if (record.GetRecordType() == LogRecordType.RowInsert)
{
int ret = this.items.InsertRow(rs);
this.rowCount++;
Debug.Assert(ret != -1);
}
else
{
throw new NotImplementedException();
}
}
}
public void WriteLog(ILogRecord log)
{
try
{
lock (log)
{
using (var connection = new SqlConnection(_connectionString))
{
using (var command = new SqlCommand("[Admon].LogRecordInsert", connection))
{
connection.Open();
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@userSystem", log.UserSystem);
command.Parameters.AddWithValue("@nameSystem", log.NameSystem);
command.Parameters.AddWithValue("@userName", log.UserName);
command.Parameters.AddWithValue("@logTimeStart", log.LogTimeStart);
command.Parameters.AddWithValue("@esquema", log.Esquema);
command.Parameters.AddWithValue("@ipOrigin", log.IpOrigin);
command.Parameters.AddWithValue("@metodo", log.Metodo);
command.Parameters.AddWithValue("@ruta", log.Ruta);
command.Parameters.AddWithValue("@host", log.Host);
command.Parameters.AddWithValue("@queryString", log.QueryString);
command.Parameters.AddWithValue("@body", log.Body);
command.Parameters.AddWithValue("@response", log.Response);
command.Parameters.AddWithValue("@statusResponse", log.ResponseStatus);
command.ExecuteNonQuery();
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public long Write(ILogRecord record)
{
lock (_lockSync)
{
if (_isClosed)
{
return(-1L);
}
if (_currentChunk.IsCompleted)
{
_currentChunk = _chunkManager.AddNewChunk();
}
var result = _currentChunk.TryAppend(record);
if (!result.Success)//文件已满
{
//结束当前文件
_currentChunk.Complete();
//新建文件
_currentChunk = _chunkManager.AddNewChunk();
result = _currentChunk.TryAppend(record);//重新写文件
if (!result.Success)
{
throw new ChunkWriteException(this.ToString(), "写记录失败");
}
}
if (!_chunkManager.IsMemoryMode && _chunkManager.Config.SyncFlush)
{
_currentChunk.Flush();
}
return(result.Position);
}
}
public List <IPDataResult> ParseW3CLog()
{
ILogRecordSet rsLP = null;
ILogRecord rowLP = null;
var result = new List <IPDataResult>();
rsLP = IISLogQuery.Execute(_logPathFileName, new string[] { "c-ip", "s-ip", "cs(Referer)" });
do
{
rowLP = rsLP.Element;
var clientIp = ExtractClientIP(rowLP.GetValue(0));
var fqdn = ExtractFQDN(rowLP.GetValue(2), rowLP.GetValue(1));
var nCalls = ExtractNCalls(rowLP.GetValue(3));
SetupResult(clientIp, fqdn, nCalls, ref result);
rsLP = rsLP.NextElement;
} while (rsLP != null);
return(result);
}
public void getLogTable(object sender, EventArgs e)
{
LogTable.Rows.Clear();
Logger l = new Logger(query.Text);
ILogRecordset result = l.executeCommand();
ILogRecord dataRow = null;
TableHeaderRow header = new TableHeaderRow();
TableHeaderCell headerCell;
for (int i = 0; i < result.getColumnCount(); i++)
{
headerCell = new TableHeaderCell();
headerCell.Text = result.getColumnName(i);
headerCell.CssClass = "forumHeader";
headerCell.Style.Add("border", "1px solid black");
header.Cells.Add(headerCell);
}
LogTable.Rows.Add(header);
while (!result.atEnd())
{
dataRow = result.getRecord();
TableRow row = new TableRow();
TableCell cell;
for (int i = 0; i < result.getColumnCount(); i++)
{
cell = new TableCell();
cell.Text = dataRow.getValue(i).ToString();
cell.Style.Add("border", "1px solid black");
row.Cells.Add(cell);
}
LogTable.Rows.Add(row);
result.moveNext();
}
}
public JObject ReadToJson(ILogRecord row)
{
var json = new JObject();
foreach (var field in this.fields)
{
if (this.columnMap.ContainsKey(field.Name))
{
object v = row.getValue(field.Name);
if (field.DataType == typeof(DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
{
json.Add(new JProperty(field.Name, v));
}
}
}
AddTimestamp(json);
return json;
}
public override void ImportToDatabase(ILogRecord record, DateTime date)
{
if (ConfigurationManager.AppSettings["AddToDatabase"] == "1")
{
using(IDnaDataReader dataReader = StoredProcedureReader.Create("impressionswrite"))
{
dataReader.AddParameter("@date", DateTime.Parse(record.getValue("DateTime").ToString()));
dataReader.AddParameter("@machine_name", record.getValue("ComputerName"));
dataReader.AddParameter("@http_method", record.getValue("Method"));
dataReader.AddParameter("@http_status", record.getValue("Status"));
dataReader.AddParameter("@url", record.getValue("URL"));
dataReader.AddParameter("@site", String.IsNullOrEmpty(record.getValue("Site").ToString()) ? "UNKNOWN" : record.getValue("Site").ToString());
dataReader.AddParameter("@count", Int32.Parse(record.getValue("count").ToString()));
dataReader.AddParameter("@min", Int32.Parse(record.getValue("min").ToString()));
dataReader.AddParameter("@max", Int32.Parse(record.getValue("max").ToString()));
dataReader.AddParameter("@avg", Int32.Parse(record.getValue("avg").ToString()));
dataReader.Execute();
}
}
}
public virtual void ImportToDatabase(ILogRecord record, DateTime date) { }
public RecordWriteResult TryAppend(ILogRecord record)
{
if (_isCompleted)
{
throw new ChunkWriteException(this.ToString(), string.Format("Cannot write to a read-only chunk, isMemoryChunk: {0}, _dataPosition: {1}", _isMemoryChunk, _dataPosition));
}
_lastActiveTime = DateTime.Now;
var writerWorkItem = _writerWorkItem;
var bufferStream = writerWorkItem.BufferStream;
var bufferWriter = writerWorkItem.BufferWriter;
var recordBuffer = default(byte[]);
if (IsFixedDataSize())
{
if (writerWorkItem.WorkingStream.Position + _chunkConfig.ChunkDataUnitSize > ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize)
{
return(RecordWriteResult.NotEnoughSpace());
}
bufferStream.Position = 0;
record.WriteTo(GlobalDataPosition, bufferWriter);
var recordLength = (int)bufferStream.Length;
if (recordLength != _chunkConfig.ChunkDataUnitSize)
{
throw new ChunkWriteException(this.ToString(), string.Format("Invalid fixed data length, expected length {0}, but was {1}", _chunkConfig.ChunkDataUnitSize, recordLength));
}
if (_cacheItems != null)
{
recordBuffer = new byte[recordLength];
Buffer.BlockCopy(bufferStream.GetBuffer(), 0, recordBuffer, 0, recordLength);
}
}
else
{
bufferStream.SetLength(4);
bufferStream.Position = 4;
record.WriteTo(GlobalDataPosition, bufferWriter);
var recordLength = (int)bufferStream.Length - 4;
bufferWriter.Write(recordLength); // write record length suffix
bufferStream.Position = 0;
bufferWriter.Write(recordLength); // write record length prefix
if (recordLength > _chunkConfig.MaxLogRecordSize)
{
throw new ChunkWriteException(this.ToString(),
string.Format("Log record at data position {0} has too large length: {1} bytes, while limit is {2} bytes",
_dataPosition, recordLength, _chunkConfig.MaxLogRecordSize));
}
if (writerWorkItem.WorkingStream.Position + recordLength + 2 * sizeof(int) > ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize)
{
return(RecordWriteResult.NotEnoughSpace());
}
if (_cacheItems != null)
{
recordBuffer = new byte[recordLength];
Buffer.BlockCopy(bufferStream.GetBuffer(), 4, recordBuffer, 0, recordLength);
}
}
var writtenPosition = _dataPosition;
var buffer = bufferStream.GetBuffer();
lock (_writeSyncObj)
{
writerWorkItem.AppendData(buffer, 0, (int)bufferStream.Length);
}
_dataPosition = (int)writerWorkItem.WorkingStream.Position - ChunkHeader.Size;
var position = ChunkHeader.ChunkDataStartPosition + writtenPosition;
if (_chunkConfig.EnableCache)
{
if (_memoryChunk != null)
{
var result = _memoryChunk.TryAppend(record);
if (!result.Success)
{
throw new ChunkWriteException(this.ToString(), "Append record to file chunk success, but append to memory chunk failed as memory space not enough, this should not be happened.");
}
else if (result.Position != position)
{
throw new ChunkWriteException(this.ToString(), string.Format("Append record to file chunk success, and append to memory chunk success, but the position is not equal, memory chunk write position: {0}, file chunk write position: {1}.", result.Position, position));
}
}
else if (_cacheItems != null && recordBuffer != null)
{
var index = writtenPosition % _chunkConfig.ChunkLocalCacheSize;
_cacheItems[index] = new CacheItem {
RecordPosition = writtenPosition, RecordBuffer = recordBuffer
};
}
}
else if (_cacheItems != null && recordBuffer != null)
{
var index = writtenPosition % _chunkConfig.ChunkLocalCacheSize;
_cacheItems[index] = new CacheItem {
RecordPosition = writtenPosition, RecordBuffer = recordBuffer
};
}
if (!_isMemoryChunk && _chunkConfig.EnableChunkWriteStatistic)
{
_chunkStatisticService.AddWriteBytes(ChunkHeader.ChunkNumber, (int)bufferStream.Length);
}
return(RecordWriteResult.Successful(position));
}
public async Task Recovery(BinaryReader reader, IPageManager pageManager, ITransaction tran)
{
// Phase I.
// Figure out what is committed.
bool passCompleted = false;
HashSet <ulong> committedTransactions = new HashSet <ulong>();
HashSet <ulong> uncommittedTransactions = new HashSet <ulong>();
while (!passCompleted)
{
if (reader.BaseStream.Position == reader.BaseStream.Length)
{
passCompleted = true;
break;
}
LogRecordType recordType = (LogRecordType)reader.ReadByte();
ILogRecord rc = null;
switch (recordType)
{
case LogRecordType.NullRecord: passCompleted = true; break;
case LogRecordType.Commit:
ulong transactionId = reader.ReadUInt64();
uncommittedTransactions.Remove(transactionId);
committedTransactions.Add(transactionId);
break;
case LogRecordType.Rollback: break;
case LogRecordType.RowModify:
rc = new UpdateRowRecord(reader);
break;
case LogRecordType.AllocatePage:
rc = new AllocatePageLogRecord(reader);
break;
case LogRecordType.RowInsert:
rc = new InsertRowRecord(reader);
break;
}
if (rc != null)
{
uncommittedTransactions.Add(rc.TransactionId());
}
}
// Phase II.
// Redo committed.
reader.BaseStream.Seek(0, SeekOrigin.Begin);
List <ILogRecord> undoTranList = new List <ILogRecord>();
passCompleted = false;
while (!passCompleted)
{
if (reader.BaseStream.Position == reader.BaseStream.Length)
{
passCompleted = true;
break;
}
LogRecordType recordType = (LogRecordType)reader.ReadByte();
ILogRecord rc = null;
switch (recordType)
{
case LogRecordType.NullRecord: passCompleted = true; break;
case LogRecordType.Commit:
ulong transactionId = reader.ReadUInt64();
break;
case LogRecordType.Rollback: break;
case LogRecordType.RowModify:
rc = new UpdateRowRecord(reader);
break;
case LogRecordType.RowInsert:
rc = new InsertRowRecord(reader);
break;
case LogRecordType.AllocatePage:
rc = new AllocatePageLogRecord(reader);
break;
}
if (rc != null)
{
if (committedTransactions.Contains(rc.TransactionId()))
{
await rc.Redo(pageManager, tran).ConfigureAwait(false);
}
else
{
undoTranList.Add(rc);
}
}
}
// Phase III.
// Undo uncommitted.
undoTranList.Reverse();
foreach (ILogRecord rc in undoTranList)
{
await rc.Undo(pageManager, tran).ConfigureAwait(false);
}
}
private Result Map(ILogRecord record)
{
var value = -1;
var dateTime = DateTime.Now;
var name = string.Empty;
var r = record.getValue(0);
if ((r is string))
{
name = Convert.ToString(r);
}
if ((r is int) || (r is decimal))
{
value = Convert.ToInt32(r);
}
r = record.getValue(1);
if ((r is string))
{
name = Convert.ToString(r);
}
if ((r is int) || (r is decimal))
{
value = Convert.ToInt32(r);
}
//Console.WriteLine(record.getValue(0).GetType());
//Console.WriteLine(record.getValue(1).GetType());
try
{
//Console.WriteLine(record.getValue(2));
}
catch (Exception ex)
{
var s = ex.Message;
}
//Console.WriteLine(string.Format("{0} {1}", record.getValue(0), record.getValue(1)));
//for (int i = 0; i < record.; i++)
//{
// if (record[i] is string)
// {
// name = Convert.ToString(record[i]);
// }
// if (record[i] is int || record[i] is decimal)
// {
// value = Convert.ToInt32(record[i]);
// }
// if (record[i] is DateTime)
// {
// dateTime = Convert.ToDateTime(record[i]);
// }
//}
if (this.Name != string.Empty && name == string.Empty)
{
name = this.Name;
}
this.Log.Debug(string.Format("Got [{1}] {0}", value, dateTime));
var result = new Result(name, dateTime, this.Path);
result.SetValue(value);
return result;
}
