private void CreateHead(int nextId)
{
var newHead = new QueueFileInfo(nextId, GetFilename(nextId));
_files.Add(nextId, newHead);
_head = newHead;
}
private void Initialize()
{
if (!Directory.Exists(_storageRoot))
{
Directory.CreateDirectory(_storageRoot);
}
var files = from filename in Directory.GetFiles(_storageRoot, "data_*.bin")
let parts = Path.GetFileNameWithoutExtension(filename).Split('_')
let id = int.Parse(parts[parts.Length - 1])
orderby id
select new QueueFileInfo(id, GetFilename(id));
_log.DebugFormat("Queue storage usage for {0}", _storageRoot);
foreach (var queueFileInfo in files)
{
if (_log.IsDebugEnabled)
{
var finfo = new System.IO.FileInfo(GetFilename(queueFileInfo.Id));
_log.DebugFormat(" {0}: {1:0.00}KB", Path.GetFileName(GetFilename(queueFileInfo.Id)), (double)finfo.Length / 1024);
}
_head = queueFileInfo;
_files.Add(queueFileInfo.Id, queueFileInfo);
MapRecords(queueFileInfo);
}
if (_files.Count == 0)
{
CreateHead(1);
}
}
private void RemoveFile(QueueFileInfo queueFileInfo)
{
queueFileInfo.Stream.Close();
queueFileInfo.Stream.Dispose();
var filename = GetFilename(queueFileInfo.Id);
try {
File.Delete(filename);
} catch (Exception e) {
_log.Warn(string.Format("unable to delete disposed file '{0}'", filename), e);
}
_files.Remove(queueFileInfo.Id);
}
private void RemoveHandle(QueueStreamHandle handle, QueueFileInfo queueFileInfo)
{
HashSet <QueueStreamHandle> handles;
if (!_recordMap.TryGetValue(handle.Id, out handles))
{
return;
}
handles.Remove(handle);
if (handles.Count != 0)
{
return;
}
_recordMap.Remove(handle.Id);
if (queueFileInfo == _head)
{
// tail caught up with head, so let's truncate the file
queueFileInfo.Stream.SetLength(0);
}
else
{
// no undeleted records in non-head file, remove file
RemoveFile(queueFileInfo);
}
if (_head.Id == 1 || _recordQueue.Count > 0)
{
return;
}
if (_recordMap.Count > 0)
{
return;
}
// the head is not the first file and there are no remaining items, so we can reset the head to the first file
RemoveFile(_head);
CreateHead(1);
}
private void MapRecords(QueueFileInfo queueFileInfo)
{
if (queueFileInfo.Stream.Length == 0)
{
return;
}
queueFileInfo.Stream.Position = 0;
var header = new byte[RecordMarker.Length];
while (true)
{
var position = queueFileInfo.Stream.Position;
var missedHeader = false;
int read;
var isDeleted = false;
while (true)
{
read = queueFileInfo.Stream.Read(header, 0, header.Length);
if (read != header.Length)
{
// end of file? WTF? ok, bail
_log.Warn("reached end of file trying to read the next record marker");
return;
}
if (ArrayUtil.Compare(header, RecordMarker) == 0)
{
break;
}
if (ArrayUtil.Compare(header, DeletedMarker) == 0)
{
isDeleted = true;
break;
}
queueFileInfo.Stream.Position = ++position;
missedHeader = true;
}
if (missedHeader)
{
_log.Warn("missed expected header and skipped corrupt data");
}
var lengthBytes = new byte[LENGTH_SIZE];
read = queueFileInfo.Stream.Read(lengthBytes, 0, LENGTH_SIZE);
if (read < LENGTH_SIZE)
{
// end of file? WTF? ok, bail
_log.Warn("reached end of file trying to read the number of bytes in the next record, skipping record");
queueFileInfo.Stream.Seek(0, SeekOrigin.Begin);
return;
}
var length = BitConverter.ToInt32(lengthBytes, 0);
if (length <= 0)
{
_log.Warn("illegal record length, must be in corrupted record, skipping record");
queueFileInfo.Stream.Position = ++position;
continue;
}
if (!isDeleted)
{
position = queueFileInfo.Stream.Position - HEADER_SIZE;
AddHandle(queueFileInfo.Id, position, length);
}
if (queueFileInfo.Stream.Seek(length, SeekOrigin.Current) == queueFileInfo.Stream.Length)
{
break;
}
}
queueFileInfo.Stream.Seek(0, SeekOrigin.Begin);
}
private void RemoveHandle(QueueStreamHandle handle, QueueFileInfo queueFileInfo)
{
HashSet<QueueStreamHandle> handles;
if(!_recordMap.TryGetValue(handle.Id, out handles)) {
return;
}
handles.Remove(handle);
if(handles.Count != 0) {
return;
}
_recordMap.Remove(handle.Id);
if(queueFileInfo == _head) {
// tail caught up with head, so let's truncate the file
queueFileInfo.Stream.SetLength(0);
} else {
// no undeleted records in non-head file, remove file
RemoveFile(queueFileInfo);
}
if(_head.Id == 1 || _recordQueue.Count > 0) {
return;
}
if(_recordMap.Count > 0) {
return;
}
// the head is not the first file and there are no remaining items, so we can reset the head to the first file
RemoveFile(_head);
CreateHead(1);
}
private void RemoveFile(QueueFileInfo queueFileInfo)
{
queueFileInfo.Stream.Close();
queueFileInfo.Stream.Dispose();
var filename = GetFilename(queueFileInfo.Id);
try {
File.Delete(filename);
} catch(Exception e) {
_log.Warn(string.Format("unable to delete disposed file '{0}'", filename), e);
}
_files.Remove(queueFileInfo.Id);
}
private void MapRecords(QueueFileInfo queueFileInfo)
{
if(queueFileInfo.Stream.Length == 0) {
return;
}
queueFileInfo.Stream.Position = 0;
var header = new byte[RecordMarker.Length];
while(true) {
var position = queueFileInfo.Stream.Position;
var missedHeader = false;
int read;
var isDeleted = false;
while(true) {
read = queueFileInfo.Stream.Read(header, 0, header.Length);
if(read != header.Length) {
// end of file? WTF? ok, bail
_log.Warn("reached end of file trying to read the next record marker");
return;
}
if(ArrayUtil.Compare(header, RecordMarker) == 0) {
break;
}
if(ArrayUtil.Compare(header, DeletedMarker) == 0) {
isDeleted = true;
break;
}
queueFileInfo.Stream.Position = ++position;
missedHeader = true;
}
if(missedHeader) {
_log.Warn("missed expected header and skipped corrupt data");
}
var lengthBytes = new byte[LENGTH_SIZE];
read = queueFileInfo.Stream.Read(lengthBytes, 0, LENGTH_SIZE);
if(read < LENGTH_SIZE) {
// end of file? WTF? ok, bail
_log.Warn("reached end of file trying to read the number of bytes in the next record, skipping record");
queueFileInfo.Stream.Seek(0, SeekOrigin.Begin);
return;
}
var length = BitConverter.ToInt32(lengthBytes, 0);
if(length <= 0) {
_log.Warn("illegal record length, must be in corrupted record, skipping record");
queueFileInfo.Stream.Position = ++position;
continue;
}
if(!isDeleted) {
position = queueFileInfo.Stream.Position - HEADER_SIZE;
AddHandle(queueFileInfo.Id, position, length);
}
if(queueFileInfo.Stream.Seek(length, SeekOrigin.Current) == queueFileInfo.Stream.Length) {
break;
}
}
queueFileInfo.Stream.Seek(0, SeekOrigin.Begin);
}
private void Initialize()
{
if(!Directory.Exists(_storageRoot)) {
Directory.CreateDirectory(_storageRoot);
}
var files = from filename in Directory.GetFiles(_storageRoot, "data_*.bin")
let parts = Path.GetFileNameWithoutExtension(filename).Split('_')
let id = int.Parse(parts[parts.Length - 1])
orderby id
select new QueueFileInfo(id, GetFilename(id));
_log.DebugFormat("Queue storage usage for {0}", _storageRoot);
foreach(var queueFileInfo in files) {
if(_log.IsDebugEnabled) {
var finfo = new System.IO.FileInfo(GetFilename(queueFileInfo.Id));
_log.DebugFormat(" {0}: {1:0.00}KB", Path.GetFileName(GetFilename(queueFileInfo.Id)), (double)finfo.Length / 1024);
}
_head = queueFileInfo;
_files.Add(queueFileInfo.Id, queueFileInfo);
MapRecords(queueFileInfo);
}
if(_files.Count == 0) {
CreateHead(1);
}
}
private void CreateHead(int nextId)
{
var newHead = new QueueFileInfo(nextId, GetFilename(nextId));
_files.Add(nextId, newHead);
_head = newHead;
}