private void CreateHead(int nextId) { var newHead = new QueueFileInfo(nextId, GetFilename(nextId)); _files.Add(nextId, newHead); _head = newHead; }
private void Initialize() { if (!Directory.Exists(_storageRoot)) { Directory.CreateDirectory(_storageRoot); } var files = from filename in Directory.GetFiles(_storageRoot, "data_*.bin") let parts = Path.GetFileNameWithoutExtension(filename).Split('_') let id = int.Parse(parts[parts.Length - 1]) orderby id select new QueueFileInfo(id, GetFilename(id)); _log.DebugFormat("Queue storage usage for {0}", _storageRoot); foreach (var queueFileInfo in files) { if (_log.IsDebugEnabled) { var finfo = new System.IO.FileInfo(GetFilename(queueFileInfo.Id)); _log.DebugFormat(" {0}: {1:0.00}KB", Path.GetFileName(GetFilename(queueFileInfo.Id)), (double)finfo.Length / 1024); } _head = queueFileInfo; _files.Add(queueFileInfo.Id, queueFileInfo); MapRecords(queueFileInfo); } if (_files.Count == 0) { CreateHead(1); } }
private void RemoveFile(QueueFileInfo queueFileInfo) { queueFileInfo.Stream.Close(); queueFileInfo.Stream.Dispose(); var filename = GetFilename(queueFileInfo.Id); try { File.Delete(filename); } catch (Exception e) { _log.Warn(string.Format("unable to delete disposed file '{0}'", filename), e); } _files.Remove(queueFileInfo.Id); }
private void RemoveHandle(QueueStreamHandle handle, QueueFileInfo queueFileInfo) { HashSet <QueueStreamHandle> handles; if (!_recordMap.TryGetValue(handle.Id, out handles)) { return; } handles.Remove(handle); if (handles.Count != 0) { return; } _recordMap.Remove(handle.Id); if (queueFileInfo == _head) { // tail caught up with head, so let's truncate the file queueFileInfo.Stream.SetLength(0); } else { // no undeleted records in non-head file, remove file RemoveFile(queueFileInfo); } if (_head.Id == 1 || _recordQueue.Count > 0) { return; } if (_recordMap.Count > 0) { return; } // the head is not the first file and there are no remaining items, so we can reset the head to the first file RemoveFile(_head); CreateHead(1); }
private void MapRecords(QueueFileInfo queueFileInfo) { if (queueFileInfo.Stream.Length == 0) { return; } queueFileInfo.Stream.Position = 0; var header = new byte[RecordMarker.Length]; while (true) { var position = queueFileInfo.Stream.Position; var missedHeader = false; int read; var isDeleted = false; while (true) { read = queueFileInfo.Stream.Read(header, 0, header.Length); if (read != header.Length) { // end of file? WTF? ok, bail _log.Warn("reached end of file trying to read the next record marker"); return; } if (ArrayUtil.Compare(header, RecordMarker) == 0) { break; } if (ArrayUtil.Compare(header, DeletedMarker) == 0) { isDeleted = true; break; } queueFileInfo.Stream.Position = ++position; missedHeader = true; } if (missedHeader) { _log.Warn("missed expected header and skipped corrupt data"); } var lengthBytes = new byte[LENGTH_SIZE]; read = queueFileInfo.Stream.Read(lengthBytes, 0, LENGTH_SIZE); if (read < LENGTH_SIZE) { // end of file? WTF? ok, bail _log.Warn("reached end of file trying to read the number of bytes in the next record, skipping record"); queueFileInfo.Stream.Seek(0, SeekOrigin.Begin); return; } var length = BitConverter.ToInt32(lengthBytes, 0); if (length <= 0) { _log.Warn("illegal record length, must be in corrupted record, skipping record"); queueFileInfo.Stream.Position = ++position; continue; } if (!isDeleted) { position = queueFileInfo.Stream.Position - HEADER_SIZE; AddHandle(queueFileInfo.Id, position, length); } if (queueFileInfo.Stream.Seek(length, SeekOrigin.Current) == queueFileInfo.Stream.Length) { break; } } queueFileInfo.Stream.Seek(0, SeekOrigin.Begin); }
private void RemoveHandle(QueueStreamHandle handle, QueueFileInfo queueFileInfo) { HashSet<QueueStreamHandle> handles; if(!_recordMap.TryGetValue(handle.Id, out handles)) { return; } handles.Remove(handle); if(handles.Count != 0) { return; } _recordMap.Remove(handle.Id); if(queueFileInfo == _head) { // tail caught up with head, so let's truncate the file queueFileInfo.Stream.SetLength(0); } else { // no undeleted records in non-head file, remove file RemoveFile(queueFileInfo); } if(_head.Id == 1 || _recordQueue.Count > 0) { return; } if(_recordMap.Count > 0) { return; } // the head is not the first file and there are no remaining items, so we can reset the head to the first file RemoveFile(_head); CreateHead(1); }
private void RemoveFile(QueueFileInfo queueFileInfo) { queueFileInfo.Stream.Close(); queueFileInfo.Stream.Dispose(); var filename = GetFilename(queueFileInfo.Id); try { File.Delete(filename); } catch(Exception e) { _log.Warn(string.Format("unable to delete disposed file '{0}'", filename), e); } _files.Remove(queueFileInfo.Id); }
private void MapRecords(QueueFileInfo queueFileInfo) { if(queueFileInfo.Stream.Length == 0) { return; } queueFileInfo.Stream.Position = 0; var header = new byte[RecordMarker.Length]; while(true) { var position = queueFileInfo.Stream.Position; var missedHeader = false; int read; var isDeleted = false; while(true) { read = queueFileInfo.Stream.Read(header, 0, header.Length); if(read != header.Length) { // end of file? WTF? ok, bail _log.Warn("reached end of file trying to read the next record marker"); return; } if(ArrayUtil.Compare(header, RecordMarker) == 0) { break; } if(ArrayUtil.Compare(header, DeletedMarker) == 0) { isDeleted = true; break; } queueFileInfo.Stream.Position = ++position; missedHeader = true; } if(missedHeader) { _log.Warn("missed expected header and skipped corrupt data"); } var lengthBytes = new byte[LENGTH_SIZE]; read = queueFileInfo.Stream.Read(lengthBytes, 0, LENGTH_SIZE); if(read < LENGTH_SIZE) { // end of file? WTF? ok, bail _log.Warn("reached end of file trying to read the number of bytes in the next record, skipping record"); queueFileInfo.Stream.Seek(0, SeekOrigin.Begin); return; } var length = BitConverter.ToInt32(lengthBytes, 0); if(length <= 0) { _log.Warn("illegal record length, must be in corrupted record, skipping record"); queueFileInfo.Stream.Position = ++position; continue; } if(!isDeleted) { position = queueFileInfo.Stream.Position - HEADER_SIZE; AddHandle(queueFileInfo.Id, position, length); } if(queueFileInfo.Stream.Seek(length, SeekOrigin.Current) == queueFileInfo.Stream.Length) { break; } } queueFileInfo.Stream.Seek(0, SeekOrigin.Begin); }
private void Initialize() { if(!Directory.Exists(_storageRoot)) { Directory.CreateDirectory(_storageRoot); } var files = from filename in Directory.GetFiles(_storageRoot, "data_*.bin") let parts = Path.GetFileNameWithoutExtension(filename).Split('_') let id = int.Parse(parts[parts.Length - 1]) orderby id select new QueueFileInfo(id, GetFilename(id)); _log.DebugFormat("Queue storage usage for {0}", _storageRoot); foreach(var queueFileInfo in files) { if(_log.IsDebugEnabled) { var finfo = new System.IO.FileInfo(GetFilename(queueFileInfo.Id)); _log.DebugFormat(" {0}: {1:0.00}KB", Path.GetFileName(GetFilename(queueFileInfo.Id)), (double)finfo.Length / 1024); } _head = queueFileInfo; _files.Add(queueFileInfo.Id, queueFileInfo); MapRecords(queueFileInfo); } if(_files.Count == 0) { CreateHead(1); } }