private byte[] ReadEntriesFromFile(PersistentQueueEntry firstEntry, long currentBufferSize)
{
byte[] buffer = new byte[currentBufferSize];
if (firstEntry.Length < 1)
{
return(buffer);
}
using (FileStream reader = new FileStream(GetDataPath(firstEntry.FileNumber),
FileMode.OpenOrCreate, FileAccess.Read, FileShare.ReadWrite))
{
reader.Position = firstEntry.Start;
int totalRead = 0;
do
{
int bytesRead = reader.Read(buffer, totalRead, buffer.Length - totalRead);
if (bytesRead == 0)
{
throw new InvalidOperationException("End of file reached while trying to read queue item");
}
totalRead += bytesRead;
} while (totalRead < buffer.Length);
}
return(buffer);
}
public PersistentQueueEntry Dequeue()
{
lock (_entries)
{
LinkedListNode <PersistentQueueEntry> first = _entries.First;
if (first == null)
{
return(null);
}
PersistentQueueEntry entry = first.Value;
if (entry.Data == null)
{
ReadAhead();
}
_entries.RemoveFirst();
// we need to create a copy so we will not hold the data
// in memory as well as the position
lock (_checkedOutEntries)
{
_checkedOutEntries.Add(new PersistentQueueEntry(entry.FileNumber, entry.Start, entry.Length));
}
return(entry);
}
}
private IEnumerable <int> ApplyTransactionOperationsInMemory(IEnumerable <PersistentQueueOperation> operations)
{
foreach (PersistentQueueOperation operation in operations)
{
switch (operation.Type)
{
case PersistentQueueOperationTypes.ENQUEUE:
PersistentQueueEntry entryToAdd = new PersistentQueueEntry(operation);
lock (_entries)
{
_entries.AddLast(entryToAdd);
}
int itemCountAddition = _countOfItemsPerFile.GetValueOrDefault(entryToAdd.FileNumber);
_countOfItemsPerFile[entryToAdd.FileNumber] = itemCountAddition + 1;
break;
case PersistentQueueOperationTypes.DEQUEUE:
PersistentQueueEntry entryToRemove = new PersistentQueueEntry(operation);
lock (_checkedOutEntries)
{
_checkedOutEntries.Remove(entryToRemove);
}
int itemCountRemoval = _countOfItemsPerFile.GetValueOrDefault(entryToRemove.FileNumber);
_countOfItemsPerFile[entryToRemove.FileNumber] = itemCountRemoval - 1;
break;
case PersistentQueueOperationTypes.REINSTATE:
PersistentQueueEntry entryToReinstate = new PersistentQueueEntry(operation);
lock (_entries)
{
_entries.AddFirst(entryToReinstate);
}
lock (_checkedOutEntries)
{
_checkedOutEntries.Remove(entryToReinstate);
}
break;
}
}
HashSet <int> filesToRemove = new HashSet <int>(
from pair in _countOfItemsPerFile
where pair.Value < 1
select pair.Key
);
foreach (int i in filesToRemove)
{
_countOfItemsPerFile.Remove(i);
}
return(filesToRemove.ToArray());
}
/// <summary>
/// This special purpose function is to work around potential issues with Mono
/// </summary>
// private static PersistentQueueEntry[] ToArray(LinkedList<PersistentQueueEntry> list)
// {
// if (list == null) return new PersistentQueueEntry[0];
// List<PersistentQueueEntry> outp = new List<PersistentQueueEntry>(25);
// LinkedListNode<PersistentQueueEntry> cur = list.First;
// while (cur != null)
// {
// outp.Add(cur.Value);
// cur = cur.Next;
// }
//
// return outp.ToArray();
// }
private static void WriteEntryToTransactionLog(Stream ms, PersistentQueueEntry entry, PersistentQueueOperationTypes operationType)
{
ms.Write(PersistentQueueUtils.OperationSeparatorBytes, 0, PersistentQueueUtils.OperationSeparatorBytes.Length);
ms.WriteByte((byte)operationType);
byte[] fileNumber = BitConverter.GetBytes(entry.FileNumber);
ms.Write(fileNumber, 0, fileNumber.Length);
byte[] start = BitConverter.GetBytes(entry.Start);
ms.Write(start, 0, start.Length);
byte[] length = BitConverter.GetBytes(entry.Length);
ms.Write(length, 0, length.Length);
}
/// <summary>
/// Try to pull data from the queue. Data is removed from the queue on `Flush()`
/// </summary>
public byte[] Dequeue()
{
PersistentQueueEntry entry = _queue.Dequeue();
if (entry == null)
{
return(null);
}
_operations.Add(new PersistentQueueOperation(
PersistentQueueOperationTypes.DEQUEUE,
entry.FileNumber,
entry.Start,
entry.Length
));
return(entry.Data);
}
/// <summary>
/// Assumes that entries has at least one entry. Should be called inside a lock.
/// </summary>
private void ReadAhead()
{
long currentBufferSize = 0;
PersistentQueueEntry firstEntry = _entries.First.Value;
PersistentQueueEntry lastEntry = firstEntry;
foreach (PersistentQueueEntry entry in _entries)
{
// we can't read ahead to another file or
// if we have unordered queue, or sparse items
if (entry != lastEntry &&
(entry.FileNumber != lastEntry.FileNumber ||
entry.Start != (lastEntry.Start + lastEntry.Length)))
{
break;
}
if (currentBufferSize + entry.Length > PersistentQueueUtils._1Megabytes)
{
break;
}
lastEntry = entry;
currentBufferSize += entry.Length;
}
if (lastEntry == firstEntry)
{
currentBufferSize = lastEntry.Length;
}
byte[] buffer = ReadEntriesFromFile(firstEntry, currentBufferSize);
int index = 0;
foreach (PersistentQueueEntry entry in _entries)
{
entry.Data = new byte[entry.Length];
Buffer.BlockCopy(buffer, index, entry.Data, 0, entry.Length);
index += entry.Length;
if (entry == lastEntry)
{
break;
}
}
}
