/// <summary>
/// Sends the specified message to the logical queue specified by <paramref name="destinationQueueName"/> by writing
/// a JSON serialied text to a file in the corresponding directory. The actual write operation is delayed until
/// the commit phase of the queue transaction unless we're non-transactional, in which case it is written immediately.
/// </summary>
public async Task Send(string destinationQueueName, TransportMessage message, ITransactionContext context)
{
var destinationDirectory = _queueRegister.EnsureQueueInitialized(destinationQueueName);
var serializedMessage = Serialize(message);
var fileName = string.Empty;
if (message.Headers.TryGetValue(Headers.DeferredUntil, out var deferTime))
{
var deferUntil = DateTime.Parse(deferTime);
fileName = _fileNameGenerator.GetDeferedFileName(deferUntil);
}
else
{
fileName = _fileNameGenerator.GetNextFileName();
}
string tempFileName = $"t{fileName.Substring(1)}";
string tempFilePath = Path.Combine(destinationDirectory, tempFileName);
context.OnCommitted(async() =>
{
// new file in async mode
// write the file with the temporary name prefix (so it could not be read while it is written)
using (var stream = new FileStream(tempFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1024 * 8, true))
{
var bytes = TransportHelper.FavoriteEncoding.GetBytes(serializedMessage);
await stream.WriteAsync(bytes, 0, bytes.Length);
}
// rename the file after the write is completed
TransportHelper.RenameFile(tempFilePath, fileName, out var _);
});
}
public string GetNextFileName()
{
Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999);
string ticks = TransportHelper.GetTimeTicks();
string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0');
return($"b{ticks}{seqnum}_{_transportId}.json");
}
/// <summary>
/// Gets the full path to the next file to be received or null if no files
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public string Dequeue(CancellationToken cancellationToken)
{
string fullPath = null;
if (!this._TryDequeue(out fullPath))
{
// allow only one thread at a time to fill the cache (it is done without locking)
if (Interlocked.CompareExchange(ref _accessCounter, 1, 0) > 0)
{
return(null);
}
try
{
// try again, maybe another thread already added items to the queue
if (!this._TryDequeue(out fullPath))
{
TimeSpan lastNoMessage = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoMessage);
if (lastNoMessage.TotalMilliseconds > BACKOFF_NOMSG_MSEC)
{
var dirName = _queueRegister.EnsureQueueInitialized(this._inputQueue);
List <string> tempStore = new List <string>();
// Don't check for defered too often
TimeSpan lastNoDefered = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoDefered);
if (lastNoDefered.TotalMilliseconds > BACKOFF_DEFERED_MSEC)
{
int deferedCnt = this._TryFillCacheDefered(tempStore, dirName, cancellationToken);
if (deferedCnt == 0)
{
this._lastNoDefered = DateTime.Now.Ticks;
}
}
int fileCnt = this._TryFillCache(tempStore, dirName, cancellationToken);
if (fileCnt == 0)
{
this._lastNoMessage = DateTime.Now.Ticks;
}
foreach (string name in tempStore.OrderBy(name => TransportHelper.GetFileUtcDate(name)))
{
this._filesQueue.Enqueue(name);
}
this._TryDequeue(out fullPath);
}
}
}
finally
{
Interlocked.CompareExchange(ref _accessCounter, 0, 1);
}
}
return(fullPath);
}
public string GetDeferedFileName(DateTime deferUntil)
{
string id = TransportHelper.GenerateID();
Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999);
string ticks = TransportHelper.GetTimeTicks(deferUntil);
string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0');
return($"d{ticks}{seqnum}_{id}.json");
}
/// <summary>
/// Constructs the file system transport to create "queues" as subdirectories of the specified base directory.
/// While it is apparent that <seealso cref="baseDirectory"/> must be a valid directory name, please note that
/// <seealso cref="_inputQueue"/> must not contain any invalid path either.
/// </summary>
public FileSystemTransport(string baseDirectory, string inputQueue)
{
if (inputQueue == null)
{
return;
}
TransportHelper.EnsureQueueNameIsValid(inputQueue);
_inputQueue = inputQueue;
_queueRegister = new QueueRegister(baseDirectory);
_fileQueue = new FileQueue(inputQueue, _queueRegister);
_fileNameGenerator = new FileNameGenerator();
}
private bool _TryDequeue(out string fullPath)
{
fullPath = null;
var dirName = _queueRegister.EnsureQueueInitialized(this._inputQueue);
string fileName;
bool loopAgain = false;
do
{
loopAgain = false;
fullPath = null;
fileName = null;
string tempPath = null;
lock (this._filesCacheLock)
{
if (this._filesQueue.TryDequeue(out fileName))
{
tempPath = Path.Combine(dirName, fileName);
this._filesCache.Remove(fileName);
}
else
{
return(false);
}
}
/*
* // Check if the file is defered (skip them if not ready for processing)
* if (fileName.StartsWith("d") && TransportHelper.GetFileUtcDate(fileName) > DateTime.Now.ToUniversalTime())
* {
* loopAgain = true;
* }
* else
*/
if (!TransportHelper.RenameToTempWithLock(tempPath, out fullPath))
{
// this file is used by somebody else (try to get another one)
loopAgain = true;
}
} while (loopAgain);
return(true);
}
private int _TryFillCacheDefered(List <string> tempStore, string dirName, CancellationToken cancellationToken)
{
DirectoryInfo info = new DirectoryInfo(dirName);
var nowUtc = DateTime.Now.ToUniversalTime();
var files = info.EnumerateFiles("d*.json").Where(p => TransportHelper.GetFileUtcDate(p.Name) <= nowUtc).OrderBy(p => p.Name).Take(DEFER_CACHE_SIZE).ToList();
int cnt = 0;
foreach (var file in files)
{
lock (this._filesCacheLock)
{
if (!this._filesCache.Contains(file.Name))
{
tempStore.Add(file.Name);
this._filesCache.Add(file.Name);
++cnt;
}
}
cancellationToken.ThrowIfCancellationRequested();
}
return(cnt);
}
/// <summary>
/// Receives the next message from the logical input queue by loading the next file from the corresponding directory,
/// deserializing it, deleting it when the transaction is committed.
/// </summary>
public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
{
TransportMessage receivedTransportMessage = null;
string fullPath = null;
bool loopAgain = false;
do
{
loopAgain = false;
receivedTransportMessage = null;
fullPath = _fileQueue.Dequeue(cancellationToken);
if (!string.IsNullOrEmpty(fullPath))
{
var jsonText = await TransportHelper.ReadAllText(fullPath);
receivedTransportMessage = Deserialize(jsonText);
if (!_CheckIsValid(fullPath, receivedTransportMessage))
{
File.Delete(fullPath);
loopAgain = true;
}
}
} while (loopAgain);
if (receivedTransportMessage != null)
{
context.OnCompleted(async() => {
await TransportHelper.DeleteFile(fullPath);
});
context.OnAborted(async() => {
TransportHelper.RenameToError(fullPath, out var _);
});
}
return(receivedTransportMessage);
}
public FileNameGenerator()
{
// Generate unique transport id
_transportId = TransportHelper.GenerateID();
}