/// <summary> /// Sends the specified message to the logical queue specified by <paramref name="destinationQueueName"/> by writing /// a JSON serialied text to a file in the corresponding directory. The actual write operation is delayed until /// the commit phase of the queue transaction unless we're non-transactional, in which case it is written immediately. /// </summary> public async Task Send(string destinationQueueName, TransportMessage message, ITransactionContext context) { var destinationDirectory = _queueRegister.EnsureQueueInitialized(destinationQueueName); var serializedMessage = Serialize(message); var fileName = string.Empty; if (message.Headers.TryGetValue(Headers.DeferredUntil, out var deferTime)) { var deferUntil = DateTime.Parse(deferTime); fileName = _fileNameGenerator.GetDeferedFileName(deferUntil); } else { fileName = _fileNameGenerator.GetNextFileName(); } string tempFileName = $"t{fileName.Substring(1)}"; string tempFilePath = Path.Combine(destinationDirectory, tempFileName); context.OnCommitted(async() => { // new file in async mode // write the file with the temporary name prefix (so it could not be read while it is written) using (var stream = new FileStream(tempFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1024 * 8, true)) { var bytes = TransportHelper.FavoriteEncoding.GetBytes(serializedMessage); await stream.WriteAsync(bytes, 0, bytes.Length); } // rename the file after the write is completed TransportHelper.RenameFile(tempFilePath, fileName, out var _); }); }
public string GetNextFileName() { Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999); string ticks = TransportHelper.GetTimeTicks(); string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0'); return($"b{ticks}{seqnum}_{_transportId}.json"); }
/// <summary> /// Gets the full path to the next file to be received or null if no files /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public string Dequeue(CancellationToken cancellationToken) { string fullPath = null; if (!this._TryDequeue(out fullPath)) { // allow only one thread at a time to fill the cache (it is done without locking) if (Interlocked.CompareExchange(ref _accessCounter, 1, 0) > 0) { return(null); } try { // try again, maybe another thread already added items to the queue if (!this._TryDequeue(out fullPath)) { TimeSpan lastNoMessage = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoMessage); if (lastNoMessage.TotalMilliseconds > BACKOFF_NOMSG_MSEC) { var dirName = _queueRegister.EnsureQueueInitialized(this._inputQueue); List <string> tempStore = new List <string>(); // Don't check for defered too often TimeSpan lastNoDefered = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoDefered); if (lastNoDefered.TotalMilliseconds > BACKOFF_DEFERED_MSEC) { int deferedCnt = this._TryFillCacheDefered(tempStore, dirName, cancellationToken); if (deferedCnt == 0) { this._lastNoDefered = DateTime.Now.Ticks; } } int fileCnt = this._TryFillCache(tempStore, dirName, cancellationToken); if (fileCnt == 0) { this._lastNoMessage = DateTime.Now.Ticks; } foreach (string name in tempStore.OrderBy(name => TransportHelper.GetFileUtcDate(name))) { this._filesQueue.Enqueue(name); } this._TryDequeue(out fullPath); } } } finally { Interlocked.CompareExchange(ref _accessCounter, 0, 1); } } return(fullPath); }
public string GetDeferedFileName(DateTime deferUntil) { string id = TransportHelper.GenerateID(); Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999); string ticks = TransportHelper.GetTimeTicks(deferUntil); string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0'); return($"d{ticks}{seqnum}_{id}.json"); }
/// <summary> /// Constructs the file system transport to create "queues" as subdirectories of the specified base directory. /// While it is apparent that <seealso cref="baseDirectory"/> must be a valid directory name, please note that /// <seealso cref="_inputQueue"/> must not contain any invalid path either. /// </summary> public FileSystemTransport(string baseDirectory, string inputQueue) { if (inputQueue == null) { return; } TransportHelper.EnsureQueueNameIsValid(inputQueue); _inputQueue = inputQueue; _queueRegister = new QueueRegister(baseDirectory); _fileQueue = new FileQueue(inputQueue, _queueRegister); _fileNameGenerator = new FileNameGenerator(); }
private bool _TryDequeue(out string fullPath) { fullPath = null; var dirName = _queueRegister.EnsureQueueInitialized(this._inputQueue); string fileName; bool loopAgain = false; do { loopAgain = false; fullPath = null; fileName = null; string tempPath = null; lock (this._filesCacheLock) { if (this._filesQueue.TryDequeue(out fileName)) { tempPath = Path.Combine(dirName, fileName); this._filesCache.Remove(fileName); } else { return(false); } } /* * // Check if the file is defered (skip them if not ready for processing) * if (fileName.StartsWith("d") && TransportHelper.GetFileUtcDate(fileName) > DateTime.Now.ToUniversalTime()) * { * loopAgain = true; * } * else */ if (!TransportHelper.RenameToTempWithLock(tempPath, out fullPath)) { // this file is used by somebody else (try to get another one) loopAgain = true; } } while (loopAgain); return(true); }
private int _TryFillCacheDefered(List <string> tempStore, string dirName, CancellationToken cancellationToken) { DirectoryInfo info = new DirectoryInfo(dirName); var nowUtc = DateTime.Now.ToUniversalTime(); var files = info.EnumerateFiles("d*.json").Where(p => TransportHelper.GetFileUtcDate(p.Name) <= nowUtc).OrderBy(p => p.Name).Take(DEFER_CACHE_SIZE).ToList(); int cnt = 0; foreach (var file in files) { lock (this._filesCacheLock) { if (!this._filesCache.Contains(file.Name)) { tempStore.Add(file.Name); this._filesCache.Add(file.Name); ++cnt; } } cancellationToken.ThrowIfCancellationRequested(); } return(cnt); }
/// <summary> /// Receives the next message from the logical input queue by loading the next file from the corresponding directory, /// deserializing it, deleting it when the transaction is committed. /// </summary> public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken) { TransportMessage receivedTransportMessage = null; string fullPath = null; bool loopAgain = false; do { loopAgain = false; receivedTransportMessage = null; fullPath = _fileQueue.Dequeue(cancellationToken); if (!string.IsNullOrEmpty(fullPath)) { var jsonText = await TransportHelper.ReadAllText(fullPath); receivedTransportMessage = Deserialize(jsonText); if (!_CheckIsValid(fullPath, receivedTransportMessage)) { File.Delete(fullPath); loopAgain = true; } } } while (loopAgain); if (receivedTransportMessage != null) { context.OnCompleted(async() => { await TransportHelper.DeleteFile(fullPath); }); context.OnAborted(async() => { TransportHelper.RenameToError(fullPath, out var _); }); } return(receivedTransportMessage); }
public FileNameGenerator() { // Generate unique transport id _transportId = TransportHelper.GenerateID(); }