/// <summary>
        /// Sends the specified message to the logical queue specified by <paramref name="destinationQueueName"/> by writing
        /// a JSON serialied text to a file in the corresponding directory. The actual write operation is delayed until
        /// the commit phase of the queue transaction unless we're non-transactional, in which case it is written immediately.
        /// </summary>
        public async Task Send(string destinationQueueName, TransportMessage message, ITransactionContext context)
        {
            var destinationDirectory = _queueRegister.EnsureQueueInitialized(destinationQueueName);

            var serializedMessage = Serialize(message);
            var fileName          = string.Empty;

            if (message.Headers.TryGetValue(Headers.DeferredUntil, out var deferTime))
            {
                var deferUntil = DateTime.Parse(deferTime);
                fileName = _fileNameGenerator.GetDeferedFileName(deferUntil);
            }
            else
            {
                fileName = _fileNameGenerator.GetNextFileName();
            }
            string tempFileName = $"t{fileName.Substring(1)}";
            string tempFilePath = Path.Combine(destinationDirectory, tempFileName);

            context.OnCommitted(async() =>
            {
                // new file in async mode
                // write the file with the temporary name prefix (so it could not be read while it is written)
                using (var stream = new FileStream(tempFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1024 * 8, true))
                {
                    var bytes = TransportHelper.FavoriteEncoding.GetBytes(serializedMessage);
                    await stream.WriteAsync(bytes, 0, bytes.Length);
                }

                // rename the file after the write is completed
                TransportHelper.RenameFile(tempFilePath, fileName, out var _);
            });
        }
        public string GetNextFileName()
        {
            Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999);
            string ticks  = TransportHelper.GetTimeTicks();
            string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0');

            return($"b{ticks}{seqnum}_{_transportId}.json");
        }
Exemple #3
0
        /// <summary>
        /// Gets the full path to the next file to be received or null if no files
        /// </summary>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public string Dequeue(CancellationToken cancellationToken)
        {
            string fullPath = null;

            if (!this._TryDequeue(out fullPath))
            {
                // allow only one thread at a time to fill the cache (it is done without locking)
                if (Interlocked.CompareExchange(ref _accessCounter, 1, 0) > 0)
                {
                    return(null);
                }
                try
                {
                    // try again, maybe another thread already added items to the queue
                    if (!this._TryDequeue(out fullPath))
                    {
                        TimeSpan lastNoMessage = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoMessage);
                        if (lastNoMessage.TotalMilliseconds > BACKOFF_NOMSG_MSEC)
                        {
                            var           dirName   = _queueRegister.EnsureQueueInitialized(this._inputQueue);
                            List <string> tempStore = new List <string>();

                            // Don't check for defered too often
                            TimeSpan lastNoDefered = TimeSpan.FromTicks(DateTime.Now.Ticks - this._lastNoDefered);
                            if (lastNoDefered.TotalMilliseconds > BACKOFF_DEFERED_MSEC)
                            {
                                int deferedCnt = this._TryFillCacheDefered(tempStore, dirName, cancellationToken);
                                if (deferedCnt == 0)
                                {
                                    this._lastNoDefered = DateTime.Now.Ticks;
                                }
                            }

                            int fileCnt = this._TryFillCache(tempStore, dirName, cancellationToken);
                            if (fileCnt == 0)
                            {
                                this._lastNoMessage = DateTime.Now.Ticks;
                            }

                            foreach (string name in tempStore.OrderBy(name => TransportHelper.GetFileUtcDate(name)))
                            {
                                this._filesQueue.Enqueue(name);
                            }

                            this._TryDequeue(out fullPath);
                        }
                    }
                }
                finally
                {
                    Interlocked.CompareExchange(ref _accessCounter, 0, 1);
                }
            }

            return(fullPath);
        }
        public string GetDeferedFileName(DateTime deferUntil)
        {
            string id = TransportHelper.GenerateID();

            Interlocked.CompareExchange(ref _incrementingCounter, 0, 99999);
            string ticks  = TransportHelper.GetTimeTicks(deferUntil);
            string seqnum = Interlocked.Increment(ref _incrementingCounter).ToString().PadLeft(5, '0');

            return($"d{ticks}{seqnum}_{id}.json");
        }
        /// <summary>
        /// Constructs the file system transport to create "queues" as subdirectories of the specified base directory.
        /// While it is apparent that <seealso cref="baseDirectory"/> must be a valid directory name, please note that
        /// <seealso cref="_inputQueue"/> must not contain any invalid path either.
        /// </summary>
        public FileSystemTransport(string baseDirectory, string inputQueue)
        {
            if (inputQueue == null)
            {
                return;
            }
            TransportHelper.EnsureQueueNameIsValid(inputQueue);
            _inputQueue = inputQueue;

            _queueRegister     = new QueueRegister(baseDirectory);
            _fileQueue         = new FileQueue(inputQueue, _queueRegister);
            _fileNameGenerator = new FileNameGenerator();
        }
Exemple #6
0
        private bool _TryDequeue(out string fullPath)
        {
            fullPath = null;
            var    dirName = _queueRegister.EnsureQueueInitialized(this._inputQueue);
            string fileName;
            bool   loopAgain = false;

            do
            {
                loopAgain = false;
                fullPath  = null;
                fileName  = null;
                string tempPath = null;

                lock (this._filesCacheLock)
                {
                    if (this._filesQueue.TryDequeue(out fileName))
                    {
                        tempPath = Path.Combine(dirName, fileName);
                        this._filesCache.Remove(fileName);
                    }
                    else
                    {
                        return(false);
                    }
                }

                /*
                 * // Check if the file is defered (skip them if not ready for processing)
                 * if (fileName.StartsWith("d") && TransportHelper.GetFileUtcDate(fileName) > DateTime.Now.ToUniversalTime())
                 * {
                 *  loopAgain = true;
                 * }
                 * else
                 */

                if (!TransportHelper.RenameToTempWithLock(tempPath, out fullPath))
                {
                    // this file is used by somebody else (try to get another one)
                    loopAgain = true;
                }
            } while (loopAgain);

            return(true);
        }
Exemple #7
0
        private int _TryFillCacheDefered(List <string> tempStore, string dirName, CancellationToken cancellationToken)
        {
            DirectoryInfo info   = new DirectoryInfo(dirName);
            var           nowUtc = DateTime.Now.ToUniversalTime();
            var           files  = info.EnumerateFiles("d*.json").Where(p => TransportHelper.GetFileUtcDate(p.Name) <= nowUtc).OrderBy(p => p.Name).Take(DEFER_CACHE_SIZE).ToList();
            int           cnt    = 0;

            foreach (var file in files)
            {
                lock (this._filesCacheLock)
                {
                    if (!this._filesCache.Contains(file.Name))
                    {
                        tempStore.Add(file.Name);
                        this._filesCache.Add(file.Name);
                        ++cnt;
                    }
                }
                cancellationToken.ThrowIfCancellationRequested();
            }

            return(cnt);
        }
        /// <summary>
        /// Receives the next message from the logical input queue by loading the next file from the corresponding directory,
        /// deserializing it, deleting it when the transaction is committed.
        /// </summary>
        public async Task <TransportMessage> Receive(ITransactionContext context, CancellationToken cancellationToken)
        {
            TransportMessage receivedTransportMessage = null;
            string           fullPath  = null;
            bool             loopAgain = false;

            do
            {
                loopAgain = false;
                receivedTransportMessage = null;
                fullPath = _fileQueue.Dequeue(cancellationToken);
                if (!string.IsNullOrEmpty(fullPath))
                {
                    var jsonText = await TransportHelper.ReadAllText(fullPath);

                    receivedTransportMessage = Deserialize(jsonText);
                    if (!_CheckIsValid(fullPath, receivedTransportMessage))
                    {
                        File.Delete(fullPath);
                        loopAgain = true;
                    }
                }
            } while (loopAgain);


            if (receivedTransportMessage != null)
            {
                context.OnCompleted(async() => {
                    await TransportHelper.DeleteFile(fullPath);
                });
                context.OnAborted(async() => {
                    TransportHelper.RenameToError(fullPath, out var _);
                });
            }
            return(receivedTransportMessage);
        }
 public FileNameGenerator()
 {
     // Generate unique transport id
     _transportId = TransportHelper.GenerateID();
 }