public bool DispatchMessage(string consumerId, string message)
{
if (string.IsNullOrEmpty(consumerId) || string.IsNullOrEmpty(message))
{
return(false);
}
// is this an echo message?
if (message.StartsWith("{\"Relation\":\"http://api.sportingsolutions.com/rels/stream/echo\""))
{
EchoManager.ProcessEcho(consumerId);
return(true);
}
if (UDAPI.Configuration.VerboseLogging)
{
_logger.DebugFormat("Update arrived for consumerId={0}", consumerId);
}
// note that TryGetValue is lock-free
ConsumerQueue c = null;
if (!_subscribers.TryGetValue(consumerId, out c) || c == null)
{
_logger.WarnFormat("Update not dispatched to consumerId={0} as it was not found", consumerId);
return(false);
}
c.Add(message);
EchoManager.ProcessEcho(consumerId);
return(true);
}
public void RunComponentsDefault()
{
TimeoutManager = new TimeoutManager();
CallbackBufferBlock = new ConsumerQueue <IActionItem>();
ActionCaseConsumer = new ConsumerQueue <IActionCase>();
RunComponents();
}
public void CreateAnd_Add()
{
ConsumerQueue <int, int> target = Create_And_Add(new CountdownEvent(0));
Assert.AreEqual(1, target.Count);
Assert.AreEqual(11, target.Key);
}
private ConsumerQueue <int, int> Create_And_Add(CountdownEvent dequeueCount)
{
var target = new ConsumerQueue <int, int>(11, w => { }, q => dequeueCount.Signal());
target.Add(1);
return(target);
}
public void RemoveSubscriber(IStreamSubscriber subscriber)
{
if (subscriber == null)
{
return;
}
ConsumerQueue c = null;
_subscribers.TryGetValue(subscriber.Consumer.Id, out c);
if (c == null)
{
_logger.WarnFormat("consumerId={0} can't be removed from the dispatcher cause it was not found", subscriber.Consumer.Id);
return;
}
try
{
// it is important to first send the disconnect event,
// and only then, remove the item from the list
c.Disconnect();
int tmp = Interlocked.Decrement(ref _subscribersCount);
_logger.InfoFormat("consumerId={0} removed from the dispatcher, count={1}", subscriber.Consumer.Id, tmp);
_subscribers.TryRemove(subscriber.Consumer.Id, out c);
}
finally
{
EchoManager.RemoveConsumer(c.Subscriber);
}
}
static void Main(string[] args)
{
var consumer = new ConsumerQueue(Constants.DocumentQueueName, Constants.DirectQueueType);
consumer.DocumentRecievedHandler += Handler;
Console.ReadKey();
consumer.Dispose();
}
/// <summary>
/// Consumer method- Polls each job from the job queue, runs the job and stores the transfer result if failed or skipped
/// </summary>
private void ConsumerRun()
{
while (true)
{
if (CancelToken.IsCancellationRequested)
{
return;
}
var job = ConsumerQueue.Poll();
if (job is PoisonJob)
{
ConsumerQueue.Add(new PoisonJob());
return;
}
var res = job.DoRun(JobLog) as SingleEntryTransferStatus;
if (res == null)
{
continue;
}
if (res.Status == SingleChunkStatus.Successful)
{
if (res.Type == EntryType.Chunk)
{
Interlocked.Increment(ref Status.ChunksTransfered);
RecordedMetadata.AddRecord($"CHUNK{TransferLog.MetaDataDelimiter}{res.Source}{TransferLog.MetaDataDelimiter}{res.ChunkId}");
}
else if (res.Type == EntryType.File)
{
// Entry size is zero for concat
if (res.EntrySize != 0)
{
Interlocked.Increment(ref Status.NonChunkedFileTransferred);
}
Interlocked.Increment(ref Status.FilesTransfered);
// For successful concat we want to flush the metadata records
AddCompleteRecord(res.Source, res.EntrySize == 0);
}
else
{
Interlocked.Increment(ref Status.DirectoriesTransferred);
AddCompleteRecord(res.Source);
}
if (res.EntrySize > 0)
{
Interlocked.Add(ref Status.SizeTransfered, res.EntrySize);
}
}
else if (res.Status == SingleChunkStatus.Failed)
{
Status.AddFailedEntries(res);
}
else
{
Status.AddSkippedEntries(res.EntryName);
}
}
}
public IStreamSubscriber GetSubscriber(string subscriberId)
{
ConsumerQueue c = null;
if (_subscribers.TryGetValue(subscriberId, out c) && c != null)
{
return(c.Subscriber);
}
return(null);
}
public void Create_Add_And_Dequeue()
{
var dequeueCount = new CountdownEvent(1);
ConsumerQueue <int, int> target = Create_And_Add(dequeueCount);
int item;
Assert.IsTrue(target.TryDequeue(out item));
Assert.AreEqual(0, target.Count);
Assert.AreEqual(1, item);
Assert.IsFalse(target.TryDequeue(out item));
Assert.AreEqual(0, target.Count);
Assert.IsTrue(dequeueCount.Wait(10));
}
private void Consume()
{
using (var connection = Connection.GetConnection())
using (var channel = connection.CreateModel())
{
string key = Shared.Constants.Constant.QueueKey;
ConsumerQueue consumerQueue = new ConsumerQueue(channel, String.Empty, key);
consumerQueue.ConsumeMessage();
Console.WriteLine(" Press [enter] to exit.");
Console.ReadLine();
}
}
public void AddSubscriber(IStreamSubscriber subscriber)
{
if (subscriber == null)
{
return;
}
int tmp = Interlocked.Increment(ref _subscribersCount);
var c = new ConsumerQueue(subscriber);
_subscribers[subscriber.Consumer.Id] = c;
EchoManager.AddConsumer(subscriber);
c.Connect();
_logger.InfoFormat("consumerId={0} added to the dispatcher, count={1}", subscriber.Consumer.Id, tmp);
}
// If the transfer process is being resumed and it was successfully transfered before then we return false- meaning
// it does not need to be transfered
protected bool AddDirectoryToConsumerQueue(string dirFullName, bool isUpload)
{
if (RecordedMetadata.EntryTransferredSuccessfulLastTime(dirFullName))
{
return(false);
}
string relativePath = GetDestDirectoryFormatted(dirFullName.Substring(SourcePath.Length));
string destPath = DestPath + GetDestDirectorySeparator() + relativePath;
ConsumerQueue.Add(new MakeDirJob(dirFullName, destPath, Client, isUpload));
if (FileTransferLog.IsDebugEnabled)
{
FileTransferLog.Debug($"FileTransfer.DirectoryProduced, {destPath}");
}
return(true);
}
public void TryAcceptWorkerMock()
{
var worker = new MockWorker();
var completed = new ManualResetEventSlim(false);
var dequeueCount = new CountdownEvent(1);
var target = new ConsumerQueue<int, int>(11, w =>
{
Assert.AreSame(worker, w);
completed.Set();
}, q => dequeueCount.Signal()); //test that correct instance passed
target.Add(1);
Task task = Task.Factory.StartNew(() => target.TryAcceptWorker(worker));
Assert.IsTrue(SpinWaitHelper.SpinWaitForCondition(() => target.IsBusy, 1000));
//should become busy when accept occurs
task.Wait();
Assert.IsTrue(dequeueCount.Wait(1000)); //completed should be called
Assert.IsTrue(completed.Wait(1000)); //completed should be called
}
/// Main method that does the upload or download
protected TransferStatus RunTransfer()
{
//Sets up the producer threads for first pass producer run and conumer threads
SetupThreads();
if (CancelToken.IsCancellationRequested)
{
return(Status);
}
//Runs the enumeration first and waits for it to end
StartProducer();
WaitForProducerToFinish();
NumProducerThreads = 1;
// After enumeration we have a better idea the sample distribution of file sizes
// In second pass we can determine how the chunking will be done for uploader or downloader
InitializesProducerThreads(FinalPassProducerRun);
if (CancelToken.IsCancellationRequested)
{
return(Status);
}
StartProducer();
//Now the consumer can run in parallel to producer
StartConsumer();
WaitForProducerToFinish();
//Stats thread should only start after enumeration is done
if (ProgressTracker != null)
{
_threadStats.Start();
}
//After thread producer ends then only stats thread should start
ConsumerQueue.Add(new PoisonJob());
for (int i = 0; i < NumConsumerThreads; i++)
{
_threadConsumer[i].Join();
}
RecordedMetadata.EndRecording(CancelToken.IsCancellationRequested);
if (ProgressTracker != null)
{
Interlocked.Increment(ref _consumerDone);
_threadStats.Join();
}
return(Status);
}
public void TryAcceptWorkerMock()
{
var worker = new MockWorker();
var completed = new ManualResetEventSlim(false);
var dequeueCount = new CountdownEvent(1);
var target = new ConsumerQueue <int, int>(11, w =>
{
Assert.AreSame(worker, w);
completed.Set();
}, q => dequeueCount.Signal()); //test that correct instance passed
target.Add(1);
Task task = Task.Factory.StartNew(() => target.TryAcceptWorker(worker));
Assert.IsTrue(SpinWaitHelper.SpinWaitForCondition(() => target.IsBusy, 1000));
//should become busy when accept occurs
task.Wait();
Assert.IsTrue(dequeueCount.Wait(1000)); //completed should be called
Assert.IsTrue(completed.Wait(1000)); //completed should be called
}
public void workerCompleted_Should_Throw_ArgumentNullException()
{
var target = new ConsumerQueue <int, int>(11, null, q => { });
}
/// Adds file or its chunks to Consumer queue
protected long AddFileToConsumerQueue(string fileFullName, long fileLength, bool isToBeChunked, out long fileSizeToTransfer)
{
string relativePath;
if (SourcePath.Equals(fileFullName)) //This is the case when the input path is a file
{
relativePath = "";
}
else
{
relativePath = fileFullName.Substring(SourcePath.Length);
relativePath = GetDestDirectoryFormatted(relativePath);
}
long numChunks = 0;
string tempChunkFolder = null;
fileSizeToTransfer = 0;
// We will never be here if the log has detected that the file is done
if (isToBeChunked)
{
//If RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) is true then the file was attempted and incomplete. Because if it is successful then we will not be here
int numChunksAlreadyTransferred = RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) ? RecordedMetadata.LoadedMetaData[fileFullName].Chunks.Count : -1;
// numChunksAlreadyTransferred is -1 means this file was not attempted before so effectively 0 chunks were done
numChunks = fileLength / ChunkSize + (fileLength % ChunkSize == 0 ? 0 : 1);
tempChunkFolder = RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) ? RecordedMetadata.LoadedMetaData[fileFullName].SegmentFolder : DestPath + relativePath + Guid.NewGuid() + "Segments";
var bulk = AssignMetaData(fileFullName, tempChunkFolder, DestPath + relativePath, fileLength, numChunks, numChunksAlreadyTransferred);
if (numChunksAlreadyTransferred == numChunks)
{// If all chunks were transferred correctly then add a CopyFileJob only. CopyFileJob will make the necessary checks and determine which state we are in
ConsumerQueue.Add(new CopyFileJob(0, bulk, Client));
}
else
{
// If this file was attempted in last transfer and it is being resumed, at the enumeration time we just add the jobs for the chunks which are not
// reported in the log file. In reality there can be different states 1) Only those reported in resume file are done 2) More than reported are done however concat wasn't started yet
// 3) All chunks are actually done and concat is half done 4) All chunks are done and concat is done. The discrepancy between the resume file and actual events is because the
// log file is written in a separate producer-consumer queue (not serialized) for perf reasons. All these cases checked and are taken care in FileMetaData.
// If we are in state 1 and 2 then the jobs are done as expected. If we are in states beyond 2 then we do the required concat job and chunks
// are reported done without any actual transfer.
for (int i = 0; i < numChunks; i++)
{
if (RecordedMetadata.EntryTransferAttemptedLastTime(fileFullName) && RecordedMetadata.LoadedMetaData[fileFullName].Chunks.Contains(i))
{
continue;
}
ConsumerQueue.Add(new CopyFileJob(i, bulk, Client));
fileSizeToTransfer += i == numChunks - 1 ? fileLength - i * ChunkSize : ChunkSize;
}
}
// Number of chunks to be uploaded for this file will be the total chunks minus the chunks already transferred
numChunks -= numChunksAlreadyTransferred < 0 ? 0 : numChunksAlreadyTransferred;
}
else
{
FileMetaData bulk = AssignMetaData(fileFullName, null, DestPath + relativePath, fileLength, numChunks);
ConsumerQueue.Add(new CopyFileJob(-1, bulk, Client));
fileSizeToTransfer = fileLength;
}
if (FileTransferLog.IsDebugEnabled)
{
FileTransferLog.Debug($"FileTransfer.FileProduced, Name: {fileFullName}, Dest: {tempChunkFolder ?? DestPath + relativePath}, Length: {fileLength}, Chunks: {numChunks}");
}
return(numChunks);
}
public void onDequeue_Should_Throw_ArgumentNullException()
{
var target = new ConsumerQueue <int, int>(11, w => { }, null);
}
public bool Start(/*HostControl hostControl*/)
{
documentQueue = new ConsumerQueue(Constants.DocumentQueueName, Constants.DirectQueueType);
documentQueue.DocumentRecievedHandler += Handler;
return(true);
}
public override void Awake()
{
base.Awake();
instance = this;
}
public FrameBufferBlock(Func <byte[], int> writeData, Func <CommandMessage, byte[]> createFrameBuffer)
{
_writeData = writeData;
_createFrameBuffer = createFrameBuffer;
_queue = new ConsumerQueue <ActionHandlerResult>();
}
public void onDequeue_Should_Throw_ArgumentNullException()
{
var target = new ConsumerQueue<int, int>(11, w => { }, null);
}
public void workerCompleted_Should_Throw_ArgumentNullException()
{
var target = new ConsumerQueue<int, int>(11, null, q => { });
}
private ConsumerQueue<int, int> Create_And_Add(CountdownEvent dequeueCount)
{
var target = new ConsumerQueue<int, int>(11, w => { }, q => dequeueCount.Signal());
target.Add(1);
return target;
}
/// <summary>
/// Adds the concat jobs for downloader
/// </summary>
/// <param name="source">Source file path</param>
/// <param name="chunkSegmentFolder">Temporary destination file name</param>
/// <param name="dest">Destination file</param>
/// <param name="totSize">Total size of the file- needed for verification of the copy</param>
/// <param name="totalChunks">Total number of chunks</param>
/// <param name="doUploadRenameOnly"></param>
internal override void AddConcatJobToQueue(string source, string chunkSegmentFolder, string dest, long totSize,
long totalChunks, bool doUploadRenameOnly = false)
{
ConsumerQueue.Add(new ConcatenateJob(source, chunkSegmentFolder, dest, Client, totSize, totalChunks, false));
}
public void AddSubscriber(IStreamSubscriber subscriber)
{
if(subscriber == null)
return;
int tmp = Interlocked.Increment(ref _subscribersCount);
var c = new ConsumerQueue(subscriber);
_subscribers[subscriber.Consumer.Id] = c;
EchoManager.AddConsumer(subscriber);
c.Connect();
_logger.InfoFormat("consumerId={0} added to the dispatcher, count={1}", subscriber.Consumer.Id, tmp);
}
