public void Start(string listenToQueue)
{
IMessageQueue downloadQueue = _messageQueueProvider.Get(listenToQueue, false);
if (downloadQueue == null)
{
return;
}
IMessageQueue dispatchQueue = _messageQueueProvider.Get(DispatchQueueName, true);
var httpClient = new HttpClient();
while (true)
{
try
{
var message = downloadQueue.Dequeue <DownloadContentMessageBody>();
if (message != null)
{
Console.Write(message.Url);
var task = httpClient.GetStreamAsync(message.Url);
task.Wait();
Console.Write("[download]");
var generatedPath = string.Format("{0}/{1}", DateTime.Now.ToString("yyyy-MM-dd"), Guid.NewGuid());
_blobClient.Upload(task.Result, generatedPath);
Console.Write("[store]");
dispatchQueue.Send(new ProcessContentMessageBody
{
OrignalUrl = message.Url,
BlobPath = generatedPath,
Context = message.Context
});
Console.WriteLine();
}
else
{
System.Threading.Thread.Sleep(10000);
}
}
catch (System.Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
public void Start(string name, string collection = null, string handler = null)
{
var setting = _configrationManager.GetSetting(name);
var discoverObject = new DiscoverObject(setting);
discoverObject.Builder = _buildIndexProvider.Get(setting.IndexBuilder);
IMessageQueue dispatchQueue = _messageQueueProvider != null?_messageQueueProvider.Get(DispatchQueueName, true) : null;
var task = discoverObject.ForEachAsync((item) => {
if (dispatchQueue != null)
{
var message = new DownloadContentMessageBody
{
Url = item.GetValue("uri").AsString
};
message.Context.Add("saveTo", collection);
message.Context.Add("handler", handler);
dispatchQueue.Send(message);
Console.WriteLine("[Dispatch]");
}
if (_action != null)
{
try
{
_action.Invoke(item);
}catch (System.Exception ex)
{
// Ignore the exception caused by external invoke
}
}
});
task.Wait();
}
public void Start(string collection, string handler)
{
IMessageQueue dispatchQueue = _messageQueueProvider != null?_messageQueueProvider.Get(DispatchQueueName, true) : null;
foreach (var resource in _resources)
{
if (dispatchQueue != null)
{
var message = new DownloadContentMessageBody
{
Url = resource
};
message.Context.Add("saveTo", collection);
message.Context.Add("handler", handler);
dispatchQueue.Send(message);
Console.WriteLine("[Dispatch]");
}
}
}
public void Start(string listenToQueueName)
{
var processQueue = _messageQueueProvider.Get(listenToQueueName, false);
if (processQueue == null)
{
return;
}
var processErrorQueue = _messageQueueProvider.Get(OnErrorQueueName, true);
Console.ForegroundColor = ConsoleColor.Blue;
var documentPersistenceCache = new Dictionary <string, IDocumentPersistence>();
while (true)
{
var message = processQueue.Dequeue <ProcessContentMessageBody>();
try
{
if (message != null)
{
using (var stream = _blobClient.Download(message.BlobPath))
{
var module = _normaliztionProvider.Activate(message.Context["handler"]);
var metadata = module.Normalize(stream, new Dictionary <string, object> {
{ "url", message.OrignalUrl }
});
if (!documentPersistenceCache.ContainsKey(message.Context["saveTo"]))
{
documentPersistenceCache[message.Context["saveTo"]] = _persistenceProvider.Get(message.Context["saveTo"]);
}
IDocumentPersistence persistence = documentPersistenceCache[message.Context["saveTo"]];
persistence.Persist(metadata);
}
Console.Write(".");
}
else
{
System.Threading.Thread.Sleep(10000);
}
}
catch (System.Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("x");
Console.WriteLine(ex.Message);
processErrorQueue.Send(new DownloadContentMessageBody
{
Url = message.OrignalUrl,
Context = message.Context
});
Console.ForegroundColor = ConsoleColor.Blue;
}
}
}