/// <summary>
/// Downloads meta-information about user account and files from Google Drive.
/// </summary>
/// <returns>Meta-information container.</returns>
public async Task <MetaInformation> DownloadMetaInformation()
{
try
{
var filesInfoDownloader = new FilesInfoDownloader(this.CancellationToken);
this.AddAsCurrentOperation(filesInfoDownloader);
var getAboutTask = new TaskWrapper <DriveService, About>(new GetAboutTask());
this.AddTask(getAboutTask);
var files = (await filesInfoDownloader.GetFilesInfo(this.service)).ToList();
var metaInformation = new MetaInformation(
await getAboutTask.Run(this.service),
files);
this.Done();
return(metaInformation);
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
this.Log.LogMessage("Operation was cancelled.");
throw;
}
this.Log.LogError("Exception when downloading metainformation", e);
return(null);
}
}
private void RunSocketServer( )
{
try
{
IPAddress ipAddress;
if (!IPAddress.TryParse(_endpoint.Host, out ipAddress))
{
return;
}
_serverSocket = new TcpListener(ipAddress, _endpoint.Port);
_serverSocket.Start( );
}
catch (Exception ex)
{
_logger.LogError("Exception on creating listener: " + ex.StackTrace + ex.Message);
}
for ( ; _doWorkSwitch;)
{
try
{
TcpClient clientSocket = _serverSocket.AcceptTcpClient( );
TaskWrapper.Run(() => ProcessClient(clientSocket));
}
catch (Exception ex)
{
_logger.LogError("Exception on trying to accept connection: " + ex.StackTrace);
}
}
}
private TaskWrapper PrepareAndSend(string deviceId, string jsonData)
{
var msg = PrepareMessage(jsonData);
var sh = new SafeAction <Message>(m => deviceClients[deviceId].SendEventAsync(msg), Logger);
return(TaskWrapper.Run(() => sh.SafeInvoke(msg)));
}
public void Start(SensorEndpoint endpoint, int messagesToSend)
{
_messagesToSend = messagesToSend;
_endpoint = endpoint;
_doWorkSwitch = true;
var sh = new SafeAction <int>(e => RunSocketAsClient(e), _logger);
TaskWrapper.Run(() => sh.SafeInvoke(CONNECTION_RETRIES));
}
private TaskWrapper PrepareAndSend(string jsonData)
{
Message msg = PrepareMessage(jsonData);
// send to the cloud asynchronously, but wait for completetion
// this is actually serializing access to the SenderLink type
var sh = new SafeAction <Message>(m => SendAmqpMessage(m), Logger);
return(TaskWrapper.Run(() => sh.SafeInvoke(msg)));
}
public void TaskWrapper_BasicTest()
{
var task = new TaskWrapper();
task.Run();
Thread.Sleep(2000);
task.Dispose();
}
public override bool Start(Func <string, int> enqueue)
{
_enqueue = enqueue;
_doWorkSwitch = true;
var sh = new SafeAction <int>((t) => RunForSerial(t), _logger);
TaskWrapper.Run(() => sh.SafeInvoke(SLEEP_TIME_BETWEEN_SCAN));
return(true);
}
public override bool Start(Func <string, int> enqueue)
{
_enqueue = enqueue;
_doWorkSwitch = true;
var sh = new SafeAction(() => Listen(PORT, BAUD_RATE), _logger);
TaskWrapper.Run(sh.SafeInvoke);
return(true);
}
public override bool Start(Func <string, int> enqueue)
{
_enqueue = enqueue;
_doWorkSwitch = true;
var sh = new SafeAction <int>((t) => RunForSocket(t), _logger);
TaskWrapper.Run(() => sh.SafeInvoke(CONNECTION_RETRIES));
return(true);
}
private void SendOutcome(Message message, Outcome outcome, object state)
{
int sent = Interlocked.Increment(ref _sentMessages);
string messageToLog = Encoding.UTF8.GetString(message.Encode( ).Buffer);
int jsonBracketIndex = messageToLog.IndexOf("{", System.StringComparison.Ordinal);
if (jsonBracketIndex > 0)
{
messageToLog = messageToLog.Substring(jsonBracketIndex);
}
jsonBracketIndex = messageToLog.LastIndexOf("}", System.StringComparison.Ordinal);
if (jsonBracketIndex > 0)
{
messageToLog = messageToLog.Substring(0, jsonBracketIndex + 1);
}
if (outcome is Accepted)
{
#if DEBUG_LOG
Logger.LogInfo("Message is accepted");
#endif
if (sent == 1)
{
_start = DateTime.Now;
}
if (Interlocked.CompareExchange(ref _sentMessages, 0, Constants.MessagesLoggingThreshold) == Constants.MessagesLoggingThreshold)
{
DateTime now = DateTime.Now;
TimeSpan elapsed = (now - _start);
_start = now;
var sh = new SafeAction <String>(s => Logger.LogInfo(s), Logger);
TaskWrapper.Run(() => sh.SafeInvoke(
String.Format("GatewayService sent {0} events to Event Hub succesfully in {1} ms ",
Constants.MessagesLoggingThreshold, elapsed.TotalMilliseconds.ToString( ))));
}
}
else
{
#if DEBUG_LOG
Logger.LogInfo("Message is rejected: " + messageToLog);
#endif
}
}
static void Main(string[] args)
{
Console.WriteLine("Main (ThreadId = " + Thread.CurrentThread.ManagedThreadId + ")");
var task = new TaskWrapper();
task.CallBack += OnCallBack;
task.Run();
while (true)
{
var action = Handlers.Take();
action();
}
}
public override bool Start(Func <string, int> enqueue)
{
_enqueue = enqueue;
_doWorkSwitch = true;
//var sh = new SafeAction<int>( ( t ) => RunSocketAsClient( t ), _logger );
var sh = new SafeAction(() => RunSocketServer( ), _logger);
TaskWrapper.Run(() => sh.SafeInvoke( ));
return(true);
}
protected virtual void DataInQueue(QueuedItem data)
{
DataInQueueEventHandler newData = OnDataInQueue;
if (newData != null)
{
var sh = new SafeAction <QueuedItem>(d => newData(d), Logger);
TaskWrapper.Run(() => sh.SafeInvoke(data));
}
//
// NO logging on production code, enable for diagnostic purposes for debugging
//
#if DEBUG_LOG
LogMessageReceived( );
#endif
}
public TaskWrapper <OperationStatus <T> > TryPop( )
{
Func <OperationStatus <T> > deque = () =>
{
T returnedItem;
bool isReturned = _Queue.TryDequeue(out returnedItem);
if (isReturned)
{
return(OperationStatusFactory.CreateSuccess <T>(returnedItem));
}
return(OperationStatusFactory.CreateError <T>(ErrorCode.NoDataReceived));
};
var sf = new SafeFunc <OperationStatus <T> >(deque, null);
return(TaskWrapper <OperationStatus <T> > .Run(() => sf.SafeInvoke()));
}
//--//
public TaskWrapper SendMessage(T data)
{
Action <T> send = (d) =>
{
lock ( _sentMessages )
{
if (_sentMessages.ContainsKey(d))
{
_sentMessages[d]++;
}
else
{
_sentMessages.Add(d, 1);
}
}
};
var sh = new SafeAction <T>((d) => send(d), null);
return(TaskWrapper.Run(() => sh.SafeInvoke(data)));
}
private void LogMessageReceived( )
{
int sent = Interlocked.Increment(ref _receivedMessages);
if (sent == 1)
{
_start = DateTime.Now;
}
if (Interlocked.CompareExchange(ref _receivedMessages, 0, Constants.MessagesLoggingThreshold) == Constants.MessagesLoggingThreshold)
{
DateTime now = DateTime.Now;
TimeSpan elapsed = (now - _start);
_start = now;
var sh = new SafeAction <String>(s => Logger.LogInfo(s), Logger);
TaskWrapper.Run(() => sh.SafeInvoke(
String.Format("GatewayService received {0} events succesfully in {1} ms ", Constants.MessagesLoggingThreshold, elapsed.TotalMilliseconds.ToString( ))));
}
}
public void Start(SensorEndpoint endpoint)
{
var sh = new SafeAction <SensorEndpoint>(e => RunSocketServer(e), _logger);
TaskWrapper.Run(() => sh.SafeInvoke(endpoint));
}
private void ThreadJob()
{
// signal that the worker thread has actually started processing the events
_operational.Set();
try
{
const int WAIT_TIMEOUT = 50; // milliseconds
// run until Stop() is called
while (_running == true)
{
try
{
// If there are no tasks to be served, wait for some events to process
// Use a timeout to prevent race conditions on the outstanding tasks count
// and the actual queue count
_doWork.WaitOne(WAIT_TIMEOUT);
_logger.Flush();
// Fish from the queue and accumulate, keep track of outstanding tasks to
// avoid accumulating too many competing tasks. Note that we are going to schedule
// one more tasks than strictly needed, so that we prevent tasks to sit in the queue
// because of the race condition on the outstanding task count (_outstandingTasks)
// and the tasks actually sitting in the queue. (*)
// To prevent this race condition, we will wait with a timeout
int count = _dataSource.Count - _outstandingTasks;
if (count == 0)
{
continue;
}
// check if we have been woken up to actually stop processing
EventBatchProcessedEventHandler eventBatchProcessed = null;
lock (_syncRoot)
{
if (_running == false)
{
return;
}
// take a snapshot of event handlers to invoke
eventBatchProcessed = OnEventsBatchProcessed;
}
// allocate a container to keep track of tasks for events in the queue
var tasks = new List <TaskWrapper>();
// process all messages that have not been processed yet
while (--count >= 0)
{
TaskWrapper <OperationStatus <TQueueItem> > t = null;
try
{
t = _dataSource.TryPop();
}
catch
{
Interlocked.Decrement(ref _outstandingTasks);
continue;
}
// increment outstanding task count
Interlocked.Increment(ref _outstandingTasks);
t.ContinueWith <TaskWrapper>(popped =>
{
// Decrement the numbers of outstanding tasks.
// (*) Note that there is a race condition because at this point in time the tasks
// is already out of the queue but we did not decrement the outstanding task count
// yet. This race condition may cause tasks to be left sitting in the queue.
// To deal with this race condition, we will wait with a timeout
Interlocked.Decrement(ref _outstandingTasks);
// because the outstanding task counter is incremented before
// adding, we should never incur a negative count
Debug.Assert(_outstandingTasks >= 0);
if (popped?.Result != null && popped.Result.IsSuccess)
{
return(_dataTarget.SendMessage(popped.Result.Result.GetDeviceId(), popped.Result.Result));
}
return(null);
});
AddToProcessed(tasks, t);
}
// alert any client about outstanding message tasks
if (eventBatchProcessed != null)
{
var sh = new SafeAction <List <TaskWrapper> >(allScheduledTasks => eventBatchProcessed(allScheduledTasks), Logger);
TaskWrapper.Run(() => sh.SafeInvoke(tasks));
}
}
catch (StackOverflowException ex) // do not hide stack overflow exceptions
{
Logger.LogError(_logMessagePrefix + ex.Message);
throw;
}
catch (OutOfMemoryException ex) // do not hide memory exceptions
{
Logger.LogError(_logMessagePrefix + ex.Message);
throw;
}
catch (Exception ex) // catch all other exceptions
{
Logger.LogError(_logMessagePrefix + ex.Message);
}
}
}
finally
{
_operational.Set();
}
}
/// <summary>
/// Downloads files.
/// </summary>
/// <param name="files">Meta-information about files to download.</param>
/// <param name="exportFormats">A dictionary with possible export formats for Google Docs files, shall be
/// received from Google Drive.</param>
/// <returns>A list of info objects about downloaded files.</returns>
public async Task <IList <DownloadedFile> > DownloadFiles(
IEnumerable <File> files,
IDictionary <string, IList <string> > exportFormats)
{
var filesList = files as IList <File> ?? files.ToList();
var weights = filesList.ToDictionary(f => f.Id, f => f.Size ?? Constants.GoogleDocsFileSize);
this.Log.LogMessage($"Files to download: {weights.Count}, overall size: {weights.Values.Sum()} bytes.");
IDictionary <File, IEnumerable <DirectoryInfo> > directories;
try
{
directories = this.CreateDirectoryStructure(filesList, this.targetDir);
}
catch (Exception e)
{
this.Log.LogError("Unexpected error while creating directory structure, aborting download.", e);
this.Error();
return(new List <DownloadedFile>());
}
Func <long, int> weight = size => (int)Math.Max(1, size / 1000000);
this.SetGuessEstimation(weights.Values.Select(weight).Sum());
var semaphore = new AsyncSemaphore(Utils.Constants.DownloadThreadsCount);
try
{
return(await AsyncHelper.ProcessInChunks(
filesList,
f =>
{
this.Status = Status.DownloadingFiles;
if (exportFormats.ContainsKey(f.MimeType))
{
var downloadTask = new TaskWrapper <File, DownloadedFile>(new DownloadGoogleDocsFileTask(
exportFormats,
semaphore,
directories,
this.service));
this.AddTask(downloadTask);
return downloadTask.Run(f).ContinueWith(t => t.Result, this.CancellationToken);
}
if (!string.IsNullOrEmpty(f.WebContentLink))
{
var binaryFileDownloader = new BinaryFileDownloader(this.CancellationToken);
this.AddAsCurrentOperation(binaryFileDownloader);
return binaryFileDownloader.DownloadBinaryFile(
this.service,
f,
semaphore,
weight(f.Size ?? 0),
directories).ContinueWith(t => t.Result, this.CancellationToken);
}
if (f.MimeType != "application/vnd.google-apps.folder")
{
this.Log.LogDebugMessage(
$"File '{f.Name}' has no web content link, can not be exported"
+ " and not a folder, skipping.");
}
return TaskEx.FromResult <DownloadedFile>(null);
},
Utils.Constants.TaskChunkSize,
this.CancellationToken));
}
catch (Exception e)
{
if (this.CancellationToken.IsCancellationRequested)
{
this.Log.LogMessage("Operation was cancelled.");
throw;
}
this.Log.LogError("Error while downloading file.", e);
this.Error();
return(new List <DownloadedFile>());
}
}
//--//
public WindowsService(ILogger logger)
{
if (logger == null)
{
throw new ArgumentException("Cannot run service without logging");
}
_logger = logger;
if (logger is TunableLogger)
{
TunableLogger.LoggingLevel loggingLevel = TunableLogger.LevelFromString(ConfigurationManager.AppSettings.Get("LoggingLevel"));
(( TunableLogger )logger).Level = (loggingLevel != TunableLogger.LoggingLevel.Undefined) ? loggingLevel : TunableLogger.LoggingLevel.Errors;
}
try
{
_THREADING.TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
// Name the Windows Service
ServiceName = Constants.WindowsServiceName;
_gatewayQueue = new GatewayQueue <QueuedItem>( );
AMQPConfig amqpConfig = Loader.GetAMQPConfig( );
if (amqpConfig == null)
{
_logger.LogError("AMQP configuration is missing");
return;
}
_AMPQSender = new AMQPSender <SensorDataContract>(
amqpConfig.AMQPSAddress,
amqpConfig.EventHubName,
amqpConfig.EventHubMessageSubject,
amqpConfig.EventHubDeviceId,
amqpConfig.EventHubDeviceDisplayName,
_logger
);
_batchSenderThread = new BatchSenderThread <QueuedItem, SensorDataContract>(
_gatewayQueue,
_AMPQSender,
null, //m => DataTransforms.AddTimeCreated(DataTransforms.SensorDataContractFromQueuedItem(m, _Logger)),
new Func <QueuedItem, string>(m => m.JsonData),
_logger);
_dataIntakeLoader = new DeviceAdapterLoader(Loader.GetSources( ), Loader.GetEndpoints( ), _logger);
TaskWrapper.Run(() => IPAddressHelper.GetIPAddressString(ref _gatewayIPAddressString));
DataTransformsConfig dataTransformsConfig = Loader.GetDataTransformsConfig( );
if (dataTransformsConfig.AttachIP || dataTransformsConfig.AttachTime)
{
Func <string, SensorDataContract> transform = (m => DataTransforms.SensorDataContractFromString(m, _logger));
if (dataTransformsConfig.AttachTime)
{
var transformPrev = transform;
transform = (m => DataTransforms.AddTimeCreated(transformPrev(m)));
}
if (dataTransformsConfig.AttachTime)
{
var transformPrev = transform;
transform = (m => DataTransforms.AddIPToLocation(transformPrev(m), _gatewayIPAddressString));
}
_gatewayTransform = (m => DataTransforms.QueuedItemFromSensorDataContract(transform(m)));
}
}
catch (Exception ex)
{
_logger.LogError("Exception creating WindowsService: " + ex.Message);
}
}
//--//
private static void InitGateway(ILogger logger)
{
if (logger == null)
{
throw new ArgumentException("Cannot run service without logging");
}
_logger = logger;
if (logger is TunableLogger)
{
TunableLogger.LoggingLevel loggingLevel = TunableLogger.LevelFromString(ConfigurationManager.AppSettings.Get("LoggingLevel"));
(( TunableLogger )logger).Level = (loggingLevel != TunableLogger.LoggingLevel.Undefined) ? loggingLevel : TunableLogger.LoggingLevel.Errors;
}
try
{
System.Threading.Tasks.TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
// Name the Windows Service
_gatewayQueue = new GatewayQueue <QueuedItem>( );
IotHubConfig iotHubConfig = Loader.GetIotHubConfig();
if (iotHubConfig == null)
{
_logger.LogError("IoT Hub connection configuration is missing");
return;
}
_MessageSender = new MessageSender <SensorDataContract>(iotHubConfig.IotHubConnectionString, _logger);
_batchSenderThread = new BatchSenderThread <QueuedItem, SensorDataContract>(
_gatewayQueue,
_MessageSender,
null, //m => DataTransforms.AddTimeCreated(DataTransforms.SensorDataContractFromQueuedItem(m, _Logger)),
new Func <QueuedItem, string>(m => m.JsonData),
_logger);
_dataIntakeLoader = new DeviceAdapterLoader(Loader.GetSources( ), Loader.GetEndpoints( ), _logger);
TaskWrapper.Run(() => IPAddressHelper.GetIPAddressString(ref _gatewayIPAddressString));
DataTransformsConfig dataTransformsConfig = Loader.GetDataTransformsConfig( );
if (dataTransformsConfig.AttachIP || dataTransformsConfig.AttachTime)
{
Func <string, SensorDataContract> transform = (m => DataTransforms.SensorDataContractFromString(m, _logger));
if (dataTransformsConfig.AttachTime)
{
var transformPrev = transform;
transform = (m => DataTransforms.AddTimeCreated(transformPrev(m)));
}
if (dataTransformsConfig.AttachTime)
{
var transformPrev = transform;
transform = (m => DataTransforms.AddIPToLocation(transformPrev(m), _gatewayIPAddressString));
}
_gatewayTransform = (m => DataTransforms.QueuedItemFromSensorDataContract(transform(m)));
}
}
catch (Exception ex)
{
_logger.LogError("Exception creating Gateway: " + ex.Message);
}
}