/// <summary>
/// A simpler alternative to the irritatingly useless FileSystemWatcher
/// </summary>
/// <param name="file">The file to monitor</param>
/// <param name="refreshPeriod">The refresh period.</param>
/// <param name="scheduler">The scheduler.</param>
/// <returns></returns>
public static IObservable<FileNotification> WatchFile(this FileInfo file, TimeSpan? refreshPeriod = null,
IScheduler scheduler = null)
{
return Observable.Create<FileNotification>(observer =>
{
var refresh = refreshPeriod ?? TimeSpan.FromMilliseconds(250);
scheduler = scheduler ?? Scheduler.Default;
FileNotification notification = null;
return scheduler.ScheduleRecurringAction(refresh, () =>
{
try
{
notification = notification == null
? new FileNotification(file)
: new FileNotification(notification);
observer.OnNext(notification);
}
catch (Exception ex)
{
notification = new FileNotification(file, ex);
observer.OnNext(notification);
}
});
}).DistinctUntilChanged();
}
public async Task Run(
[QueueTrigger("notifications", Connection = "AzureWebJobsStorage")]
string myQueueItem,
ILogger log)
{
FileNotification notification = null;
try
{
notification = _fileNotificationSerializer.Deserialize(myQueueItem);
var parameters = new DocumentCreateParameters
{
FileName = notification.FileName,
DocumentSummary = notification.DocumentSummary,
DocumentData = await _fileStore.DownloadFile(notification.FileName)
};
_documentService.Create(parameters);
log.LogInformation($"File '{notification.FileName}' is processed. Info: {myQueueItem}");
}
catch (Exception e)
{
log.LogError($"File '{notification?.FileName ?? string.Empty}' processing error. Info: {myQueueItem}", e);
}
}
public void Notify()
{
var file = Path.GetTempFileName();
File.Delete(file);
var info = new FileInfo(file);
var scheduler = new TestScheduler();
FileNotification result = null;
using (info.WatchFile(TimeSpan.FromSeconds(1), scheduler).Subscribe(x => result = x))
{
scheduler.AdvanceBySeconds(1);
result.NotificationType.Should().Be(FileNotificationType.Missing);
File.AppendAllLines(file, Enumerable.Range(1, 10).Select(i => i.ToString()));
scheduler.AdvanceBySeconds(1);
result.NotificationType.Should().Be(FileNotificationType.Created);
result.NotificationType.Should().NotBe(0);
File.AppendAllLines(file, Enumerable.Range(11, 10).Select(i => i.ToString()));
scheduler.AdvanceBySeconds(1);
result.NotificationType.Should().Be(FileNotificationType.Changed);
File.Delete(file);
scheduler.AdvanceBySeconds(1);
result.NotificationType.Should().Be(FileNotificationType.Missing);
}
}
private static async Task <FileNotification> VerifyFileNotification(Tuple <string, string> deviceInfo)
{
FileNotification fileNotification = null;
Stopwatch sw = new Stopwatch();
sw.Start();
while (sw.Elapsed.Minutes < 2)
{
// Receive the file notification from queue
fileNotification = await fileNotificationReceiver.ReceiveAsync(TimeSpan.FromSeconds(20));
if (fileNotification != null)
{
if (fileNotification.DeviceId == deviceInfo.Item1)
{
await fileNotificationReceiver.CompleteAsync(fileNotification);
break;
}
await fileNotificationReceiver.AbandonAsync(fileNotification);
fileNotification = null;
}
}
sw.Stop();
return(fileNotification);
}
private static async Task <FileNotification> VerifyFileNotification(FileNotificationReceiver <FileNotification> fileNotificationReceiver, string deviceId)
{
FileNotification fileNotification = null;
Stopwatch sw = new Stopwatch();
sw.Start();
while (sw.Elapsed.TotalMinutes < 2)
{
// Receive the file notification from queue
fileNotification = await fileNotificationReceiver.ReceiveAsync(TimeSpan.FromSeconds(20)).ConfigureAwait(false);
if (fileNotification != null)
{
if (fileNotification.DeviceId == deviceId)
{
await fileNotificationReceiver.CompleteAsync(fileNotification).ConfigureAwait(false);
break;
}
await fileNotificationReceiver.AbandonAsync(fileNotification).ConfigureAwait(false);
fileNotification = null;
}
}
sw.Stop();
return(fileNotification);
}
public async Task StartListenAsync()
{
try
{
FileNotificationReceiver <FileNotification> fileNotificationReceiver = _serviceClient.GetFileNotificationReceiver();
while (true)
{
FileNotification fileNotification = null;
try
{
fileNotification = await fileNotificationReceiver.ReceiveAsync();
if (fileNotification != null)
{
await SendToWebAppAsync(fileNotification);
await fileNotificationReceiver.CompleteAsync(fileNotification);
}
}
catch (Exception ex)
{
await fileNotificationReceiver.AbandonAsync(fileNotification);
}
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}
public void SkipFirstLine()
{
var firstLine = "This is the first line";
var secondLine = "This is the second line";
var startPosition = (firstLine + Environment.NewLine).Length;
FileNotification result = null;
var scheduler = new TestScheduler();
using (var file = new TestFile())
using (file.Info
.WatchFile(scheduler: scheduler)
.ScanFrom(startPosition, scheduler: scheduler)
.Subscribe(x => result = x))
{
file.Append(firstLine);
file.Append(secondLine);
scheduler.AdvanceByMilliSeconds(1000);
var lines = File.ReadAllLines(result.FullName).ToArray();
lines.Should().HaveCount(1);
lines.Single().Should().Be(secondLine);
}
}
private static async Task StartReceivingLoopAsync()
{
s_log.WriteLine("Starting receiving file notification loop...");
CancellationToken cancellationToken = new CancellationTokenSource(s_duration).Token;
while (!cancellationToken.IsCancellationRequested)
{
try
{
FileNotification fileNotification = await s_fileNotificationReceiver.ReceiveAsync(s_interval).ConfigureAwait(false);
if (fileNotification != null)
{
string key = RetrieveKey(fileNotification.BlobName);
s_fileNotifications.TryAdd(key, fileNotification);
s_log.WriteLine($"File notification received deviceId={fileNotification.DeviceId}, blobName={fileNotification.BlobName}.");
await s_fileNotificationReceiver.AbandonAsync(fileNotification).ConfigureAwait(false);
}
}
catch (Exception)
{
s_log.WriteLine("Ignore any exception while receiving/abandon file upload notification.");
}
}
s_log.WriteLine("End receiving file notification loop.");
}
public FileScanResult(FileNotification notification, LineIndex[] matchingLines, int totalLines, int endOfTail, int index)
{
Notification = notification;
MatchingLines = matchingLines;
TotalLines = totalLines;
EndOfTail = endOfTail;
Index = index;
}
public FileScanResult(FileNotification notification, LineIndex[] matchingLines, int totalLines, int endOfTail,int index)
{
Notification = notification;
MatchingLines = matchingLines;
TotalLines = totalLines;
EndOfTail = endOfTail;
Index = index;
}
public async Task BlobIsLoad(FileNotification fileBlob)
{
await _serverService.NotifyPhotoArrivedToDeviceAsync(fileBlob.DeviceId, fileBlob.BlobName);
int threshold = await GetThresholdAsync(fileBlob.DeviceId);
byte[] photo = await _blobService.GetPhotoAsync(fileBlob.BlobName);
Clients.All.photoLoaded(fileBlob.DeviceId, threshold, photo);
}
private async Task UploadFileDisconnectTransport(
Client.TransportType transport,
string filename,
string faultType,
string reason,
int delayInSec,
int durationInSec = 0,
int retryDurationInMilliSec = FaultInjection.RecoveryTimeMilliseconds)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(DevicePrefix).ConfigureAwait(false);
ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);
FileNotificationReceiver <FileNotification> notificationReceiver = serviceClient.GetFileNotificationReceiver();
DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;
Task fileuploadTask;
Task <FileNotification> verifyTask;
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
verifyTask = VerifyFileNotification(notificationReceiver, testDevice.Id);
fileuploadTask = deviceClient.UploadToBlobAsync(filename, fileStreamSource);
try
{
await
deviceClient.SendEventAsync(FaultInjection.ComposeErrorInjectionProperties(faultType, reason,
delayInSec, durationInSec)).ConfigureAwait(false);
}
catch (Exception)
{
// catch and ignore exceptions resulted from error injection and continue to
// check result of the file upload status
}
await Task.WhenAll(fileuploadTask, verifyTask).ConfigureAwait(false);
}
FileNotification fileNotification = await verifyTask.ConfigureAwait(false);
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
Assert.AreEqual(testDevice.Id + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
private async Task SendToWebAppAsync(FileNotification fileNotification)
{
if (fileNotification != null)
{
try
{
await _blobHubProxy.Invoke("BlobIsLoad", fileNotification);
Console.WriteLine("Send Blob To WebApp" + fileNotification.BlobName);
}
catch (Exception ex)
{
Console.WriteLine("Send Blob To WebApp ERR: " + ex.Message);
throw;
}
}
}
public void EmptyForEmptyFile()
{
FileNotification result = null;
var scheduler = new TestScheduler();
using (var file = new TestFile())
using (file.Info
.WatchFile(scheduler: scheduler)
.ScanFrom(0, scheduler: scheduler)
.Subscribe(x => result = x))
{
scheduler.AdvanceByMilliSeconds(1000);
var lines = File.ReadAllLines(result.FullName).ToArray();
lines.Should().HaveCount(0);
}
}
private async Task UploadFile(Client.TransportType transport, string filename, bool x509auth = false)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(
DevicePrefix,
x509auth?TestDeviceType.X509 : TestDeviceType.Sasl).ConfigureAwait(false);
DeviceClient deviceClient;
if (x509auth)
{
X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();
var auth = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);
deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, auth, transport);
}
else
{
deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
}
using (deviceClient)
using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
FileNotificationReceiver <FileNotification> notificationReceiver = serviceClient.GetFileNotificationReceiver();
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
await deviceClient.UploadToBlobAsync(filename, fileStreamSource).ConfigureAwait(false);
}
FileNotification fileNotification = await VerifyFileNotification(notificationReceiver, testDevice.Id).ConfigureAwait(false);
// The following checks allow running these tests multiple times in parallel.
// Notifications for one of the test-run instances may be received by the other test-run.
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
_log.WriteLine($"TestDevice: '{testDevice.Id}', blobName: '{fileNotification.BlobName}', size: {fileNotification.BlobSizeInBytes}");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
}
async Task uploadFileDisconnectTransport(Client.TransportType transport, string filename, string faultType, string reason, int delayInSec,
int durationInSec = 0, int retryDurationInMilliSec = 24000)
{
Tuple <string, string> deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
DeviceClient deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
deviceClient.OperationTimeoutInMilliseconds = (uint)retryDurationInMilliSec;
Task fileuploadTask;
Task <FileNotification> verifyTask;
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
verifyTask = VerifyFileNotification(deviceInfo);
fileuploadTask = deviceClient.UploadToBlobAsync(filename, fileStreamSource);
try
{
await
deviceClient.SendEventAsync(TestUtil.ComposeErrorInjectionProperties(faultType, reason,
delayInSec, durationInSec)).ConfigureAwait(false);
}
catch (Exception)
{
// catch and ignore exceptions resulted from error injection and continue to
// check result of the file upload status
}
await Task.WhenAll(fileuploadTask, verifyTask).ConfigureAwait(false);
}
FileNotification fileNotification = await verifyTask.ConfigureAwait(false);
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
Assert.AreEqual(deviceInfo.Item1 + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync().ConfigureAwait(false);
await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager).ConfigureAwait(false);
}
private async Task UploadFile(Client.TransportType transport, string filename, bool x509auth = false)
{
TestDevice testDevice = await TestDevice.GetTestDeviceAsync(
DevicePrefix,
x509auth ? TestDeviceType.X509 : TestDeviceType.Sasl).ConfigureAwait(false);
DeviceClient deviceClient;
if (x509auth)
{
X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();
var auth = new DeviceAuthenticationWithX509Certificate(testDevice.Id, cert);
deviceClient = DeviceClient.Create(testDevice.IoTHubHostName, auth, transport);
}
else
{
deviceClient = DeviceClient.CreateFromConnectionString(testDevice.ConnectionString, transport);
}
using(deviceClient)
using (ServiceClient serviceClient = ServiceClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString))
{
FileNotificationReceiver<FileNotification> notificationReceiver = serviceClient.GetFileNotificationReceiver();
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
await deviceClient.UploadToBlobAsync(filename, fileStreamSource).ConfigureAwait(false);
}
FileNotification fileNotification = await VerifyFileNotification(notificationReceiver, testDevice.Id).ConfigureAwait(false);
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
Assert.AreEqual(testDevice.Id + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync().ConfigureAwait(false);
await serviceClient.CloseAsync().ConfigureAwait(false);
}
}
public void StartAfterEndOfFileShouldReturnNothing()
{
FileNotification result = null;
var scheduler = new TestScheduler();
using (var file = new TestFile())
using (file.Info
.WatchFile(scheduler: scheduler)
.ScanFrom(10000, scheduler: scheduler)
.Subscribe(x => result = x))
{
file.Append("A");
file.Append("B");
file.Append("C");
scheduler.AdvanceByMilliSeconds(1000);
var lines = File.ReadAllLines(result.FullName).ToArray();
lines.Should().HaveCount(0);
}
}
async Task uploadFile(Client.TransportType transport, string filename, bool x509auth = false)
{
DeviceClient deviceClient;
Tuple <string, string> deviceInfo;
if (x509auth)
{
deviceInfo = TestUtil.CreateDeviceWithX509(DevicePrefix, hostName, registryManager);
string certBase64 = Environment.GetEnvironmentVariable("IOTHUB_X509_PFX_CERTIFICATE");
Byte[] buff = Convert.FromBase64String(certBase64);
var cert = new X509Certificate2(buff);
var auth = new DeviceAuthenticationWithX509Certificate(deviceInfo.Item1, cert);
deviceClient = DeviceClient.Create(deviceInfo.Item2, auth, transport);
}
else
{
deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
}
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
await deviceClient.UploadToBlobAsync(filename, fileStreamSource);
}
FileNotification fileNotification = await VerifyFileNotification(deviceInfo);
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
Assert.AreEqual(deviceInfo.Item1 + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync();
TestUtil.RemoveDevice(deviceInfo.Item1, registryManager);
}
async Task uploadFile(Client.TransportType transport, string filename, bool x509auth = false)
{
DeviceClient deviceClient;
Tuple <string, string> deviceInfo;
if (x509auth)
{
deviceInfo = TestUtil.CreateDeviceWithX509(DevicePrefix, hostName, registryManager);
X509Certificate2 cert = Configuration.IoTHub.GetCertificateWithPrivateKey();
var auth = new DeviceAuthenticationWithX509Certificate(deviceInfo.Item1, cert);
deviceClient = DeviceClient.Create(deviceInfo.Item2, auth, transport);
}
else
{
deviceInfo = TestUtil.CreateDevice(DevicePrefix, hostName, registryManager);
deviceClient = DeviceClient.CreateFromConnectionString(deviceInfo.Item2, transport);
}
using (FileStream fileStreamSource = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
await deviceClient.UploadToBlobAsync(filename, fileStreamSource).ConfigureAwait(false);
}
FileNotification fileNotification = await VerifyFileNotification(deviceInfo).ConfigureAwait(false);
Assert.IsNotNull(fileNotification, "FileNotification is not received.");
Assert.AreEqual(deviceInfo.Item1 + "/" + filename, fileNotification.BlobName, "Uploaded file name mismatch in notifications");
Assert.AreEqual(new FileInfo(filename).Length, fileNotification.BlobSizeInBytes, "Uploaded file size mismatch in notifications");
Assert.IsFalse(string.IsNullOrEmpty(fileNotification.BlobUri), "File notification blob uri is null or empty");
await deviceClient.CloseAsync().ConfigureAwait(false);
await TestUtil.RemoveDeviceAsync(deviceInfo.Item1, registryManager).ConfigureAwait(false);
}
private async Task ReceiveFileUploadNotifications(string targetDeviceId, CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(targetDeviceId))
{
_logger.LogInformation($"Target device is specified, will only complete matching notifications.");
}
_logger.LogInformation($"Listening for file upload notifications from the service.");
FileNotificationReceiver <FileNotification> notificationReceiver = _serviceClient.GetFileNotificationReceiver();
int totalNotificationsReceived = 0;
int totalNotificationsCompleted = 0;
int totalNotificationsAbandoned = 0;
while (!cancellationToken.IsCancellationRequested)
{
try
{
FileNotification fileUploadNotification = await notificationReceiver.ReceiveAsync(s_notificationReceiverTimeout);
if (fileUploadNotification == null)
{
_logger.LogInformation($"Did not receive any notification after {s_notificationReceiverTimeout.TotalSeconds} seconds.");
continue;
}
totalNotificationsReceived++;
_logger.LogInformation($"Received file upload notification.");
_logger.LogInformation($"\tDeviceId: {fileUploadNotification.DeviceId ?? "N/A"}.");
_logger.LogInformation($"\tFileName: {fileUploadNotification.BlobName ?? "N/A"}.");
_logger.LogInformation($"\tEnqueueTimeUTC: {fileUploadNotification.EnqueuedTimeUtc}.");
_logger.LogInformation($"\tBlobSizeInBytes: {fileUploadNotification.BlobSizeInBytes}.");
// If the targetDeviceId is set and does not match the notification's origin, ignore it by abandoning the notification.
// Completing a notification will remove that notification from the service's queue so it won't be delivered to any other receiver again.
// Abandoning a notification will put it back on the queue to be re-delivered to receivers. This is mostly used when multiple receivers
// are configured and each receiver is only interested in notifications from a particular device/s.
if (!string.IsNullOrWhiteSpace(targetDeviceId) &&
!string.Equals(fileUploadNotification.DeviceId, targetDeviceId, StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation($"Marking notification for {fileUploadNotification.DeviceId} as Abandoned.");
await notificationReceiver.AbandonAsync(fileUploadNotification);
_logger.LogInformation($"Successfully marked the notification for device {fileUploadNotification.DeviceId} as Abandoned.");
totalNotificationsAbandoned++;
}
else
{
_logger.LogInformation($"Marking notification for {fileUploadNotification.DeviceId} as Completed.");
await notificationReceiver.CompleteAsync(fileUploadNotification);
_logger.LogInformation($"Successfully marked the notification for device {fileUploadNotification.DeviceId} as Completed.");
totalNotificationsCompleted++;
}
}
catch (Exception e) when((e is IotHubException) || (e is DeviceMessageLockLostException))
{
_logger.LogWarning($"Caught a recoverable exception, will retry: {e.Message} - {e}");
}
}
_logger.LogInformation($"Total Notifications Received: {totalNotificationsReceived}.");
_logger.LogInformation($"Total Notifications Marked as Completed: {totalNotificationsCompleted}.");
_logger.LogInformation($"Total Notifications Marked as Abandoned: {totalNotificationsAbandoned}.");
}
public async Task OnFileDeletionAsync(FileNotification notification)
{
await DeleteAsync(notification.FileId);
}
