public void TestZipUnzipRandomSucceeds()
{
//// ARRANGE
var source = new byte[992];
var r = new Random(42);
r.NextBytes(source);
//// ACT
var zippedSource = GZipHelper.Zip(source);
Assert.AreEqual(1015, zippedSource.Length);
var unzipped = GZipHelper.Unzip(zippedSource);
//// ASSERT
Assert.AreEqual(992, unzipped.Length);
for (int i = 0; i < 992; i++)
{
Assert.AreEqual(source[i], unzipped[i]);
}
}
public void TestZipUnzipSucceeds()
{
//// ARRANGE
var source = new byte[992];
for (int i = 0; i < source.Length; i++)
{
source[i] = 42;
}
//// ACT
var zippedSource = GZipHelper.Zip(source);
Assert.AreEqual(29, zippedSource.Length);
var unzipped = GZipHelper.Unzip(zippedSource);
//// ASSERT
Assert.AreEqual(992, unzipped.Length);
for (int i = 0; i < 992; i++)
{
Assert.AreEqual(source[i], unzipped[i]);
}
}
public void DownloadFiles(string baseDirectoryUri,
IList <string> blobs,
string destination,
string storageConnectionString,
CancellationToken cancelToken,
Action <string> fileDownloadedAction,
Action <string, Exception> fileFailedAction,
Action completedAction)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var baseBlobLength = 0;
if (!string.IsNullOrEmpty(baseDirectoryUri))
{
baseBlobLength = baseDirectoryUri.Length + (baseDirectoryUri.EndsWith("/") ? 0 : 1);
}
else
{
var shortestBlob = blobs.OrderBy(s => s.Length).First();
baseBlobLength = shortestBlob.LastIndexOf("/") + 1;
}
try
{
Parallel.ForEach(
blobs,
new ParallelOptions {
MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken
},
(blobName, loopState) =>
{
var blob = blobClient.GetBlobReference(blobName);
var blockBlob = blob.ToBlockBlob;
var destPath = Path.Combine(destination, blobName.Substring(baseBlobLength));
var fileDownloaded = false;
while (!fileDownloaded && !loopState.IsStopped)
{
try
{
blob.FetchAttributes();
var isGZip = blob.Properties.ContentEncoding != null && blob.Properties.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase);
if (File.Exists(destPath))
{
var hash = string.Empty;
if (isGZip)
{
using (var fileStream = GZipHelper.Zip(destPath))
{
hash = HashHelper.ComputeMD5(fileStream);
}
}
else
{
using (var fileStream = File.Open(destPath, FileMode.Open))
{
hash = HashHelper.ComputeMD5(fileStream);
}
}
if (blob.Attributes.Properties.ContentMD5 == hash)
{
fileDownloaded = true;
}
else
{
//delete obsolete existing file
File.Delete(destPath);
}
}
if (!fileDownloaded)
{
Directory.CreateDirectory(Path.GetDirectoryName(destPath));
if (isGZip)
{
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream, new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
});
stream.Seek(0, SeekOrigin.Begin);
GZipHelper.Unzip(stream, destPath);
}
}
else
{
blob.DownloadToFile(destPath, new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
});
}
fileDownloaded = true;
}
if (!loopState.IsStopped && fileDownloaded && fileDownloadedAction != null)
{
fileDownloadedAction(blobName);
}
}
catch (Exception ex)
{
if (!loopState.IsStopped && fileFailedAction != null)
{
fileFailedAction(blobName, ex);
}
Thread.Sleep(ArgSettings.RetryInterval);
}
}
});
if (completedAction != null)
{
completedAction();
}
}
catch (OperationCanceledException)
{
}
}
/// <summary>
/// This method is called whenever the module is sent a message from the EdgeHub.
/// It just pipe the messages without any change.
/// It prints all the incoming messages.
/// </summary>
private static async Task <MessageResponse> PipeMessageInputOne(Message message, object userContext)
{
int counterValue = Interlocked.Increment(ref counter);
var moduleClient = userContext as ModuleClient;
if (moduleClient == null)
{
throw new InvalidOperationException("UserContext doesn't contain " + "expected values");
}
var connectionDeviceId = message.ConnectionDeviceId;
var connectionModuleId = message.ConnectionModuleId;
byte[] messageBytes = message.GetBytes();
if (message.ContentEncoding == "gzip" &&
message.ContentType == "application/zip")
{
var zippedLength = messageBytes.Length;
messageBytes = GZipHelper.Unzip(messageBytes);
System.Console.WriteLine($"Uncompressed from GZIP {zippedLength} bytes to {messageBytes.Length} bytes");
}
if (message.ContentEncoding == "deflate" &&
message.ContentType == "application/zip")
{
var zippedLength = messageBytes.Length;
messageBytes = DeflateHelper.Unzip(messageBytes);
System.Console.WriteLine($"Uncompressed from Deflate {zippedLength} bytes to {messageBytes.Length} bytes");
}
string messageString = Encoding.UTF8.GetString(messageBytes);
Console.WriteLine($"-> Received echo message: {counterValue}, Body: '{messageString}'");
if (!string.IsNullOrEmpty(messageString))
{
var messageBody = JsonConvert.DeserializeObject(messageString);
var moduleOutput = _moduleOutputs.GetModuleOutput("output1");
if (moduleOutput != null)
{
moduleOutput.Properties.Clear();
foreach (var prop in message.Properties)
{
moduleOutput.Properties.Add(prop.Key, prop.Value);
Console.WriteLine($"Property added: key:'{prop.Key}' value:'{prop.Value}'");
}
if (!string.IsNullOrEmpty(connectionDeviceId))
{
Console.WriteLine($"connectionDeviceId: '{connectionDeviceId}'");
}
if (!string.IsNullOrEmpty(connectionModuleId))
{
Console.WriteLine($"ConnectionModuleId: '{connectionModuleId}'");
}
await moduleOutput.SendMessage(messageBody);
Console.WriteLine("Received message echoed");
}
}
return(MessageResponse.Completed);
}
