Пример #1
0
        public void TestZipUnzipRandomSucceeds()
        {
            //// ARRANGE

            var source = new byte[992];

            var r = new Random(42);

            r.NextBytes(source);

            //// ACT

            var zippedSource = GZipHelper.Zip(source);

            Assert.AreEqual(1015, zippedSource.Length);

            var unzipped = GZipHelper.Unzip(zippedSource);

            //// ASSERT

            Assert.AreEqual(992, unzipped.Length);

            for (int i = 0; i < 992; i++)
            {
                Assert.AreEqual(source[i], unzipped[i]);
            }
        }
Пример #2
0
        public void TestZipUnzipSucceeds()
        {
            //// ARRANGE

            var source = new byte[992];

            for (int i = 0; i < source.Length; i++)
            {
                source[i] = 42;
            }

            //// ACT

            var zippedSource = GZipHelper.Zip(source);

            Assert.AreEqual(29, zippedSource.Length);

            var unzipped = GZipHelper.Unzip(zippedSource);

            //// ASSERT

            Assert.AreEqual(992, unzipped.Length);

            for (int i = 0; i < 992; i++)
            {
                Assert.AreEqual(source[i], unzipped[i]);
            }
        }
Пример #3
0
        public void DownloadFiles(string baseDirectoryUri,
                                  IList <string> blobs,
                                  string destination,
                                  string storageConnectionString,
                                  CancellationToken cancelToken,
                                  Action <string> fileDownloadedAction,
                                  Action <string, Exception> fileFailedAction,
                                  Action completedAction)
        {
            var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
            var blobClient     = storageAccount.CreateCloudBlobClient();
            var baseBlobLength = 0;

            if (!string.IsNullOrEmpty(baseDirectoryUri))
            {
                baseBlobLength = baseDirectoryUri.Length + (baseDirectoryUri.EndsWith("/") ? 0 : 1);
            }
            else
            {
                var shortestBlob = blobs.OrderBy(s => s.Length).First();
                baseBlobLength = shortestBlob.LastIndexOf("/") + 1;
            }

            try
            {
                Parallel.ForEach(
                    blobs,
                    new ParallelOptions {
                    MaxDegreeOfParallelism = ArgSettings.Parallelism, CancellationToken = cancelToken
                },
                    (blobName, loopState) =>
                {
                    var blob           = blobClient.GetBlobReference(blobName);
                    var blockBlob      = blob.ToBlockBlob;
                    var destPath       = Path.Combine(destination, blobName.Substring(baseBlobLength));
                    var fileDownloaded = false;

                    while (!fileDownloaded && !loopState.IsStopped)
                    {
                        try
                        {
                            blob.FetchAttributes();
                            var isGZip = blob.Properties.ContentEncoding != null && blob.Properties.ContentEncoding.Equals("gzip", StringComparison.OrdinalIgnoreCase);

                            if (File.Exists(destPath))
                            {
                                var hash = string.Empty;

                                if (isGZip)
                                {
                                    using (var fileStream = GZipHelper.Zip(destPath))
                                    {
                                        hash = HashHelper.ComputeMD5(fileStream);
                                    }
                                }
                                else
                                {
                                    using (var fileStream = File.Open(destPath, FileMode.Open))
                                    {
                                        hash = HashHelper.ComputeMD5(fileStream);
                                    }
                                }

                                if (blob.Attributes.Properties.ContentMD5 == hash)
                                {
                                    fileDownloaded = true;
                                }
                                else
                                {
                                    //delete obsolete existing file
                                    File.Delete(destPath);
                                }
                            }

                            if (!fileDownloaded)
                            {
                                Directory.CreateDirectory(Path.GetDirectoryName(destPath));

                                if (isGZip)
                                {
                                    using (var stream = new MemoryStream())
                                    {
                                        blob.DownloadToStream(stream, new BlobRequestOptions()
                                        {
                                            RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
                                        });

                                        stream.Seek(0, SeekOrigin.Begin);

                                        GZipHelper.Unzip(stream, destPath);
                                    }
                                }
                                else
                                {
                                    blob.DownloadToFile(destPath, new BlobRequestOptions()
                                    {
                                        RetryPolicy = RetryPolicies.Retry(ArgSettings.RetryCount, TimeSpan.FromMilliseconds(ArgSettings.RetryInterval))
                                    });
                                }


                                fileDownloaded = true;
                            }

                            if (!loopState.IsStopped && fileDownloaded && fileDownloadedAction != null)
                            {
                                fileDownloadedAction(blobName);
                            }
                        }
                        catch (Exception ex)
                        {
                            if (!loopState.IsStopped && fileFailedAction != null)
                            {
                                fileFailedAction(blobName, ex);
                            }

                            Thread.Sleep(ArgSettings.RetryInterval);
                        }
                    }
                });

                if (completedAction != null)
                {
                    completedAction();
                }
            }
            catch (OperationCanceledException)
            {
            }
        }
Пример #4
0
        /// <summary>
        /// This method is called whenever the module is sent a message from the EdgeHub.
        /// It just pipe the messages without any change.
        /// It prints all the incoming messages.
        /// </summary>
        private static async Task <MessageResponse> PipeMessageInputOne(Message message, object userContext)
        {
            int counterValue = Interlocked.Increment(ref counter);

            var moduleClient = userContext as ModuleClient;

            if (moduleClient == null)
            {
                throw new InvalidOperationException("UserContext doesn't contain " + "expected values");
            }

            var connectionDeviceId = message.ConnectionDeviceId;

            var connectionModuleId = message.ConnectionModuleId;

            byte[] messageBytes = message.GetBytes();

            if (message.ContentEncoding == "gzip" &&
                message.ContentType == "application/zip")
            {
                var zippedLength = messageBytes.Length;
                messageBytes = GZipHelper.Unzip(messageBytes);
                System.Console.WriteLine($"Uncompressed from GZIP {zippedLength} bytes to {messageBytes.Length} bytes");
            }

            if (message.ContentEncoding == "deflate" &&
                message.ContentType == "application/zip")
            {
                var zippedLength = messageBytes.Length;
                messageBytes = DeflateHelper.Unzip(messageBytes);
                System.Console.WriteLine($"Uncompressed from Deflate {zippedLength} bytes to {messageBytes.Length} bytes");
            }

            string messageString = Encoding.UTF8.GetString(messageBytes);

            Console.WriteLine($"-> Received echo message: {counterValue}, Body: '{messageString}'");

            if (!string.IsNullOrEmpty(messageString))
            {
                var messageBody = JsonConvert.DeserializeObject(messageString);

                var moduleOutput = _moduleOutputs.GetModuleOutput("output1");

                if (moduleOutput != null)
                {
                    moduleOutput.Properties.Clear();

                    foreach (var prop in message.Properties)
                    {
                        moduleOutput.Properties.Add(prop.Key, prop.Value);

                        Console.WriteLine($"Property added: key:'{prop.Key}' value:'{prop.Value}'");
                    }

                    if (!string.IsNullOrEmpty(connectionDeviceId))
                    {
                        Console.WriteLine($"connectionDeviceId: '{connectionDeviceId}'");
                    }

                    if (!string.IsNullOrEmpty(connectionModuleId))
                    {
                        Console.WriteLine($"ConnectionModuleId: '{connectionModuleId}'");
                    }

                    await moduleOutput.SendMessage(messageBody);

                    Console.WriteLine("Received message echoed");
                }
            }

            return(MessageResponse.Completed);
        }