예제 #1
0
        public void TestSharpCompressWithEmptyStream()
        {
            ResetScratch();

            MemoryStream stream = new NonSeekableMemoryStream();

            using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
            {
                zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
                zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
            }

            stream = new MemoryStream(stream.ToArray());
            File.WriteAllBytes(Path.Combine(SCRATCH_FILES_PATH, "foo.zip"), stream.ToArray());

            using (IReader zipReader = ZipReader.Open(stream))
            {
                while (zipReader.MoveToNextEntry())
                {
                    using (EntryStream entry = zipReader.OpenEntryStream())
                    {
                        MemoryStream tempStream = new MemoryStream();
                        const int    bufSize    = 0x1000;
                        byte[]       buf        = new byte[bufSize];
                        int          bytesRead  = 0;
                        while ((bytesRead = entry.Read(buf, 0, bufSize)) > 0)
                        {
                            tempStream.Write(buf, 0, bytesRead);
                        }
                    }
                }
            }
        }
예제 #2
0
        public void TestSharpCompressWithEmptyStream()
        {
            MemoryStream stream = new NonSeekableMemoryStream();

            using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
            {
                zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
                zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
            }

            stream = new MemoryStream(stream.ToArray());
            File.WriteAllBytes("foo.zip", stream.ToArray());

            using (var zipArchive = ZipArchive.Open(stream))
            {
                foreach (var entry in zipArchive.Entries)
                {
                    using (var entryStream = entry.OpenEntryStream())
                    {
                        MemoryStream tempStream = new MemoryStream();
                        const int    bufSize    = 0x1000;
                        byte[]       buf        = new byte[bufSize];
                        int          bytesRead  = 0;
                        while ((bytesRead = entryStream.Read(buf, 0, bufSize)) > 0)
                        {
                            tempStream.Write(buf, 0, bytesRead);
                        }
                    }
                }
            }
        }
        public void TestSharpCompressWithEmptyStream()
        {
            MemoryStream stream = new NonSeekableMemoryStream();

            using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
            {
                zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
                zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
            }

            stream = new MemoryStream(stream.ToArray());
            File.WriteAllBytes("foo.zip", stream.ToArray());

            using (var zipArchive = ZipArchive.Open(stream))
            {
                foreach(var entry in zipArchive.Entries)
                {
                    using (var entryStream = entry.OpenEntryStream())
                    {
                        MemoryStream tempStream = new MemoryStream();
                        const int bufSize = 0x1000;
                        byte[] buf = new byte[bufSize];
                        int bytesRead = 0;
                        while ((bytesRead = entryStream.Read(buf, 0, bufSize)) > 0)
                            tempStream.Write(buf, 0, bytesRead);
                    }
                }
            }
        }
        public void IHashFunction_SpeedTest_MultipleItems_ComputeHashAsync_Stream_NonSeekable()
        {
            IHashFunction_SpeedTest_MultipleItems((sw, testHashFunction, count, testBytes) => {
                var testHashFunctionAsync = testHashFunction as IHashFunctionAsync;

                if (testHashFunctionAsync == null)
                {
                    return(false);
                }

                using (var ms = new NonSeekableMemoryStream(testBytes))
                {
                    sw.Start();

                    for (int x = 0; x < count; ++x)
                    {
                        testHashFunctionAsync.ComputeHashAsync(ms)
                        .Wait();

                        ms.Real_Seek(0, SeekOrigin.Begin);
                    }

                    sw.Stop();
                }

                return(true);
            });
        }
예제 #5
0
        public void TestSharpCompressWithEmptyStream()
        {
            MemoryStream stream = new NonSeekableMemoryStream();

            using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
            {
                zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
                zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
            }

            stream = new MemoryStream(stream.ToArray());
            File.WriteAllBytes(Path.Combine(SCRATCH_FILES_PATH, "foo.zip"), stream.ToArray());

            using (IReader zipReader = ZipReader.Open(stream))
            {
                while (zipReader.MoveToNextEntry())
                {
                    using (EntryStream entry = zipReader.OpenEntryStream())
                    {
                        MemoryStream tempStream = new MemoryStream();
                        const int bufSize = 0x1000;
                        byte[] buf = new byte[bufSize];
                        int bytesRead = 0;
                        while ((bytesRead = entry.Read(buf, 0, bufSize)) > 0)
                            tempStream.Write(buf, 0, bytesRead);
                    }
                }
            }
        }
        public void IHashFunction_SpeedTest_SingleBlock_ComputeHash_Stream_NonSeekable()
        {
            IHashFunction_SpeedTest_SingleBlock((sw, testHashFunction, testBytes) => {
                using (var ms = new NonSeekableMemoryStream(testBytes))
                {
                    sw.Start();
                    {
                        testHashFunction.ComputeHash(ms);
                    }
                    sw.Stop();
                }

                return(true);
            });
        }
예제 #7
0
        public void IHashFunction_ComputeHash_Stream_NonSeekable_MatchesKnownValues()
        {
            foreach (var knownValue in KnownValues)
            {
                var hf = CreateHashFunction(knownValue.HashSize);

                using (var ms = new NonSeekableMemoryStream(knownValue.TestValue))
                {
                    var hashResults = hf.ComputeHash(ms);

                    Assert.Equal(
                        knownValue.ExpectedValue.Take((hf.HashSize + 7) / 8),
                        hashResults);
                }
            }
        }
예제 #8
0
        public async Task NonSeekableMemoryStream_To_MemoryStream(bool sourceIsExposable, bool disposeSource)
        {
            byte[] input       = CreateByteArray(8192);
            var    source      = new NonSeekableMemoryStream(input, sourceIsExposable);
            var    destination = new MemoryStream();

            await StreamToStreamCopy.CopyAsync(source, destination, 4096, disposeSource);

            Assert.NotEqual(disposeSource, source.CanRead);
            if (!disposeSource)
            {
                Assert.Equal(input.Length, source.Position);
            }

            Assert.Equal(input, destination.ToArray());
            Assert.Equal(input.Length, destination.Position);
            Assert.Equal(input.Length, destination.Length);
        }
예제 #9
0
        public async Task NonSeekableMemoryStream_To_MemoryStream(bool sourceIsExposable, bool disposeSource)
        {
            byte[] input = CreateByteArray(8192);
            var source = new NonSeekableMemoryStream(input, sourceIsExposable);
            var destination = new MemoryStream();

            await StreamToStreamCopy.CopyAsync(source, destination, 4096, disposeSource);

            Assert.NotEqual(disposeSource, source.CanRead);
            if (!disposeSource)
            {
                Assert.Equal(input.Length, source.Position);
            }

            Assert.Equal(input, destination.ToArray());
            Assert.Equal(input.Length, destination.Position);
            Assert.Equal(input.Length, destination.Length);
        }
        public void IHashFunction_SpeedTest_MultipleItems_ComputeHash_Stream_NonSeekable()
        {
            IHashFunction_SpeedTest_MultipleItems((sw, testHashFunction, count, testBytes) => {
                using (var ms = new NonSeekableMemoryStream(testBytes))
                {
                    sw.Start();

                    for (int x = 0; x < count; ++x)
                    {
                        testHashFunction.ComputeHash(ms);

                        ms.Real_Seek(0, SeekOrigin.Begin);
                    }

                    sw.Stop();
                }

                return(true);
            });
        }
        public void IHashFunction_SpeedTest_SingleBlock_ComputeHashAsync_Stream_NonSeekable()
        {
            IHashFunction_SpeedTest_SingleBlock((sw, testHashFunction, testBytes) => {
                var testHashFunctionAsync = testHashFunction as IHashFunctionAsync;

                if (testHashFunctionAsync == null)
                {
                    return(false);
                }


                using (var ms = new NonSeekableMemoryStream(testBytes))
                {
                    sw.Start();
                    {
                        testHashFunctionAsync.ComputeHashAsync(ms)
                        .Wait();
                    }
                    sw.Stop();
                }

                return(true);
            });
        }
예제 #12
0
        static async Task Main(string[] args)
        {
            var connectionString = Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING");

            if (string.IsNullOrEmpty(connectionString))
            {
                throw new InvalidOperationException("Environment variable STORAGE_CONNECTION_STRING is not set");
            }

            Log($"ITERATIONS: {_iterations}");
            Log($"UPLOAD_SIZE: {_uploadSize}");
            Log($"BUFFER_SIZE: {_bufferSize}");
            Log($"MAX_CONCURRENCY: {_maxConcurrency}");

            // Enable SDK logging (with timestamps)
            using var azureListener = new AzureEventSourceListener(
                      (eventData, text) => Log(String.Format("[{1}] {0}: {2}", eventData.EventSource.Name, eventData.Level, text)),
                      EventLevel.Verbose);

            // Enable System.Net logging
            using var httpListener    = new LogEventListener("Microsoft-System-Net-Http");
            using var socketsListener = new LogEventListener("Microsoft-System-Net-Sockets");

            var containerName = $"container{DateTime.Now.Ticks}";

            // Test custom transport with shorter timeout
            var containerClient = new BlobContainerClient(connectionString, containerName, new BlobClientOptions()
            {
                Transport = new HttpClientTransport(new HttpClient()
                {
                    Timeout = _httpClientTimeout
                })
            });

            Log($"Creating container {containerName}");
            await containerClient.CreateAsync();

            Log($"Created container {containerName}");

            var randomBuffer = new byte[_uploadSize];

            new Random(0).NextBytes(randomBuffer);
            var randomStream = new NonSeekableMemoryStream(randomBuffer);

            for (var i = 0; i < _iterations; i++)
            {
                try
                {
                    Log($"Iteration {i}");

                    var blobName   = $"blob{DateTime.Now.Ticks}";
                    var blobClient = containerClient.GetBlobClient(blobName);

                    randomStream.Seek(0, SeekOrigin.Begin);

                    Log($"Uploading blob {blobName}");
                    await blobClient.UploadAsync(randomStream, transferOptions : new StorageTransferOptions()
                    {
                        MaximumConcurrency    = _maxConcurrency,
                        MaximumTransferLength = _bufferSize
                    });

                    Log($"Uploaded blob {blobName}");

                    Log($"Deleting blob {blobName}");
                    await blobClient.DeleteAsync();

                    Log($"Deleted blob {blobName}");
                }
                catch (Exception e)
                {
                    Log(e);
                }
            }

            Log($"Deleting container {containerName}");
            await containerClient.DeleteAsync();

            Log($"Deleted container {containerName}");
        }
예제 #13
0
        public async Task Tests()
        {
            List <TestCase> testCases = new List <TestCase>
            {
                new TestCase("test_null_0.avro", 57, o => Assert.IsNull(o)),                                                // null
                new TestCase("test_null_1.avro", 60, o => Assert.AreEqual(true, (bool)o)),                                  // bool
                new TestCase("test_null_2.avro", 59, o => Assert.AreEqual("adsfasdf09809dsf-=adsf", (string)o)),            // string
                new TestCase("test_null_3.avro", 58, o => Assert.AreEqual(Encoding.UTF8.GetBytes("12345abcd"), (byte[])o)), // byte[]
                new TestCase("test_null_4.avro", 56, o => Assert.AreEqual(1234, (int)o)),                                   // int
                new TestCase("test_null_5.avro", 57, o => Assert.AreEqual(1234L, (long)o)),                                 // long
                new TestCase("test_null_6.avro", 58, o => Assert.AreEqual(1234.0, (float)o)),                               // float
                new TestCase("test_null_7.avro", 59, o => Assert.AreEqual(1234.0, (double)o)),                              // fouble
                // Not supported today.
                //new TestCase("test_null_8.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("B"), (byte[])o)), // fixed
                new TestCase("test_null_9.avro", 106, o => Assert.AreEqual("B", (string)o)), // enum
                // Not supported today.
                // new TestCase("test_null_10.avro", o => Assert.AreEqual(new List<long>() { 1, 2, 3 }, (List<long>)o)), // array
                new TestCase("test_null_11.avro", 84, o => Assert.AreEqual(
                                 new Dictionary <string, int>()
                {
                    { "a", 1 }, { "b", 3 }, { "c", 2 }
                }, (Dictionary <string, object>)o)),                          // dictionary
                new TestCase("test_null_12.avro", 77, o => Assert.IsNull(o)), // union
                new TestCase("test_null_13.avro", 129, o =>                   // record
                {
                    Dictionary <string, object> expected = new Dictionary <string, object>()
                    {
                        { "$schema", "Test" }, { "f", 5 }
                    };
                    Dictionary <string, object> actual = (Dictionary <string, object>)o;
                    Assert.AreEqual(expected.Count, actual.Count);
                    foreach (KeyValuePair <string, object> keyValuePair in actual)
                    {
                        Assert.AreEqual(expected[keyValuePair.Key], keyValuePair.Value);
                    }
                })
            };

            // seekable streams
            foreach (TestCase testCase in testCases)
            {
                // Arrange
                using FileStream stream = File.OpenRead(
                          $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}");
                AvroReader avroReader = new AvroReader(stream);

                // Act
                object o = await avroReader.Next(async : true).ConfigureAwait(false);

                testCase.Predicate(o);
                Assert.AreEqual(testCase.BlockOffset, avroReader.BlockOffset);
            }

            // non-seekable streams
            foreach (TestCase testCase in testCases)
            {
                // Arrange
                string filePath = $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}";
                using FileStream stream = File.OpenRead(filePath);
                using NonSeekableMemoryStream nonSeekableMemoryStream = new NonSeekableMemoryStream();
                await stream.CopyToAsync(nonSeekableMemoryStream);

                nonSeekableMemoryStream.Reset();
                AvroReader avroReader = new AvroReader(nonSeekableMemoryStream);

                // Act
                object o = await avroReader.Next(async : true).ConfigureAwait(false);

                testCase.Predicate(o);
                Assert.AreEqual(testCase.BlockOffset, avroReader.BlockOffset);
            }
        }
예제 #14
0
파일: StreamTests.cs 프로젝트: srvuppu/bond
        protected override InputStream MakeInputStream(ArraySegment <byte> buffer)
        {
            var ms = new NonSeekableMemoryStream(buffer);

            return(new InputStream(ms, InternalBufferSize));
        }