public void TestSharpCompressWithEmptyStream()
{
ResetScratch();
MemoryStream stream = new NonSeekableMemoryStream();
using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
{
zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
}
stream = new MemoryStream(stream.ToArray());
File.WriteAllBytes(Path.Combine(SCRATCH_FILES_PATH, "foo.zip"), stream.ToArray());
using (IReader zipReader = ZipReader.Open(stream))
{
while (zipReader.MoveToNextEntry())
{
using (EntryStream entry = zipReader.OpenEntryStream())
{
MemoryStream tempStream = new MemoryStream();
const int bufSize = 0x1000;
byte[] buf = new byte[bufSize];
int bytesRead = 0;
while ((bytesRead = entry.Read(buf, 0, bufSize)) > 0)
{
tempStream.Write(buf, 0, bytesRead);
}
}
}
}
}
public void TestSharpCompressWithEmptyStream()
{
MemoryStream stream = new NonSeekableMemoryStream();
using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
{
zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
}
stream = new MemoryStream(stream.ToArray());
File.WriteAllBytes("foo.zip", stream.ToArray());
using (var zipArchive = ZipArchive.Open(stream))
{
foreach (var entry in zipArchive.Entries)
{
using (var entryStream = entry.OpenEntryStream())
{
MemoryStream tempStream = new MemoryStream();
const int bufSize = 0x1000;
byte[] buf = new byte[bufSize];
int bytesRead = 0;
while ((bytesRead = entryStream.Read(buf, 0, bufSize)) > 0)
{
tempStream.Write(buf, 0, bytesRead);
}
}
}
}
}
public void TestSharpCompressWithEmptyStream()
{
MemoryStream stream = new NonSeekableMemoryStream();
using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
{
zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
}
stream = new MemoryStream(stream.ToArray());
File.WriteAllBytes("foo.zip", stream.ToArray());
using (var zipArchive = ZipArchive.Open(stream))
{
foreach(var entry in zipArchive.Entries)
{
using (var entryStream = entry.OpenEntryStream())
{
MemoryStream tempStream = new MemoryStream();
const int bufSize = 0x1000;
byte[] buf = new byte[bufSize];
int bytesRead = 0;
while ((bytesRead = entryStream.Read(buf, 0, bufSize)) > 0)
tempStream.Write(buf, 0, bytesRead);
}
}
}
}
public void IHashFunction_SpeedTest_MultipleItems_ComputeHashAsync_Stream_NonSeekable()
{
IHashFunction_SpeedTest_MultipleItems((sw, testHashFunction, count, testBytes) => {
var testHashFunctionAsync = testHashFunction as IHashFunctionAsync;
if (testHashFunctionAsync == null)
{
return(false);
}
using (var ms = new NonSeekableMemoryStream(testBytes))
{
sw.Start();
for (int x = 0; x < count; ++x)
{
testHashFunctionAsync.ComputeHashAsync(ms)
.Wait();
ms.Real_Seek(0, SeekOrigin.Begin);
}
sw.Stop();
}
return(true);
});
}
public void TestSharpCompressWithEmptyStream()
{
MemoryStream stream = new NonSeekableMemoryStream();
using (IWriter zipWriter = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate))
{
zipWriter.Write("foo.txt", new MemoryStream(new byte[0]));
zipWriter.Write("foo2.txt", new MemoryStream(new byte[10]));
}
stream = new MemoryStream(stream.ToArray());
File.WriteAllBytes(Path.Combine(SCRATCH_FILES_PATH, "foo.zip"), stream.ToArray());
using (IReader zipReader = ZipReader.Open(stream))
{
while (zipReader.MoveToNextEntry())
{
using (EntryStream entry = zipReader.OpenEntryStream())
{
MemoryStream tempStream = new MemoryStream();
const int bufSize = 0x1000;
byte[] buf = new byte[bufSize];
int bytesRead = 0;
while ((bytesRead = entry.Read(buf, 0, bufSize)) > 0)
tempStream.Write(buf, 0, bytesRead);
}
}
}
}
public void IHashFunction_SpeedTest_SingleBlock_ComputeHash_Stream_NonSeekable()
{
IHashFunction_SpeedTest_SingleBlock((sw, testHashFunction, testBytes) => {
using (var ms = new NonSeekableMemoryStream(testBytes))
{
sw.Start();
{
testHashFunction.ComputeHash(ms);
}
sw.Stop();
}
return(true);
});
}
public void IHashFunction_ComputeHash_Stream_NonSeekable_MatchesKnownValues()
{
foreach (var knownValue in KnownValues)
{
var hf = CreateHashFunction(knownValue.HashSize);
using (var ms = new NonSeekableMemoryStream(knownValue.TestValue))
{
var hashResults = hf.ComputeHash(ms);
Assert.Equal(
knownValue.ExpectedValue.Take((hf.HashSize + 7) / 8),
hashResults);
}
}
}
public async Task NonSeekableMemoryStream_To_MemoryStream(bool sourceIsExposable, bool disposeSource)
{
byte[] input = CreateByteArray(8192);
var source = new NonSeekableMemoryStream(input, sourceIsExposable);
var destination = new MemoryStream();
await StreamToStreamCopy.CopyAsync(source, destination, 4096, disposeSource);
Assert.NotEqual(disposeSource, source.CanRead);
if (!disposeSource)
{
Assert.Equal(input.Length, source.Position);
}
Assert.Equal(input, destination.ToArray());
Assert.Equal(input.Length, destination.Position);
Assert.Equal(input.Length, destination.Length);
}
public async Task NonSeekableMemoryStream_To_MemoryStream(bool sourceIsExposable, bool disposeSource)
{
byte[] input = CreateByteArray(8192);
var source = new NonSeekableMemoryStream(input, sourceIsExposable);
var destination = new MemoryStream();
await StreamToStreamCopy.CopyAsync(source, destination, 4096, disposeSource);
Assert.NotEqual(disposeSource, source.CanRead);
if (!disposeSource)
{
Assert.Equal(input.Length, source.Position);
}
Assert.Equal(input, destination.ToArray());
Assert.Equal(input.Length, destination.Position);
Assert.Equal(input.Length, destination.Length);
}
public void IHashFunction_SpeedTest_MultipleItems_ComputeHash_Stream_NonSeekable()
{
IHashFunction_SpeedTest_MultipleItems((sw, testHashFunction, count, testBytes) => {
using (var ms = new NonSeekableMemoryStream(testBytes))
{
sw.Start();
for (int x = 0; x < count; ++x)
{
testHashFunction.ComputeHash(ms);
ms.Real_Seek(0, SeekOrigin.Begin);
}
sw.Stop();
}
return(true);
});
}
public void IHashFunction_SpeedTest_SingleBlock_ComputeHashAsync_Stream_NonSeekable()
{
IHashFunction_SpeedTest_SingleBlock((sw, testHashFunction, testBytes) => {
var testHashFunctionAsync = testHashFunction as IHashFunctionAsync;
if (testHashFunctionAsync == null)
{
return(false);
}
using (var ms = new NonSeekableMemoryStream(testBytes))
{
sw.Start();
{
testHashFunctionAsync.ComputeHashAsync(ms)
.Wait();
}
sw.Stop();
}
return(true);
});
}
static async Task Main(string[] args)
{
var connectionString = Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING");
if (string.IsNullOrEmpty(connectionString))
{
throw new InvalidOperationException("Environment variable STORAGE_CONNECTION_STRING is not set");
}
Log($"ITERATIONS: {_iterations}");
Log($"UPLOAD_SIZE: {_uploadSize}");
Log($"BUFFER_SIZE: {_bufferSize}");
Log($"MAX_CONCURRENCY: {_maxConcurrency}");
// Enable SDK logging (with timestamps)
using var azureListener = new AzureEventSourceListener(
(eventData, text) => Log(String.Format("[{1}] {0}: {2}", eventData.EventSource.Name, eventData.Level, text)),
EventLevel.Verbose);
// Enable System.Net logging
using var httpListener = new LogEventListener("Microsoft-System-Net-Http");
using var socketsListener = new LogEventListener("Microsoft-System-Net-Sockets");
var containerName = $"container{DateTime.Now.Ticks}";
// Test custom transport with shorter timeout
var containerClient = new BlobContainerClient(connectionString, containerName, new BlobClientOptions()
{
Transport = new HttpClientTransport(new HttpClient()
{
Timeout = _httpClientTimeout
})
});
Log($"Creating container {containerName}");
await containerClient.CreateAsync();
Log($"Created container {containerName}");
var randomBuffer = new byte[_uploadSize];
new Random(0).NextBytes(randomBuffer);
var randomStream = new NonSeekableMemoryStream(randomBuffer);
for (var i = 0; i < _iterations; i++)
{
try
{
Log($"Iteration {i}");
var blobName = $"blob{DateTime.Now.Ticks}";
var blobClient = containerClient.GetBlobClient(blobName);
randomStream.Seek(0, SeekOrigin.Begin);
Log($"Uploading blob {blobName}");
await blobClient.UploadAsync(randomStream, transferOptions : new StorageTransferOptions()
{
MaximumConcurrency = _maxConcurrency,
MaximumTransferLength = _bufferSize
});
Log($"Uploaded blob {blobName}");
Log($"Deleting blob {blobName}");
await blobClient.DeleteAsync();
Log($"Deleted blob {blobName}");
}
catch (Exception e)
{
Log(e);
}
}
Log($"Deleting container {containerName}");
await containerClient.DeleteAsync();
Log($"Deleted container {containerName}");
}
public async Task Tests()
{
List <TestCase> testCases = new List <TestCase>
{
new TestCase("test_null_0.avro", 57, o => Assert.IsNull(o)), // null
new TestCase("test_null_1.avro", 60, o => Assert.AreEqual(true, (bool)o)), // bool
new TestCase("test_null_2.avro", 59, o => Assert.AreEqual("adsfasdf09809dsf-=adsf", (string)o)), // string
new TestCase("test_null_3.avro", 58, o => Assert.AreEqual(Encoding.UTF8.GetBytes("12345abcd"), (byte[])o)), // byte[]
new TestCase("test_null_4.avro", 56, o => Assert.AreEqual(1234, (int)o)), // int
new TestCase("test_null_5.avro", 57, o => Assert.AreEqual(1234L, (long)o)), // long
new TestCase("test_null_6.avro", 58, o => Assert.AreEqual(1234.0, (float)o)), // float
new TestCase("test_null_7.avro", 59, o => Assert.AreEqual(1234.0, (double)o)), // fouble
// Not supported today.
//new TestCase("test_null_8.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("B"), (byte[])o)), // fixed
new TestCase("test_null_9.avro", 106, o => Assert.AreEqual("B", (string)o)), // enum
// Not supported today.
// new TestCase("test_null_10.avro", o => Assert.AreEqual(new List<long>() { 1, 2, 3 }, (List<long>)o)), // array
new TestCase("test_null_11.avro", 84, o => Assert.AreEqual(
new Dictionary <string, int>()
{
{ "a", 1 }, { "b", 3 }, { "c", 2 }
}, (Dictionary <string, object>)o)), // dictionary
new TestCase("test_null_12.avro", 77, o => Assert.IsNull(o)), // union
new TestCase("test_null_13.avro", 129, o => // record
{
Dictionary <string, object> expected = new Dictionary <string, object>()
{
{ "$schema", "Test" }, { "f", 5 }
};
Dictionary <string, object> actual = (Dictionary <string, object>)o;
Assert.AreEqual(expected.Count, actual.Count);
foreach (KeyValuePair <string, object> keyValuePair in actual)
{
Assert.AreEqual(expected[keyValuePair.Key], keyValuePair.Value);
}
})
};
// seekable streams
foreach (TestCase testCase in testCases)
{
// Arrange
using FileStream stream = File.OpenRead(
$"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}");
AvroReader avroReader = new AvroReader(stream);
// Act
object o = await avroReader.Next(async : true).ConfigureAwait(false);
testCase.Predicate(o);
Assert.AreEqual(testCase.BlockOffset, avroReader.BlockOffset);
}
// non-seekable streams
foreach (TestCase testCase in testCases)
{
// Arrange
string filePath = $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}";
using FileStream stream = File.OpenRead(filePath);
using NonSeekableMemoryStream nonSeekableMemoryStream = new NonSeekableMemoryStream();
await stream.CopyToAsync(nonSeekableMemoryStream);
nonSeekableMemoryStream.Reset();
AvroReader avroReader = new AvroReader(nonSeekableMemoryStream);
// Act
object o = await avroReader.Next(async : true).ConfigureAwait(false);
testCase.Predicate(o);
Assert.AreEqual(testCase.BlockOffset, avroReader.BlockOffset);
}
}
protected override InputStream MakeInputStream(ArraySegment <byte> buffer)
{
var ms = new NonSeekableMemoryStream(buffer);
return(new InputStream(ms, InternalBufferSize));
}
