private static async Task SmallAsync()
{
var url = "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.3/newtonsoft.json.10.0.3.nupkg";
using (var httpClient = new HttpClient())
{
var httpZipProvider = new HttpZipProvider(httpClient);
using (var zipDirectoryReader = await httpZipProvider.GetReaderAsync(new Uri(url)))
{
var zipDirectory = await zipDirectoryReader.ReadAsync();
Console.WriteLine("Top 5 ZIP entries by compressed size:");
var entries = zipDirectory
.Entries
.OrderByDescending(x => x.GetCompressedSize())
.Take(5)
.ToList();
for (var i = 0; i < entries.Count; i++)
{
Console.WriteLine($"{i + 1}. {entries[i].GetName()}");
}
}
}
}
public GetReaderAsync()
{
_getResponse = request => new HttpResponseMessage(HttpStatusCode.NotFound);
_httpClient = new HttpClient(new TestMessageHandler(r => _getResponse(r)));
_target = new HttpZipProvider(_httpClient);
_requestUri = new Uri("http://example/foo.zip");
}
public async Task UsesETagWhenRequiringAnETag()
{
// Arrange
var fileName = "System.IO.Compression/refzipfiles/normal.zip";
using (var server = TestUtility.GetTestServer(TestUtility.TestDataDirectory))
using (var client = server.CreateClient())
{
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), fileName);
var target = new HttpZipProvider(client)
{
ETagBehavior = ETagBehavior.Required
};
// Act
var reader = await target.GetReaderAsync(requestUri);
// Assert
var actual = await reader.ReadAsync();
var expected = await TestUtility.ReadWithMiniZipAsync(TestUtility.BufferTestData(fileName));
TestUtility.VerifyJsonEquals(expected.Data, actual);
}
}
public async Task WithSelfUsingHttpRangeReader(string path)
{
// Arrange
using (var memoryStream = TestUtility.BufferTestData(path))
using (var server = TestUtility.GetTestServer(TestUtility.TestDataDirectory))
using (var client = server.CreateClient())
{
var httpZipProvider = new HttpZipProvider(client)
{
ETagBehavior = ETagBehavior.Required,
FirstBufferSize = 1,
SecondBufferSize = 1,
BufferGrowthExponent = 2,
};
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), path);
using (var bufferedRangeStream = await httpZipProvider.GetStreamAsync(requestUri))
{
// Act
var a = await TestUtility.ReadWithMiniZipAsync(memoryStream);
var b = await TestUtility.ReadWithMiniZipAsync(bufferedRangeStream);
// Assert
TestUtility.VerifyJsonEquals(a.Data, b.Data);
Assert.Equal(a.Success, b.Success);
Assert.Equal(a.Exception?.Message, b.Exception?.Message);
Assert.Equal(a.Exception?.GetType(), b.Exception?.GetType());
}
}
}
public async Task CanReadSignatureFile(string id, string version)
{
using (var testDirectory = TestDirectory.Create())
{
var packageUri = await NuGetUtility.GetNupkgUrlAsync(id, version);
using (var httpClient = new HttpClient())
{
var httpZipProvider = new HttpZipProvider(httpClient)
{
RequireAcceptRanges = false,
};
using (var reader = await httpZipProvider.GetReaderAsync(packageUri))
{
// Read the ZIP directory from the .nupkg URL.
var zipDirectory = await reader.ReadAsync();
// Find the signature entry
var entry = zipDirectory.Entries.Single(x => x.GetName() == ".signature.p7s");
// Read the signature file
var signatureBytes = await reader.ReadUncompressedFileDataAsync(zipDirectory, entry);
// Decode the signature file
var cms = new SignedCms();
cms.Decode(signatureBytes);
Assert.NotEmpty(cms.Certificates);
}
}
}
}
public async Task HandlesChangingETagProperly(ETagBehavior etagBehavior, bool success)
{
// Arrange
using (var directory = TestDirectory.Create())
{
var fileName = "normal.zip";
var serverDir = Path.Combine(directory, TestUtility.TestServerDirectory);
var filePath = Path.Combine(serverDir, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
using (var server = TestUtility.GetTestServer(
serverDir,
etags: true,
middleware: async(context, next) =>
{
await next.Invoke();
File.SetLastWriteTimeUtc(filePath, DateTime.UtcNow);
}))
using (var client = server.CreateClient())
{
File.Copy(
Path.Combine(TestUtility.TestDataDirectory, "System.IO.Compression/refzipfiles/normal.zip"),
filePath);
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), fileName);
var target = new HttpZipProvider(client)
{
ETagBehavior = etagBehavior
};
var reader = await target.GetReaderAsync(requestUri);
// Act & Assert
if (success)
{
var actual = await reader.ReadAsync();
var expected = await TestUtility.ReadWithMiniZipAsync(TestUtility.BufferTestData(filePath));
TestUtility.VerifyJsonEquals(expected.Data, actual);
}
else
{
var ex = await Assert.ThrowsAsync <MiniZipHttpException>(() => reader.ReadAsync());
Assert.StartsWith(
"The HTTP response did not have the expected status code HTTP 206 Partial Content. The response was 412 Precondition Failed.",
ex.Message);
Assert.Equal(HttpStatusCode.PreconditionFailed, ex.StatusCode);
Assert.Equal("Precondition Failed", ex.ReasonPhrase);
Assert.StartsWith("HTTP/", ex.DebugResponse);
}
}
}
}
public async Task CanGatherAndRecreateNuGetPackageCentralDirectory()
{
using (var testDirectory = TestDirectory.Create())
{
// Discover the .nupkg URL.
var sourceRepository = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json");
var serviceIndex = await sourceRepository.GetResourceAsync <ServiceIndexResourceV3>();
var packageBaseAddress = serviceIndex.GetServiceEntryUri(ServiceTypes.PackageBaseAddress);
var id = "Newtonsoft.Json".ToLowerInvariant();
var version = NuGetVersion.Parse("9.0.1").ToNormalizedString().ToLowerInvariant();
var packageUri = new Uri(packageBaseAddress, $"{id}/{version}/{id}.{version}.nupkg");
ZipDirectory zipDirectoryA;
string mzipPath;
using (var httpClient = new HttpClient())
{
var httpZipProvider = new HttpZipProvider(httpClient);
using (var reader = await httpZipProvider.GetReaderAsync(packageUri))
{
// Read the ZIP directory from the .nupkg URL.
zipDirectoryA = await reader.ReadAsync();
// Save the .mzip to the test directory.
mzipPath = Path.Combine(testDirectory, $"{id}.{version}.mzip");
using (var fileStream = new FileStream(mzipPath, FileMode.Create))
{
var mzipFormat = new MZipFormat();
await mzipFormat.WriteAsync(reader.Stream, fileStream);
}
}
}
// Read the .mzip back from disk.
ZipDirectory zipDirectoryB;
using (var fileStream = new FileStream(mzipPath, FileMode.Open))
{
var mzipFormat = new MZipFormat();
using (var mzipStream = await mzipFormat.ReadAsync(fileStream))
using (var reader = new ZipDirectoryReader(mzipStream))
{
zipDirectoryB = await reader.ReadAsync();
}
}
// Compare the results.
TestUtility.VerifyJsonEquals(zipDirectoryA, zipDirectoryB);
}
}
public async Task RejectsChangedLength()
{
// Arrange
using (var directory = TestDirectory.Create())
{
var fileName = "empty.zip";
var serverDir = Path.Combine(directory, TestUtility.TestServerDirectory);
var filePath = Path.Combine(serverDir, fileName);
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
using (var server = TestUtility.GetTestServer(
serverDir,
etags: true,
middleware: async(context, next) =>
{
await next.Invoke();
File.Delete(filePath);
File.Copy(
Path.Combine(TestUtility.TestDataDirectory, "System.IO.Compression/refzipfiles/normal.zip"),
filePath);
}))
using (var client = server.CreateClient())
{
File.Copy(
Path.Combine(TestUtility.TestDataDirectory, "System.IO.Compression/refzipfiles/empty.zip"),
filePath);
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), fileName);
var target = new HttpZipProvider(client)
{
ETagBehavior = ETagBehavior.Ignore
};
var reader = await target.GetReaderAsync(requestUri);
// Act & Assert
var ex = await Assert.ThrowsAsync <MiniZipHttpException>(() => reader.ReadAsync());
Assert.StartsWith(
"The length of the ZIP file fetched over HTTP changed from the expected 2671162 bytes to 22 bytes.",
ex.Message);
Assert.Equal(HttpStatusCode.PartialContent, ex.StatusCode);
Assert.Equal("Partial Content", ex.ReasonPhrase);
Assert.StartsWith("HTTP/", ex.DebugResponse);
}
}
}
public async Task CanGatherAndRecreateNuGetPackageCentralDirectory(string id, string version)
{
using (var testDirectory = TestDirectory.Create())
{
var packageUri = await NuGetUtility.GetNupkgUrlAsync(id, version);
ZipDirectory zipDirectoryA;
string mzipPath;
using (var httpClient = new HttpClient())
{
var httpZipProvider = new HttpZipProvider(httpClient)
{
RequireAcceptRanges = false,
};
using (var reader = await httpZipProvider.GetReaderAsync(packageUri))
{
// Read the ZIP directory from the .nupkg URL.
zipDirectoryA = await reader.ReadAsync();
// Save the .mzip to the test directory.
mzipPath = Path.Combine(testDirectory, $"{id}.{version}.mzip");
using (var fileStream = new FileStream(mzipPath, FileMode.Create))
{
var mzipFormat = new MZipFormat();
await mzipFormat.WriteAsync(reader.Stream, fileStream);
}
}
}
// Read the .mzip back from disk.
ZipDirectory zipDirectoryB;
using (var fileStream = new FileStream(mzipPath, FileMode.Open))
{
var mzipFormat = new MZipFormat();
using (var mzipStream = await mzipFormat.ReadAsync(fileStream))
using (var reader = new ZipDirectoryReader(mzipStream))
{
zipDirectoryB = await reader.ReadAsync();
}
}
// Compare the results.
TestUtility.VerifyJsonEquals(zipDirectoryA, zipDirectoryB);
}
}
public async Task ReturnsHeadersAsProperties()
{
// Arrange
var fileName = "System.IO.Compression/refzipfiles/normal.zip";
using (var server = TestUtility.GetTestServer(TestUtility.TestDataDirectory))
using (var client = server.CreateClient())
{
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), fileName);
var target = new HttpZipProvider(client);
// Act
var reader = await target.GetReaderAsync(requestUri);
// Assert
Assert.Equal("2671162", Assert.Single(reader.Properties["Content-Length"]));
Assert.Equal("application/x-zip-compressed", Assert.Single(reader.Properties["Content-Type"]));
}
}
public async Task FailsWhenRequiredETagIsMissing()
{
// Arrange
var fileName = "System.IO.Compression/refzipfiles/normal.zip";
using (var server = TestUtility.GetTestServer(TestUtility.TestDataDirectory, etags: false))
using (var client = server.CreateClient())
{
var requestUri = new Uri(new Uri(server.BaseAddress, TestUtility.TestServerDirectory + "/"), fileName);
var target = new HttpZipProvider(client)
{
ETagBehavior = ETagBehavior.Required
};
// Act & Assert
var ex = await Assert.ThrowsAsync <MiniZipException>(() => target.GetReaderAsync(requestUri));
Assert.Equal("An ETag header is required when using ETagBehavior.Required.", ex.Message);
}
}
private static async Task LargeAsync()
{
// Use the top 5 NuGet packages as an example.
var urls = new[]
{
"https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.3/newtonsoft.json.10.0.3.nupkg",
"https://api.nuget.org/v3-flatcontainer/nunit/3.9.0/nunit.3.9.0.nupkg",
"https://api.nuget.org/v3-flatcontainer/entityframework/6.2.0/entityframework.6.2.0.nupkg",
"https://api.nuget.org/v3-flatcontainer/jquery/3.2.1/jquery.3.2.1.nupkg",
"https://api.nuget.org/v3-flatcontainer/htmlagilitypack/1.6.7/htmlagilitypack.1.6.7.nupkg",
};
// Set up and HTTP client that logs HTTP requests, to help clarify this example.
using (var httpClientHandler = new HttpClientHandler())
using (var loggingHandler = new LoggingHandler {
InnerHandler = httpClientHandler
})
using (var httpClient = new HttpClient(loggingHandler))
{
// This provider uses and HTTP client and initializes a ZipDirectoryReader from a URL. This URL
// must support HEAD method, Content-Length response header, and Range request header.
var httpZipProvider = new HttpZipProvider(httpClient);
foreach (var url in urls)
{
Console.WriteLine(new string('=', 40));
Console.WriteLine();
// Initialize the reader. This performs a HEAD request to determine if the length of the
// ZIP file and whether the URL supports Range requests.
using (var reader = await httpZipProvider.GetReaderAsync(new Uri(url)))
{
// Read the ZIP file by requesting just the Central Directory part of the .zip.
var zipDirectory = await reader.ReadAsync();
// At this point, we known all about the entries of the .zip file include name, compressed
// size, and relative offset in the .zip file.
Console.WriteLine("Top 5 ZIP entries by compressed size:");
var entries = zipDirectory
.Entries
.OrderByDescending(x => x.GetCompressedSize())
.Take(5)
.ToList();
for (var i = 0; i < entries.Count; i++)
{
Console.WriteLine($"{i + 1}. {entries[i].GetName()} ({entries[i].GetCompressedSize():N0} bytes)");
}
}
Console.WriteLine();
}
Console.WriteLine(new string('=', 40));
Console.WriteLine();
// Summarize the work done. For NuGet packages, it is very common to download less than 1% of the
// package content while determining the .zip entry metadata.
var ratio = ((double)loggingHandler.TotalResponseBodyBytes) / loggingHandler.TotalContentLength;
Console.WriteLine($"Total ZIP files checked: {urls.Length:N0}");
Console.WriteLine($"Total HTTP requests: {loggingHandler.TotalRequests:N0}");
Console.WriteLine($"Total Content-Length bytes: {loggingHandler.TotalContentLength:N0}");
Console.WriteLine($"Actual downloaded bytes: {loggingHandler.TotalResponseBodyBytes:N0}"); // well, sort of...
Console.WriteLine($"Downloaded %: {Math.Round(ratio * 100, 3):0.000}%");
}
}
public async Task Run()
{
var name1 = "CloudBlockBlob and ZipArchive";
var average1 = await ExecuteTestsAsync(
name1,
async (url) =>
{
var blobClient = new CloudBlobClient(
new Uri("https://example"),
new FixUpBlobStorageHandler
{
InnerHandler = new HttpClientDelegatingHandler(_client),
});
blobClient.DefaultRequestOptions.DisableContentMD5Validation = true;
var blob = new CloudBlockBlob(url, blobClient);
using (var stream = await blob.OpenReadAsync())
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read))
{
var entries = zipArchive.Entries.ToList();
}
});
var name2 = "CloudBlockBlob and ZipDirectoryReader";
var average2 = await ExecuteTestsAsync(
name2,
async (url) =>
{
var blobClient = new CloudBlobClient(
new Uri("https://example"),
new FixUpBlobStorageHandler
{
InnerHandler = new HttpClientDelegatingHandler(_client),
});
blobClient.DefaultRequestOptions.DisableContentMD5Validation = true;
var blob = new CloudBlockBlob(url, blobClient);
using (var stream = await blob.OpenReadAsync())
using (var zipDirectoryReader = new ZipDirectoryReader(stream))
{
var zipDirectory = await zipDirectoryReader.ReadAsync();
}
});
var name3 = "HttpZipProvider";
var average3 = await ExecuteTestsAsync(
name3,
async (url) =>
{
// Read the entries using HttpClient and MiniZip.
var httpZipProvider = new HttpZipProvider(_client);
using (var reader = await httpZipProvider.GetReaderAsync(url))
{
var zipDirectory = await reader.ReadAsync();
}
});
Assert.True(average2 < average1, $"'{name2}' should be less than '{name1}'.");
Assert.True(average3 < average1, $"'{name3}' should be less than '{name1}'.");
// This is what we want. Ideally the stream provided by HttpZipProvider performs better than CloudBlockBlob.
Assert.True(average2 < average3, $"'{name2}' should be less than '{name3}'.");
}