public OperationCompressor(ICompressionAlgorithm compressionAlgorithm, ClusterOptions clusterOptions,
ILogger <OperationCompressor> logger)
{
_compressionAlgorithm = compressionAlgorithm ?? throw new ArgumentNullException(nameof(compressionAlgorithm));
_clusterOptions = clusterOptions ?? throw new ArgumentNullException(nameof(clusterOptions));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
private static void WriteBlockStream(ICompressionAlgorithm compressionAlgorithm, CacheHeader header,
DemoCustomHeader customHeader, MemoryStream ms, string s)
{
using (var blockStream = new BlockStream(compressionAlgorithm, ms, CompressionMode.Compress, true))
using (var writer = new ExtendedBinaryWriter(blockStream))
{
CheckReadException(blockStream);
blockStream.WriteHeader(header.Write);
var bp = new BlockStream.BlockPosition();
// detect that we have written a block
blockStream.GetBlockPosition(bp);
writer.WriteOptAscii(s);
blockStream.GetBlockPosition(bp);
// here we write a test string that won't invoke a new block
blockStream.GetBlockPosition(customHeader.DemoPosition);
writer.WriteOptAscii(SmallString);
blockStream.GetBlockPosition(bp);
Assert.Equal(customHeader.DemoPosition.FileOffset, blockStream.Position);
blockStream.Flush();
// this will be flushed during dispose
writer.WriteOptAscii(FinalString);
}
}
/// <summary>
/// constructor
/// </summary>
/// <param name="compressionAlgorithm">The algorithm used for compression and decompression.</param>
/// <param name="stream">The stream to compress or decompress.</param>
/// <param name="compressionMode">One of the enumeration values that indicates whether to compress or decompress the stream.</param>
/// <param name="leaveStreamOpen">true to leave the stream open after disposing the object; otherwise, false.</param>
public BlockStream(ICompressionAlgorithm compressionAlgorithm, Stream stream, CompressionMode compressionMode,
bool leaveStreamOpen = false)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
_stream = stream;
_isCompressor = compressionMode == CompressionMode.Compress;
_leaveStreamOpen = leaveStreamOpen;
_block = new Block(compressionAlgorithm);
// sanity check: make sure we can use the stream for reading or writing
if (_isCompressor && !_stream.CanWrite)
{
throw new ArgumentException("A stream lacking write capability was provided to the block GZip compressor.");
}
if (!_isCompressor && !_stream.CanRead)
{
throw new ArgumentException("A stream lacking read capability was provided to the block GZip decompressor.");
}
if (_isCompressor)
{
_writer = new BinaryWriter(_stream, Encoding.UTF8, true);
}
}
public DefaultConfigurator AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
_configuration.AddCompressionAlgorithm(algorithm);
return this;
}
public IServiceBusConfigurationBuilder AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
configuration.AddCompressionAlgorithm(algorithm);
return this;
}
public DefaultConfigurator AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
_configuration.AddCompressionAlgorithm(algorithm);
return(this);
}
public ServiceBusConfigurator AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
configuration.AddCompressionAlgorithm(algorithm);
return(this);
}
public Message Decompress(ICompressionAlgorithm packer)
{
var result = packer.Decompress(Content);
Size = 0;
AddBytes(result);
return(this);
}
/// <summary>
/// constructor
/// </summary>
protected SaBlock(ICompressionAlgorithm compressionAlgorithm, int size)
{
CompressionAlgorithm = compressionAlgorithm;
UncompressedBlock = new byte[size];
var compressedBlockSize = compressionAlgorithm.GetCompressedBufferBounds(size);
CompressedBlock = new byte[compressedBlockSize];
Header = new BlockHeader();
}
public NsaBlock(ICompressionAlgorithm compressionAlgorithm, int size)
{
_compressionAlgorithm = compressionAlgorithm;
_uncompressedBlock = new byte[size];
_writer = new ExtendedBinaryWriter(new MemoryStream(_uncompressedBlock));
int compressedBlockSize = compressionAlgorithm.GetCompressedBufferBounds(size);
_compressedBlock = new byte[compressedBlockSize];
}
/// <summary>
/// constructor
/// </summary>
private PhylopReader()
{
_phylopIntervals = new List <PhylopInterval>();
_currentIntervalIndex = -1;
_scores = new short[PhylopCommon.MaxIntervalLength];
_scoreBytes = new byte[PhylopCommon.MaxIntervalLength * 2];
_qlz = new QuickLZ();
}
/// <summary>
/// constructor
/// </summary>
public Block(ICompressionAlgorithm compressionAlgorithm, int size = DefaultSize)
{
_compressionAlgorithm = compressionAlgorithm;
Offset = 0;
_size = size;
_uncompressedBlock = new byte[_size];
_compressedBlockSize = compressionAlgorithm.GetCompressedBufferBounds(_size);
_compressedBlock = new byte[_compressedBlockSize];
_header = new BlockHeader();
}
private static MemoryStream GetBlockStream(ICompressionAlgorithm compressionAlgorithm, byte[] bytes, int numBytes, out int copyLength)
{
var ms = new MemoryStream();
var writeBlock = new Block(compressionAlgorithm);
copyLength = writeBlock.CopyTo(bytes, 0, numBytes);
writeBlock.Write(ms);
writeBlock.WriteEof(ms);
return(ms);
}
/// <summary>
/// constructor
/// </summary>
public Block(ICompressionAlgorithm compressionAlgorithm)
{
_compressionAlgorithm = compressionAlgorithm;
Offset = 0;
_uncompressedBlock = new byte[Size];
_compressedBlockSize = compressionAlgorithm.GetCompressedBufferBounds(Size);
_compressedBlock = new byte[_compressedBlockSize];
_header = new BlockHeader();
}
public static Stream Compress(this ICompressionAlgorithm algorithm, Stream stream)
{
Guard.AgainstNull(algorithm, nameof(algorithm));
Guard.AgainstNull(stream, nameof(stream));
using (var ms = new MemoryStream())
{
stream.CopyTo(ms);
return(new MemoryStream(algorithm.Compress(ms.ToArray())));
}
}
internal CompressedStream(
Stream baseStream,
StreamOperationMode mode,
ICompressionAlgorithm compressionAlgorithm,
IChecksumAlgorithm checksumAlgorithm)
: base(baseStream, mode)
{
if (compressionAlgorithm == null)
{
throw new ArgumentNullException(nameof(compressionAlgorithm));
}
this.Initialize(baseStream, compressionAlgorithm, checksumAlgorithm);
}
public CompressedStream(
Stream baseStream,
StreamOperationMode mode,
CompressionSettings settings,
bool useCrc32,
EncryptionSettings encryptionSettings)
: base(baseStream, mode)
{
baseStream = encryptionSettings != null ? (Stream) new CryptoStream(baseStream, mode, PlatformSettings.Manager.GetCryptoProvider(encryptionSettings)) : baseStream;
ICompressionAlgorithm compressionAlgorithm = ZipHelper.GetCompressionAlgorithm(settings);
IChecksumAlgorithm checksumAlgorithm = useCrc32 ? (IChecksumAlgorithm) new Crc32() : (IChecksumAlgorithm) new Adler32();
this.Initialize(baseStream, compressionAlgorithm, checksumAlgorithm);
}
private static void AssertAlgorithm(ICompressionAlgorithm algorithm, string text)
{
Assert.AreEqual(text,
Encoding.UTF8.GetString(algorithm.Decompress(algorithm.Compress(Encoding.UTF8.GetBytes(text)))));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)))
using (var compressed = algorithm.Compress(stream))
using (var decompressed = algorithm.Decompress(compressed))
using (var decompressedStream = new MemoryStream())
{
decompressed.CopyTo(decompressedStream);
Assert.AreEqual(text, Encoding.UTF8.GetString(decompressedStream.ToArray()));
}
}
/// <summary>
/// constructor
/// </summary>
private PhylopWriter(string refSeqName, DataSourceVersion version, GenomeAssembly genomeAssembly,
int intervalLength = PhylopCommon.MaxIntervalLength)
{
_scores = new short[intervalLength];
_scoreBytes = new byte[intervalLength * 2];
_intervalLength = intervalLength;
_refSeqName = refSeqName;
_version = version;
_compressor = new QuickLZ();
ChromosomeIntervals = new List <PhylopInterval>();
_genomeAssembly = genomeAssembly;
var requiredBufferSize = _compressor.GetCompressedBufferBounds(_scoreBytes.Length);
_compressedBuffer = new byte[requiredBufferSize];
}
private void Initialize(
Stream baseStream,
ICompressionAlgorithm compressionAlgorithm,
IChecksumAlgorithm checksumAlgorithm)
{
this.BaseStream = baseStream;
this.algorithm = compressionAlgorithm;
this.ChecksumAlgorithm = checksumAlgorithm;
switch (this.Mode)
{
case StreamOperationMode.Read:
this.Transform = this.algorithm.CreateDecompressor();
break;
case StreamOperationMode.Write:
this.Transform = this.algorithm.CreateCompressor();
break;
}
}
private async Task <bool> ArchiveExtraction(ICompressionAlgorithm compressionAlgorithm, string fileType)
{
var archive = await _workingDir.GetFileAsync(ArchiveName + fileType);
Assert.IsNotNull(archive);
var outputFolder = await _workingDir.CreateFolderAsync("simpleZipUiTempOutput");
Assert.IsNotNull(outputFolder);
// extract archive
Assert.IsTrue(await compressionAlgorithm.Decompress(archive, outputFolder));
var files = await outputFolder.GetFilesAsync();
Assert.IsNotNull(files);
if (files.Count == 1)
{
foreach (var file in files)
{
using (var streamReader = new StreamReader(await file.OpenStreamForReadAsync()))
{
var line = streamReader.ReadLine();
Assert.IsNotNull(line);
if (!line.Equals(FileText))
{
Assert.Fail("Files do not match.");
}
}
}
}
else
{
Assert.Fail("Archive not properly created.");
}
await outputFolder.DeleteAsync(); // clean up when done
return(true);
}
protected override void Dispose(bool disposing)
{
if (this.IsDisposed)
{
return;
}
try
{
if (!this.HasFlushedFinalBlock)
{
this.FlushFinalBlock();
}
this.compressedSize = this.CompressedSize;
}
finally
{
this.algorithm = (ICompressionAlgorithm)null;
base.Dispose(disposing);
}
}
// ReSharper disable once UnusedParameter.Local
private static void ReadFromBlockStream(ICompressionAlgorithm compressionAlgorithm, MemoryStream ms, string expectedRandomString)
{
using (var blockStream = new BlockStream(compressionAlgorithm, ms, CompressionMode.Decompress))
using (var reader = new ExtendedBinaryReader(blockStream))
{
CheckWriteException(blockStream);
// grab the header
var header = GetHeader(blockStream, out var customHeader);
Assert.Equal(ExpectedGenomeAssembly, header.GenomeAssembly);
// sequential string check
CheckString(reader, expectedRandomString);
CheckString(reader, SmallString);
CheckString(reader, FinalString);
// random access string check
blockStream.SetBlockPosition(customHeader.DemoPosition);
CheckString(reader, SmallString);
}
}
private static void WriteBlockStream(ICompressionAlgorithm compressionAlgorithm, DemoHeader header,
DemoCustomHeader customHeader, Stream ms, string s)
{
using (var blockStream = new BlockStream(compressionAlgorithm, ms, CompressionMode.Compress, true))
using (var writer = new ExtendedBinaryWriter(blockStream))
{
CheckReadException(blockStream);
blockStream.WriteHeader(header.Write);
writer.WriteOptAscii(s);
(customHeader.FileOffset, customHeader.InternalOffset) = blockStream.GetBlockPosition();
Assert.Equal(customHeader.FileOffset, blockStream.Position);
writer.WriteOptAscii(SmallString);
blockStream.Flush();
// this will be flushed during dispose
writer.WriteOptAscii(FinalString);
}
}
private async Task <bool> PerformArchiveOperations(ICompressionAlgorithm compressionAlgorithm, string fileType, WriterOptions options)
{
return(await Task.Run(async() =>
{
var tempFile = await _workingDir.CreateFileAsync("tempFile");
using (var streamWriter = new StreamWriter(await tempFile.OpenStreamForWriteAsync()))
{
streamWriter.AutoFlush = true;
streamWriter.WriteLine(FileText);
}
_files = await _workingDir.GetFilesAsync();
Assert.IsNotNull(_files);
Assert.AreEqual(_files.Count, 1); // check if file exists
var archive = await _workingDir.CreateFileAsync(ArchiveName + fileType);
Assert.IsTrue(await compressionAlgorithm.Compress(_files, archive, _workingDir, options));
return await ArchiveExtraction(compressionAlgorithm, fileType); // extract archive after creation
}));
}
// ReSharper disable once UnusedParameter.Local
private static void ReadFromBlockStream(ICompressionAlgorithm compressionAlgorithm, Stream ms, string expectedRandomString)
{
// grab the header
var header = DemoHeader.Read(ms);
Assert.Equal(ExpectedAssembly, header.Assembly);
using (var blockStream = new BlockStream(compressionAlgorithm, ms, CompressionMode.Decompress))
using (var reader = new ExtendedBinaryReader(blockStream))
{
CheckWriteException(blockStream);
// sequential string check
CheckString(reader, expectedRandomString);
CheckString(reader, SmallString);
CheckString(reader, FinalString);
// random access string check
blockStream.SetBlockPosition(header.Custom.FileOffset, header.Custom.InternalOffset);
//reader.Reset();
CheckString(reader, SmallString);
}
}
public CompressionConcernBuilder Add(ICompressionAlgorithm algorithm)
{
_Algorithms.Add(algorithm);
return(this);
}
public void AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
_compressionAlgorithms.Add(algorithm);
}
/// <summary>
/// constructor
/// </summary>
public SaReadBlock(ICompressionAlgorithm compressionAlgorithm, int size = SaWriteBlock.DefaultBlockSize)
: base(compressionAlgorithm, size)
{
}
public SaWriteBlock(ICompressionAlgorithm compressionAlgorithm, int size = DefaultBlockSize)
: base(compressionAlgorithm, size)
{
_blockPositions = new List <ISaIndexOffset>();
}
public CompressionContext(ICompressionAlgorithm strategy)
{
this.strategy = strategy;
}
public void AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, nameof(algorithm));
_compressionAlgorithms.Add(algorithm);
}
public void AddCompressionAlgorithm(ICompressionAlgorithm algorithm)
{
Guard.AgainstNull(algorithm, "algorithm");
compressionAlgorithms.Add(algorithm);
}
