public bool Create(Guid id, string filename, byte[] data)
{
ICompressionProvider compressionProvider = Factory.GetCompressionProvider(configurationProvider.CompressionProviderType, configurationProvider);
IEncryptionProvider encryptionProvider = Factory.GetEncryptionProvider(configurationProvider.EncryptionProviderType, configurationProvider);
int originalSize = data.Length;
Dictionary <string, string> metaData = new Dictionary <string, string>();
byte[] compressedData = compressionProvider.Compress(data);
if ((double)compressedData.Length / data.Length <= CompressionThreshold)
{
metaData.Add(CompressionKey, configurationProvider.CompressionProviderType.ToString());
data = compressedData;
}
data = encryptionProvider.Encrypt(data);
if (!(encryptionProvider is PassThroughEncryptionProvider))
{
metaData.Add(EncryptionKey, configurationProvider.EncryptionProviderType.ToString());
}
if (originalSize != data.Length)
{
metaData.Add(OriginalFileSizeKey, originalSize.ToString());
}
return(storageProvider.Create(id, filename, data, metaData));
}
public RabbitService(
Options options,
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null,
ILoggerFactory loggerFactory = null,
Func <PublishReceipt, ValueTask> processReceiptAsync = null)
{
Guard.AgainstNull(options, nameof(options));
Guard.AgainstNull(serializationProvider, nameof(serializationProvider));
LogHelper.LoggerFactory = loggerFactory;
Options = options;
ChannelPool = new ChannelPool(Options);
SerializationProvider = serializationProvider;
EncryptionProvider = encryptionProvider;
CompressionProvider = compressionProvider;
Publisher = new Publisher(ChannelPool, SerializationProvider, EncryptionProvider, CompressionProvider);
Topologer = new Topologer(ChannelPool);
Options.ApplyGlobalConsumerOptions();
BuildConsumers();
Publisher
.StartAutoPublishAsync(processReceiptAsync)
.GetAwaiter()
.GetResult();
BuildConsumerTopology()
.GetAwaiter()
.GetResult();
}
public CuratorFrameworkImpl(CuratorFrameworkFactory.Builder builder)
{
IZookeeperFactory localZookeeperFactory = makeZookeeperFactory(builder.GetZookeeperFactory());
this.client = new CuratorZookeeperClient(localZookeeperFactory,
builder.GetEnsembleProvider(),
builder.GetSessionTimeoutMs(),
builder.GetConnectionTimeoutMs(),
new WatchedWatcher(this),
builder.GetRetryPolicy(),
builder.CanBeReadOnly());
listeners = new ListenerContainer<ICuratorListener>();
unhandledErrorListeners = new ListenerContainer<IUnhandledErrorListener>();
backgroundOperations = new DelayQueue<OperationAndData<object>>();
@namespace = new NamespaceImpl(this, builder.GetNamespace());
maxCloseWaitMs = builder.GetMaxCloseWaitMs();
connectionStateManager = new ConnectionStateManager(this, builder.GetEventQueueSize());
compressionProvider = builder.GetCompressionProvider();
aclProvider = builder.GetAclProvider();
state = new AtomicReference<CuratorFrameworkState>(CuratorFrameworkState.LATENT);
useContainerParentsIfAvailable = builder.UseContainerParentsIfAvailable();
byte[] builderDefaultData = builder.GetDefaultData();
byte[] destDefaults = new byte[builderDefaultData.Length];
Array.Copy(builderDefaultData, destDefaults, builderDefaultData.Length);
defaultData = (builderDefaultData != null)
? destDefaults
: new byte[0];
authInfos = buildAuths(builder);
failedDeleteManager = new FailedDeleteManager(this);
namespaceWatcherMap = new NamespaceWatcherMap(this);
namespaceFacadeCache = new NamespaceFacadeCache(this);
}
private static async Task SetupAsync()
{
var letterTemplate = new Letter("", "TestRabbitServiceQueue", null, new LetterMetadata());
var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel));
_logger = loggerFactory.CreateLogger <ConsumerDataflow <WorkState> >();
_hashingProvider = new Argon2ID_HashingProvider();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_metricsProvider = new NullMetricsProvider();
_encryptionProvider = new AesGcmEncryptionProvider(hashKey);
_compressionProvider = new LZ4PickleProvider();
_serializationProvider = new Utf8JsonProvider();
_rabbitService = new RabbitService(
"Config.json",
_serializationProvider,
_encryptionProvider,
_compressionProvider,
loggerFactory);
await _rabbitService
.Topologer
.CreateQueueAsync("TestRabbitServiceQueue")
.ConfigureAwait(false);
}
public CuratorFrameworkImpl(CuratorFrameworkFactory.Builder builder)
{
IZookeeperFactory localZookeeperFactory = makeZookeeperFactory(builder.GetZookeeperFactory());
this.client = new CuratorZookeeperClient(localZookeeperFactory,
builder.GetEnsembleProvider(),
builder.GetSessionTimeoutMs(),
builder.GetConnectionTimeoutMs(),
new WatchedWatcher(this),
builder.GetRetryPolicy(),
builder.CanBeReadOnly());
listeners = new ListenerContainer <ICuratorListener>();
unhandledErrorListeners = new ListenerContainer <IUnhandledErrorListener>();
backgroundOperations = new DelayQueue <OperationAndData <object> >();
@namespace = new NamespaceImpl(this, builder.GetNamespace());
maxCloseWaitMs = builder.GetMaxCloseWaitMs();
connectionStateManager = new ConnectionStateManager(this, builder.GetEventQueueSize());
compressionProvider = builder.GetCompressionProvider();
aclProvider = builder.GetAclProvider();
state = new AtomicReference <CuratorFrameworkState>(CuratorFrameworkState.LATENT);
useContainerParentsIfAvailable = builder.UseContainerParentsIfAvailable();
byte[] builderDefaultData = builder.GetDefaultData();
byte[] destDefaults = new byte[builderDefaultData.Length];
Array.Copy(builderDefaultData, destDefaults, builderDefaultData.Length);
defaultData = (builderDefaultData != null)
? destDefaults
: new byte[0];
authInfos = buildAuths(builder);
failedDeleteManager = new FailedDeleteManager(this);
namespaceWatcherMap = new NamespaceWatcherMap(this);
namespaceFacadeCache = new NamespaceFacadeCache(this);
}
public ProviderCandidate(string encodingName, double quality, int priority, ICompressionProvider provider)
{
EncodingName = encodingName;
Quality = quality;
Priority = priority;
Provider = provider;
}
private ICompressionProvider ResolveCompressionProvider()
{
if (!_providerCreated)
{
_providerCreated = true;
_compressionProvider = _provider.GetCompressionProvider(_context);
}
return(_compressionProvider);
}
internal VectorTransferClient(string clientId, Socket clientSocket)
{
Contract.Assert(clientId != null);
Contract.Assert(clientSocket != null);
_clientId = clientId;
_socket = SetupSocket(clientSocket);
_endpoint = clientSocket.RemoteEndPoint;
_compression = new DeflateCompressionProvider();
}
public DataTransformer(
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null)
{
Guard.AgainstNull(serializationProvider, nameof(serializationProvider));
_serializationProvider = serializationProvider;
_encryptionProvider = encryptionProvider;
_compressionProvider = compressionProvider;
}
public void Setup()
{
Enumerable.Repeat <byte>(0xFF, 1000).ToArray().CopyTo(Payload1, 0);
Enumerable.Repeat <byte>(0xAA, 1000).ToArray().CopyTo(Payload1, 1000);
Enumerable.Repeat <byte>(0x1A, 1000).ToArray().CopyTo(Payload1, 2000);
Enumerable.Repeat <byte>(0xAF, 1000).ToArray().CopyTo(Payload1, 3000);
Enumerable.Repeat <byte>(0x01, 1000).ToArray().CopyTo(Payload1, 4000);
CompressionProvider = new RecyclableDeflateProvider();
CompressedPayload1 = CompressionProvider.Compress(Payload1).ToArray();
}
public LZ4PickleTests(ITestOutputHelper output)
{
_output = output;
Enumerable.Repeat <byte>(0xFF, 1000).ToArray().CopyTo(_data, 0);
Enumerable.Repeat <byte>(0xAA, 1000).ToArray().CopyTo(_data, 1000);
Enumerable.Repeat <byte>(0x1A, 1000).ToArray().CopyTo(_data, 2000);
Enumerable.Repeat <byte>(0xAF, 1000).ToArray().CopyTo(_data, 3000);
Enumerable.Repeat <byte>(0x01, 1000).ToArray().CopyTo(_data, 4000);
_provider = new LZ4PickleProvider();
_compressedData = _provider.Compress(_data).ToArray();
}
public void Setup()
{
Enumerable.Repeat <byte>(0xFF, 1000).ToArray().CopyTo(Payload1, 0);
Enumerable.Repeat <byte>(0xAA, 1000).ToArray().CopyTo(Payload1, 1000);
Enumerable.Repeat <byte>(0x1A, 1000).ToArray().CopyTo(Payload1, 2000);
Enumerable.Repeat <byte>(0xAF, 1000).ToArray().CopyTo(Payload1, 3000);
Enumerable.Repeat <byte>(0x01, 1000).ToArray().CopyTo(Payload1, 4000);
RecyclableManager.ConfigureNewStaticManagerWithDefaults();
RecyclableProvider = new RecyclableGzipProvider();
CompressedPayload1 = RecyclableProvider.Compress(Payload1).ToArray();
}
public Publisher(
RabbitOptions options,
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null)
: this(
new ChannelPool(options),
serializationProvider,
encryptionProvider,
compressionProvider)
{
}
public BackupManager(ICompressionProvider compressionProvider,
IBackupProvider backupProvider,
ICryptoProvider cryptoProvider,
IUploadProvider uploadProvider,
ILogger logger)
{
_compressionProvider = compressionProvider;
_backupProvider = backupProvider;
_cryptoProvider = cryptoProvider;
_uploadProvider = uploadProvider;
_logger = logger;
}
private static byte[] Decompress(ICompressionProvider compressionProvider, byte[] messageData)
{
var output = new MemoryStream();
var content = new MemoryStream(messageData.AsMemory(5).ToArray());
var decompressionStream = compressionProvider.CreateDecompressionStream(content);
decompressionStream.CopyTo(output);
var result = output.ToArray();
return(result);
}
public static void Setup()
{
Enumerable.Repeat <byte>(0xFF, 1000).ToArray().CopyTo(_data, 0);
Enumerable.Repeat <byte>(0xAA, 1000).ToArray().CopyTo(_data, 1000);
Enumerable.Repeat <byte>(0x1A, 1000).ToArray().CopyTo(_data, 2000);
Enumerable.Repeat <byte>(0xAF, 1000).ToArray().CopyTo(_data, 3000);
Enumerable.Repeat <byte>(0x01, 1000).ToArray().CopyTo(_data, 4000);
MyClass.ByteData = _data;
_serializationProvider = new Utf8JsonProvider();
_gzipProvider = new GzipProvider();
_deflateProvider = new DeflateProvider();
_serializedData = _serializationProvider.Serialize(MyClass).ToArray();
}
public CompressionTests(ITestOutputHelper output)
{
_output = output;
Enumerable.Repeat <byte>(0xFF, 1000).ToArray().CopyTo(_compressableData, 0);
Enumerable.Repeat <byte>(0xAA, 1000).ToArray().CopyTo(_compressableData, 1000);
Enumerable.Repeat <byte>(0x1A, 1000).ToArray().CopyTo(_compressableData, 2000);
Enumerable.Repeat <byte>(0xAF, 1000).ToArray().CopyTo(_compressableData, 3000);
Enumerable.Repeat <byte>(0x01, 1000).ToArray().CopyTo(_compressableData, 4000);
_gzipProvider = new GzipProvider();
_brotliProvider = new BrotliProvider();
_deflateProvider = new DeflateProvider();
_lz4StreamProvider = new LZ4StreamProvider();
_lz4PickleProvider = new LZ4PickleProvider();
}
public RabbitService(
RabbitOptions options,
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null,
ILoggerFactory loggerFactory = null,
Func <IPublishReceipt, ValueTask> processReceiptAsync = null) : this(
new ChannelPool(options),
serializationProvider,
encryptionProvider,
compressionProvider,
loggerFactory,
processReceiptAsync)
{
}
public Publisher(
IChannelPool channelPool,
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null)
{
Guard.AgainstNull(channelPool, nameof(channelPool));
Guard.AgainstNull(serializationProvider, nameof(serializationProvider));
Options = channelPool.Options;
_logger = LogHelper.GetLogger <Publisher>();
_serializationProvider = serializationProvider;
if (Options.PublisherOptions.Encrypt && encryptionProvider == null)
{
_encrypt = false;
_logger.LogWarning("Encryption disabled, encryptionProvider provided was null.");
}
else if (Options.PublisherOptions.Encrypt)
{
_encrypt = true;
_encryptionProvider = encryptionProvider;
}
if (Options.PublisherOptions.Compress && compressionProvider == null)
{
_compress = false;
_logger.LogWarning("Compression disabled, compressionProvider provided was null.");
}
else if (Options.PublisherOptions.Compress)
{
_compress = true;
_compressionProvider = compressionProvider;
}
_channelPool = channelPool;
_receiptBuffer = Channel.CreateBounded <IPublishReceipt>(
new BoundedChannelOptions(1024)
{
SingleWriter = false,
SingleReader = false,
FullMode = BoundedChannelFullMode.DropOldest, // never block
});
_withHeaders = Options.PublisherOptions.WithHeaders;
_createPublishReceipts = Options.PublisherOptions.CreatePublishReceipts;
_waitForConfirmation = TimeSpan.FromMilliseconds(Options.PublisherOptions.WaitForConfirmationTimeoutInMilliseconds);
}
public HttpServer(string serverId = null, int port = DefaultPort, string host = "127.0.0.1")
{
_socketEndpoint = GetEndpoint(host, port);
_baseEndpoint = GetBaseUri(port, host);
_serverId = serverId ?? Util.GenerateId();
_completedHandle = new ManualResetEvent(false);
SetupSocket();
_status = ServerStatus.Stopped;
_compression = new DeflateCompressionProvider();
RequestTimeout = TimeSpan.FromMinutes(2);
}
public static async Task Main()
{
_hashingProvider = new Argon2IDHasher();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_encryptionProvider = new AesGcmEncryptionProvider(hashKey, _hashingProvider.Type);
_compressionProvider = new GzipProvider();
_serializationProvider = new Utf8JsonProvider(StandardResolver.Default);
LogHelper.LoggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information));
await Console.Out.WriteLineAsync("CookedRabbit.Core StressTest v1.00").ConfigureAwait(false);
await Console.Out.WriteLineAsync("- StressTest setting everything up...").ConfigureAwait(false);
var setupFailed = false;
try
{ await SetupAsync().ConfigureAwait(false); }
catch (Exception ex)
{
setupFailed = true;
await Console.Out.WriteLineAsync($"- StressTest failed with exception {ex.Message}.").ConfigureAwait(false);
}
if (!setupFailed)
{
await Console.Out.WriteLineAsync("- StressTest starting!").ConfigureAwait(false);
try
{
await StartStressTestAsync()
.ConfigureAwait(false);
}
catch (Exception ex)
{ await Console.Out.WriteLineAsync($"- StressTest failed with exception {ex.Message}.").ConfigureAwait(false); }
}
await Console.Out.WriteLineAsync($"- Press any key to begin shutdown.").ConfigureAwait(false);
Console.ReadKey();
await ShutdownAsync();
await Console.Out.WriteLineAsync($"- Press any key to close.").ConfigureAwait(false);
Console.ReadKey();
}
public byte[] Read(Guid id, string filename)
{
Dictionary <string, string> metaData;
byte[] data = storageProvider.Read(id, filename, out metaData);
if (metaData.ContainsKey(EncryptionKey))
{
IEncryptionProvider encryptionProvider = Factory.GetEncryptionProvider((EncryptionProviderType)Enum.Parse(typeof(EncryptionProviderType), metaData[EncryptionKey]), configurationProvider);
data = encryptionProvider.Decrypt(data);
}
if (metaData.ContainsKey(CompressionKey))
{
ICompressionProvider compressionProvider = Factory.GetCompressionProvider((CompressionProviderType)Enum.Parse(typeof(CompressionProviderType), metaData[CompressionKey]), configurationProvider);
data = compressionProvider.Decompress(data);
}
return(data);
}
public ConsumerDataflowService(
IConfiguration config,
ILoggerFactory logger,
IRabbitService rabbitService,
ISerializationProvider serializationProvider,
ICompressionProvider compressionProvider,
IEncryptionProvider encryptionProvider,
IMetricsProvider metricsProvider)
{
_config = config;
_logger = logger.CreateLogger <ConsumerDataflowService>();
_rabbitService = rabbitService;
_serializationProvider = serializationProvider;
_compressionProvider = compressionProvider;
_encryptionProvider = encryptionProvider;
_metricsProvider = metricsProvider;
}
protected CuratorFrameworkImpl(CuratorFrameworkImpl parent)
{
client = parent.client;
listeners = parent.listeners;
unhandledErrorListeners = parent.unhandledErrorListeners;
maxCloseWaitMs = parent.maxCloseWaitMs;
backgroundOperations = parent.backgroundOperations;
connectionStateManager = parent.connectionStateManager;
defaultData = parent.defaultData;
failedDeleteManager = parent.failedDeleteManager;
compressionProvider = parent.compressionProvider;
aclProvider = parent.aclProvider;
namespaceFacadeCache = parent.namespaceFacadeCache;
@namespace = new NamespaceImpl(this, null);
state = parent.state;
authInfos = parent.authInfos;
_useContainerParentsIfAvailable = parent._useContainerParentsIfAvailable;
}
public RabbitService(
string fileNamePath,
ISerializationProvider serializationProvider,
IEncryptionProvider encryptionProvider = null,
ICompressionProvider compressionProvider = null,
ILoggerFactory loggerFactory = null, Func <PublishReceipt, ValueTask> processReceiptAsync = null)
: this(
JsonFileReader
.ReadFileAsync <Options>(fileNamePath)
.GetAwaiter()
.GetResult(),
serializationProvider,
encryptionProvider,
compressionProvider,
loggerFactory,
processReceiptAsync)
{
}
public TransferHandle(
string path,
Verb verb,
string clientId,
Socket clientSocket,
bool disposeSocket,
ICompressionProvider compression,
Action<TransferHandle> onConnect)
{
ClientSocket = clientSocket;
ClientId = clientId;
Path = path;
OnConnect = onConnect ?? (h => { });
Id = Util.GenerateId();
BufferSize = 1024;
Verb = verb;
_compression = compression;
_disposeSocket = disposeSocket;
}
private async Task <RabbitService> SetupAsync()
{
_hashingProvider = new Argon2IDHasher();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_encryptionProvider = new AesGcmEncryptionProvider(hashKey, _hashingProvider.Type);
_compressionProvider = new GzipProvider();
_serializationProvider = new Utf8JsonProvider(StandardResolver.Default);
var letterTemplate = new Letter("", "TestRabbitServiceQueue", null, new LetterMetadata());
var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(Program.LogLevel));
_logger = loggerFactory.CreateLogger <ConsumerPipelineMicroservice>();
var rabbitService = new RabbitService(
"Config.json",
_serializationProvider,
_encryptionProvider,
_compressionProvider,
loggerFactory);
await rabbitService
.Topologer
.CreateQueueAsync("TestRabbitServiceQueue")
.ConfigureAwait(false);
for (var i = 0L; i < _targetCount; i++)
{
var letter = letterTemplate.Clone();
letter.Body = JsonSerializer.SerializeToUtf8Bytes(new Message {
StringMessage = $"Sensitive ReceivedLetter {i}", MessageId = i
});
letter.LetterId = (ulong)i;
await rabbitService
.Publisher
.PublishAsync(letter, true, true)
.ConfigureAwait(false);
}
return(rabbitService);
}
public static async Task Main()
{
_hashingProvider = new Argon2IDHasher();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_encryptionProvider = new AesGcmEncryptionProvider(hashKey, _hashingProvider.Type);
_compressionProvider = new GzipProvider();
_serializationProvider = new Utf8JsonProvider(StandardResolver.Default);
_rabbitService = new RabbitService(
"Config.json",
_serializationProvider,
_encryptionProvider,
_compressionProvider,
null);
await RunSimpleClientWithEncryptionAsync()
.ConfigureAwait(false);
await RunDataExecutionEngineAsync()
.ConfigureAwait(false);
}
private static async Task SetupAsync()
{
var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().SetMinimumLevel(LogLevel));
_hashingProvider = new Argon2IDHasher();
var hashKey = await _hashingProvider.GetHashKeyAsync("passwordforencryption", "saltforencryption", 32).ConfigureAwait(false);
_encryptionProvider = new AesGcmEncryptionProvider(hashKey, _hashingProvider.Type);
_compressionProvider = new LZ4PickleProvider();
_serializationProvider = new Utf8JsonProvider();
_rabbitService = new RabbitService(
"Config.json",
_serializationProvider,
_encryptionProvider,
_compressionProvider,
loggerFactory);
await _rabbitService
.Topologer
.CreateQueueAsync("TestRabbitServiceQueue")
.ConfigureAwait(false);
}
public RabbitFixture()
{
CompressionProvider = new GzipProvider();
HashingProvider = new Argon2ID_HashingProvider();
HashKey = HashingProvider.GetHashKey(Passphrase, Salt, 32);
EncryptionProvider = new AesGcmEncryptionProvider(HashKey);
SerializationProvider = new Utf8JsonProvider();
Options = JsonFileReader.ReadFileAsync <RabbitOptions>("RabbitMQ\\TestConfig.json").GetAwaiter().GetResult();
RabbitService = new RabbitService(
Options,
SerializationProvider,
EncryptionProvider,
CompressionProvider,
LoggerFactory
.Create(
builder => builder.AddConsole().SetMinimumLevel(LogLevel.Information)));
ChannelPool = RabbitService.ChannelPool;
Topologer = RabbitService.Topologer;
Publisher = RabbitService.Publisher;
}
public MessageInspector(Compress compress)
{
_compress = compress;
_compressionProvider = new SharpCompressionProvider();
}
protected CuratorFrameworkImpl(CuratorFrameworkImpl parent)
{
client = parent.client;
listeners = parent.listeners;
unhandledErrorListeners = parent.unhandledErrorListeners;
maxCloseWaitMs = parent.maxCloseWaitMs;
backgroundOperations = parent.backgroundOperations;
connectionStateManager = parent.connectionStateManager;
defaultData = parent.defaultData;
failedDeleteManager = parent.failedDeleteManager;
compressionProvider = parent.compressionProvider;
aclProvider = parent.aclProvider;
namespaceFacadeCache = parent.namespaceFacadeCache;
@namespace = new NamespaceImpl(this, null);
state = parent.state;
authInfos = parent.authInfos;
_useContainerParentsIfAvailable = parent._useContainerParentsIfAvailable;
}
public VectorTransferClient(string clientId = null, int port = VectorTransferServer.DefaultTcpPort, string host = "127.0.0.1")
{
_clientId = clientId ?? Util.GenerateId();
_endpoint = VectorTransferServer.GetEndpoint(host, port);
_compression = new DeflateCompressionProvider();
}
/// <param name="compressionProvider">Compression algorithm provider.</param>
/// <param name="threadsNumber">Number of available threads.</param>
/// <param name="memory">The maximum size of the used RAM.</param>
public FileCompressor(ICompressionProvider compressionProvider, int threadsNumber, ulong memory) : this(threadsNumber, memory)
{
_compressionProvider = compressionProvider;
}
public void Open(string pathIn)
{
_inputPath = pathIn;
_inputDir = Path.GetDirectoryName(_inputPath) ?? _inputPath;
if (_inputDir == "") _inputDir = ".";
if (_inputDir == pathIn && CUEProcessorPlugins.ripper != null)
{
ICDRipper ripper = Activator.CreateInstance(CUEProcessorPlugins.ripper) as ICDRipper;
try
{
ripper.Open(pathIn[0]);
if (ripper.TOC.AudioTracks > 0)
{
OpenCD(ripper);
int driveOffset;
if (!AccurateRipVerify.FindDriveReadOffset(_ripper.ARName, out driveOffset))
throw new Exception("Failed to find drive read offset for drive" + _ripper.ARName);
_ripper.DriveOffset = driveOffset;
//LookupAlbumInfo();
return;
}
}
catch
{
ripper.Dispose();
_ripper = null;
throw;
}
}
TextReader sr;
if (Directory.Exists(pathIn))
throw new Exception("is a directory");
//{
// if (cueDir + Path.DirectorySeparatorChar != pathIn && cueDir != pathIn)
// throw new Exception("Input directory must end on path separator character.");
// string cueSheet = null;
// string[] audioExts = new string[] { "*.wav", "*.flac", "*.wv", "*.ape", "*.m4a", "*.tta" };
// for (i = 0; i < audioExts.Length && cueSheet == null; i++)
// cueSheet = CUESheet.CreateDummyCUESheet(pathIn, audioExts[i]);
// if (_config.udc1Extension != null && cueSheet == null)
// cueSheet = CUESheet.CreateDummyCUESheet(pathIn, "*." + _config.udc1Extension);
// if (cueSheet == null)
// throw new Exception("Input directory doesn't contain supported audio files.");
// sr = new StringReader(cueSheet);
// List<CUEToolsSourceFile> logFiles = new List<CUEToolsSourceFile>();
// foreach (string logPath in Directory.GetFiles(pathIn, "*.log"))
// logFiles.Add(new CUEToolsSourceFile(logPath, new StreamReader(logPath, CUESheet.Encoding)));
// CUEToolsSourceFile selectedLogFile = ChooseFile(logFiles, null, false);
// _eacLog = selectedLogFile != null ? selectedLogFile.contents : null;
//}
else if (CUEProcessorPlugins.arcp_fmt.Contains(Path.GetExtension(pathIn).ToLower().Trim('.')))
{
_archive = null;
foreach (Type type in CUEProcessorPlugins.arcp)
{
CompressionProviderClass archclass = Attribute.GetCustomAttribute(type, typeof(CompressionProviderClass)) as CompressionProviderClass;
if (archclass.Extension == Path.GetExtension(pathIn).ToLower().Trim('.'))
{
_archive = Activator.CreateInstance(type, pathIn) as ICompressionProvider;
break;
}
}
if (_archive == null)
throw new Exception("archive type not supported.");
_isArchive = true;
_archiveContents = new List<string>();
_archive.PasswordRequired += new EventHandler<CompressionPasswordRequiredEventArgs>(unzip_PasswordRequired);
_archive.ExtractionProgress += new EventHandler<CompressionExtractionProgressEventArgs>(unzip_ExtractionProgress);
foreach (string f in _archive.Contents)
_archiveContents.Add(f);
_logFiles = new List<CUEToolsSourceFile>();
List<CUEToolsSourceFile> cueFiles = new List<CUEToolsSourceFile>();
foreach (string s in _archiveContents)
{
if (Path.GetExtension(s).ToLower() == ".cue" || Path.GetExtension(s).ToLower() == ".log")
{
Stream archiveStream = OpenArchive(s, false);
CUEToolsSourceFile sourceFile = new CUEToolsSourceFile(s, new StreamReader(archiveStream, CUESheet.Encoding));
archiveStream.Close();
if (Path.GetExtension(s).ToLower() == ".cue")
cueFiles.Add(sourceFile);
else
_logFiles.Add(sourceFile);
}
}
CUEToolsSourceFile selectedCUEFile = ChooseFile(cueFiles, null, true);
if (selectedCUEFile == null || selectedCUEFile.contents == "")
throw new Exception("Input archive doesn't contain a usable cue sheet.");
_defaultLog = Path.GetFileNameWithoutExtension(selectedCUEFile.path);
_archiveCUEpath = Path.GetDirectoryName(selectedCUEFile.path);
string cueText = selectedCUEFile.contents;
if (_config.autoCorrectFilenames)
{
string extension;
cueText = CorrectAudioFilenames(_config, _archiveCUEpath, cueText, false, _archiveContents, out extension);
}
sr = new StringReader(cueText);
if (_logFiles.Count == 1)
_eacLog = _logFiles[0].contents;
}
else if (Path.GetExtension(pathIn).ToLower() == ".cue")
{
if (_config.autoCorrectFilenames)
sr = new StringReader(CorrectAudioFilenames(_config, pathIn, false));
else
sr = new StreamReader(pathIn, CUESheet.Encoding);
_logFiles = new List<CUEToolsSourceFile>();
_defaultLog = Path.GetFileNameWithoutExtension(pathIn);
foreach (string logPath in Directory.GetFiles(_inputDir, "*.log"))
try { _logFiles.Add(new CUEToolsSourceFile(logPath, new StreamReader(logPath, CUESheet.Encoding))); }
catch { }
}
else if (Path.GetExtension(pathIn).ToLower() == ".m3u")
{
string cueSheet = CUESheet.CreateDummyCUESheet(_config, pathIn);
sr = new StringReader(cueSheet);
_logFiles = new List<CUEToolsSourceFile>();
_defaultLog = Path.GetFileNameWithoutExtension(pathIn);
foreach (string logPath in Directory.GetFiles(_inputDir, "*.log"))
try { _logFiles.Add(new CUEToolsSourceFile(logPath, new StreamReader(logPath, CUESheet.Encoding))); }
catch { }
}
else
{
string extension = Path.GetExtension(pathIn).ToLower();
sr = null;
CUEToolsFormat fmt;
if (!extension.StartsWith(".") || !_config.formats.TryGetValue(extension.Substring(1), out fmt) || !fmt.allowLossless)
throw new Exception("Unknown input format.");
if (fmt.allowEmbed)
{
string cuesheetTag = null;
TagLib.File fileInfo;
GetSampleLength(pathIn, out fileInfo);
NameValueCollection tags = Tagging.Analyze(fileInfo);
cuesheetTag = tags.Get("CUESHEET");
_accurateRipId = tags.Get("ACCURATERIPID");
_eacLog = tags.Get("LOG");
if (_eacLog == null) _eacLog = tags.Get("LOGFILE");
if (_eacLog == null) _eacLog = tags.Get("EACLOG");
if (cuesheetTag != null)
{
sr = new StringReader(cuesheetTag);
_hasEmbeddedCUESheet = true;
}
}
if (!_hasEmbeddedCUESheet)
{
string cueSheet = CUESheet.CreateDummyCUESheet(_config, pathIn);
if (cueSheet == null)
throw new Exception("Input file doesn't seem to contain a cue sheet or be part of an album.");
sr = new StringReader(cueSheet);
_logFiles = new List<CUEToolsSourceFile>();
foreach (string logPath in Directory.GetFiles(_inputDir, "*.log"))
try { _logFiles.Add(new CUEToolsSourceFile(logPath, new StreamReader(logPath, CUESheet.Encoding))); }
catch { }
}
}
OpenCUE(sr);
}
public void Close()
{
if (_progress != null)
{
_progress.cueSheet = null;
_progress = null;
}
if (_archive != null)
_archive.Close();
_archive = null;
if (_ripper != null)
{
//_ripper.ReadProgress -= new EventHandler<ReadProgressArgs>(CDReadProgress);
_ripper.Close();
}
_ripper = null;
}
public void CompressUsing(ICompressionProvider compressionProvider)
{
Contract.Assert(compressionProvider != null);
_compression = compressionProvider;
}
private static async Task<Stream> ReceiveData(TransferHandle state, ICompressionProvider compression)
{
var context = await new AsyncSocketWriterReader(state.ClientSocket, state.BufferSize)
.ReadAsync();
return compression.DecompressFrom(context.ReceivedData);
}
/**
* @param compressionProvider the compression provider
* @return this
*/
public Builder CompressionProvider(ICompressionProvider compressionProvider)
{
this.compressionProvider = compressionProvider;
return this;
}
public ConsumerDataflow <TState> SetCompressionProvider(ICompressionProvider provider)
{
Guard.AgainstNull(provider, nameof(provider));
_compressProvider = provider;
return(this);
}
private static async Task Send(TransferHandle state, DataBatch transferDoc, ICompressionProvider compression)
{
var buffer = new byte[state.BufferSize];
using (var ms = new MemoryStream())
using (var cs = compression.CompressTo(ms))
{
transferDoc.Save(cs);
cs.Flush();
cs.Dispose();
ms.Flush();
ms.Position = 0;
var sockWriter = new AsyncSocketWriterReader(state.ClientSocket, state.BufferSize);
var sent = await sockWriter.WriteAsync(ms);
DebugOutput.LogVerbose("Sent {0} bytes", sent);
}
}
private static async Task <StreamContent> CreateResponseContentCore <TResponse>(TResponse[] responses, ICompressionProvider compressionProvider)
where TResponse : IMessage <TResponse>
{
var ms = new MemoryStream();
foreach (var response in responses)
{
await WriteResponseAsync(ms, response, compressionProvider);
}
ms.Seek(0, SeekOrigin.Begin);
var streamContent = new StreamContent(ms);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/grpc");
return(streamContent);
}
public static async Task WriteResponseAsync <TResponse>(Stream ms, TResponse response, ICompressionProvider compressionProvider)
where TResponse : IMessage <TResponse>
{
var compress = false;
byte[] data;
if (compressionProvider != null)
{
compress = true;
var output = new MemoryStream();
var compressionStream = compressionProvider.CreateCompressionStream(output, System.IO.Compression.CompressionLevel.Fastest);
var compressedData = response.ToByteArray();
compressionStream.Write(compressedData, 0, compressedData.Length);
compressionStream.Flush();
compressionStream.Dispose();
data = output.ToArray();
}
else
{
data = response.ToByteArray();
}
await ResponseUtils.WriteHeaderAsync(ms, data.Length, compress, CancellationToken.None);
#if NET5_0_OR_GREATER
await ms.WriteAsync(data);
#else
await ms.WriteAsync(data, 0, data.Length);
#endif
}
