public TrinketDtpServerImpl(IStreamFactory streamFactory, BootstrapConfigurationGenerator bootstrapConfigurationGenerator, IDtpNode transportNode, TrinketStartupConfiguration startupConfiguration)
{
this.streamFactory = streamFactory;
this.bootstrapConfigurationGenerator = bootstrapConfigurationGenerator;
this.transportNode = transportNode;
this.startupConfiguration = startupConfiguration;
}
public void Setup()
{
_capturedEvents = new EventCapturer()
.Capture<CommandStartedEvent>()
.Capture<CommandSucceededEvent>()
.Capture<CommandFailedEvent>();
_streamFactory = Substitute.For<IStreamFactory>();
_endPoint = new DnsEndPoint("localhost", 27017);
var serverId = new ServerId(new ClusterId(), _endPoint);
_connectionInitializer = Substitute.For<IConnectionInitializer>();
_connectionInitializer.InitializeConnectionAsync(null, CancellationToken.None)
.ReturnsForAnyArgs(Task.FromResult(new ConnectionDescription(
new ConnectionId(serverId),
new IsMasterResult(new BsonDocument()),
new BuildInfoResult(new BsonDocument("version", "2.6.3")))));
_subject = new BinaryConnection(
serverId: serverId,
endPoint: _endPoint,
settings: new ConnectionSettings(),
streamFactory: _streamFactory,
connectionInitializer: _connectionInitializer,
eventSubscriber: _capturedEvents);
_stream = new BlockingMemoryStream();
_streamFactory.CreateStreamAsync(null, CancellationToken.None)
.ReturnsForAnyArgs(Task.FromResult<Stream>(_stream));
_subject.OpenAsync(CancellationToken.None).Wait();
_capturedEvents.Clear();
_operationIdDisposer = EventContext.BeginOperation();
}
// constructors
public BinaryConnection(ServerId serverId, EndPoint endPoint, ConnectionSettings settings, IStreamFactory streamFactory, IConnectionInitializer connectionInitializer, IEventSubscriber eventSubscriber)
{
Ensure.IsNotNull(serverId, nameof(serverId));
_endPoint = Ensure.IsNotNull(endPoint, nameof(endPoint));
_settings = Ensure.IsNotNull(settings, nameof(settings));
_streamFactory = Ensure.IsNotNull(streamFactory, nameof(streamFactory));
_connectionInitializer = Ensure.IsNotNull(connectionInitializer, nameof(connectionInitializer));
Ensure.IsNotNull(eventSubscriber, nameof(eventSubscriber));
_backgroundTaskCancellationTokenSource = new CancellationTokenSource();
_backgroundTaskCancellationToken = _backgroundTaskCancellationTokenSource.Token;
_connectionId = new ConnectionId(serverId);
_receiveLock = new SemaphoreSlim(1);
_sendLock = new SemaphoreSlim(1);
_state = new InterlockedInt32(State.Initial);
_commandEventHelper = new CommandEventHelper(eventSubscriber);
eventSubscriber.TryGetEventHandler(out _failedEventHandler);
eventSubscriber.TryGetEventHandler(out _closingEventHandler);
eventSubscriber.TryGetEventHandler(out _closedEventHandler);
eventSubscriber.TryGetEventHandler(out _openingEventHandler);
eventSubscriber.TryGetEventHandler(out _openedEventHandler);
eventSubscriber.TryGetEventHandler(out _failedOpeningEventHandler);
eventSubscriber.TryGetEventHandler(out _receivingMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _receivedMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _failedReceivingMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _sendingMessagesEventHandler);
eventSubscriber.TryGetEventHandler(out _sentMessagesEventHandler);
eventSubscriber.TryGetEventHandler(out _failedSendingMessagesEvent);
}
public XslTransformer(IStreamFactory streamFactory)
{
if (streamFactory == null)
throw new ArgumentNullException("streamFactory");
this.mStreamFactory = streamFactory;
}
public ProjectFileParser(IStreamFactory streamFactory)
{
if (streamFactory == null)
throw new ArgumentNullException("streamFactory");
this.mStreamFactory = streamFactory;
}
public void Setup()
{
_capturedEvents = new EventCapturer();
_streamFactory = Substitute.For<IStreamFactory>();
_endPoint = new DnsEndPoint("localhost", 27017);
var serverId = new ServerId(new ClusterId(), _endPoint);
_connectionInitializer = Substitute.For<IConnectionInitializer>();
_connectionInitializer.InitializeConnection(null, CancellationToken.None)
.ReturnsForAnyArgs(new ConnectionDescription(
new ConnectionId(serverId),
new IsMasterResult(new BsonDocument()),
new BuildInfoResult(new BsonDocument("version", "2.6.3"))));
_connectionInitializer.InitializeConnectionAsync(null, CancellationToken.None)
.ReturnsForAnyArgs(Task.FromResult(new ConnectionDescription(
new ConnectionId(serverId),
new IsMasterResult(new BsonDocument()),
new BuildInfoResult(new BsonDocument("version", "2.6.3")))));
_subject = new BinaryConnection(
serverId: serverId,
endPoint: _endPoint,
settings: new ConnectionSettings(),
streamFactory: _streamFactory,
connectionInitializer: _connectionInitializer,
eventSubscriber: _capturedEvents);
}
public AnnouncingHtmlOutputUrlProcessor(
IValidatorWrapper validator,
IRunnerContext context,
IFileNameGenerator fileNameGenerator,
ResourceCopier resourceCopier,
IValidatorReportWriterFactory reportWriterFactory,
IStreamFactory streamFactory,
IHtmlIndexFileWriter htmlIndexFileWriter)
{
if (validator == null)
throw new ArgumentNullException("validator");
if (context == null)
throw new ArgumentNullException("context");
if (fileNameGenerator == null)
throw new ArgumentNullException("fileNameGenerator");
if (resourceCopier == null)
throw new ArgumentNullException("resourceCopier");
if (reportWriterFactory == null)
throw new ArgumentNullException("reportWriterFactory");
if (streamFactory == null)
throw new ArgumentNullException("streamFactory");
if (htmlIndexFileWriter == null)
throw new ArgumentNullException("htmlIndexFileWriter");
this.mValidator = validator;
this.mContext = context;
this.mFileNameGenerator = fileNameGenerator;
this.mResourceCopier = resourceCopier;
this.mReportWriterFactory = reportWriterFactory;
this.mStreamFactory = streamFactory;
this.mHtmlIndexFileWriter = htmlIndexFileWriter;
}
// constructors
public BinaryConnectionFactory(ConnectionSettings settings, IStreamFactory streamFactory, IEventSubscriber eventSubscriber)
{
_settings = Ensure.IsNotNull(settings, nameof(settings));
_streamFactory = Ensure.IsNotNull(streamFactory, nameof(streamFactory));
_eventSubscriber = Ensure.IsNotNull(eventSubscriber, nameof(eventSubscriber));
_connectionInitializer = new ConnectionInitializer(settings.ApplicationName);
}
public SyncNowCommand(IStorageRepository storageRepository, ILocalBus localBus, IStreamFactory streamFactory,
ITestRunImportResultsReaderFactory resultsReaderFactory)
{
_storageRepository = storageRepository;
_localBus = localBus;
_streamFactory = streamFactory;
_resultsReaderFactory = resultsReaderFactory;
}
public SitemapsParser(IHttpClient httpClient, IStreamFactory streamFactory)
{
if (httpClient == null)
throw new ArgumentNullException("httpClient");
if (streamFactory == null)
throw new ArgumentNullException("streamFactory");
this.mHttpClient = httpClient;
this.mStreamFactory = streamFactory;
}
public void Initialize()
{
_factoryStub = MockRepository.GenerateStub<IStreamFactory>();
_textWriterMock = MockRepository.GenerateMock<TextWriter>();
_streamMock = MockRepository.GenerateMock<Stream>();
_factoryStub.Stub(f => f.CreateDictionaryStreamWriter($"{Filepath}.{CompressingIndexSerializer.DictionaryFileSuffix}"))
.Return(_textWriterMock);
_factoryStub.Stub(f => f.CreateDocumentIdsStream($"{Filepath}.{CompressingIndexSerializer.PostingsFileSuffix}", FileMode.Create)).Return(_streamMock);
_encoderStub = MockRepository.GenerateStub<INumberEncoder>();
_numberLengthReducerStub = MockRepository.GenerateStub<INumberLengthReducer>();
_target = new CompressingIndexSerializer(_factoryStub, _encoderStub, _numberLengthReducerStub);
}
public void Initialize()
{
_directory = Substitute.For<IDirectory>();
_file = Substitute.For<IFile>();
_path = Substitute.For<IPath>();
_path.SetPathDefaultBehavior();
_tagHelper = Substitute.For<ITagHelper>();
_streamFactory = Substitute.For<IStreamFactory>();
_fileStream = Substitute.For<IFileStream>();
_streamFactory.CreateFileStream(Arg.Any<string>(), Arg.Any<FileMode>(), Arg.Any<FileAccess>(), Arg.Any<FileShare>()).Returns(_fileStream);
_geolocHelper = Substitute.For<IGeolocationHelper>();
_target = new FileSystemHelperTester(_directory, _file, _path, _tagHelper, _streamFactory, _geolocHelper);
}
public HtmlValidator(IHttpClient httpClient, IStreamFactory streamFactory, ResourceCopier resourceCopier, IValidatorSoap12ResponseParser soapResponseParser)
{
if (httpClient == null)
throw new ArgumentNullException("httpClient");
if (streamFactory == null)
throw new ArgumentNullException("streamFactory");
if (resourceCopier == null)
throw new ArgumentNullException("resourceCopier");
if (soapResponseParser == null)
throw new ArgumentNullException("soapResponseParser");
this.mHttpClient = httpClient;
this.mStreamFactory = streamFactory;
this.mResourceCopier = resourceCopier;
this.mSoapResponseParser = soapResponseParser;
}
/// <summary>
/// Constructor for opening an emulated listener socket using
/// the Yaler reverse HTTP protocol.
/// Preconditions
/// host != null
/// host.Length > 0
/// port > 0
/// port leq 65536
/// relayDomain != null
/// relayDomain.Length >= 11
/// </summary>
/// <param name="host">Internet address or domain name of the
/// relay server.</param>
/// <param name="port">Port number of the relay service.</param>
/// <param name="relayDomain">Yaler relay domain.</param>
public YalerStreamListener(string relayHostName, int relayPort, IStreamFactory relay, string relayDomain, string relaySecretKey)
{
Contract.Requires(relayHostName != null);
Contract.Requires(relayHostName.Length > 0);
Contract.Requires(relayPort >= 0);
Contract.Requires(relayPort <= 65535);
Contract.Requires(relay != null);
Contract.Requires(relayDomain != null);
Contract.Requires(relayDomain.Length >= 11);
Contract.Requires(relaySecretKey != null);
Contract.Requires(relaySecretKey.Length > 0);
LocalHostName = relayHostName;
LocalPort = relayPort;
this.relay = relay;
Domain = relayDomain;
this.secretKey = relaySecretKey;
LocalUrl = "http://" + LocalHostName;
if (LocalPort != 80)
{
LocalUrl = LocalUrl + ':' + LocalPort;
}
LocalUrl = LocalUrl + '/' + relayDomain;
}
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList();
return(vself);
}
public WebSocketSharpListener(ILogger logger, ICertificate certificate, IMemoryStreamFactory memoryStreamProvider, ITextEncoding textEncoding, INetworkManager networkManager, ISocketFactory socketFactory, ICryptoProvider cryptoProvider, IStreamFactory streamFactory, bool enableDualMode, Func <HttpListenerContext, IHttpRequest> httpRequestFactory)
{
_logger = logger;
_certificate = certificate;
_memoryStreamProvider = memoryStreamProvider;
_textEncoding = textEncoding;
_networkManager = networkManager;
_socketFactory = socketFactory;
_cryptoProvider = cryptoProvider;
_streamFactory = streamFactory;
_enableDualMode = enableDualMode;
_httpRequestFactory = httpRequestFactory;
}
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found)
{
var result = _vfsCache.Get(hash.ToArray());
if (result == null)
{
found = null;
return(false);
}
var data = IndexedVirtualFile.Read(new MemoryStream(result));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
found.Name = path;
found.Hash = hash;
return(true);
}
public FileSystemHelperTester(IDirectory directory, IFile file, IPath path, ITagHelper tagHelper, IStreamFactory streamFactory, IGeolocationHelper geolocHelper)
: base(directory, file, path, tagHelper, streamFactory, geolocHelper)
{
}
public HttpListenerHost(IServerApplicationHost applicationHost,
ILogger logger,
IServerConfigurationManager config,
string serviceName,
string defaultRedirectPath, INetworkManager networkManager, IMemoryStreamFactory memoryStreamProvider, ITextEncoding textEncoding, ISocketFactory socketFactory, ICryptoProvider cryptoProvider, IJsonSerializer jsonSerializer, IXmlSerializer xmlSerializer, IEnvironmentInfo environment, ICertificate certificate, IStreamFactory streamFactory, Func <Type, Func <string, object> > funcParseFn, bool enableDualModeSockets, IFileSystem fileSystem)
{
Instance = this;
_appHost = applicationHost;
DefaultRedirectPath = defaultRedirectPath;
_networkManager = networkManager;
_memoryStreamProvider = memoryStreamProvider;
_textEncoding = textEncoding;
_socketFactory = socketFactory;
_cryptoProvider = cryptoProvider;
_jsonSerializer = jsonSerializer;
_xmlSerializer = xmlSerializer;
_environment = environment;
_certificate = certificate;
_streamFactory = streamFactory;
_funcParseFn = funcParseFn;
_enableDualModeSockets = enableDualModeSockets;
_fileSystem = fileSystem;
_config = config;
_logger = logger;
RequestFilters = new List <Action <IRequest, IResponse, object> >();
ResponseFilters = new List <Action <IRequest, IResponse, object> >();
}
public BinaryConnectionFactory(ConnectionSettings settings, IStreamFactory streamFactory, IConnectionListener listener)
{
_settings = Ensure.IsNotNull(settings, "settings");
_streamFactory = Ensure.IsNotNull(streamFactory, "streamFactory");
_listener = listener;
}
public IStreamFactory wrapStreamFactory(IStreamFactory source)
{
return(new TailStreamFactory(streamInformation, source, limit));
}
// constructors
public BinaryConnectionFactory(ConnectionSettings settings, IStreamFactory streamFactory, IEventSubscriber eventSubscriber)
{
_settings = Ensure.IsNotNull(settings, "settings");
_streamFactory = Ensure.IsNotNull(streamFactory, "streamFactory");
_eventSubscriber = Ensure.IsNotNull(eventSubscriber, "eventSubscriber");
}
public SslStreamFactory(SslStreamSettings settings, IStreamFactory wrapped)
{
_settings = Ensure.IsNotNull(settings, "settings");
_wrapped = Ensure.IsNotNull(wrapped, "wrapped");
}
public BinaryConnectionFactory(ConnectionSettings settings, IStreamFactory streamFactory, IConnectionListener listener)
{
_settings = Ensure.IsNotNull(settings, "settings");
_streamFactory = Ensure.IsNotNull(streamFactory, "streamFactory");
_listener = listener;
}
public ExtractedMemoryFile(IStreamFactory factory)
{
_factory = factory;
}
public LoadSolutionCommand(IStreamFactory streamFactory, DargonNodeFactory dargonNodeFactory)
{
this.streamFactory = streamFactory;
this.dargonNodeFactory = dargonNodeFactory;
}
public BlockSigner(IHashProvider hashProvider, IStreamFactory streamFactory)
{
_hashProvider = hashProvider;
_streamFactory = streamFactory;
}
private HttpConnection(ILogger logger, ISocket sock, EndPointListener epl, bool secure, ICertificate cert, ICryptoProvider cryptoProvider, IStreamFactory streamFactory, IMemoryStreamFactory memoryStreamFactory, ITextEncoding textEncoding)
{
_logger = logger;
this.sock = sock;
this.epl = epl;
this.secure = secure;
this.cert = cert;
_cryptoProvider = cryptoProvider;
_memoryStreamFactory = memoryStreamFactory;
_textEncoding = textEncoding;
_streamFactory = streamFactory;
}
public CompressingIndexSerializer(IStreamFactory factory, INumberEncoder encoder, INumberLengthReducer numberLengthReducer)
{
_factory = factory;
_encoder = encoder;
_numberLengthReducer = numberLengthReducer;
}
public CompressCommand(IStreamFactory streamFactory, ILogger logger, ICompressorSettings settings)
{
_streamFactory = streamFactory ?? throw new ArgumentNullException(nameof(streamFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
}
public JsonExporter(IStreamFactory streamFactory)
{
this.streamFactory = streamFactory;
}
public MockStreamWriter(IStreamFactory streamFactory, Action <System.IO.StreamWriter, object> writeRecord, IRecordFormatter recordFormatter = null)
: base(streamFactory, recordFormatter)
{
this.writeRecord = writeRecord;
}
public SslStreamFactory(SslStreamSettings settings, IStreamFactory wrapped)
{
_settings = Ensure.IsNotNull(settings, nameof(settings));
_wrapped = Ensure.IsNotNull(wrapped, nameof(wrapped));
}
public SslStreamFactory(SslStreamSettings settings, IStreamFactory wrapped)
{
_settings = Ensure.IsNotNull(settings, nameof(settings));
_wrapped = Ensure.IsNotNull(wrapped, nameof(wrapped));
}
public TrinketDtpServerFactoryImpl(IStreamFactory streamFactory, IDtpNodeFactory transportNodeFactory, BootstrapConfigurationGenerator bootstrapConfigurationGenerator)
{
this.streamFactory = streamFactory;
this.transportNodeFactory = transportNodeFactory;
this.bootstrapConfigurationGenerator = bootstrapConfigurationGenerator;
}
public PofStreamsFactoryImpl(IThreadingProxy threadingProxy, IStreamFactory streamFactory, IPofSerializer serializer)
{
this.threadingProxy = threadingProxy;
this.streamFactory = streamFactory;
this.serializer = serializer;
}
public AutomaticTestRunImportListener(IStorageRepository storageRepository, ILocalBus localBus, IStreamFactory streamFactory,
ITestRunImportResultsReaderFactory resultsReaderFactory, IActivityLogger log)
{
_storageRepository = storageRepository;
_localBus = localBus;
_streamFactory = streamFactory;
_resultsReaderFactory = resultsReaderFactory;
_log = log;
}
public MockHtmlOutputUrlProcessor(
IRunnerContext context,
IValidatorWrapper validator,
IValidatorReportWriterFactory reportWriterFactory,
IFileNameGenerator fileNameGenerator,
ResourceCopier resourceCopier,
IStreamFactory streamFactory,
IHtmlIndexFileWriter htmlIndexFileWriter)
{
this.mContext = context;
this.mValidator = validator;
this.mReportWriterFactory = reportWriterFactory;
this.mFileNameGenerator = fileNameGenerator;
this.mResourceCopier = resourceCopier;
this.mStreamFactory = streamFactory;
this.mHtmlIndexFileWriter = htmlIndexFileWriter;
}
public IHttpProcessor GetHttpProcessor(ITcpClientWrapper tcpClient,
IStreamFactory streamFactory, Action <string> getHandler,
Action <string, IMemoryStreamWrapper, IStreamWriterWrapper> postHandler)
{
return(new HttpProcessor(tcpClient, streamFactory, getHandler, postHandler));
}
public void setSource(StreamInformation streamInformation, IStreamFactory streamFactory)
{
this.streamInformation = streamInformation;
this.streamFactory = streamFactory;
}
public TextStreamWriter(IStreamFactory streamFactory, IRecordFormatter recordFormatter = null)
: base(streamFactory, recordFormatter)
{
}
public async Task BroadcastFile(string fileName,
IByteSender byteSender,
IStreamFactory streamFactory,
IScheduler scheduler,
float headerRebroadcastSeconds = 1,
CancellationToken cancellationToken = default)
{
if (headerRebroadcastSeconds < 0)
{
throw new ArgumentException(nameof(headerRebroadcastSeconds));
}
var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var token = cancellationTokenSource.Token;
var reader = streamFactory.CreateReader(fileName);
const int byteToMegabyteFactor = 1000000;
var pageSize = 8 * byteToMegabyteFactor;
var pageObservable = reader.PageObservable(pageSize, token);
const long broadcastIdByteCount = 16; //guid bytes
const long fudgeAmount = 100; //the hash alg chosen has to be equal to or smaller than this
var packetSize = byteSender.MaximumTransmittableBytes - broadcastIdByteCount - fudgeAmount;
var partialPacketCache = new ConcurrentDictionary <long, byte[]>();
var payloadObservable = pageObservable
.ObserveOn(scheduler)
.SelectMany(page =>
{
var firstPacketIndex = page.PageIndex / packetSize;
var hasFirstFragmented = (page.PageIndex % packetSize) != 0;
var packetList = new List <PayloadWrapper>();
var firstPartialPacketIndex = hasFirstFragmented
? firstPacketIndex
: (long?)null;
var secondPacketPayloadIndex = (firstPartialPacketIndex + 1) * packetSize;
var firstPartialLength = (secondPacketPayloadIndex - page.PageIndex) ?? 0;
if (hasFirstFragmented)
{
var partialBuffer = new byte[firstPartialLength];
Array.Copy(page.Bytes, partialBuffer, firstPartialLength);
var firstPartialPacket = partialBuffer;
var firstPayload = new PayloadWrapper()
{
PayloadIndex = firstPartialPacketIndex.Value, bytes = firstPartialPacket
};
CachePartialPacket(partialPacketCache, firstPayload, packetList);
}
var firstFullPacketIndex = hasFirstFragmented
? firstPacketIndex + 1
: firstPacketIndex;
var lastPageByteIndex = page.Bytes.Length - 1;
var lastBytePayloadIndex = page.PageIndex + lastPageByteIndex;
var lastPacketIndex = lastBytePayloadIndex / packetSize;
var lastPartialLength = (page.Bytes.Length - firstPartialLength) % packetSize;
var hasLastPartialPacket = lastPacketIndex > firstPacketIndex &&
(lastPartialLength > 0);
var lastFullPacketIndex = hasLastPartialPacket
? lastPacketIndex - 1
: lastPacketIndex;
//todo: consider parallel foreach
for (long packetIndex = firstFullPacketIndex; packetIndex <= lastFullPacketIndex; packetIndex++)
{
var packetBuffer = new byte[packetSize];
var startPageIndex = ((packetIndex - firstFullPacketIndex) * packetSize) + firstPartialLength;
Array.Copy(page.Bytes, startPageIndex, packetBuffer, 0, packetSize);
var payload = new PayloadWrapper()
{
PayloadIndex = packetIndex, bytes = packetBuffer
};
packetList.Add(payload);
}
if (hasLastPartialPacket)
{
var partialBuffer = new byte[lastPartialLength];
var lastPartialPageIndex = page.Bytes.Length - lastPartialLength;
var lastPartialPacketIndex = lastPacketIndex;
Array.Copy(page.Bytes, lastPartialPageIndex, partialBuffer, 0, lastPartialLength);
var lastPayload = new PayloadWrapper()
{
PayloadIndex = lastPartialPacketIndex, bytes = partialBuffer
};
CachePartialPacket(partialPacketCache, lastPayload, packetList);
}
return(packetList.AsEnumerable());
})
.Concat(partialPacketCache
.Select(kvp => new PayloadWrapper {
PayloadIndex = kvp.Key, bytes = kvp.Value
}).ToObservable());
var broadcastId = Guid.NewGuid();
var md5 = MD5.Create();
var serializedPayloadObservable = payloadObservable
.Select(payloadWrapper =>
{
byte[] serializedPayload;
var hash = md5.ComputeHash(payloadWrapper.bytes);
var protoMessage = new ProtoMessage(broadcastId, payloadWrapper.PayloadIndex, payloadWrapper.bytes,
null, null,
null, hash);
using (var memoryStream = new MemoryStream())
{
Serializer.Serialize(memoryStream,
protoMessage);
serializedPayload = memoryStream.ToArray();
}
return(serializedPayload);
})
.Finally(() => { md5.Dispose(); })
.Publish();
byte[] serializedHeader;
var packetCount = (long)Math.Ceiling((double)reader.Length / packetSize);
var rebroadcastTime = TimeSpan.FromSeconds(headerRebroadcastSeconds);
using (var memoryStream = new MemoryStream())
{
Serializer.Serialize(memoryStream,
new ProtoMessage(broadcastId, null, null, packetSize, fileName, packetCount, null));
serializedHeader = memoryStream.ToArray();
}
var headerObservable =
GetHeaderObservable(serializedHeader, rebroadcastTime, serializedPayloadObservable, scheduler);
var packetObservable = headerObservable
.Merge(serializedPayloadObservable);
serializedPayloadObservable.Connect();
var sendObservable = packetObservable
.ObserveOn(scheduler)
.SelectMany((array, index) =>
{
return(Observable.FromAsync(async c =>
{
await byteSender.Send(array).ConfigureAwait(false);
return Unit.Default;
}));
});
await sendObservable
.ToTask(token).ConfigureAwait(false);
}
public MappingChecker(IStreamFactory streamFactory, ITestCaseResolverFactory resolverFactory, ITestRunImportResultsReaderFactory resultsReaderFactory)
{
_streamFactory = streamFactory;
_resolverFactory = resolverFactory;
_resultsReaderFactory = resultsReaderFactory;
}
public StoragePluginsService(IStoragePluginsRepository storageRepository, IStreamFactory streamFactory)
{
_storagePluginsRepository = storageRepository;
_streamFactory = streamFactory;
}
public TrackStreamer(IStreamFactory streamFactory)
{
_streamFactory = streamFactory;
}
public StreamPool(IStreamFactory factory, bool appendOnly)
{
_factory = factory;
_writer = new Lazy <Stream>(() => _factory.GetStream(true, appendOnly));
}
public StreamReader(IStreamFactory streamFactory, IRecordMapper recordMapper = null)
: base(recordMapper)
{
this.StreamFactory = streamFactory;
this.disposeStream = true;
}
private HttpConnection(ILogger logger, IAcceptSocket socket, EndPointListener epl, bool secure, ICertificate cert, ICryptoProvider cryptoProvider, IStreamFactory streamFactory, IMemoryStreamFactory memoryStreamFactory, ITextEncoding textEncoding, IFileSystem fileSystem, IEnvironmentInfo environment)
{
_logger = logger;
this._socket = socket;
this._epl = epl;
this.secure = secure;
this.cert = cert;
_cryptoProvider = cryptoProvider;
_memoryStreamFactory = memoryStreamFactory;
_textEncoding = textEncoding;
_fileSystem = fileSystem;
_environment = environment;
_streamFactory = streamFactory;
}
public HttpRequestMiddleware(AppFunc next, IStreamFactory streamFactory)
{
_next = next;
_streamFactory = streamFactory;
}
public static async Task <HttpConnection> Create(ILogger logger, IAcceptSocket sock, EndPointListener epl, bool secure, ICertificate cert, ICryptoProvider cryptoProvider, IStreamFactory streamFactory, IMemoryStreamFactory memoryStreamFactory, ITextEncoding textEncoding, IFileSystem fileSystem, IEnvironmentInfo environment)
{
var connection = new HttpConnection(logger, sock, epl, secure, cert, cryptoProvider, streamFactory, memoryStreamFactory, textEncoding, fileSystem, environment);
await connection.InitStream().ConfigureAwait(false);
return(connection);
}
public LoadSolutionCommand(IStreamFactory streamFactory, DargonNodeFactory dargonNodeFactory)
{
this.streamFactory = streamFactory;
this.dargonNodeFactory = dargonNodeFactory;
}
// constructors
public BinaryConnection(ServerId serverId, EndPoint endPoint, ConnectionSettings settings, IStreamFactory streamFactory, IConnectionInitializer connectionInitializer, IEventSubscriber eventSubscriber)
{
Ensure.IsNotNull(serverId, nameof(serverId));
_endPoint = Ensure.IsNotNull(endPoint, nameof(endPoint));
_settings = Ensure.IsNotNull(settings, nameof(settings));
_streamFactory = Ensure.IsNotNull(streamFactory, nameof(streamFactory));
_connectionInitializer = Ensure.IsNotNull(connectionInitializer, nameof(connectionInitializer));
Ensure.IsNotNull(eventSubscriber, nameof(eventSubscriber));
_connectionId = new ConnectionId(serverId);
_receiveLock = new SemaphoreSlim(1);
_sendLock = new SemaphoreSlim(1);
_state = new InterlockedInt32(State.Initial);
_commandEventHelper = new CommandEventHelper(eventSubscriber);
eventSubscriber.TryGetEventHandler(out _failedEventHandler);
eventSubscriber.TryGetEventHandler(out _closingEventHandler);
eventSubscriber.TryGetEventHandler(out _closedEventHandler);
eventSubscriber.TryGetEventHandler(out _openingEventHandler);
eventSubscriber.TryGetEventHandler(out _openedEventHandler);
eventSubscriber.TryGetEventHandler(out _failedOpeningEventHandler);
eventSubscriber.TryGetEventHandler(out _receivingMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _receivedMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _failedReceivingMessageEventHandler);
eventSubscriber.TryGetEventHandler(out _sendingMessagesEventHandler);
eventSubscriber.TryGetEventHandler(out _sentMessagesEventHandler);
eventSubscriber.TryGetEventHandler(out _failedSendingMessagesEvent);
_compressorSource = new CompressorSource(settings.Compressors);
}
public async Task Listen(IByteReceiver byteReceiver,
IStreamFactory streamFactory,
IScheduler scheduler,
string destinationPath = "",
CancellationToken cancellationToken = default)
{
var basePath = string.IsNullOrWhiteSpace(destinationPath)
? ""
: destinationPath;
if (basePath != "")
{
Directory.CreateDirectory(basePath); //does not throw an exception if it does not exist
}
var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var token = cancellationTokenSource.Token;
var packetBufferObservable = new Subject <byte[]>();
var protoMessageObservable = packetBufferObservable
.ObserveOn(scheduler)
.Select(bytes =>
{
ProtoMessage message;
using (var memoryStream = new MemoryStream(bytes))
{
var protoMessage = Serializer.Deserialize <ProtoMessage>(memoryStream);
message = protoMessage;
}
return(message);
});
var headerCache = new ConcurrentDictionary <Guid, Header>();
var payloadCache = new ConcurrentDictionary <Guid, List <ProtoMessage> >();
var fileWriteRequestSubject = new Subject <FileWriteWrapper>();
var headerObservable = protoMessageObservable
.Where(message => !string.IsNullOrWhiteSpace(message.FileName) && message.PayloadCount.HasValue);
var payloadObservable = protoMessageObservable
.Where(message => message.PayloadIndex.HasValue && message.Payload != null);
string GetFileName(ProtoMessage header)
{
return($"{System.DateTime.Now:HH-mm-ss-ffff}.{header.FileName}");
}
Header ConvertToHeader(ProtoMessage header)
{
return(new Header
{
BroadcastId = header.GetBroadcastId(), FileName = GetFileName(header),
PayloadCount = header.PayloadCount.Value, PayloadMaxBytes = header.PayloadMaxSize.Value
});
}
void HandleCachedPayloads(Guid guid, Header header1)
{
if (payloadCache.TryRemove(guid, out var payloads))
{
var p = payloads.ToArray(); //todo:figure out why this bombed - List changed exception
payloads.Clear();
foreach (var payload in p)
{
fileWriteRequestSubject.OnNext(new FileWriteWrapper()
{
Header = header1, Payload = payload
});
}
}
}
headerObservable
.ObserveOn(scheduler)
.Subscribe(protoHeader => headerCache.AddOrUpdate(protoHeader.GetBroadcastId(),
bcid =>
{
//todo: decide if header.payloadCount X headerMaxPayloadSize is too much
var header = ConvertToHeader(protoHeader);
HandleCachedPayloads(bcid, header);
return(header);
},
(bcid, header) =>
{
HandleCachedPayloads(bcid, header);
return(header);
})
);
var md5 = MD5.Create();
bool CheckHash(ProtoMessage payload)
{
var expected = payload.Hash;
if (expected == null || expected.Length == 0)
{
return(false);
}
var actual = md5.ComputeHash(payload.Payload);
if (actual.Length == 0 || expected.Length != actual.Length)
{
return(false);
}
for (var index = 0; index < expected.Length; index++)
{
var expectedByte = expected[index];
var actualByte = actual[index];
if (expectedByte != actualByte)
{
return(false);
}
}
return(true);
}
payloadObservable
.ObserveOn(scheduler)
.Subscribe(payload =>
{
if (!CheckHash(payload))
{
throw new InvalidDataException("Hash check failed");
//todo: return error statistics to the caller
}
if (headerCache.TryGetValue(payload.GetBroadcastId(), out var header))
{
fileWriteRequestSubject.OnNext(new FileWriteWrapper()
{
Header = header, Payload = payload
});
}
else
{
payloadCache.AddOrUpdate(payload.GetBroadcastId(),
bcid =>
{
var list = new List <ProtoMessage>();
list.Add(payload);
return(list);
},
(bcid, list) =>
{
list.Add(payload);
return(list);
});
}
});
var writeCompleteObservable = fileWriteRequestSubject
//.ObserveOn(observableScheduler)
.Select(writeRequest =>
{
if (writeRequest.Payload.Payload.Length > writeRequest.Header.PayloadMaxBytes)
{
throw new ArgumentException("Payload exceeds max byte length");
}
return(Observable.FromAsync(async() =>
{
var fullPath = Path.Combine(basePath, writeRequest.Header.FileName);
using var writer = streamFactory.CreateWriter(fullPath);
var writeIndex = writeRequest.Payload.PayloadIndex.Value *
writeRequest.Header.PayloadMaxBytes;
await writer.Write(writeIndex, writeRequest.Payload.Payload, token).ConfigureAwait(false);
return writeRequest.Header;
}));
}).Merge(1);
var fileWriteObservable = writeCompleteObservable
.ObserveOn(scheduler)
.GroupBy(w => w.BroadcastId);
const int fileCompleteTimeout = 10;
var fileTimeout = TimeSpan.FromSeconds(fileCompleteTimeout);
var fileStoppedObservable = CreateTimeoutObservable(fileTimeout, fileWriteObservable, scheduler);
var fileStoppedSub = fileStoppedObservable.Subscribe(header =>
{
OnBroadcastEnded(new BroadcastResult(header.BroadcastId, header.FileName));
});
token.Register(() => fileStoppedSub.Dispose());
void OnBytesReceived(object sender, IBytesReceived bytesReceived)
{
packetBufferObservable.OnNext(bytesReceived.Bytes);
}
byteReceiver.BytesReceived += OnBytesReceived;
cancellationToken.Register(() => { byteReceiver.BytesReceived -= OnBytesReceived; });
await byteReceiver.Listen(cancellationToken);
}
public SslStreamFactory(SslStreamSettings settings, IStreamFactory wrapped)
{
_settings = Ensure.IsNotNull(settings, "settings");
_wrapped = Ensure.IsNotNull(wrapped, "wrapped");
}
public HttpStreamRequest(DatadogHttpClient client, Uri uri, IStreamFactory streamFactory)
{
_uri = uri;
_client = client;
_streamFactory = streamFactory;
}
public XmlPersister( IXmlSerializer serializer, IStreamFactory streamfactory, int milliSecondsBetweenSerialization )
{
_serializer = serializer;
_streamfactory = streamfactory;
_milliSecondsBetweenSerialization = milliSecondsBetweenSerialization;
}
public TrinketSpawnerImpl(IStreamFactory streamFactory, IPofSerializer pofSerializer, ExeggutorService exeggutorService)
{
this.streamFactory = streamFactory;
this.pofSerializer = pofSerializer;
this.exeggutorService = exeggutorService;
}
// constructors
public BinaryConnectionFactory(ConnectionSettings settings, IStreamFactory streamFactory, IEventSubscriber eventSubscriber)
{
_settings = Ensure.IsNotNull(settings, nameof(settings));
_streamFactory = Ensure.IsNotNull(streamFactory, nameof(streamFactory));
_eventSubscriber = Ensure.IsNotNull(eventSubscriber, nameof(eventSubscriber));
}
public StreamFactory(IStreamFactory innerFactory, TcpStreamSettings settings)
{
_innerFactory = innerFactory ?? throw new ArgumentNullException(nameof(innerFactory));
_settings = settings ?? new TcpStreamSettings();
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
IPath relPath, int depth = 0)
{
Hash hash = default;
if (extractedFile is NativeFileStreamFactory)
{
hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync();
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.xxHashAsync();
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
{
return(vself);
}
await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = stream.Length,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
}
// Can't extract, so return
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
return(self);
}
try
{
var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));
self.Children = list.Values.ToImmutableList();
}
catch (EndOfStreamException)
{
return(self);
}
catch (Exception)
{
Utils.Log($"Error while examining the contents of {relPath.FileName}");
throw;
}
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().Write(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
return(self);
}
