public ByteConnection(IByteSender sender, IByteReceiver receiver)
{
this.sender = sender;
this.receiver = receiver;
receiver.Received.Subscribe(HandleReceived);
Received = new MessageQueue<byte[]>();
sender.Error += HandleError;
receiver.Error += HandleError;
}
public KeekStream(Stream innerStream, bool leaveInnerStreamOpen, bool isFragmented = false)
{
_innerStream = innerStream ?? throw new ArgumentException(nameof(innerStream));
_byteSender = new TransparentByteSender(_innerStream);
_leaveInnerStreamOpen = leaveInnerStreamOpen;
_accumulator = new ByteAccumulator();
_readLock = new object();
_isFragmented = isFragmented;
_byteSender = _isFragmented ?
(IByteSender) new TransparentByteSender(_innerStream) :
(IByteSender) new FragmentedByteSender(_innerStream);
}
public async Task BroadcastFile(string fileName,
IByteSender byteSender,
IStreamFactory streamFactory,
float headerRebroadcastSeconds = 1,
CancellationToken cancellationToken = default)
{
var taskPoolScheduler =
Scheduler.Default; //new TaskPoolScheduler(new TaskFactory(cancellationToken));
await BroadcastFile(fileName,
byteSender,
streamFactory,
taskPoolScheduler,
headerRebroadcastSeconds,
cancellationToken);
}
public async Task BroadcastFile(string fileName,
IByteSender byteSender,
IStreamFactory streamFactory,
IScheduler scheduler,
float headerRebroadcastSeconds = 1,
CancellationToken cancellationToken = default)
{
if (headerRebroadcastSeconds < 0)
{
throw new ArgumentException(nameof(headerRebroadcastSeconds));
}
var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var token = cancellationTokenSource.Token;
var reader = streamFactory.CreateReader(fileName);
const int byteToMegabyteFactor = 1000000;
var pageSize = 8 * byteToMegabyteFactor;
var pageObservable = reader.PageObservable(pageSize, token);
const long broadcastIdByteCount = 16; //guid bytes
const long fudgeAmount = 100; //the hash alg chosen has to be equal to or smaller than this
var packetSize = byteSender.MaximumTransmittableBytes - broadcastIdByteCount - fudgeAmount;
var partialPacketCache = new ConcurrentDictionary <long, byte[]>();
var payloadObservable = pageObservable
.ObserveOn(scheduler)
.SelectMany(page =>
{
var firstPacketIndex = page.PageIndex / packetSize;
var hasFirstFragmented = (page.PageIndex % packetSize) != 0;
var packetList = new List <PayloadWrapper>();
var firstPartialPacketIndex = hasFirstFragmented
? firstPacketIndex
: (long?)null;
var secondPacketPayloadIndex = (firstPartialPacketIndex + 1) * packetSize;
var firstPartialLength = (secondPacketPayloadIndex - page.PageIndex) ?? 0;
if (hasFirstFragmented)
{
var partialBuffer = new byte[firstPartialLength];
Array.Copy(page.Bytes, partialBuffer, firstPartialLength);
var firstPartialPacket = partialBuffer;
var firstPayload = new PayloadWrapper()
{
PayloadIndex = firstPartialPacketIndex.Value, bytes = firstPartialPacket
};
CachePartialPacket(partialPacketCache, firstPayload, packetList);
}
var firstFullPacketIndex = hasFirstFragmented
? firstPacketIndex + 1
: firstPacketIndex;
var lastPageByteIndex = page.Bytes.Length - 1;
var lastBytePayloadIndex = page.PageIndex + lastPageByteIndex;
var lastPacketIndex = lastBytePayloadIndex / packetSize;
var lastPartialLength = (page.Bytes.Length - firstPartialLength) % packetSize;
var hasLastPartialPacket = lastPacketIndex > firstPacketIndex &&
(lastPartialLength > 0);
var lastFullPacketIndex = hasLastPartialPacket
? lastPacketIndex - 1
: lastPacketIndex;
//todo: consider parallel foreach
for (long packetIndex = firstFullPacketIndex; packetIndex <= lastFullPacketIndex; packetIndex++)
{
var packetBuffer = new byte[packetSize];
var startPageIndex = ((packetIndex - firstFullPacketIndex) * packetSize) + firstPartialLength;
Array.Copy(page.Bytes, startPageIndex, packetBuffer, 0, packetSize);
var payload = new PayloadWrapper()
{
PayloadIndex = packetIndex, bytes = packetBuffer
};
packetList.Add(payload);
}
if (hasLastPartialPacket)
{
var partialBuffer = new byte[lastPartialLength];
var lastPartialPageIndex = page.Bytes.Length - lastPartialLength;
var lastPartialPacketIndex = lastPacketIndex;
Array.Copy(page.Bytes, lastPartialPageIndex, partialBuffer, 0, lastPartialLength);
var lastPayload = new PayloadWrapper()
{
PayloadIndex = lastPartialPacketIndex, bytes = partialBuffer
};
CachePartialPacket(partialPacketCache, lastPayload, packetList);
}
return(packetList.AsEnumerable());
})
.Concat(partialPacketCache
.Select(kvp => new PayloadWrapper {
PayloadIndex = kvp.Key, bytes = kvp.Value
}).ToObservable());
var broadcastId = Guid.NewGuid();
var md5 = MD5.Create();
var serializedPayloadObservable = payloadObservable
.Select(payloadWrapper =>
{
byte[] serializedPayload;
var hash = md5.ComputeHash(payloadWrapper.bytes);
var protoMessage = new ProtoMessage(broadcastId, payloadWrapper.PayloadIndex, payloadWrapper.bytes,
null, null,
null, hash);
using (var memoryStream = new MemoryStream())
{
Serializer.Serialize(memoryStream,
protoMessage);
serializedPayload = memoryStream.ToArray();
}
return(serializedPayload);
})
.Finally(() => { md5.Dispose(); })
.Publish();
byte[] serializedHeader;
var packetCount = (long)Math.Ceiling((double)reader.Length / packetSize);
var rebroadcastTime = TimeSpan.FromSeconds(headerRebroadcastSeconds);
using (var memoryStream = new MemoryStream())
{
Serializer.Serialize(memoryStream,
new ProtoMessage(broadcastId, null, null, packetSize, fileName, packetCount, null));
serializedHeader = memoryStream.ToArray();
}
var headerObservable =
GetHeaderObservable(serializedHeader, rebroadcastTime, serializedPayloadObservable, scheduler);
var packetObservable = headerObservable
.Merge(serializedPayloadObservable);
serializedPayloadObservable.Connect();
var sendObservable = packetObservable
.ObserveOn(scheduler)
.SelectMany((array, index) =>
{
return(Observable.FromAsync(async c =>
{
await byteSender.Send(array).ConfigureAwait(false);
return Unit.Default;
}));
});
await sendObservable
.ToTask(token).ConfigureAwait(false);
}
public DirectoryResquestSender(IByteSender sender)
{
this._sender = sender;
}
public FileSender(IByteSender sender)
{
this._sender = sender;
}
public MockByteTransporter(IByteSender sender, int mtu = 1400)
{
_sender = sender;
MaximumTransmittableBytes = mtu;
}
