Example #1
0
        public IEnumerable <TStream> Fetch(string symbol, string column, DateTime start, DateTime end)
        {
            var bTree = GetBTree(symbol, column);

            foreach (var entry in bTree.Search(start, end).ToList())
            {
                if (start < entry.Key || !entry.Value.FilePath.FileExists())
                {
                    // Load from start to entry.Key
                    foreach (var stream in CollectAndFetch(symbol, column, start, entry.Key.AddTicks(-1)))
                    {
                        yield return(stream);
                    }

                    Persist(_filePathProvider.GetMetaFilePath(symbol /*, column*/), bTree);
                }

                var reader = _streamFactory.CreateReader(entry.Value.FilePath);
                if (reader != null)
                {
                    yield return(reader);
                }
                start = entry.Value.End;
            }

            if (start < end)
            {
                foreach (var stream in CollectAndFetch(symbol, column, start, end))
                {
                    yield return(stream);
                }

                Persist(_filePathProvider.GetMetaFilePath(symbol /*, column*/), bTree);
            }
        }
Example #2
0
        public async Task BroadcastFile(string fileName,
                                        IByteSender byteSender,
                                        IStreamFactory streamFactory,
                                        IScheduler scheduler,
                                        float headerRebroadcastSeconds      = 1,
                                        CancellationToken cancellationToken = default)
        {
            if (headerRebroadcastSeconds < 0)
            {
                throw new ArgumentException(nameof(headerRebroadcastSeconds));
            }

            var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var token = cancellationTokenSource.Token;

            var reader = streamFactory.CreateReader(fileName);

            const int byteToMegabyteFactor = 1000000;
            var       pageSize             = 8 * byteToMegabyteFactor;

            var pageObservable = reader.PageObservable(pageSize, token);

            const long broadcastIdByteCount = 16;  //guid bytes
            const long fudgeAmount          = 100; //the hash alg chosen has to be equal to or smaller than this
            var        packetSize           = byteSender.MaximumTransmittableBytes - broadcastIdByteCount - fudgeAmount;
            var        partialPacketCache   = new ConcurrentDictionary <long, byte[]>();
            var        payloadObservable    = pageObservable
                                              .ObserveOn(scheduler)
                                              .SelectMany(page =>
            {
                var firstPacketIndex   = page.PageIndex / packetSize;
                var hasFirstFragmented = (page.PageIndex % packetSize) != 0;
                var packetList         = new List <PayloadWrapper>();

                var firstPartialPacketIndex = hasFirstFragmented
                        ? firstPacketIndex
                        : (long?)null;
                var secondPacketPayloadIndex = (firstPartialPacketIndex + 1) * packetSize;
                var firstPartialLength       = (secondPacketPayloadIndex - page.PageIndex) ?? 0;

                if (hasFirstFragmented)
                {
                    var partialBuffer = new byte[firstPartialLength];
                    Array.Copy(page.Bytes, partialBuffer, firstPartialLength);
                    var firstPartialPacket = partialBuffer;
                    var firstPayload       = new PayloadWrapper()
                    {
                        PayloadIndex = firstPartialPacketIndex.Value, bytes = firstPartialPacket
                    };
                    CachePartialPacket(partialPacketCache, firstPayload, packetList);
                }

                var firstFullPacketIndex = hasFirstFragmented
                        ? firstPacketIndex + 1
                        : firstPacketIndex;

                var lastPageByteIndex    = page.Bytes.Length - 1;
                var lastBytePayloadIndex = page.PageIndex + lastPageByteIndex;
                var lastPacketIndex      = lastBytePayloadIndex / packetSize;
                var lastPartialLength    = (page.Bytes.Length - firstPartialLength) % packetSize;
                var hasLastPartialPacket = lastPacketIndex > firstPacketIndex &&
                                           (lastPartialLength > 0);
                var lastFullPacketIndex = hasLastPartialPacket
                        ? lastPacketIndex - 1
                        : lastPacketIndex;

                //todo: consider parallel foreach
                for (long packetIndex = firstFullPacketIndex; packetIndex <= lastFullPacketIndex; packetIndex++)
                {
                    var packetBuffer   = new byte[packetSize];
                    var startPageIndex = ((packetIndex - firstFullPacketIndex) * packetSize) + firstPartialLength;
                    Array.Copy(page.Bytes, startPageIndex, packetBuffer, 0, packetSize);
                    var payload = new PayloadWrapper()
                    {
                        PayloadIndex = packetIndex, bytes = packetBuffer
                    };
                    packetList.Add(payload);
                }

                if (hasLastPartialPacket)
                {
                    var partialBuffer          = new byte[lastPartialLength];
                    var lastPartialPageIndex   = page.Bytes.Length - lastPartialLength;
                    var lastPartialPacketIndex = lastPacketIndex;
                    Array.Copy(page.Bytes, lastPartialPageIndex, partialBuffer, 0, lastPartialLength);
                    var lastPayload = new PayloadWrapper()
                    {
                        PayloadIndex = lastPartialPacketIndex, bytes = partialBuffer
                    };
                    CachePartialPacket(partialPacketCache, lastPayload, packetList);
                }

                return(packetList.AsEnumerable());
            })
                                              .Concat(partialPacketCache
                                                      .Select(kvp => new PayloadWrapper {
                PayloadIndex = kvp.Key, bytes = kvp.Value
            }).ToObservable());

            var broadcastId = Guid.NewGuid();

            var md5 = MD5.Create();
            var serializedPayloadObservable = payloadObservable
                                              .Select(payloadWrapper =>
            {
                byte[] serializedPayload;
                var hash         = md5.ComputeHash(payloadWrapper.bytes);
                var protoMessage = new ProtoMessage(broadcastId, payloadWrapper.PayloadIndex, payloadWrapper.bytes,
                                                    null, null,
                                                    null, hash);
                using (var memoryStream = new MemoryStream())
                {
                    Serializer.Serialize(memoryStream,
                                         protoMessage);
                    serializedPayload = memoryStream.ToArray();
                }

                return(serializedPayload);
            })
                                              .Finally(() => { md5.Dispose(); })
                                              .Publish();

            byte[] serializedHeader;
            var    packetCount     = (long)Math.Ceiling((double)reader.Length / packetSize);
            var    rebroadcastTime = TimeSpan.FromSeconds(headerRebroadcastSeconds);

            using (var memoryStream = new MemoryStream())
            {
                Serializer.Serialize(memoryStream,
                                     new ProtoMessage(broadcastId, null, null, packetSize, fileName, packetCount, null));
                serializedHeader = memoryStream.ToArray();
            }

            var headerObservable =
                GetHeaderObservable(serializedHeader, rebroadcastTime, serializedPayloadObservable, scheduler);

            var packetObservable = headerObservable
                                   .Merge(serializedPayloadObservable);

            serializedPayloadObservable.Connect();

            var sendObservable = packetObservable
                                 .ObserveOn(scheduler)
                                 .SelectMany((array, index) =>
            {
                return(Observable.FromAsync(async c =>
                {
                    await byteSender.Send(array).ConfigureAwait(false);
                    return Unit.Default;
                }));
            });
            await sendObservable
            .ToTask(token).ConfigureAwait(false);
        }