private void OutputAllEvents(PartitionEntry partition) { // Output all collected intervals foreach (var kvp in partition.eventMap) { var index = FastDictionary2 <ActiveEvent, int> .IteratorStart; while (kvp.Value.Iterate(ref index)) { var outevt = kvp.Value.entries[index].key; for (int j = 0; j < kvp.Value.entries[index].value; j++) { var ind = this.output.Count++; this.output.vsync.col[ind] = kvp.Key; this.output.vother.col[ind] = outevt.End; this.output.key.col[ind] = outevt.Key; this.output[ind] = outevt.Payload; this.output.hash.col[ind] = outevt.Hash; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } }
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, long end, ref TLeft payload) { // Create end edges for all joined right edges. var edges = partition.rightEdgeMap.Find(partition.hash); int index; while (edges.Next(out index)) { AddToBatch( currentTime, StreamEvent.InfinitySyncTime, partition, ref payload, ref partition.rightEdgeMap.Values[index].Payload); } // Create end edges for all joined right intervals. var intervals = partition.rightIntervalMap.Find(partition.hash); while (intervals.Next(out index)) { long rightEnd = partition.rightIntervalMap.Values[index].End; AddToBatch( currentTime, end < rightEnd ? end : rightEnd, partition, ref payload, ref partition.rightIntervalMap.Values[index].Payload); } }
private void LeaveTime(PartitionEntry partition) { var leftIntervals = partition.leftIntervalMap.TraverseInvisible(); while (leftIntervals.Next(out int index, out int hash)) { CreateOutputForStartInterval( partition, partition.currTime, ref partition.leftIntervalMap.Values[index].Key, ref partition.leftIntervalMap.Values[index].Payload, hash); leftIntervals.MakeVisible(); } var rightIntervals = partition.rightIntervalMap.TraverseInvisible(); while (rightIntervals.Next(out int index, out int hash)) { CreateOutputForStartInterval( partition, partition.currTime, ref partition.rightIntervalMap.Values[index].Key, ref partition.rightIntervalMap.Values[index].Payload, hash); rightIntervals.MakeVisible(); } }
private void ProcessLeftEvent(PartitionEntry partition, long start, long end, ref TKey key, TLeft payload, int hash) { var lim = partition.leftIntervalMap; var lem = partition.leftEdgeMap; if (start < end) { // Row is a start edge or interval. var leph = partition.leftEndPointHeap; bool isInterval = end < StreamEvent.InfinitySyncTime; if (isInterval) { bool isFullyOutputtable = partition.nextRightTime >= end; if (isFullyOutputtable) { // Output full interval. AddToBatch(start, end, ref key, ref payload, hash); } else { // Insert into map to remember interval. int mapIndex = lim.Insert(hash); // Insert into heap to schedule removal at endpoint. int heapIndex = leph.Insert(end, mapIndex); // Set value in map, also remembering heap's index. lim.Values[mapIndex].Initialize(start, ref key, ref payload, heapIndex); // Output start edge. AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payload, hash); } } else { int index = lem.Insert(hash); lem.Values[index].Populate(start, ref key, ref payload); // Output start edge. AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payload, hash); } } else { // Row is an end edge. var leftEvents = lem.Find(hash); while (leftEvents.Next(out int index)) { if (AreSame(end, ref key, ref payload, ref lem.Values[index])) { // Output end edge. AddToBatch(start, end, ref key, ref payload, hash); // Remove from leftMap. leftEvents.Remove(); break; } } } }
private void ReachTime(PartitionEntry partition) { while (partition.endPointHeap.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index)) { if (index >= 0) { // Endpoint is left interval ending. CreateOutputForEndInterval( partition, endPointTime, partition.leftIntervalMap.Values[index].Start, ref partition.leftIntervalMap.Values[index].Payload, partition.leftIntervalMap.GetHash(index)); partition.leftIntervalMap.Remove(index); } else { // Endpoint is right interval ending. index = ~index; CreateOutputForEndInterval( partition, endPointTime, partition.rightIntervalMap.Values[index].Start, ref partition.rightIntervalMap.Values[index].Payload, partition.rightIntervalMap.GetHash(index)); partition.rightIntervalMap.Remove(index); } } }
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, long end, ref TRight payload, int hash) { // Create end edges for all joined left edges. var edges = partition.leftEdgeMap.Find(hash); int index; while (edges.Next(out index)) { AddToBatch( currentTime, StreamEvent.InfinitySyncTime, ref partition.key, ref partition.leftEdgeMap.Values[index].Payload, ref payload, hash); } // Create end edges for all joined left intervals. var intervals = partition.leftIntervalMap.Find(hash); while (intervals.Next(out index)) { long leftEnd = partition.leftIntervalMap.Values[index].End; AddToBatch( currentTime, end < leftEnd ? end : leftEnd, ref partition.key, ref partition.leftIntervalMap.Values[index].Payload, ref payload, hash); } }
private void CreateOutputForEndEdge(PartitionEntry partition, long currentTime, long start, ref TRight payload, int hash) { // Create end edges for all joined left edges. var edges = partition.leftEdgeMap.Find(hash); int index; while (edges.Next(out index)) { long leftStart = partition.leftEdgeMap.Values[index].Start; AddToBatch( currentTime, start > leftStart ? start : leftStart, ref partition.key, ref partition.leftEdgeMap.Values[index].Payload, ref payload, hash); } // Create end edges for all joined left intervals. var intervals = partition.leftIntervalMap.Find(hash); while (intervals.Next(out index)) { long leftStart = partition.leftIntervalMap.Values[index].Start; AddToBatch( currentTime, start > leftStart ? start : leftStart, ref partition.key, ref partition.leftIntervalMap.Values[index].Payload, ref payload, hash); } }
private void OnPunctuation(PartitionEntry partition, long syncTime) { if (syncTime > partition.lastSyncTime) { /* Issue start edges for held aggregates */ foreach (int iter1 in partition.heldAggregates) { var iter1entry = this.aggregateByKey.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(iter1entry.value.state); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } // Time has moved forward, clear the held aggregates partition.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime partition.lastSyncTime = syncTime; } }
private void CreateOutputForEndEdge(PartitionEntry partition, long currentTime, long start, ref TLeft payload) { // Create end edges for all joined right edges. var edges = partition.rightEdgeMap.Find(partition.hash); int index; while (edges.Next(out index)) { long rightStart = partition.rightEdgeMap.Values[index].Start; AddToBatch( currentTime, start > rightStart ? start : rightStart, partition, ref payload, ref partition.rightEdgeMap.Values[index].Payload); } // Create end edges for all joined right intervals. var intervals = partition.rightIntervalMap.Find(partition.hash); while (intervals.Next(out index)) { long rightStart = partition.rightIntervalMap.Values[index].Start; AddToBatch( currentTime, start > rightStart ? start : rightStart, partition, ref payload, ref partition.rightIntervalMap.Values[index].Payload); } }
private void OnPunctuation(PartitionEntry partition, long syncTime) { if (syncTime > partition.lastSyncTime) { AdvanceTime(partition, syncTime); } }
private void UpdateTime(PartitionEntry partition, long time) { if (time > partition.currTime) { LeaveTime(partition); partition.currTime = time; } }
private void ProcessRightEvent(PartitionEntry partition, long start, TRight payload) { int index = partition.rightIntervalMap.Insert(partition.hash); partition.rightIntervalMap.Values[index].Populate(start, ref payload); CreateOutputForStartInterval(partition, start, ref payload); partition.endPointHeap.Insert(start + this.rightDuration, ~index); }
private void ProcessLeftEvent(PartitionEntry partition, long start, ref TGroupKey key, TLeft payload, int hash) { int index = partition.leftIntervalMap.Insert(hash); partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload); CreateOutputForStartInterval(partition, start, ref key, ref payload, hash); partition.endPointHeap.Insert(start + this.leftDuration, index); }
private static void UpdateNextRightTime(PartitionEntry partition, long time) { partition.nextRightTime = time; if (time == StreamEvent.InfinitySyncTime) { partition.isRightComplete = true; } }
private void UpdateTime(PartitionEntry partition, long time) { if (time > partition.currTime) { partition.currTime = time; ReachTime(partition); } }
public void writeUpdatePartitionEntries(Stream iostr, VoldemortFilter filter, PartitionEntry entry, string store) { VoldemortAdminRequest request = new VoldemortAdminRequest(); request.type = AdminRequestType.UPDATE_PARTITION_ENTRIES; request.update_partition_entries = new UpdatePartitionEntriesRequest(); request.update_partition_entries.filter = filter; request.update_partition_entries.partition_entry = entry; request.update_partition_entries.store = store; Serializer.SerializeWithLengthPrefix(iostr, request, PrefixStyle.Fixed32BigEndian); }
public async Task WhenNewPartitionIsCreated_Then_ItIsRetrievable() { string partitionName = "test"; await _fixture.PartitionStore.AddPartitionAsync(partitionName); PartitionEntry partition = await _fixture.PartitionStore.GetPartitionAsync(partitionName); Assert.NotNull(partition); }
private void ProcessRightEvent(PartitionEntry partition, long start, long end, ref TKey key, int hash) { if (start >= end) { // Row is an end edge, which we don't care about because the start edge would have already // removed all joining left events. return; } // Mark any matching left intervals as no longer active. var lim = partition.leftIntervalMap; var lem = partition.leftEdgeMap; var leph = partition.leftEndPointHeap; var leftIntervals = lim.Find(hash); while (leftIntervals.Next(out int index)) { long leftStart = lim.Values[index].Start; if (leftStart < start && this.keyComparerEquals(key, lim.Values[index].Key)) { // Output end edge. AddToBatch( start, leftStart, ref lim.Values[index].Key, ref lim.Values[index].Payload, hash); // Remove from heap and map. leph.Remove(lim.Values[index].HeapIndex); leftIntervals.Remove(); } } // Remove any matching left edges. var leftEdges = lem.Find(hash); while (leftEdges.Next(out int index)) { long leftStart = lem.Values[index].Start; if (leftStart < start && this.keyComparerEquals(key, lem.Values[index].Key)) { // Output end edge. AddToBatch( start, leftStart, ref lem.Values[index].Key, ref lem.Values[index].Payload, hash); // Remove left event. leftEdges.Remove(); } } }
private static void UpdateNextRightTime(PartitionEntry partition, long time) { partition.nextRightTime = time; if (time == StreamEvent.InfinitySyncTime && partition.rightEdgeMap.IsInvisibleEmpty && partition.rightIntervalMap.IsInvisibleEmpty && partition.rightIntervalMap.IsEmpty) { partition.isRightComplete = true; } }
private void ReachTime(PartitionEntry partition) { while (partition.endPointHeap.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index)) { if (index >= 0) { partition.leftIntervalMap.Remove(index); } else { partition.rightIntervalMap.Remove(~index); } } }
private void ProcessLeftEvent(PartitionEntry partition, long start, ref TGroupKey key, TLeft payload, int hash) { bool processable = partition.nextRightTime > start; if (processable) { int index = partition.leftIntervalMap.Insert(hash); partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload); CreateOutputForStartInterval(partition, start, ref key, ref payload, hash); } else { int index = partition.leftIntervalMap.InsertInvisible(hash); partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload); } }
private void ProcessRightEvent(PartitionEntry partition, long start, TRight payload) { bool processable = partition.nextLeftTime > start; if (processable) { int index = partition.rightIntervalMap.Insert(partition.hash); partition.rightIntervalMap.Values[index].Populate(start, ref payload); CreateOutputForStartInterval(partition, start, ref payload); } else { int index = partition.rightIntervalMap.InsertInvisible(partition.hash); partition.rightIntervalMap.Values[index].Populate(start, ref payload); } }
private bool FindOnRight(PartitionEntry partition, ref TKey key, int hash, out int index) { var rm = partition.rightMap; var rightEvents = rm.Find(hash); while (rightEvents.Next(out int rightIndex)) { if (this.keyComparerEquals(key, rm.Values[rightIndex].Key)) { index = rightIndex; return(true); } } index = 0; return(false); }
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, ref TRight payload) { // Create end edges for all joined left intervals. var intervals = partition.leftIntervalMap.Find(partition.hash); while (intervals.Next(out var index)) { long rightEnd = currentTime + this.rightDuration; long leftEnd = partition.leftIntervalMap.Values[index].Start + this.leftDuration; AddToBatch( currentTime, rightEnd < leftEnd ? rightEnd : leftEnd, partition, ref partition.leftIntervalMap.Values[index].Payload, ref payload); } }
private void OutputCompletedIntervals(PartitionEntry partition) { // Output all completed intervals var delList = new List <long>(); foreach (var kvp in partition.eventMap) { var index = FastDictionary2 <ActiveEvent, int> .IteratorStart; while (kvp.Value.Iterate(ref index)) { var outevt = kvp.Value.entries[index].key; if (outevt.End == StreamEvent.InfinitySyncTime) { foreach (var key in delList) { partition.eventMap.Remove(key); } return; } for (int j = 0; j < kvp.Value.entries[index].value; j++) { var ind = this.output.Count++; this.output.vsync.col[ind] = kvp.Key; this.output.vother.col[ind] = outevt.End; this.output.key.col[ind] = outevt.Key; this.output[ind] = outevt.Payload; this.output.hash.col[ind] = outevt.Hash; partition.lastSyncTime = kvp.Key; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } kvp.Value.Remove(outevt); } this.dictPool.Return(kvp.Value); delList.Add(kvp.Key); } foreach (var key in delList) { partition.eventMap.Remove(key); } }
static void fixupPartitionEntry(ref PartitionEntry part, long diskLength) { if (part.Type != MbrPartitionType.GptProtective) { return; } long lastSector = (diskLength / Program.SECTOR_SIZE) - 1; long numberOfSectors = lastSector - part.FirstSectorLba; if (numberOfSectors >= (long)uint.MaxValue) { part.NumberOfSectors = uint.MaxValue; } else { part.NumberOfSectors = (uint)numberOfSectors; } }
private void MakeMatchingLeftInvisible(PartitionEntry partition, long time, ref TKey key, int hash) { // Make matching left intervals invisible. var lem = partition.leftEdgeMap; var lim = partition.leftIntervalMap; var leftEvents = lim.Find(hash); while (leftEvents.Next(out int index)) { if (this.keyComparerEquals(key, lim.Values[index].Key)) { // Output end edge. AddToBatch( time, lim.Values[index].CurrentStart, ref lim.Values[index].Key, ref lim.Values[index].Payload, hash); // Mark left event as not visible. lim.Values[index].CurrentStart = NotActive; } } // Make matching left edges invisible. leftEvents = lem.Find(hash); while (leftEvents.Next(out int index)) { if (this.keyComparerEquals(key, lem.Values[index].Key)) { // Output end edge. AddToBatch( time, lem.Values[index].CurrentStart, ref lem.Values[index].Key, ref lem.Values[index].Payload, hash); // Mark left event as not visible. lem.Values[index].CurrentStart = NotActive; } } }
private void ReachTime(PartitionEntry partition) { // Carry-out all interval endpoints for left intervals that end prior or at new current time. var lim = partition.leftIntervalMap; var leph = partition.leftEndPointHeap; while (leph.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index)) { // Output end edge. AddToBatch( endPointTime, lim.Values[index].Start, ref lim.Values[index].Key, ref lim.Values[index].Payload, lim.GetHash(index)); // Remove from leftMap. lim.Remove(index); } }
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, ref TKey key, ref TLeft payload, int hash) { // Create end edges for all joined right intervals. var intervals = partition.rightIntervalMap.Find(hash); while (intervals.Next(out var index)) { if (this.keyComparerEquals(key, partition.rightIntervalMap.Values[index].Key)) { long leftEnd = currentTime + this.leftDuration; long rightEnd = partition.rightIntervalMap.Values[index].Start + this.rightDuration; AddToBatch( currentTime, leftEnd < rightEnd ? leftEnd : rightEnd, ref key, ref payload, ref partition.rightIntervalMap.Values[index].Payload, hash); } } }
public DicomRequestContext( string method, Uri uri, Uri baseUri, string correlationId, IDictionary <string, StringValues> requestHeaders, IDictionary <string, StringValues> responseHeaders) { EnsureArg.IsNotNullOrWhiteSpace(method, nameof(method)); EnsureArg.IsNotNull(uri, nameof(uri)); EnsureArg.IsNotNull(baseUri, nameof(baseUri)); EnsureArg.IsNotNull(responseHeaders, nameof(responseHeaders)); Method = method; Uri = uri; BaseUri = baseUri; CorrelationId = correlationId; RequestHeaders = requestHeaders; ResponseHeaders = responseHeaders; DataPartitionEntry = new PartitionEntry(DefaultPartition.Key, DefaultPartition.Name); }
private void ProcessRightEvent(PartitionEntry partition, long start, long end, ref TKey key, int hash) { if (start < end) { // Row is a start edge or interval. var rm = partition.rightMap; if (FindOnRight(partition, ref key, hash, out int index)) { // Corresponding key already exists in map, so any joining on left and already not active. rm.Values[index].Count++; } else { // First instance of this key, so insert and make any joining left entries not active. index = rm.Insert(hash); rm.Values[index].Initialize(ref key); MakeMatchingLeftInvisible(partition, start, ref key, hash); } if (end != StreamEvent.InfinitySyncTime) { // Row is an interval, so schedule removal of interval. var reph = partition.rightEndPointHeap; reph.Insert(end, index); } } else { // Row is an end edge. // Queue for removal when time advances. var ree = partition.rightEndEdges; int index = ree.Push(); ree.Values[index].Populate(ref key, hash); } }