private void OutputAllEvents(PartitionEntry partition)
{
// Output all collected intervals
foreach (var kvp in partition.eventMap)
{
var index = FastDictionary2 <ActiveEvent, int> .IteratorStart;
while (kvp.Value.Iterate(ref index))
{
var outevt = kvp.Value.entries[index].key;
for (int j = 0; j < kvp.Value.entries[index].value; j++)
{
var ind = this.output.Count++;
this.output.vsync.col[ind] = kvp.Key;
this.output.vother.col[ind] = outevt.End;
this.output.key.col[ind] = outevt.Key;
this.output[ind] = outevt.Payload;
this.output.hash.col[ind] = outevt.Hash;
if (this.output.Count == Config.DataBatchSize)
{
FlushContents();
}
}
}
}
}
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, long end, ref TLeft payload)
{
// Create end edges for all joined right edges.
var edges = partition.rightEdgeMap.Find(partition.hash);
int index;
while (edges.Next(out index))
{
AddToBatch(
currentTime,
StreamEvent.InfinitySyncTime,
partition,
ref payload,
ref partition.rightEdgeMap.Values[index].Payload);
}
// Create end edges for all joined right intervals.
var intervals = partition.rightIntervalMap.Find(partition.hash);
while (intervals.Next(out index))
{
long rightEnd = partition.rightIntervalMap.Values[index].End;
AddToBatch(
currentTime,
end < rightEnd ? end : rightEnd,
partition,
ref payload,
ref partition.rightIntervalMap.Values[index].Payload);
}
}
private void LeaveTime(PartitionEntry partition)
{
var leftIntervals = partition.leftIntervalMap.TraverseInvisible();
while (leftIntervals.Next(out int index, out int hash))
{
CreateOutputForStartInterval(
partition,
partition.currTime,
ref partition.leftIntervalMap.Values[index].Key,
ref partition.leftIntervalMap.Values[index].Payload,
hash);
leftIntervals.MakeVisible();
}
var rightIntervals = partition.rightIntervalMap.TraverseInvisible();
while (rightIntervals.Next(out int index, out int hash))
{
CreateOutputForStartInterval(
partition,
partition.currTime,
ref partition.rightIntervalMap.Values[index].Key,
ref partition.rightIntervalMap.Values[index].Payload,
hash);
rightIntervals.MakeVisible();
}
}
private void ProcessLeftEvent(PartitionEntry partition, long start, long end, ref TKey key, TLeft payload, int hash)
{
var lim = partition.leftIntervalMap;
var lem = partition.leftEdgeMap;
if (start < end)
{
// Row is a start edge or interval.
var leph = partition.leftEndPointHeap;
bool isInterval = end < StreamEvent.InfinitySyncTime;
if (isInterval)
{
bool isFullyOutputtable = partition.nextRightTime >= end;
if (isFullyOutputtable)
{
// Output full interval.
AddToBatch(start, end, ref key, ref payload, hash);
}
else
{
// Insert into map to remember interval.
int mapIndex = lim.Insert(hash);
// Insert into heap to schedule removal at endpoint.
int heapIndex = leph.Insert(end, mapIndex);
// Set value in map, also remembering heap's index.
lim.Values[mapIndex].Initialize(start, ref key, ref payload, heapIndex);
// Output start edge.
AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payload, hash);
}
}
else
{
int index = lem.Insert(hash);
lem.Values[index].Populate(start, ref key, ref payload);
// Output start edge.
AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payload, hash);
}
}
else
{
// Row is an end edge.
var leftEvents = lem.Find(hash);
while (leftEvents.Next(out int index))
{
if (AreSame(end, ref key, ref payload, ref lem.Values[index]))
{
// Output end edge.
AddToBatch(start, end, ref key, ref payload, hash);
// Remove from leftMap.
leftEvents.Remove();
break;
}
}
}
}
private void ReachTime(PartitionEntry partition)
{
while (partition.endPointHeap.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index))
{
if (index >= 0)
{
// Endpoint is left interval ending.
CreateOutputForEndInterval(
partition,
endPointTime,
partition.leftIntervalMap.Values[index].Start,
ref partition.leftIntervalMap.Values[index].Payload,
partition.leftIntervalMap.GetHash(index));
partition.leftIntervalMap.Remove(index);
}
else
{
// Endpoint is right interval ending.
index = ~index;
CreateOutputForEndInterval(
partition,
endPointTime,
partition.rightIntervalMap.Values[index].Start,
ref partition.rightIntervalMap.Values[index].Payload,
partition.rightIntervalMap.GetHash(index));
partition.rightIntervalMap.Remove(index);
}
}
}
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, long end, ref TRight payload, int hash)
{
// Create end edges for all joined left edges.
var edges = partition.leftEdgeMap.Find(hash);
int index;
while (edges.Next(out index))
{
AddToBatch(
currentTime,
StreamEvent.InfinitySyncTime,
ref partition.key,
ref partition.leftEdgeMap.Values[index].Payload,
ref payload,
hash);
}
// Create end edges for all joined left intervals.
var intervals = partition.leftIntervalMap.Find(hash);
while (intervals.Next(out index))
{
long leftEnd = partition.leftIntervalMap.Values[index].End;
AddToBatch(
currentTime,
end < leftEnd ? end : leftEnd,
ref partition.key,
ref partition.leftIntervalMap.Values[index].Payload,
ref payload,
hash);
}
}
private void CreateOutputForEndEdge(PartitionEntry partition, long currentTime, long start, ref TRight payload, int hash)
{
// Create end edges for all joined left edges.
var edges = partition.leftEdgeMap.Find(hash);
int index;
while (edges.Next(out index))
{
long leftStart = partition.leftEdgeMap.Values[index].Start;
AddToBatch(
currentTime,
start > leftStart ? start : leftStart,
ref partition.key,
ref partition.leftEdgeMap.Values[index].Payload,
ref payload,
hash);
}
// Create end edges for all joined left intervals.
var intervals = partition.leftIntervalMap.Find(hash);
while (intervals.Next(out index))
{
long leftStart = partition.leftIntervalMap.Values[index].Start;
AddToBatch(
currentTime,
start > leftStart ? start : leftStart,
ref partition.key,
ref partition.leftIntervalMap.Values[index].Payload,
ref payload,
hash);
}
}
private void OnPunctuation(PartitionEntry partition, long syncTime)
{
if (syncTime > partition.lastSyncTime)
{
/* Issue start edges for held aggregates */
foreach (int iter1 in partition.heldAggregates)
{
var iter1entry = this.aggregateByKey.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.computeResult(iter1entry.value.state);
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
// Time has moved forward, clear the held aggregates
partition.heldAggregates.Clear();
// Since sync time changed, set lastSyncTime
partition.lastSyncTime = syncTime;
}
}
private void CreateOutputForEndEdge(PartitionEntry partition, long currentTime, long start, ref TLeft payload)
{
// Create end edges for all joined right edges.
var edges = partition.rightEdgeMap.Find(partition.hash);
int index;
while (edges.Next(out index))
{
long rightStart = partition.rightEdgeMap.Values[index].Start;
AddToBatch(
currentTime,
start > rightStart ? start : rightStart,
partition,
ref payload,
ref partition.rightEdgeMap.Values[index].Payload);
}
// Create end edges for all joined right intervals.
var intervals = partition.rightIntervalMap.Find(partition.hash);
while (intervals.Next(out index))
{
long rightStart = partition.rightIntervalMap.Values[index].Start;
AddToBatch(
currentTime,
start > rightStart ? start : rightStart,
partition,
ref payload,
ref partition.rightIntervalMap.Values[index].Payload);
}
}
private void OnPunctuation(PartitionEntry partition, long syncTime)
{
if (syncTime > partition.lastSyncTime)
{
AdvanceTime(partition, syncTime);
}
}
private void UpdateTime(PartitionEntry partition, long time)
{
if (time > partition.currTime)
{
LeaveTime(partition);
partition.currTime = time;
}
}
private void ProcessRightEvent(PartitionEntry partition, long start, TRight payload)
{
int index = partition.rightIntervalMap.Insert(partition.hash);
partition.rightIntervalMap.Values[index].Populate(start, ref payload);
CreateOutputForStartInterval(partition, start, ref payload);
partition.endPointHeap.Insert(start + this.rightDuration, ~index);
}
private void ProcessLeftEvent(PartitionEntry partition, long start, ref TGroupKey key, TLeft payload, int hash)
{
int index = partition.leftIntervalMap.Insert(hash);
partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload);
CreateOutputForStartInterval(partition, start, ref key, ref payload, hash);
partition.endPointHeap.Insert(start + this.leftDuration, index);
}
private static void UpdateNextRightTime(PartitionEntry partition, long time)
{
partition.nextRightTime = time;
if (time == StreamEvent.InfinitySyncTime)
{
partition.isRightComplete = true;
}
}
private void UpdateTime(PartitionEntry partition, long time)
{
if (time > partition.currTime)
{
partition.currTime = time;
ReachTime(partition);
}
}
public void writeUpdatePartitionEntries(Stream iostr, VoldemortFilter filter, PartitionEntry entry, string store)
{
VoldemortAdminRequest request = new VoldemortAdminRequest();
request.type = AdminRequestType.UPDATE_PARTITION_ENTRIES;
request.update_partition_entries = new UpdatePartitionEntriesRequest();
request.update_partition_entries.filter = filter;
request.update_partition_entries.partition_entry = entry;
request.update_partition_entries.store = store;
Serializer.SerializeWithLengthPrefix(iostr, request, PrefixStyle.Fixed32BigEndian);
}
public async Task WhenNewPartitionIsCreated_Then_ItIsRetrievable()
{
string partitionName = "test";
await _fixture.PartitionStore.AddPartitionAsync(partitionName);
PartitionEntry partition = await _fixture.PartitionStore.GetPartitionAsync(partitionName);
Assert.NotNull(partition);
}
private void ProcessRightEvent(PartitionEntry partition, long start, long end, ref TKey key, int hash)
{
if (start >= end)
{
// Row is an end edge, which we don't care about because the start edge would have already
// removed all joining left events.
return;
}
// Mark any matching left intervals as no longer active.
var lim = partition.leftIntervalMap;
var lem = partition.leftEdgeMap;
var leph = partition.leftEndPointHeap;
var leftIntervals = lim.Find(hash);
while (leftIntervals.Next(out int index))
{
long leftStart = lim.Values[index].Start;
if (leftStart < start && this.keyComparerEquals(key, lim.Values[index].Key))
{
// Output end edge.
AddToBatch(
start,
leftStart,
ref lim.Values[index].Key,
ref lim.Values[index].Payload,
hash);
// Remove from heap and map.
leph.Remove(lim.Values[index].HeapIndex);
leftIntervals.Remove();
}
}
// Remove any matching left edges.
var leftEdges = lem.Find(hash);
while (leftEdges.Next(out int index))
{
long leftStart = lem.Values[index].Start;
if (leftStart < start && this.keyComparerEquals(key, lem.Values[index].Key))
{
// Output end edge.
AddToBatch(
start,
leftStart,
ref lem.Values[index].Key,
ref lem.Values[index].Payload,
hash);
// Remove left event.
leftEdges.Remove();
}
}
}
private static void UpdateNextRightTime(PartitionEntry partition, long time)
{
partition.nextRightTime = time;
if (time == StreamEvent.InfinitySyncTime &&
partition.rightEdgeMap.IsInvisibleEmpty &&
partition.rightIntervalMap.IsInvisibleEmpty &&
partition.rightIntervalMap.IsEmpty)
{
partition.isRightComplete = true;
}
}
private void ReachTime(PartitionEntry partition)
{
while (partition.endPointHeap.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index))
{
if (index >= 0)
{
partition.leftIntervalMap.Remove(index);
}
else
{
partition.rightIntervalMap.Remove(~index);
}
}
}
private void ProcessLeftEvent(PartitionEntry partition, long start, ref TGroupKey key, TLeft payload, int hash)
{
bool processable = partition.nextRightTime > start;
if (processable)
{
int index = partition.leftIntervalMap.Insert(hash);
partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload);
CreateOutputForStartInterval(partition, start, ref key, ref payload, hash);
}
else
{
int index = partition.leftIntervalMap.InsertInvisible(hash);
partition.leftIntervalMap.Values[index].Populate(start, ref key, ref payload);
}
}
private void ProcessRightEvent(PartitionEntry partition, long start, TRight payload)
{
bool processable = partition.nextLeftTime > start;
if (processable)
{
int index = partition.rightIntervalMap.Insert(partition.hash);
partition.rightIntervalMap.Values[index].Populate(start, ref payload);
CreateOutputForStartInterval(partition, start, ref payload);
}
else
{
int index = partition.rightIntervalMap.InsertInvisible(partition.hash);
partition.rightIntervalMap.Values[index].Populate(start, ref payload);
}
}
private bool FindOnRight(PartitionEntry partition, ref TKey key, int hash, out int index)
{
var rm = partition.rightMap;
var rightEvents = rm.Find(hash);
while (rightEvents.Next(out int rightIndex))
{
if (this.keyComparerEquals(key, rm.Values[rightIndex].Key))
{
index = rightIndex;
return(true);
}
}
index = 0;
return(false);
}
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, ref TRight payload)
{
// Create end edges for all joined left intervals.
var intervals = partition.leftIntervalMap.Find(partition.hash);
while (intervals.Next(out var index))
{
long rightEnd = currentTime + this.rightDuration;
long leftEnd = partition.leftIntervalMap.Values[index].Start + this.leftDuration;
AddToBatch(
currentTime,
rightEnd < leftEnd ? rightEnd : leftEnd,
partition,
ref partition.leftIntervalMap.Values[index].Payload,
ref payload);
}
}
private void OutputCompletedIntervals(PartitionEntry partition)
{
// Output all completed intervals
var delList = new List <long>();
foreach (var kvp in partition.eventMap)
{
var index = FastDictionary2 <ActiveEvent, int> .IteratorStart;
while (kvp.Value.Iterate(ref index))
{
var outevt = kvp.Value.entries[index].key;
if (outevt.End == StreamEvent.InfinitySyncTime)
{
foreach (var key in delList)
{
partition.eventMap.Remove(key);
}
return;
}
for (int j = 0; j < kvp.Value.entries[index].value; j++)
{
var ind = this.output.Count++;
this.output.vsync.col[ind] = kvp.Key;
this.output.vother.col[ind] = outevt.End;
this.output.key.col[ind] = outevt.Key;
this.output[ind] = outevt.Payload;
this.output.hash.col[ind] = outevt.Hash;
partition.lastSyncTime = kvp.Key;
if (this.output.Count == Config.DataBatchSize)
{
FlushContents();
}
}
kvp.Value.Remove(outevt);
}
this.dictPool.Return(kvp.Value);
delList.Add(kvp.Key);
}
foreach (var key in delList)
{
partition.eventMap.Remove(key);
}
}
static void fixupPartitionEntry(ref PartitionEntry part, long diskLength)
{
if (part.Type != MbrPartitionType.GptProtective)
{
return;
}
long lastSector = (diskLength / Program.SECTOR_SIZE) - 1;
long numberOfSectors = lastSector - part.FirstSectorLba;
if (numberOfSectors >= (long)uint.MaxValue)
{
part.NumberOfSectors = uint.MaxValue;
}
else
{
part.NumberOfSectors = (uint)numberOfSectors;
}
}
private void MakeMatchingLeftInvisible(PartitionEntry partition, long time, ref TKey key, int hash)
{
// Make matching left intervals invisible.
var lem = partition.leftEdgeMap;
var lim = partition.leftIntervalMap;
var leftEvents = lim.Find(hash);
while (leftEvents.Next(out int index))
{
if (this.keyComparerEquals(key, lim.Values[index].Key))
{
// Output end edge.
AddToBatch(
time,
lim.Values[index].CurrentStart,
ref lim.Values[index].Key,
ref lim.Values[index].Payload,
hash);
// Mark left event as not visible.
lim.Values[index].CurrentStart = NotActive;
}
}
// Make matching left edges invisible.
leftEvents = lem.Find(hash);
while (leftEvents.Next(out int index))
{
if (this.keyComparerEquals(key, lem.Values[index].Key))
{
// Output end edge.
AddToBatch(
time,
lem.Values[index].CurrentStart,
ref lem.Values[index].Key,
ref lem.Values[index].Payload,
hash);
// Mark left event as not visible.
lem.Values[index].CurrentStart = NotActive;
}
}
}
private void ReachTime(PartitionEntry partition)
{
// Carry-out all interval endpoints for left intervals that end prior or at new current time.
var lim = partition.leftIntervalMap;
var leph = partition.leftEndPointHeap;
while (leph.TryGetNextInclusive(partition.currTime, out long endPointTime, out int index))
{
// Output end edge.
AddToBatch(
endPointTime,
lim.Values[index].Start,
ref lim.Values[index].Key,
ref lim.Values[index].Payload,
lim.GetHash(index));
// Remove from leftMap.
lim.Remove(index);
}
}
private void CreateOutputForStartInterval(PartitionEntry partition, long currentTime, ref TKey key, ref TLeft payload, int hash)
{
// Create end edges for all joined right intervals.
var intervals = partition.rightIntervalMap.Find(hash);
while (intervals.Next(out var index))
{
if (this.keyComparerEquals(key, partition.rightIntervalMap.Values[index].Key))
{
long leftEnd = currentTime + this.leftDuration;
long rightEnd = partition.rightIntervalMap.Values[index].Start + this.rightDuration;
AddToBatch(
currentTime,
leftEnd < rightEnd ? leftEnd : rightEnd,
ref key,
ref payload,
ref partition.rightIntervalMap.Values[index].Payload,
hash);
}
}
}
public DicomRequestContext(
string method,
Uri uri,
Uri baseUri,
string correlationId,
IDictionary <string, StringValues> requestHeaders,
IDictionary <string, StringValues> responseHeaders)
{
EnsureArg.IsNotNullOrWhiteSpace(method, nameof(method));
EnsureArg.IsNotNull(uri, nameof(uri));
EnsureArg.IsNotNull(baseUri, nameof(baseUri));
EnsureArg.IsNotNull(responseHeaders, nameof(responseHeaders));
Method = method;
Uri = uri;
BaseUri = baseUri;
CorrelationId = correlationId;
RequestHeaders = requestHeaders;
ResponseHeaders = responseHeaders;
DataPartitionEntry = new PartitionEntry(DefaultPartition.Key, DefaultPartition.Name);
}
private void ProcessRightEvent(PartitionEntry partition, long start, long end, ref TKey key, int hash)
{
if (start < end)
{
// Row is a start edge or interval.
var rm = partition.rightMap;
if (FindOnRight(partition, ref key, hash, out int index))
{
// Corresponding key already exists in map, so any joining on left and already not active.
rm.Values[index].Count++;
}
else
{
// First instance of this key, so insert and make any joining left entries not active.
index = rm.Insert(hash);
rm.Values[index].Initialize(ref key);
MakeMatchingLeftInvisible(partition, start, ref key, hash);
}
if (end != StreamEvent.InfinitySyncTime)
{
// Row is an interval, so schedule removal of interval.
var reph = partition.rightEndPointHeap;
reph.Insert(end, index);
}
}
else
{
// Row is an end edge.
// Queue for removal when time advances.
var ree = partition.rightEndEdges;
int index = ree.Push();
ree.Values[index].Populate(ref key, hash);
}
}
