public T FindMedianEndpoint(List <Interval <T> > intervals)
{
var sortedSet = new BDSkipList <T, Interval <T> >();
foreach (var interval in intervals)
{
try
{
sortedSet.Add(interval.Min.Value, interval);
}
catch
{
//Bend.BDSkilpList rasise exception on duplicatation
}
try
{
sortedSet.Add(interval.Max.Value, interval);
}
catch
{
//Bend.BDSkilpList rasise exception on duplicatation
}
}
int medianIndex = sortedSet.Count / 2;
if (sortedSet.Count > 0)
{
return(sortedSet.ToList()[medianIndex].Key);
}
return(default(T));
}
public static void Block_Perftest(IBlockTestFactory factory)
{
// iterate through blocksizes, randomly generating input data, and then doing some
// random key queries to see how fast retrieval is
int[] block_sizes = { 2 * 1024, 40 * 1024, 100 * 1024, 512 * 1025, 2 * 1024 * 1024 };
int[] value_sizes = { 10, 30, 100, 1000, 10000 };
int[] num_levels = { 2, 3, 4 };
int[,] perf_results = new int[block_sizes.Length, value_sizes.Length];
int READ_COUNT = 3000;
Random rnd = new Random((int)DateTime.Now.ToBinary());
foreach (int key_part_count in num_levels) {
System.Console.WriteLine("--");
foreach (int block_size in block_sizes) {
foreach (int value_size in value_sizes) {
if (value_size > (block_size / 8)) {
// we want at least 8 values
continue;
}
System.GC.Collect();
// setup the block for encoding
ISegmentBlockEncoder enc = factory.makeEncoder();
MemoryStream ms = new MemoryStream();
enc.setStream(ms);
int curblock_size = 0;
// do the sorted block create.. we nest it so we can dispose the SkipList
{
var sorted_input = new BDSkipList<RecordKey, RecordUpdate>();
// first create the sorted input
while (curblock_size < block_size) {
// generate a random key
RecordKey key = new RecordKey();
for (int i = 0; i < key_part_count; i++) {
key.appendParsedKey("" + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF));
}
// generate a random value
byte[] data = new byte[value_size];
for (int i = 0; i < value_size; i++) {
data[i] = (byte)rnd.Next(40, 50);
}
RecordUpdate upd = RecordUpdate.WithPayload(data);
curblock_size += key.encode().Length;
curblock_size += value_size;
sorted_input.Add(key, upd);
}
// encode the block
foreach (var kvp in sorted_input) {
enc.add(kvp.Key, kvp.Value);
}
enc.flush();
sorted_input = null; // free the skiplist
}
// init the decoder
ISegmentBlockDecoder dec = factory.makeDecoder(new BlockAccessor(ms.ToArray()));
int num_misses = 0;
System.GC.Collect(); // force GC so it may not happen during the test
// perform random access test
DateTime start = DateTime.Now;
for (int i = 0; i < READ_COUNT; i++) {
RecordKey key = new RecordKey();
for (int ki = 0; ki < key_part_count; ki++) {
key.appendParsedKey("" + rnd.Next(8) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF));
}
try {
dec.FindNext(key, true);
} catch (KeyNotFoundException) {
num_misses++;
// System.Console.WriteLine("misfetch: {0}", key);
// no problem, but this shouuld be small
}
}
double duration_ms = (DateTime.Now - start).TotalMilliseconds;
double reads_per_second = (READ_COUNT * 1000.0) / (duration_ms);
System.Console.WriteLine("BlockSize src{0,10} final{6,10} ratio ({7:0.000}), ValueSize {1,6}, Keyparts {5,3}, {2,6} reads in {3,10:0.0}ms, {8,6} misses, {4,9:0.00} read/sec",
curblock_size, value_size, READ_COUNT, duration_ms, reads_per_second,
key_part_count, ms.Length, ((double)ms.Length / (double)curblock_size) * (double)100.0, num_misses);
}
}
}
}
protected override void OnPaint(PaintEventArgs e)
{
Graphics dc = e.Graphics;
// how to tell we are in design mode
// http://msdn.microsoft.com/en-us/magazine/cc164048.aspx
if (this.Site != null && this.Site.DesignMode) {
dc.Clear(Color.Beige);
return;
}
if (segments == null) {
return;
}
Size regionsize = this.ClientSize;
// compute the data I need first...
var segments_by_generation = new Dictionary<uint, List<SegmentDescriptor>>();
var unique_keys = new BDSkipList<RecordKey, int>();
uint max_gen = 0;
foreach (SegmentDescriptor segdesc in segments) {
unique_keys[segdesc.start_key] = 1;
unique_keys[segdesc.end_key] = 1;
try {
segments_by_generation[segdesc.generation].Add(segdesc);
} catch (KeyNotFoundException) {
var listofsegs = new List<SegmentDescriptor>();
listofsegs.Add(segdesc);
segments_by_generation[segdesc.generation] = listofsegs;
max_gen = segdesc.generation > max_gen ? segdesc.generation : max_gen;
}
}
// figure out what the "maximum number of segments in a column" is, so we can compute segment height
float segment_height = 50;
if (unique_keys.Count > 0) {
segment_height = Math.Max(1, (float)regionsize.Height / (float)unique_keys.Count); // make sure segment heights are at least 1
}
// now draw stuff!
Pen BluePen = new Pen(Color.Blue, 1);
Pen RangeLinePen = new Pen(Color.LightGray, 1);
Pen GenPen = new Pen(Color.LightGreen, 1);
dc.Clear(Color.White);
// assign y-locations to keys
if (unique_keys.Count == 0) {
return;
}
float y_loc = 0;
var key_to_position_map = new Dictionary<RecordKey, float>();
foreach (var key in unique_keys.Keys) {
key_to_position_map[key] = y_loc;
y_loc += segment_height;
}
var segs_in_merge = new HashSet<string>();
if (lastmerge != null) {
foreach (var seg in lastmerge.source_segs) {
segs_in_merge.Add(seg.ToString());
}
foreach (var seg in lastmerge.target_segs) {
segs_in_merge.Add(seg.ToString());
}
}
int cur_x = 20;
for (uint generation = 0; generation <= max_gen; generation++) {
// generation vertical lines
dc.DrawLine(GenPen, cur_x - 10, 0, cur_x - 10, regionsize.Height);
if (!segments_by_generation.ContainsKey(generation)) {
// generation is empty!
cur_x += 10;
continue;
}
foreach (var seg in segments_by_generation[generation]) {
int y_top = (int)key_to_position_map[seg.start_key];
int y_bottom =(int)key_to_position_map[seg.end_key];
int mid_y = (y_top + y_bottom) / 2;
int y_mid_top = (int)(mid_y - (float)segment_height / 2);
// color the inside of the box.
if (lastmerge != null && segs_in_merge.Contains(seg.ToString())) {
// if it's currently merging, color it blue
var fill = new SolidBrush(Color.Red);
dc.FillRectangle(fill, cur_x, y_mid_top, BLOCK_WIDTH, segment_height);
} else {
// pseudorandom color based on a hash of the start/end keys of a block
var h1 = Math.Abs(seg.start_key.ToString().GetHashCode());
var h2 = Math.Abs(seg.end_key.ToString().GetHashCode());
// vary the alpha and colors based on key hashes %N+K makes sure alpha is in a good range
var fill = new SolidBrush(Color.FromArgb((h1 % 60) + 80, (h1 + h2) & 0xff, (h2 >> 8) & 0xff, h2 & 0xff));
dc.FillRectangle(fill, cur_x, y_mid_top, BLOCK_WIDTH, segment_height);
}
if (seg.keyrangeContainsSegmentPointers()) {
dc.FillRectangle(new SolidBrush(Color.Black), cur_x+BLOCK_WIDTH-5, y_mid_top, 5, segment_height);
}
// when we get a lot of blocks, the outline covers 100% of the inside color
//dc.DrawRectangle(BluePen, cur_x, y_mid_top, 50, segment_height);
if (generation != 0 && (y_bottom != y_top + segment_height)) {
dc.DrawLine(RangeLinePen, cur_x, y_mid_top, cur_x - 10, y_top);
dc.DrawLine(RangeLinePen, cur_x, y_mid_top + segment_height, cur_x - 10, y_bottom);
}
}
// reset for next time through the loop
cur_x += 20 + BLOCK_WIDTH;
}
// paint some other stuff
{
int y_pos = 0 + 20;
long memsize = GC.GetTotalMemory(false);
Brush bb = new SolidBrush(Color.Black);
String msg = String.Format("Memsize: {0:0.00} MB", ((double)memsize / 1024 / 1024));
SizeF msgsz = dc.MeasureString(msg, this.Font);
dc.DrawString(msg, this.Font, bb,
new Point(regionsize.Width - (int)msgsz.Width, y_pos + (int)msgsz.Height));
y_pos += (int)msgsz.Height + 10;
int max_gc_generation = GC.MaxGeneration;
msg = "GC generations: " + (max_gc_generation + 1);
msgsz = dc.MeasureString(msg, this.Font);
dc.DrawString(msg, this.Font, bb, new Point(regionsize.Width - (int)msgsz.Width, y_pos + (int)msgsz.Height));
y_pos += (int)msgsz.Height + 10;
for (int x = 0; x <= max_gc_generation; x++) {
int collection_count = GC.CollectionCount(x);
msg = String.Format("[{0}] = [{1}]",x,collection_count);
msgsz = dc.MeasureString(msg, this.Font);
dc.DrawString(msg, this.Font, bb, new Point(regionsize.Width - (int)msgsz.Width, y_pos + (int)msgsz.Height));
y_pos += (int)msgsz.Height + 10;
}
}
}
private void _organizeLogSegments(RootBlockLogSegment[] segments)
{
BDSkipList<long, RootBlockLogSegment> order_segments = new BDSkipList<long, RootBlockLogSegment>();
// (2) scan and organize the log-segments (so they are in proper order)
foreach (RootBlockLogSegment seg in segments) {
Stream logsegment = regionmgr.readRegionAddr(seg.logsegment_start).getNewAccessStream();
BinaryReader br = new BinaryReader(logsegment);
LogPacketHeader hdr = Util.readStruct<LogPacketHeader>(br);
if (hdr.magic != LogPacketHeader.LOG_MAGIC) {
abortCorrupt(String.Format("_organizeLogSegments() found corrupt log segment! magic => {0}", hdr.magic));
}
if (hdr.cmddata_length == 0 && hdr.checksum == 0) {
empty_log_segments.Add(seg);
} else {
order_segments.Add(hdr.curLWSN, seg);
}
logsegment.Close();
}
// now insert the active segments in proper order..
foreach (var kvp in order_segments.scanForward(ScanRange<long>.All())) {
Console.WriteLine("active segment: {0}", kvp.Key);
active_log_segments.Add(kvp.Value);
}
}
private void INTERNAL_segmentWalkForNextKey(
IComparable<RecordKey> startkeytest,
bool direction_is_forward,
ISortedSegment curseg_raw,
RangeKey curseg_rangekey,
IScannableDictionary<RecordKey, RecordData> handledIndexRecords,
int maxgen,
IScannableDictionary<RecordKey, RecordData> recordsBeingAssembled,
bool equal_ok,
SegmentWalkStats stats)
{
// TODO: convert all ISortedSegments to be IScannable
IScannable<RecordKey, RecordUpdate> curseg = (IScannable<RecordKey, RecordUpdate>)curseg_raw;
stats.segmentWalkInvocations++;
// first look in this segment for a next-key **IF** it may contain one
if (curseg_rangekey.directlyContainsKey(startkeytest)) {
// we need to keep looking until we find a live record, as we need all the deletion tombstones
// between startkey and the next live record.
IEnumerable<KeyValuePair<RecordKey, RecordUpdate>> seg_scanner;
if (direction_is_forward) {
seg_scanner = curseg.scanForward(
new ScanRange<RecordKey>(
startkeytest,
new ScanRange<RecordKey>.maxKey(),
null));
} else {
seg_scanner = curseg.scanBackward(
new ScanRange<RecordKey>(
new ScanRange<RecordKey>.minKey(),
startkeytest,
null));
}
foreach (var kvp in seg_scanner) {
if (!equal_ok) { // have ">" test vs ">="
if (startkeytest.CompareTo(kvp.Key) == 0) {
continue;
}
}
RecordData partial_record;
stats.rowAccumulate_TryGet++;
if (!recordsBeingAssembled.TryGetValue(kvp.Key, out partial_record)) {
partial_record = new RecordData(RecordDataState.NOT_PROVIDED, kvp.Key);
recordsBeingAssembled[kvp.Key] = partial_record;
} else {
stats.rowDuplicatesAppeared++;
}
partial_record.applyUpdate(kvp.Value);
stats.rowUpdatesApplied++;
#if DEBUG_SEGMENT_ACCUMULATION
for (int depth = 10; depth > maxgen; depth--) { Console.Write(" "); }
Console.WriteLine("accumulated update: {0}", kvp);
#endif
if (partial_record.State != RecordDataState.DELETED) {
// we accumulated to at least one live record, so stop adding potential records
break;
}
}
}
// find all generation range references that are relevant for this key
// .. make a note of which ones are "current"
if (curseg_rangekey.directlyContainsKey(GEN_KEY_PREFIX)) {
BDSkipList<RecordKey, RecordUpdate> todo_list = new BDSkipList<RecordKey, RecordUpdate>();
for (int i = maxgen - 1; i >= 0; i--) {
stats.segmentRangeRowScansPerformed++;
foreach (KeyValuePair<RecordKey, RecordUpdate> rangerow in RangeKey.findAllEligibleRangeRows(curseg, startkeytest, i, stats)) {
// see if it is new for our handledIndexRecords dataset
RecordData partial_rangedata;
stats.segmentAccumulate_TryGet++;
if (!handledIndexRecords.TryGetValue(rangerow.Key, out partial_rangedata)) {
partial_rangedata = new RecordData(RecordDataState.NOT_PROVIDED, rangerow.Key);
handledIndexRecords[rangerow.Key] = partial_rangedata;
}
if ((partial_rangedata.State == RecordDataState.INCOMPLETE) ||
(partial_rangedata.State == RecordDataState.NOT_PROVIDED)) {
// we're suppilying new data for this index record
partial_rangedata.applyUpdate(rangerow.Value);
stats.segmentUpdatesApplied++;
// because we're suppilying new data, we should add this to our
// private TODO list if it is a FULL update, NOT a tombstone
if (rangerow.Value.type == RecordUpdateTypes.FULL) {
#if DEBUG_SEGMENT_RANGE_WALK
for (int depth = 10; depth > maxgen; depth--) { Console.Write(" "); }
Console.WriteLine("adding SegmentRangeRow: {0}", rangerow);
#endif
todo_list.Add(rangerow);
}
}
}
}
// now repeat the walk through our todo list:
foreach (KeyValuePair<RecordKey, RecordUpdate> rangepointer in todo_list.scanBackward(null)) {
if (rangepointer.Value.type == RecordUpdateTypes.DELETION_TOMBSTONE) {
// skip deletion tombstones
stats.segmentDeletionTombstonesSkipped++;
continue;
}
SegmentReader next_seg = segmentReaderFromRow(rangepointer);
RangeKey next_seg_rangekey = RangeKey.decodeFromRecordKey(rangepointer.Key);
#if DEBUG_SEGMENT_WALK
for (int depth = 10; depth > maxgen; depth--) { Console.Write(" "); }
Console.WriteLine("..WalkForNextKey descending to: {0}", rangepointer);
#endif
// RECURSE
INTERNAL_segmentWalkForNextKey(
startkeytest,
direction_is_forward,
next_seg,
next_seg_rangekey,
handledIndexRecords,
maxgen - 1,
recordsBeingAssembled,
equal_ok,
stats);
}
// now repeat the walk of range references in this segment, this time actually descending
}
}
public GetStatus getNextRecord_LowLevel_OLD(
IComparable<RecordKey> lowkey,
bool direction_is_forward,
ref RecordKey key,
ref RecordData record,
bool equal_ok,
bool tombstone_ok)
{
SegmentWalkStats stats = new SegmentWalkStats();
BDSkipList<RecordKey, RecordData> handledIndexRecords = new BDSkipList<RecordKey, RecordData>();
BDSkipList<RecordKey, RecordData> recordsBeingAssembled = new BDSkipList<RecordKey, RecordData>();
#if DEBUG_SEGMENT_WALK
Console.WriteLine("getNextRecord_LowLevel({0})", lowkey);
#endif
SegmentMemoryBuilder[] layers;
// snapshot the working segment layers
lock (this.store.segmentlayers) {
layers = this.store.segmentlayers.ToArray();
}
DateTime start = DateTime.Now;
// TODO: fix this super-hack to do "minKey/maxKey"
foreach (SegmentMemoryBuilder layer in layers) {
if (layer.RowCount == 0) {
continue;
}
// use the first and last records in the segment as the rangekeys
var segrk = RangeKey.newSegmentRangeKey(
layer.FindNext(null, true).Key,
layer.FindPrev(null, true).Key,
num_generations_persisted);
INTERNAL_segmentWalkForNextKey(
lowkey,
direction_is_forward,
layer,
segrk,
handledIndexRecords,
num_generations_persisted,
recordsBeingAssembled,
equal_ok,
stats: stats);
}
DateTime end = DateTime.Now;
#if DEBUG_SEGMENT_WALK_COUNTERS
Console.WriteLine("getNextRecord({0}) took {1}ms", lowkey, (((end-start).TotalMilliseconds)));
Console.WriteLine(stats);
#endif
// now check the assembled records list
try {
IEnumerable<KeyValuePair<RecordKey, RecordData>> assembled_candidates;
if (direction_is_forward) {
assembled_candidates = recordsBeingAssembled.scanForward(null);
} else {
assembled_candidates = recordsBeingAssembled.scanBackward(null);
}
foreach (var kvp in assembled_candidates) {
if (kvp.Value.State == RecordDataState.FULL) {
key = kvp.Key;
record = kvp.Value;
return GetStatus.PRESENT;
} else if (kvp.Value.State == RecordDataState.DELETED) {
if (tombstone_ok) {
key = kvp.Key;
record = kvp.Value;
return GetStatus.PRESENT;
}
} else {
throw new Exception("invalid record state in getNextRecord, record assembly processing: " +
kvp.Value.State + " k:" + kvp.Key + " v:" + kvp.Value);
}
}
return GetStatus.MISSING;
} catch (KeyNotFoundException) {
return GetStatus.MISSING;
}
}
private void INTERNAL_segmentWalkCursorSetupForNextKey(
IComparable<RecordKey> startkeytest,
bool direction_is_forward,
ISortedSegment curseg_raw,
RangeKey curseg_rangekey,
IScannableDictionary<RecordKey, RecordData> handledIndexRecords,
int maxgen,
IScannableDictionary<RangeKey, IScannable<RecordKey, RecordUpdate>> segmentsWithRecords,
bool equal_ok,
SegmentWalkStats stats)
{
// TODO: convert all ISortedSegments to be IScannable
IScannable<RecordKey, RecordUpdate> curseg = (IScannable<RecordKey, RecordUpdate>)curseg_raw;
stats.segmentWalkInvocations++;
// first look in this segment for a next-key **IF** it may contain one
if (curseg_rangekey.directlyContainsKey(startkeytest)) {
// add the current segment to the list of segments with records.
segmentsWithRecords.Add(curseg_rangekey,curseg);
}
// find all generation range references that are relevant for this key
// .. make a note of which ones are "current"
if (curseg_rangekey.directlyContainsKey(GEN_KEY_PREFIX)) {
BDSkipList<RecordKey, RecordUpdate> todo_list = new BDSkipList<RecordKey, RecordUpdate>();
if (curseg_rangekey.generation > stats.handlingGeneration) {
throw new Exception("cursor segup generation priority inversion");
}
stats.handlingGeneration = curseg_rangekey.generation;
for (int i = maxgen - 1; i >= 0; i--) {
stats.segmentRangeRowScansPerformed++;
foreach (KeyValuePair<RecordKey, RecordUpdate> rangerow in RangeKey.findAllEligibleRangeRows(curseg, startkeytest, i, stats)) {
// see if it is new for our handledIndexRecords dataset
RecordData partial_rangedata;
stats.segmentAccumulate_TryGet++;
if (!handledIndexRecords.TryGetValue(rangerow.Key, out partial_rangedata)) {
partial_rangedata = new RecordData(RecordDataState.NOT_PROVIDED, rangerow.Key);
handledIndexRecords[rangerow.Key] = partial_rangedata;
}
if ((partial_rangedata.State == RecordDataState.INCOMPLETE) ||
(partial_rangedata.State == RecordDataState.NOT_PROVIDED)) {
// we're suppilying new data for this index record
partial_rangedata.applyUpdate(rangerow.Value);
stats.segmentUpdatesApplied++;
// because we're suppilying new data, we should add this to our
// private TODO list if it is a FULL update, NOT a tombstone
if (rangerow.Value.type == RecordUpdateTypes.FULL) {
#if DEBUG_SEGMENT_RANGE_WALK
for (int depth = 10; depth > maxgen; depth--) { Console.Write(" "); }
Console.WriteLine("adding SegmentRangeRow: {0}", rangerow);
#endif
todo_list.Add(rangerow);
}
}
}
}
// now repeat the walk through our todo list:
foreach (KeyValuePair<RecordKey, RecordUpdate> rangepointer in todo_list.scanBackward(null)) {
if (rangepointer.Value.type == RecordUpdateTypes.DELETION_TOMBSTONE) {
// skip deletion tombstones
stats.segmentDeletionTombstonesSkipped++;
continue;
}
SegmentReader next_seg = segmentReaderFromRow(rangepointer);
RangeKey next_seg_rangekey = RangeKey.decodeFromRecordKey(rangepointer.Key);
#if DEBUG_SEGMENT_WALK
for (int depth = 10; depth > maxgen; depth--) { Console.Write(" "); }
Console.WriteLine("..WalkForNextKey descending to: {0}", rangepointer);
#endif
// RECURSE
INTERNAL_segmentWalkCursorSetupForNextKey(
startkeytest,
direction_is_forward,
next_seg,
next_seg_rangekey,
handledIndexRecords,
maxgen - 1,
segmentsWithRecords,
equal_ok,
stats);
}
// now repeat the walk of range references in this segment, this time actually descending
}
}