public static DiskNode LoadNode(LockedNodeFileStream backingStream, long offset, bool memoryManaged = false)
{
if (offset == 0)
{
return(null);
}
if (!CacheNodes)
{
if (RootNode is null)
{
RootNode = new DiskNode(backingStream, DiskNode.HEADER_BYTES);
foreach (long l in RootNode.ChildOffsets)
{
SmallCache.Add(l);
}
}
if (offset == DiskNode.HEADER_BYTES)
{
return(RootNode);
}
if (!RootNodes.TryGetValue(offset, out DiskNode thisNode))
{
thisNode = new DiskNode(backingStream, offset);
if (SmallCache.Contains(offset) || SmallCache.Contains(thisNode.ParentOffset))
{
RootNodes.TryAdd(offset, thisNode);
}
}
return(thisNode);
}
if (MemoryManaged.TryGetValue(offset, out DiskNode dn))
{
return(dn);
}
else if (memoryManaged && CurrentCacheCount < MaxCacheCount)
{
dn = new DiskNode(_backingStream, offset);
CurrentCacheCount++;
MemoryManaged.Add(offset, dn);
return(dn);
}
if (!Cache.TryGetValue(offset, out DiskNode toReturn))
{
toReturn = new DiskNode(backingStream, offset);
if (CurrentCacheCount < MaxCacheCount && Cache.TryAdd(offset, toReturn))
{
CurrentCacheCount++;
}
}
return(toReturn ?? new DiskNode(_backingStream, offset));
}
public OptimizedRootNode(DiskNode source, DataSourceSettings settings)
{
if (source is null)
{
throw new ArgumentNullException(nameof(source));
}
Settings = settings;
int MaxHeader = 0;
IEnumerable <DiskNode> Parents = source.Next.Cast <DiskNode>().Where(n => !(n is null));
IEnumerable <DiskNode> Children = Parents.SelectMany(n => n.Next.Cast <DiskNode>()).Where(n => !(n is null));
foreach (DiskNode n in Parents)
{
MaxHeader = Math.Max(MaxHeader, n.ChildHeader);
}
int[] MaxValues = new int[MaxHeader + 1];
next = new List <INodeBlock> [MaxValues.Length][];
foreach (DiskNode n in Parents)
{
MaxValues[n.ChildHeader] = Math.Max(MaxValues[n.ChildHeader], n.ChildCount + n.SkipChildren);
}
for (int header = 0; header <= MaxHeader; header++)
{
next[header] = new List <INodeBlock> [MaxValues[header]];
for (int value = 0; value < MaxValues[header]; value++)
{
next[header][value] = new List <INodeBlock>();
}
}
int cbCount = 0;
foreach (DiskNode c in Children)
{
if (c.NextOffset == 0)
{
next[c.Header][c.Value].Add(c);
NoCache.Add(c.Offset);
}
else
{
next[c.Header][c.Value].Add(new NodeBlock()
{
NextOffset = c.NextOffset, Offset = c.Offset, Index = cbCount++
});
c.Clear();
}
}
CachedBytes = new ByteCache[cbCount];
DiskNode.FlushCache();
GC.Collect();
}
public async Task Preload(string FilePath)
{
await Task.Run(() =>
{
try
{
Dictionary <long, NodeBlock> nodeblocks = new Dictionary <long, NodeBlock>();
for (int header = 0; header < next.Length; header++)
{
for (int value = 0; value < next[header].Length; value++)
{
foreach (INodeBlock inb in next[header][value])
{
if (inb is NodeBlock nb)
{
nodeblocks.Add(nb.Offset, nb);
}
}
}
}
using (LockedNodeFileStream ns = new LockedNodeFileStream(new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.RandomAccess)))
{
using (FileStream fs = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.RandomAccess))
{
using (FileStream fsn = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.RandomAccess))
{
byte[] offsetBytes = new byte[DiskNode.HEADER_BYTES];
fs.Read(offsetBytes, 0, offsetBytes.Length);
long JsonOffset = offsetBytes.GetLong(0);
long SortOffset = offsetBytes.GetLong(8);
fs.Seek(SortOffset, SeekOrigin.Begin);
ulong freeMem = SystemInterop.Memory.Status.ullAvailPhys;
while (freeMem > this.Settings.MinFreeMemory + this.Settings.RangeFreeMemory)
{
for (int i = 0; i < this.Settings.PreloadChunkSize / 2; i++)
{
if (fs.Position >= JsonOffset)
{
return;
}
byte[] thisNodeBytes = new byte[5];
fs.Read(thisNodeBytes, 0, thisNodeBytes.Length);
long offset = thisNodeBytes.GetInt40();
if (!NoCache.Contains(offset))
{
if (!nodeblocks.TryGetValue(offset, out NodeBlock nb))
{
continue;
}
DiskNode dn = new DiskNode(ns, offset);
if (dn.NextOffset != 0)
{
long bLength = dn.NextOffset - dn.Offset;
ByteCache cachedBytes = new ByteCache()
{
Data = new byte[bLength]
};
fsn.Seek(dn.Offset, SeekOrigin.Begin);
fsn.Read(cachedBytes.Data, 0, (int)bLength);
CachedBytes[nb.Index] = cachedBytes;
freeMem -= (ulong)cachedBytes.Data.Length;
}
}
if (freeMem < this.Settings.MinFreeMemory + this.Settings.RangeFreeMemory)
{
return;
}
}
freeMem = SystemInterop.Memory.Status.ullAvailPhys;
}
}
}
}
}
catch (Exception ex)
{
if (Debugger.IsAttached)
{
Debugger.Break();
}
}
});
}
public override void Evaluate(Evaluation e, long routeKey, bool MultiThread = true)
{
ExecutingEvaluations++;
try
{
if (e is null)
{
throw new ArgumentNullException(nameof(e));
}
string FilePath = DiskNode._backingStream.FilePath;
object streamLock = new object();
using (FileStream fs = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.RandomAccess))
{
List <INodeBlock> matchingNodes = new List <INodeBlock>(LastMatchAmount);
for (int header = 0; header < next.Length; header++)
{
int value = e.DataRow[header];
if (next[header].Length > value)
{
matchingNodes.AddRange(next[header][value]);
}
}
LastMatchAmount = Math.Max(LastMatchAmount, matchingNodes.Count);
void Evaluate(INodeBlock n)
{
if (n is DiskNode dn)
{
dn.Evaluate(e, 0, MultiThread);
}
else if (n is NodeBlock nb)
{
long bLength = n.NextOffset - n.Offset;
if (!ArrayPool.TryDequeue(out byte[] backingData) || backingData.Length < bLength)
{
backingData = new byte[bLength];
}
ByteCache cachedBytes = CachedBytes[nb.Index];
if (cachedBytes.Data is null)
{
cachedBytes = new ByteCache()
{
Data = new byte[bLength]
};
lock (streamLock)
{
fs.Seek(n.Offset, SeekOrigin.Begin);
fs.Read(cachedBytes.Data, 0, (int)bLength);
}
CachedBytes[nb.Index] = cachedBytes;
}
cachedBytes.SetLast();
cachedBytes.Data.CopyTo(backingData, 0);
DiskNode nn = new DiskNode(backingData, n.Offset, n.Offset);
nn.Evaluate(e, 0, MultiThread);
}
}
if (MultiThread)
{
Parallel.ForEach(matchingNodes, Evaluate);
}
else
{
foreach (INodeBlock nb in matchingNodes)
{
Evaluate(nb);
}
}
}
}
finally
{
ExecutingEvaluations--;
FlushMemory();
}
}
