public void PerformGlobalSplit(int leaf, int lteChild, int gtChild, SplitInfo splitInfo)
{
Assert.True(_isInitEnv);
Assert.True(_isInitTreeLearner);
Assert.True(_isInitIteration);
return;
}
private static List <SplitInfo> GetInnerSplitInfos(SplitInfo sinfo)
{
List <SplitInfo> innerSplitInfos = new List <SplitInfo>();
for (int i = sinfo.StartIndex; i < sinfo.NextIndex; i++)
{
int length = Math.Min(7, sinfo.NextIndex - i - 1);
for (int j = 1; j <= length; j++)
{
string text = sinfo.Context.Substring(i, j);
if (IsWord(text))
{
SplitInfo item = new SplitInfo
{
Context = sinfo.Context,
IsWord = true,
Length = j,
StartIndex = i
};
innerSplitInfos.Add(item);
}
}
}
return(innerSplitInfos);
}
private List <SplitInfo> GetDivisionInfo(EditorControl rootCtrl)
{
List <SplitInfo> splitInfoTbl = new List <SplitInfo>();
if (null == rootCtrl)
{
return(splitInfoTbl);
}
EditorCtrlComposite rootComp = rootCtrl as EditorCtrlComposite;
if (null == rootComp)
{
return(splitInfoTbl);
}
if (rootCtrl is SpliterCtrl)
{
SplitInfo newInfo = new SplitInfo();
newInfo.SpliterID = rootCtrl.CtrlID;
newInfo.SpliterPath = rootCtrl.GetCtrlIDPath();
newInfo.SplitOffset = rootCtrl.layoutConstraint.spliterOffset;
newInfo.SpliterOffsetInv = rootCtrl.layoutConstraint.spliterOffsetInv;
splitInfoTbl.Add(newInfo);
}
foreach (var item in rootComp.children)
{
splitInfoTbl.AddRange(GetDivisionInfo(item));
}
return(splitInfoTbl);
}
private void ConvertTreeToGraph(int globalFeatureIndex, int iteration)
{
SplitInfo splitinfo = _leafSplitCandidates.FeatureSplitInfo[globalFeatureIndex];
_subGraph.Splits[globalFeatureIndex][iteration].SplitPoint = splitinfo.Threshold;
_subGraph.Splits[globalFeatureIndex][iteration].LteValue = GamTrainerOptions.LearningRate * splitinfo.LteOutput;
_subGraph.Splits[globalFeatureIndex][iteration].GtValue = GamTrainerOptions.LearningRate * splitinfo.GTOutput;
}
/// <summary>
/// Get the split (dicing) information for a specified original node.
/// </summary>
/// <param name="bn"></param>
/// <returns></returns>
public SplitInfo GetSplitInfo(BNode bn)
{
SplitInfo ret;
if (this.splits.TryGetValue(bn, out ret) == false)
{
ret = new SplitInfo(bn);
this.splits.Add(bn, ret);
}
return(ret);
}
public List <SplitInfo> GetSplitInfo(string path)
{
if (path.Contains(".pack"))
{
List <SplitInfo> SplitList = new List <SplitInfo>();
StreamReader reader = new StreamReader(path);
SplitInfo splitInfo = null;
string imgFileName = "";
while (!reader.EndOfStream)
{
string readStr = reader.ReadLine();
if (readStr.Contains(".png"))
{
imgFileName = readStr;
}
if (readStr.Contains("repeat:") || readStr.Contains("index:"))
{
var name = reader.ReadLine();
if (string.IsNullOrEmpty(name))
{
continue;
}
splitInfo = new SplitInfo();
splitInfo.name = name;
splitInfo.imgName = imgFileName;
}
if (readStr.Contains("xy:"))
{
var values = readStr.Split(':');
var valueStrs = values[1].Split(',');
splitInfo.x = int.Parse(valueStrs[0]);
splitInfo.y = int.Parse(valueStrs[1]);
}
if (readStr.Contains("size:"))
{
var values = readStr.Split(':');
var valueStrs = values[1].Split(',');
splitInfo.width = int.Parse(valueStrs[0]);
splitInfo.height = int.Parse(valueStrs[1]);
SplitList.Add(splitInfo);
splitInfo = null;
}
}
return(SplitList);
}
return(null);
}
private int splitFile(int splits)
{
byte[] file = null;
file = File.ReadAllBytes(this.inputPath);
int chunkSize = file.Length / splits;
byte[] buffer = new byte[chunkSize];
List <byte> extraBuffer = new List <byte>();
int splitIndex = 1;
int fileIndex = 0;
using (Stream input = File.OpenRead(this.inputPath))
{
while (input.Position < input.Length)
{
int bytesRead = input.Read(buffer, 0, chunkSize);
byte extraByte = buffer[chunkSize - 1];
while (extraByte != '\n')
{
int flag = input.ReadByte();
if (flag == -1)
{
break;
}
extraByte = (byte)flag;
extraBuffer.Add(extraByte);
}
byte[] extraBufferArray = extraBuffer.ToArray();
SplitInfo splitinfo = new SplitInfo(fileIndex, buffer.Length + extraBufferArray.Length);
this.splitsDictionary.Add(splitIndex, splitinfo);
Array.Clear(buffer, 0, chunkSize);
extraBuffer.Clear();
fileIndex += bytesRead;
splitIndex++;
}
return(splitIndex - 1);
}
}
private static SplitInfo[] GetSplitInfos(FileInfo file, long splitSize)
{
long l = file.Length;
double t = l / (double)splitSize;
int n = (int)Math.Ceiling((double)t);
var rtn = new SplitInfo[n];
int i = 0;
while (l > 0)
{
rtn[i++] = new SplitInfo(file.Length - l, (int)Math.Min(l, splitSize));
l -= splitSize;
}
return(rtn);
}
/// <summary>
/// Initialize data for the SplitCollection and its use.
/// </summary>
/// <param name="dstLoop">The loop to create new diced nodes in.</param>
/// <param name="delCollisions">The segment intersections detected between two islands.</param>
/// <param name="createdSubdivs">A dictionary that gets filled in with reorganized data
/// from delCollisions.</param>
public void SetupFromCollisionData(
BLoop dstLoop,
List <Utils.BezierSubdivSample> delCollisions,
Dictionary <Utils.NodeTPos, BNode> createdSubdivs)
{
foreach (Utils.BezierSubdivSample bss in delCollisions)
{
Vector2 pos = bss.a.node.CalculatetPoint(bss.a.lEst);
BNode newSubNode = new BNode(dstLoop, pos);
dstLoop.nodes.Add(newSubNode);
createdSubdivs.Add(bss.GetTPosA(), newSubNode);
createdSubdivs.Add(bss.GetTPosB(), newSubNode);
SplitInfo sia = this.GetSplitInfo(bss.a.node);
sia.AddEntry(bss.a.lEst, newSubNode);
SplitInfo sib = this.GetSplitInfo(bss.b.node);
sib.AddEntry(bss.b.lEst, newSubNode);
}
}
public static string getTag(SplitInfo type)
{
switch (type)
{
case SplitInfo.SplitInfo:
return("split-info");
case SplitInfo.FileName:
return("file-name");
case SplitInfo.FileCRC:
return("file-crc");
case SplitInfo.DateCreated:
return("cr-date");
case SplitInfo.FilesCount:
return("files-count");
case SplitInfo.TransUnitIndex:
return("index");
case SplitInfo.File:
return("file");
case SplitInfo.TransUnits:
return("trans-units");
case SplitInfo.Words:
return("words");
case SplitInfo.FileNameAttr:
return("name");
default:
return("");
}
}
public SplitInfoException(SplitInfo si)
{
splitInfo = si;
}
public ImportFromOtherBundle(SplitInfo fromSplit, SourceFile fromFile, string[] name)
{
FromSplit = fromSplit;
FromFile = fromFile;
Name = name;
}
void IParallelTraining.PerformGlobalSplit(int leaf, int lteChild, int gtChild, SplitInfo splitInfo)
{
return;
}
private void FindBestCategoricalSplit(BinInfo parent, double parentInformation, BinInfo[] bins, SplitInfo split,
bool doesDefaultExist, GbmAlgorithmSettings algorithmSettings)
{
throw new NotImplementedException();
}
private void ScanFromLeft(BinInfo parentT, double parentTGain, BinInfo[] binsT, SplitInfo split, bool doesDefaultExist, GbmAlgorithmSettings algorithmSettings)
{
throw new NotImplementedException();
}
public void PerformGlobalSplit(int leaf, int lteChild, int gtChild, SplitInfo splitInfo)
{
return;
}
/// <summary>
/// Copies a <paramref name="source"/> file to a <paramref name="destination"/> splitting it into segments each sized <paramref name="splitSize"/>
/// </summary>
/// <param name="source">Source <see cref="FileInfo"/> to copy</param>
/// <param name="destination">Destination <see cref="FileInfo"/> to copy to</param>
/// <param name="splitSize">Size of split segments</param>
/// <param name="onWrite">Action will be called everytime theres an advance in bytes with the number of bytes as parameter, to track progress. Can be null.</param>
public static void Copy(FileInfo source, FileInfo destination, long splitSize, Action <long>?onWrite = null)
{
var sourceSplit = new SplitFile(source, splitSize);
var destSplit = new SplitFile(destination, splitSize);
int bound;
if (sourceSplit.SplitInfos.Length > destSplit.SplitInfos.Length)
{
// If source is bigger than destination copy from source at last destination pos to end
bound = destSplit.SplitInfos.Length;
SplitInfo infoSource = sourceSplit.SplitInfos[bound];
using var sourceStream = new FileStream(sourceSplit.File.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
using var destStream = new FileStream(destSplit.File.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
sourceStream.Position = infoSource.StartIndex;
destStream.Position = infoSource.StartIndex;
int len = (int)(source.Length - infoSource.StartIndex);
StreamCopy(sourceStream, destStream, len, onWrite);
}
else if (sourceSplit.SplitInfos.Length < destSplit.SplitInfos.Length || sourceSplit.SplitInfos[^ 1].Length < destSplit.SplitInfos[^ 1].Length)
{
// If destination is bigger than source throw away the diff at the end (set length of dest to src)
bound = sourceSplit.SplitInfos.Length;
using var destStream = new FileStream(destSplit.File.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
destStream.SetLength(source.Length);
}
else
{
// they are the same length w/e
bound = sourceSplit.SplitInfos.Length;
}
// loop to bound, check each split for equality and copy over if needed
Parallel.For(0, bound, (i) =>
{
SplitInfo infoSource = sourceSplit.SplitInfos[i];
SplitInfo infoDest = destSplit.SplitInfos[i];
using var sourceStream = new FileStream(sourceSplit.File.FullName, FileMode.Open, FileAccess.Read, FileShare.Read);
using var destStream = new FileStream(destSplit.File.FullName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
sourceStream.Position = infoSource.StartIndex;
destStream.Position = infoSource.StartIndex;
// if they are not the same length or not equal copy from src
if (infoSource.Length != infoDest.Length || !FastCompare.Equals(sourceStream, destStream, infoSource.Length))
{
sourceStream.Position = infoSource.StartIndex;
destStream.Position = infoSource.StartIndex;
StreamCopy(sourceStream, destStream, infoSource.Length, onWrite);
}
else
{
onWrite?.Invoke(infoSource.Length);
}
});
}
