public static int GetTotalSampleCount(string file)
{
using (var stream = new FileStream(file, FileMode.Open, FileAccess.Read))
{
int count = 0;
while (stream.Position < stream.Length)
{
var dss = MsgPackSerializer.Deserialize(stream);
count += dss.SampleCount;
}
return(count);
}
}
protected override void BeginProcessing()
{
var path = IO.GetAbsolutePath(this, Path);
using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read))
{
for (var i = 0; i < TotalCount; ++i)
{
var dss = MsgPackSerializer.Deserialize(stream);
WriteObject(dss);
if (stream.Length == stream.Position)
{
break;
}
}
}
}
protected override void BeginProcessing()
{
var path = IO.GetAbsolutePath(this, Path);
var fileMode = Append ? FileMode.Append : FileMode.Create;
using (var stream = new FileStream(path, fileMode, FileAccess.Write))
{
var count = DataSourceSet.SampleCount;
for (var i = 0; i < count; i += SplitSize)
{
var size = Math.Min(SplitSize, count - i);
if (OmitFraction && size < SplitSize)
{
break;
}
var chunk = DataSourceSet.Subset(i, size);
MsgPackSerializer.Serialize(chunk, stream);
}
}
}
protected override void BeginProcessing()
{
Path = IO.GetAbsolutePath(this, Path);
var total = MsgPackTools.GetTotalSampleCount(Path);
// Convert ratios into sample counts
if (ParameterSetName == "ratios")
{
SampleCounts = new int[Ratios.Length];
for (var i = 0; i < Ratios.Length; ++i)
{
if (Ratios[i] == -1)
{
continue;
}
SampleCounts[i] = (int)(Ratios[i] * total);
}
}
// Fix sample counts
if (OutFiles.Length == SampleCounts.Length)
{
for (var i = 0; i < SampleCounts.Length; ++i)
{
if (SampleCounts[i] == -1)
{
SampleCounts[i] = total - (SampleCounts.Sum() + 1);
break;
}
}
}
else if (OutFiles.Length == SampleCounts.Length + 1)
{
var lastCount = total - SampleCounts.Sum();
var newSamples = new int[SampleCounts.Length + 1];
SampleCounts.CopyTo(newSamples, 0);
newSamples[newSamples.Length - 1] = lastCount;
SampleCounts = newSamples;
}
else
{
WriteError(new ErrorRecord(new ArgumentException("The number of SampleCounts/Ratios should match the number of OutFiles"), "", ErrorCategory.InvalidArgument, null));
return;
}
for (var i = 0; i < SampleCounts.Length; ++i)
{
WriteVerbose(string.Format("{0}-th sample count: {1}", i, SampleCounts[i]));
}
// Get absolute paths
for (var i = 0; i < OutFiles.Length; ++i)
{
OutFiles[i] = IO.GetAbsolutePath(this, OutFiles[i]);
}
// Write partial files
var dssEnum = MsgPackTools.ReadDataSourceSet(Path, total, SplitCount).GetEnumerator();
for (var i = 0; i < OutFiles.Length; ++i)
{
using (var outStream = new FileStream(OutFiles[i], FileMode.Create, FileAccess.Write))
{
var count = 0;
while (count < SampleCounts[i])
{
if (!dssEnum.MoveNext())
{
break;
}
var dss = dssEnum.Current;
count += dss.SampleCount;
MsgPackSerializer.Serialize(dss, outStream);
}
}
}
}