public DataView GetView(DataSourceSet set, int subset)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset);
return(dataView);
}
public void TestMultipleObjects()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3 }, new int[] { 3, 1, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
var dss2 = new DataSourceSet();
var b = DataSourceFactory.Create(new float[] { 4, 5, 6, 7 }, new int[] { 2, 2, 1 });
dss2.Add("b", b);
var stream = new MemoryStream();
MsgPackSerializer.Serialize(dss, stream);
MsgPackSerializer.Serialize(dss2, stream);
stream.Position = 0;
var result = MsgPackSerializer.Deserialize(stream);
var result2 = MsgPackSerializer.Deserialize(stream);
Assert.AreEqual(1, result.Features.Count);
var x = result["a"];
CollectionAssert.AreEqual(new int[] { 3, 1, 1 }, x.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 1, 2, 3 }, x.Data.ToArray());
Assert.AreEqual(1, result2.Features.Count);
var y = result2["b"];
CollectionAssert.AreEqual(new int[] { 2, 2, 1 }, y.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 4, 5, 6, 7 }, y.Data.ToArray());
}
public void TestDataSourceCTFBuilider()
{
var writer = new StringWriter();
var ds1 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }, new int[] { 2, 3, 3 });
var ds2 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5 }, new int[] { 2, 1, 3 });
var dss = new DataSourceSet();
dss.Features.Add("data", ds1);
dss.Features.Add("label", ds2);
DataSourceSetCTFBuilder.Write(writer, dss, true);
var s = writer.ToString();
var expected =
"0\t|data 0 1\t|label 0 1\r\n" +
"0\t|data 2 3\r\n" +
"0\t|data 4 5\r\n" +
"1\t|data 6 7\t|label 2 3\r\n" +
"1\t|data 8 9\r\n" +
"1\t|data 10 11\r\n" +
"2\t|data 12 13\t|label 4 5\r\n" +
"2\t|data 14 15\r\n" +
"2\t|data 16 17";
Assert.AreEqual(expected, s);
}
public static Expression GetMappingOrNull(
MappingCreationContext context,
out bool isConditional)
{
var mappingData = context.MappingData;
var derivedTypeDataSources = DerivedComplexTypeDataSourcesFactory.CreateFor(mappingData);
if (derivedTypeDataSources.None())
{
isConditional = false;
return(null);
}
var derivedTypeDataSourceSet = DataSourceSet.For(
derivedTypeDataSources,
mappingData,
ValueExpressionBuilders.ValueSequence);
isConditional =
derivedTypeDataSources.Last().IsConditional&&
!mappingData.MapperData.TargetType.IsAbstract();
if (!isConditional)
{
context.MappingComplete = true;
}
return(derivedTypeDataSourceSet.BuildValue());
}
public void ProcessTraceAsFolder()
{
// Input data
string[] lttngData = { @"..\..\..\..\TestData\LTTng\lttng-kernel-trace.ctf" };
string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
using (var zipFile = ZipFile.OpenRead(lttngData[0]))
{
zipFile.ExtractToDirectory(tempDirectory);
}
using (var dataSourceSet = DataSourceSet.Create())
{
var ds = new DirectoryDataSource(tempDirectory);
dataSourceSet.AddDataSource(ds);
// Approach #1 - Engine - Doesn't test tables UI but tests processing
using (var runtime = Engine.Create(new EngineCreateInfo(dataSourceSet.AsReadOnly())))
{
//
// We do not assert that any cookers are enabled since we did not explicitly enable cookers here
//
Assert.IsTrue(ds.IsDirectory());
Assert.IsTrue(runtime.AvailableTables.Count() >= 1);
}
}
Directory.Delete(tempDirectory, true);
}
public static IMemberPopulator WithoutRegistration(
IChildMemberMappingData mappingData,
DataSourceSet dataSources,
Expression populateCondition = null)
{
return(new MemberPopulator(dataSources, populateCondition));
}
public void TestMsgPackSamplerMultipleFiles()
{
var files = new string[2];
for (var i = 0; i < 2; ++i)
{
files[i] = Path.GetTempFileName();
var a = DataSourceFactory.Create(new float[] { i, i + 1, i + 2 }, new int[] { 3, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
using (var stream = new FileStream(files[i], FileMode.Create, FileAccess.Write))
{
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(files, 3, false, -1, 10, false, 10))
{
Assert.AreEqual(2, sampler.SampleCountPerEpoch);
var batch = sampler.GetNextMinibatch();
CollectionAssert.AreEqual(new int[] { 3, 3 }, batch.Features["a"].Shape.Dimensions.ToArray());
var values = DataSourceFactory.FromValue(batch.Features["a"]);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 1, 2, 3, 0, 1, 2 }, values.TypedData);
}
}
public void MoveRowToSubset(DataRow row, DataSourceSet set, int subset)
{
row["@SET"] = set;
row["@SUBSET"] = subset;
HasChanges = true;
}
private void copySelection(DataGridViewRow viewRow, DataSourceSet set)
{
DataRow row = (viewRow.DataBoundItem as DataRowView).Row;
if (row != null)
{
tableSource.CopyRowToSet(row, set);
}
}
private static IDataSourceSet CreateNullDataSourceSet(
IMemberMapperData mapperData,
Func <QualifiedMember, string> commentFactory)
{
return(DataSourceSet.For(
new NullDataSource(
ReadableExpression.Comment(commentFactory.Invoke(mapperData.TargetMember))),
mapperData));
}
private void moveSelection(DataGridViewRow viewRow, DataSourceSet set, int subset)
{
DataRow row = (viewRow.DataBoundItem as DataRowView).Row;
if (row != null)
{
tableSource.MoveRowToSubset(row, set, subset);
}
}
public static IDataSourceSet CreateFor(IObjectMappingData rootMappingData)
{
var rootDataSourceFactory = _mappingRootDataSourceFactories
.First(rootMappingData, (rmd, mef) => mef.IsFor(rmd));
var rootDataSource = rootDataSourceFactory.CreateFor(rootMappingData);
return(DataSourceSet.For(rootDataSource, rootMappingData.MapperData));
}
public static IDataSourceSet CreateFor(DataSourceFindContext findContext)
{
var validDataSources = EnumerateDataSources(findContext).ToArray();
return(DataSourceSet.For(
validDataSources,
findContext,
ValueExpressionBuilders.ValueTree));
}
public object GetData(DataSourceSet set, int subset)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset);
DataTable newTable = dataView.ToTable(false);
return(newTable);
}
public object GetData(string[] columns, DataSourceSet set)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}'", (int)set);
DataTable newTable = dataView.ToTable(false, columns);
return(newTable);
}
private static IDataSource GetMapFromConditionOrDefaultDataSource(
IObjectMappingData declaredTypeMappingData,
Expression condition,
DerivedSourceTypeCheck derivedSourceCheck,
IEnumerable <TypePairGroup> typePairGroups,
Type targetType)
{
var declaredTypeMapperData = declaredTypeMappingData.MapperData;
var typePairDataSources = new List <IDataSource>();
Expression derivedTypeMapping;
IObjectMappingData derivedTypeMappingData;
foreach (var typePairGroup in typePairGroups)
{
var typePairsCondition =
declaredTypeMapperData.GetTypePairsCondition(typePairGroup.TypePairs) ??
declaredTypeMapperData.GetTargetValidCheckOrNull(typePairGroup.DerivedTargetType);
derivedTypeMapping = GetReturnMappingResultExpression(
declaredTypeMappingData,
derivedSourceCheck.TypedVariable,
typePairGroup.DerivedTargetType,
out derivedTypeMappingData);
var typePairDataSource = new DerivedComplexTypeDataSource(
derivedTypeMappingData.MapperData.SourceMember,
typePairsCondition,
derivedTypeMapping);
typePairDataSources.Add(typePairDataSource);
}
var derivedTargetTypeDataSources = DataSourceSet.For(
typePairDataSources,
declaredTypeMapperData,
ValueExpressionBuilders.ValueSequence);
derivedTypeMapping = GetReturnMappingResultExpression(
declaredTypeMappingData,
derivedSourceCheck.TypedVariable,
targetType,
out derivedTypeMappingData);
var derivedTypeMappings = Block(
derivedTargetTypeDataSources.BuildValue(),
derivedTypeMapping);
return(new DerivedComplexTypeDataSource(
derivedTypeMappingData.MapperData.SourceMember,
derivedSourceCheck,
condition,
derivedTypeMappings,
declaredTypeMapperData));
}
public static IMemberPopulator WithRegistration(
IChildMemberMappingData mappingData,
DataSourceSet dataSources,
Expression populateCondition)
{
var memberPopulation = WithoutRegistration(mappingData, dataSources, populateCondition);
memberPopulation.MapperData.RegisterTargetMemberDataSourcesIfRequired(dataSources);
return(memberPopulation);
}
public MemberPopulation(
IMemberMapperData mapperData,
DataSourceSet dataSources,
Expression populateCondition = null)
{
MapperData = mapperData;
_dataSources = dataSources;
_populateCondition = populateCondition;
mapperData.Parent.RegisterTargetMemberDataSourcesIfRequired(mapperData.TargetMember, dataSources);
}
public static IMemberPopulator NoDataSources(MemberPopulationContext context)
{
var noDataSourcesMessage = CreateNullPopulatorDescription(
GetNoDataSourcesMessage,
context.MemberMapperData);
var noDataSource = new NullDataSource(noDataSourcesMessage);
var noDataSources = DataSourceSet.For(noDataSource, context);
context.MemberMapperData.RegisterTargetMemberDataSources(noDataSources);
return(CreateNullMemberPopulator(noDataSourcesMessage, context, (msg, md) => msg));
}
/// <summary>
/// Imports only matching columns from a DataTable
/// </summary>
/// <param name="table"></param>
public void Import(DataTable table, DataSourceSet set)
{
// Add a new column to the to-be-imported DataTable
// to determine which will be the default set when importing
DataColumn col = new DataColumn("@SET", typeof(DataSourceSet));
col.DefaultValue = set;
table.Columns.Add(col);
// Merge the two
dataTable.Merge(table, true, MissingSchemaAction.Ignore);
HasChanges = true;
}
public void CopyRowToSet(DataRow row, DataSourceSet set)
{
DataRow newRow = row.Table.NewRow();
foreach (DataColumn col in row.Table.Columns)
{
newRow[col] = row[col];
}
newRow["@SET"] = set;
row.Table.Rows.Add(newRow);
HasChanges = true;
}
public void TestReadDataSourceSet()
{
const int NUM_SAMPLES = 10;
DataSourceSet dss;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_SAMPLES; ++i)
{
var a = DataSourceFactory.Create(new float[] { i, i * 10, i * 100 }, new int[] { 3, 1, 1 });
dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
var total = MsgPackTools.GetTotalSampleCount(file);
Assert.AreEqual(NUM_SAMPLES, total);
var reader = MsgPackTools.ReadDataSourceSet(file, total, 4).GetEnumerator();
var hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(4, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 0, 0, 0, 1, 10, 100, 2, 20, 200, 3, 30, 300 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(4, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 4, 40, 400, 5, 50, 500, 6, 60, 600, 7, 70, 700 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(2, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 2 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 8, 80, 800, 9, 90, 900 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(false, hasNext);
}
private Expression GetPopulation(
Expression elementMapping,
QualifiedMember dictionaryEntryMember,
IObjectMappingData mappingData)
{
var elementMapperData = new ChildMemberMapperData(dictionaryEntryMember, MapperData);
var sourceMember = mappingData.MapperData.SourceMember;
var mappingDataSource = new AdHocDataSource(sourceMember, elementMapping);
var dataSourceSetInfo = new SimpleDataSourceSetInfo(mappingData.MappingContext, elementMapperData);
var mappingDataSources = DataSourceSet.For(mappingDataSource, dataSourceSetInfo);
var populator = new MemberPopulator(mappingDataSources, elementMapperData);
var populationExpression = populator.GetPopulation();
return(populationExpression);
}
private Expression GetPopulation(
Expression elementMapping,
QualifiedMember dictionaryEntryMember,
IObjectMappingData mappingData)
{
var elementMapperData = new ChildMemberMapperData(dictionaryEntryMember, MapperData);
var sourceMember = mappingData.MapperData.SourceMember;
var mappingDataSource = new AdHocDataSource(sourceMember, elementMapping);
var mappingDataSources = DataSourceSet.For(mappingDataSource, elementMapperData);
var populationExpression = MemberPopulator
.WithoutRegistration(mappingDataSources)
.GetPopulation();
return(populationExpression);
}
public void TestMsgPackSamplerSlicing2()
{
const int NUM_SAMPLES = 300;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_SAMPLES; ++i)
{
var a = DataSourceFactory.Create(new float[] { i, i, i }, new int[] { 1, 1, 3 });
var dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(file, 5, false, 7, 10, false, 100))
{
int count = 0;
for (var i = 0; i < 10; ++i)
{
for (var j = 0; j < NUM_SAMPLES * 3 / 5; ++j)
{
var batch = sampler.GetNextMinibatch();
Assert.AreEqual(1, batch.Features.Count);
Assert.IsTrue(batch.Features.ContainsKey("a"));
var value = batch["a"];
CollectionAssert.AreEqual(new int[] { 1, 1, 5 }, value.Shape.Dimensions.ToArray());
var ds = DataSourceFactory.FromValue(value).ToArray();
Debug.WriteLine(string.Join(", ", ds));
CollectionAssert.AreEqual(new float[]
{
j * 5 / 3,
(j * 5 + 1) / 3,
(j * 5 + 2) / 3,
(j * 5 + 3) / 3,
(j * 5 + 4) / 3
}, ds);
count += 5;
}
}
}
}
private void moveSelection(DataSourceSet set, int subset)
{
// If none is selected, we move only the current
if (dgvTable.SelectedRows.Count == 0)
{
moveSelection(dgvTable.CurrentRow, set, subset);
}
else
{
foreach (DataGridViewRow viewRow in dgvTable.SelectedRows)
{
moveSelection(viewRow, set, subset);
}
}
// this.BindingSource.ResetBindings(false);
}
private static Expression BuildConditions(DataSourceSet dataSources)
{
var conditions = default(Expression);
foreach (var dataSource in dataSources.Filter(ds => ds.IsConditional))
{
if (conditions == null)
{
conditions = dataSource.Condition;
continue;
}
conditions = Expression.OrElse(conditions, dataSource.Condition);
}
return(conditions);
}
public void TestMsgPackSamplerRandomize()
{
const int NUM_CHUNKS = 30;
const int CHUNK_SIZE = 6;
const int MINIBATCH_SIZE = 2;
var data = new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 };
int FEATURE_DIM = data.Length / CHUNK_SIZE;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_CHUNKS; ++i)
{
var a = DataSourceFactory.Create(data, new int[] { FEATURE_DIM, CHUNK_SIZE });
var dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(file, MINIBATCH_SIZE, true, 10, 100, false, 100))
{
for (var i = 0; i < NUM_CHUNKS; ++i)
{
var values = new float[data.Length];
for (var j = 0; j < data.Length; j += FEATURE_DIM * MINIBATCH_SIZE)
{
var batch = sampler.GetNextMinibatch();
var value = DataSourceFactory.FromValue(batch["a"]);
CollectionAssert.AreEqual(new int[] { FEATURE_DIM, MINIBATCH_SIZE }, value.Shape.Dimensions.ToArray());
for (var k = 0; k < FEATURE_DIM * MINIBATCH_SIZE; ++k)
{
values[j + k] = value[k];
}
}
CollectionAssert.AreNotEqual(data, values);
var sorted = values.ToList();
sorted.Sort();
CollectionAssert.AreEqual(data, sorted);
}
}
}
private void copySelection(DataSourceSet set)
{
// If none is selected, we copy only the current
if (dgvTable.SelectedRows.Count == 0)
{
copySelection(dgvTable.CurrentRow, set);
}
else
{
// Otherwise we copy all the selected rows.
foreach (DataGridViewRow viewRow in dgvTable.SelectedRows)
{
copySelection(viewRow, set);
}
}
//this.BindingSource.ResetBindings(false);
}
protected override bool ShortCircuitMapping(MappingCreationContext context, out Expression mapping)
{
var derivedTypeDataSources = DerivedComplexTypeDataSourcesFactory.CreateFor(context.MappingData);
if (derivedTypeDataSources.None())
{
return(base.ShortCircuitMapping(context, out mapping));
}
var derivedTypeDataSourceSet = DataSourceSet.For(
derivedTypeDataSources,
context.MapperData,
ValueExpressionBuilders.ValueSequence);
mapping = derivedTypeDataSourceSet.BuildValue();
if (derivedTypeDataSources.Last().IsConditional)
{
context.MappingExpressions.Add(mapping);
return(false);
}
var shortCircuitReturns = GetShortCircuitReturns(context.MappingData).ToArray();
if (shortCircuitReturns.Any())
{
context.MappingExpressions.AddRange(shortCircuitReturns);
}
if (mapping.NodeType == ExpressionType.Goto)
{
mapping = ((GotoExpression)mapping).Value;
context.MappingExpressions.Add(context.MapperData.GetReturnLabel(mapping));
}
else
{
context.MappingExpressions.Add(mapping);
context.MappingExpressions.Add(context.MapperData.GetReturnLabel(mapping.Type.ToDefaultExpression()));
}
mapping = Expression.Block(context.MappingExpressions);
return(true);
}
public DataView GetView(DataSourceSet set, int subset)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset);
return dataView;
}
private void copySelection(DataSourceSet set)
{
// If none is selected, we copy only the current
if (dgvTable.SelectedRows.Count == 0)
{
copySelection(dgvTable.CurrentRow, set);
}
else
{
// Otherwise we copy all the selected rows.
foreach (DataGridViewRow viewRow in dgvTable.SelectedRows)
{
copySelection(viewRow, set);
}
}
//this.BindingSource.ResetBindings(false);
}
private void copySelection(DataGridViewRow viewRow, DataSourceSet set)
{
DataRow row = (viewRow.DataBoundItem as DataRowView).Row;
if (row != null)
tableSource.CopyRowToSet(row, set);
}
public void MoveRowToSubset(DataRow row, DataSourceSet set, int subset)
{
row["@SET"] = set;
row["@SUBSET"] = subset;
HasChanges = true;
}
/// <summary>
/// Imports only matching columns from a DataTable
/// </summary>
/// <param name="table"></param>
public void Import(DataTable table, DataSourceSet set)
{
// Add a new column to the to-be-imported DataTable
// to determine which will be the default set when importing
DataColumn col = new DataColumn("@SET", typeof(DataSourceSet));
col.DefaultValue = set;
table.Columns.Add(col);
// Merge the two
dataTable.Merge(table, true, MissingSchemaAction.Ignore);
HasChanges = true;
}
public object GetData(DataSourceSet set, int subset)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset);
DataTable newTable = dataView.ToTable(false);
return newTable;
}
public object GetData(string[] columns, DataSourceSet set)
{
DataView dataView = new DataView(this.dataTable);
dataView.RowFilter = String.Format("[@SET]='{0}'", (int)set);
DataTable newTable = dataView.ToTable(false, columns);
return newTable;
}
public void CopyRowToSet(DataRow row, DataSourceSet set)
{
DataRow newRow = row.Table.NewRow();
foreach (DataColumn col in row.Table.Columns)
{
newRow[col] = row[col];
}
newRow["@SET"] = set;
row.Table.Rows.Add(newRow);
HasChanges = true;
}
private void moveSelection(DataSourceSet set, int subset)
{
// If none is selected, we move only the current
if (dgvTable.SelectedRows.Count == 0)
{
moveSelection(dgvTable.CurrentRow, set, subset);
}
else
{
foreach (DataGridViewRow viewRow in dgvTable.SelectedRows)
{
moveSelection(viewRow, set, subset);
}
}
// this.BindingSource.ResetBindings(false);
}
private void moveSelection(DataGridViewRow viewRow, DataSourceSet set, int subset)
{
DataRow row = (viewRow.DataBoundItem as DataRowView).Row;
if (row != null)
tableSource.MoveRowToSubset(row, set, subset);
}
