public DataView GetView(DataSourceSet set, int subset) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset); return(dataView); }
public void TestMultipleObjects() { var a = DataSourceFactory.Create(new float[] { 1, 2, 3 }, new int[] { 3, 1, 1 }); var dss = new DataSourceSet(); dss.Add("a", a); var dss2 = new DataSourceSet(); var b = DataSourceFactory.Create(new float[] { 4, 5, 6, 7 }, new int[] { 2, 2, 1 }); dss2.Add("b", b); var stream = new MemoryStream(); MsgPackSerializer.Serialize(dss, stream); MsgPackSerializer.Serialize(dss2, stream); stream.Position = 0; var result = MsgPackSerializer.Deserialize(stream); var result2 = MsgPackSerializer.Deserialize(stream); Assert.AreEqual(1, result.Features.Count); var x = result["a"]; CollectionAssert.AreEqual(new int[] { 3, 1, 1 }, x.Shape.Dimensions); CollectionAssert.AreEqual(new float[] { 1, 2, 3 }, x.Data.ToArray()); Assert.AreEqual(1, result2.Features.Count); var y = result2["b"]; CollectionAssert.AreEqual(new int[] { 2, 2, 1 }, y.Shape.Dimensions); CollectionAssert.AreEqual(new float[] { 4, 5, 6, 7 }, y.Data.ToArray()); }
public void TestDataSourceCTFBuilider() { var writer = new StringWriter(); var ds1 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }, new int[] { 2, 3, 3 }); var ds2 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5 }, new int[] { 2, 1, 3 }); var dss = new DataSourceSet(); dss.Features.Add("data", ds1); dss.Features.Add("label", ds2); DataSourceSetCTFBuilder.Write(writer, dss, true); var s = writer.ToString(); var expected = "0\t|data 0 1\t|label 0 1\r\n" + "0\t|data 2 3\r\n" + "0\t|data 4 5\r\n" + "1\t|data 6 7\t|label 2 3\r\n" + "1\t|data 8 9\r\n" + "1\t|data 10 11\r\n" + "2\t|data 12 13\t|label 4 5\r\n" + "2\t|data 14 15\r\n" + "2\t|data 16 17"; Assert.AreEqual(expected, s); }
public static Expression GetMappingOrNull( MappingCreationContext context, out bool isConditional) { var mappingData = context.MappingData; var derivedTypeDataSources = DerivedComplexTypeDataSourcesFactory.CreateFor(mappingData); if (derivedTypeDataSources.None()) { isConditional = false; return(null); } var derivedTypeDataSourceSet = DataSourceSet.For( derivedTypeDataSources, mappingData, ValueExpressionBuilders.ValueSequence); isConditional = derivedTypeDataSources.Last().IsConditional&& !mappingData.MapperData.TargetType.IsAbstract(); if (!isConditional) { context.MappingComplete = true; } return(derivedTypeDataSourceSet.BuildValue()); }
public void ProcessTraceAsFolder() { // Input data string[] lttngData = { @"..\..\..\..\TestData\LTTng\lttng-kernel-trace.ctf" }; string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); using (var zipFile = ZipFile.OpenRead(lttngData[0])) { zipFile.ExtractToDirectory(tempDirectory); } using (var dataSourceSet = DataSourceSet.Create()) { var ds = new DirectoryDataSource(tempDirectory); dataSourceSet.AddDataSource(ds); // Approach #1 - Engine - Doesn't test tables UI but tests processing using (var runtime = Engine.Create(new EngineCreateInfo(dataSourceSet.AsReadOnly()))) { // // We do not assert that any cookers are enabled since we did not explicitly enable cookers here // Assert.IsTrue(ds.IsDirectory()); Assert.IsTrue(runtime.AvailableTables.Count() >= 1); } } Directory.Delete(tempDirectory, true); }
public static IMemberPopulator WithoutRegistration( IChildMemberMappingData mappingData, DataSourceSet dataSources, Expression populateCondition = null) { return(new MemberPopulator(dataSources, populateCondition)); }
public void TestMsgPackSamplerMultipleFiles() { var files = new string[2]; for (var i = 0; i < 2; ++i) { files[i] = Path.GetTempFileName(); var a = DataSourceFactory.Create(new float[] { i, i + 1, i + 2 }, new int[] { 3, 1 }); var dss = new DataSourceSet(); dss.Add("a", a); using (var stream = new FileStream(files[i], FileMode.Create, FileAccess.Write)) { MsgPackSerializer.Serialize(dss, stream); } } using (var sampler = new MsgPackSampler(files, 3, false, -1, 10, false, 10)) { Assert.AreEqual(2, sampler.SampleCountPerEpoch); var batch = sampler.GetNextMinibatch(); CollectionAssert.AreEqual(new int[] { 3, 3 }, batch.Features["a"].Shape.Dimensions.ToArray()); var values = DataSourceFactory.FromValue(batch.Features["a"]); CollectionAssert.AreEqual(new float[] { 0, 1, 2, 1, 2, 3, 0, 1, 2 }, values.TypedData); } }
public void MoveRowToSubset(DataRow row, DataSourceSet set, int subset) { row["@SET"] = set; row["@SUBSET"] = subset; HasChanges = true; }
private void copySelection(DataGridViewRow viewRow, DataSourceSet set) { DataRow row = (viewRow.DataBoundItem as DataRowView).Row; if (row != null) { tableSource.CopyRowToSet(row, set); } }
private static IDataSourceSet CreateNullDataSourceSet( IMemberMapperData mapperData, Func <QualifiedMember, string> commentFactory) { return(DataSourceSet.For( new NullDataSource( ReadableExpression.Comment(commentFactory.Invoke(mapperData.TargetMember))), mapperData)); }
private void moveSelection(DataGridViewRow viewRow, DataSourceSet set, int subset) { DataRow row = (viewRow.DataBoundItem as DataRowView).Row; if (row != null) { tableSource.MoveRowToSubset(row, set, subset); } }
public static IDataSourceSet CreateFor(IObjectMappingData rootMappingData) { var rootDataSourceFactory = _mappingRootDataSourceFactories .First(rootMappingData, (rmd, mef) => mef.IsFor(rmd)); var rootDataSource = rootDataSourceFactory.CreateFor(rootMappingData); return(DataSourceSet.For(rootDataSource, rootMappingData.MapperData)); }
public static IDataSourceSet CreateFor(DataSourceFindContext findContext) { var validDataSources = EnumerateDataSources(findContext).ToArray(); return(DataSourceSet.For( validDataSources, findContext, ValueExpressionBuilders.ValueTree)); }
public object GetData(DataSourceSet set, int subset) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset); DataTable newTable = dataView.ToTable(false); return(newTable); }
public object GetData(string[] columns, DataSourceSet set) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}'", (int)set); DataTable newTable = dataView.ToTable(false, columns); return(newTable); }
private static IDataSource GetMapFromConditionOrDefaultDataSource( IObjectMappingData declaredTypeMappingData, Expression condition, DerivedSourceTypeCheck derivedSourceCheck, IEnumerable <TypePairGroup> typePairGroups, Type targetType) { var declaredTypeMapperData = declaredTypeMappingData.MapperData; var typePairDataSources = new List <IDataSource>(); Expression derivedTypeMapping; IObjectMappingData derivedTypeMappingData; foreach (var typePairGroup in typePairGroups) { var typePairsCondition = declaredTypeMapperData.GetTypePairsCondition(typePairGroup.TypePairs) ?? declaredTypeMapperData.GetTargetValidCheckOrNull(typePairGroup.DerivedTargetType); derivedTypeMapping = GetReturnMappingResultExpression( declaredTypeMappingData, derivedSourceCheck.TypedVariable, typePairGroup.DerivedTargetType, out derivedTypeMappingData); var typePairDataSource = new DerivedComplexTypeDataSource( derivedTypeMappingData.MapperData.SourceMember, typePairsCondition, derivedTypeMapping); typePairDataSources.Add(typePairDataSource); } var derivedTargetTypeDataSources = DataSourceSet.For( typePairDataSources, declaredTypeMapperData, ValueExpressionBuilders.ValueSequence); derivedTypeMapping = GetReturnMappingResultExpression( declaredTypeMappingData, derivedSourceCheck.TypedVariable, targetType, out derivedTypeMappingData); var derivedTypeMappings = Block( derivedTargetTypeDataSources.BuildValue(), derivedTypeMapping); return(new DerivedComplexTypeDataSource( derivedTypeMappingData.MapperData.SourceMember, derivedSourceCheck, condition, derivedTypeMappings, declaredTypeMapperData)); }
public static IMemberPopulator WithRegistration( IChildMemberMappingData mappingData, DataSourceSet dataSources, Expression populateCondition) { var memberPopulation = WithoutRegistration(mappingData, dataSources, populateCondition); memberPopulation.MapperData.RegisterTargetMemberDataSourcesIfRequired(dataSources); return(memberPopulation); }
public MemberPopulation( IMemberMapperData mapperData, DataSourceSet dataSources, Expression populateCondition = null) { MapperData = mapperData; _dataSources = dataSources; _populateCondition = populateCondition; mapperData.Parent.RegisterTargetMemberDataSourcesIfRequired(mapperData.TargetMember, dataSources); }
public static IMemberPopulator NoDataSources(MemberPopulationContext context) { var noDataSourcesMessage = CreateNullPopulatorDescription( GetNoDataSourcesMessage, context.MemberMapperData); var noDataSource = new NullDataSource(noDataSourcesMessage); var noDataSources = DataSourceSet.For(noDataSource, context); context.MemberMapperData.RegisterTargetMemberDataSources(noDataSources); return(CreateNullMemberPopulator(noDataSourcesMessage, context, (msg, md) => msg)); }
/// <summary> /// Imports only matching columns from a DataTable /// </summary> /// <param name="table"></param> public void Import(DataTable table, DataSourceSet set) { // Add a new column to the to-be-imported DataTable // to determine which will be the default set when importing DataColumn col = new DataColumn("@SET", typeof(DataSourceSet)); col.DefaultValue = set; table.Columns.Add(col); // Merge the two dataTable.Merge(table, true, MissingSchemaAction.Ignore); HasChanges = true; }
public void CopyRowToSet(DataRow row, DataSourceSet set) { DataRow newRow = row.Table.NewRow(); foreach (DataColumn col in row.Table.Columns) { newRow[col] = row[col]; } newRow["@SET"] = set; row.Table.Rows.Add(newRow); HasChanges = true; }
public void TestReadDataSourceSet() { const int NUM_SAMPLES = 10; DataSourceSet dss; var file = Path.GetTempFileName(); using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write)) { for (var i = 0; i < NUM_SAMPLES; ++i) { var a = DataSourceFactory.Create(new float[] { i, i * 10, i * 100 }, new int[] { 3, 1, 1 }); dss = new DataSourceSet(); dss.Add("a", a); MsgPackSerializer.Serialize(dss, stream); } } var total = MsgPackTools.GetTotalSampleCount(file); Assert.AreEqual(NUM_SAMPLES, total); var reader = MsgPackTools.ReadDataSourceSet(file, total, 4).GetEnumerator(); var hasNext = reader.MoveNext(); Assert.AreEqual(true, hasNext); dss = reader.Current; Assert.AreEqual(4, dss.SampleCount); CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions); CollectionAssert.AreEqual(new float[] { 0, 0, 0, 1, 10, 100, 2, 20, 200, 3, 30, 300 }, dss["a"].Data.ToArray()); hasNext = reader.MoveNext(); Assert.AreEqual(true, hasNext); dss = reader.Current; Assert.AreEqual(4, dss.SampleCount); CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions); CollectionAssert.AreEqual(new float[] { 4, 40, 400, 5, 50, 500, 6, 60, 600, 7, 70, 700 }, dss["a"].Data.ToArray()); hasNext = reader.MoveNext(); Assert.AreEqual(true, hasNext); dss = reader.Current; Assert.AreEqual(2, dss.SampleCount); CollectionAssert.AreEqual(new int[] { 3, 1, 2 }, dss["a"].Shape.Dimensions); CollectionAssert.AreEqual(new float[] { 8, 80, 800, 9, 90, 900 }, dss["a"].Data.ToArray()); hasNext = reader.MoveNext(); Assert.AreEqual(false, hasNext); }
private Expression GetPopulation( Expression elementMapping, QualifiedMember dictionaryEntryMember, IObjectMappingData mappingData) { var elementMapperData = new ChildMemberMapperData(dictionaryEntryMember, MapperData); var sourceMember = mappingData.MapperData.SourceMember; var mappingDataSource = new AdHocDataSource(sourceMember, elementMapping); var dataSourceSetInfo = new SimpleDataSourceSetInfo(mappingData.MappingContext, elementMapperData); var mappingDataSources = DataSourceSet.For(mappingDataSource, dataSourceSetInfo); var populator = new MemberPopulator(mappingDataSources, elementMapperData); var populationExpression = populator.GetPopulation(); return(populationExpression); }
private Expression GetPopulation( Expression elementMapping, QualifiedMember dictionaryEntryMember, IObjectMappingData mappingData) { var elementMapperData = new ChildMemberMapperData(dictionaryEntryMember, MapperData); var sourceMember = mappingData.MapperData.SourceMember; var mappingDataSource = new AdHocDataSource(sourceMember, elementMapping); var mappingDataSources = DataSourceSet.For(mappingDataSource, elementMapperData); var populationExpression = MemberPopulator .WithoutRegistration(mappingDataSources) .GetPopulation(); return(populationExpression); }
public void TestMsgPackSamplerSlicing2() { const int NUM_SAMPLES = 300; var file = Path.GetTempFileName(); using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write)) { for (var i = 0; i < NUM_SAMPLES; ++i) { var a = DataSourceFactory.Create(new float[] { i, i, i }, new int[] { 1, 1, 3 }); var dss = new DataSourceSet(); dss.Add("a", a); MsgPackSerializer.Serialize(dss, stream); } } using (var sampler = new MsgPackSampler(file, 5, false, 7, 10, false, 100)) { int count = 0; for (var i = 0; i < 10; ++i) { for (var j = 0; j < NUM_SAMPLES * 3 / 5; ++j) { var batch = sampler.GetNextMinibatch(); Assert.AreEqual(1, batch.Features.Count); Assert.IsTrue(batch.Features.ContainsKey("a")); var value = batch["a"]; CollectionAssert.AreEqual(new int[] { 1, 1, 5 }, value.Shape.Dimensions.ToArray()); var ds = DataSourceFactory.FromValue(value).ToArray(); Debug.WriteLine(string.Join(", ", ds)); CollectionAssert.AreEqual(new float[] { j * 5 / 3, (j * 5 + 1) / 3, (j * 5 + 2) / 3, (j * 5 + 3) / 3, (j * 5 + 4) / 3 }, ds); count += 5; } } } }
private void moveSelection(DataSourceSet set, int subset) { // If none is selected, we move only the current if (dgvTable.SelectedRows.Count == 0) { moveSelection(dgvTable.CurrentRow, set, subset); } else { foreach (DataGridViewRow viewRow in dgvTable.SelectedRows) { moveSelection(viewRow, set, subset); } } // this.BindingSource.ResetBindings(false); }
private static Expression BuildConditions(DataSourceSet dataSources) { var conditions = default(Expression); foreach (var dataSource in dataSources.Filter(ds => ds.IsConditional)) { if (conditions == null) { conditions = dataSource.Condition; continue; } conditions = Expression.OrElse(conditions, dataSource.Condition); } return(conditions); }
public void TestMsgPackSamplerRandomize() { const int NUM_CHUNKS = 30; const int CHUNK_SIZE = 6; const int MINIBATCH_SIZE = 2; var data = new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; int FEATURE_DIM = data.Length / CHUNK_SIZE; var file = Path.GetTempFileName(); using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write)) { for (var i = 0; i < NUM_CHUNKS; ++i) { var a = DataSourceFactory.Create(data, new int[] { FEATURE_DIM, CHUNK_SIZE }); var dss = new DataSourceSet(); dss.Add("a", a); MsgPackSerializer.Serialize(dss, stream); } } using (var sampler = new MsgPackSampler(file, MINIBATCH_SIZE, true, 10, 100, false, 100)) { for (var i = 0; i < NUM_CHUNKS; ++i) { var values = new float[data.Length]; for (var j = 0; j < data.Length; j += FEATURE_DIM * MINIBATCH_SIZE) { var batch = sampler.GetNextMinibatch(); var value = DataSourceFactory.FromValue(batch["a"]); CollectionAssert.AreEqual(new int[] { FEATURE_DIM, MINIBATCH_SIZE }, value.Shape.Dimensions.ToArray()); for (var k = 0; k < FEATURE_DIM * MINIBATCH_SIZE; ++k) { values[j + k] = value[k]; } } CollectionAssert.AreNotEqual(data, values); var sorted = values.ToList(); sorted.Sort(); CollectionAssert.AreEqual(data, sorted); } } }
private void copySelection(DataSourceSet set) { // If none is selected, we copy only the current if (dgvTable.SelectedRows.Count == 0) { copySelection(dgvTable.CurrentRow, set); } else { // Otherwise we copy all the selected rows. foreach (DataGridViewRow viewRow in dgvTable.SelectedRows) { copySelection(viewRow, set); } } //this.BindingSource.ResetBindings(false); }
protected override bool ShortCircuitMapping(MappingCreationContext context, out Expression mapping) { var derivedTypeDataSources = DerivedComplexTypeDataSourcesFactory.CreateFor(context.MappingData); if (derivedTypeDataSources.None()) { return(base.ShortCircuitMapping(context, out mapping)); } var derivedTypeDataSourceSet = DataSourceSet.For( derivedTypeDataSources, context.MapperData, ValueExpressionBuilders.ValueSequence); mapping = derivedTypeDataSourceSet.BuildValue(); if (derivedTypeDataSources.Last().IsConditional) { context.MappingExpressions.Add(mapping); return(false); } var shortCircuitReturns = GetShortCircuitReturns(context.MappingData).ToArray(); if (shortCircuitReturns.Any()) { context.MappingExpressions.AddRange(shortCircuitReturns); } if (mapping.NodeType == ExpressionType.Goto) { mapping = ((GotoExpression)mapping).Value; context.MappingExpressions.Add(context.MapperData.GetReturnLabel(mapping)); } else { context.MappingExpressions.Add(mapping); context.MappingExpressions.Add(context.MapperData.GetReturnLabel(mapping.Type.ToDefaultExpression())); } mapping = Expression.Block(context.MappingExpressions); return(true); }
public DataView GetView(DataSourceSet set, int subset) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset); return dataView; }
private void copySelection(DataSourceSet set) { // If none is selected, we copy only the current if (dgvTable.SelectedRows.Count == 0) { copySelection(dgvTable.CurrentRow, set); } else { // Otherwise we copy all the selected rows. foreach (DataGridViewRow viewRow in dgvTable.SelectedRows) { copySelection(viewRow, set); } } //this.BindingSource.ResetBindings(false); }
private void copySelection(DataGridViewRow viewRow, DataSourceSet set) { DataRow row = (viewRow.DataBoundItem as DataRowView).Row; if (row != null) tableSource.CopyRowToSet(row, set); }
public void MoveRowToSubset(DataRow row, DataSourceSet set, int subset) { row["@SET"] = set; row["@SUBSET"] = subset; HasChanges = true; }
/// <summary> /// Imports only matching columns from a DataTable /// </summary> /// <param name="table"></param> public void Import(DataTable table, DataSourceSet set) { // Add a new column to the to-be-imported DataTable // to determine which will be the default set when importing DataColumn col = new DataColumn("@SET", typeof(DataSourceSet)); col.DefaultValue = set; table.Columns.Add(col); // Merge the two dataTable.Merge(table, true, MissingSchemaAction.Ignore); HasChanges = true; }
public object GetData(DataSourceSet set, int subset) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}' AND [@SUBSET]='{1}'", (int)set, subset); DataTable newTable = dataView.ToTable(false); return newTable; }
public object GetData(string[] columns, DataSourceSet set) { DataView dataView = new DataView(this.dataTable); dataView.RowFilter = String.Format("[@SET]='{0}'", (int)set); DataTable newTable = dataView.ToTable(false, columns); return newTable; }
public void CopyRowToSet(DataRow row, DataSourceSet set) { DataRow newRow = row.Table.NewRow(); foreach (DataColumn col in row.Table.Columns) { newRow[col] = row[col]; } newRow["@SET"] = set; row.Table.Rows.Add(newRow); HasChanges = true; }
private void moveSelection(DataSourceSet set, int subset) { // If none is selected, we move only the current if (dgvTable.SelectedRows.Count == 0) { moveSelection(dgvTable.CurrentRow, set, subset); } else { foreach (DataGridViewRow viewRow in dgvTable.SelectedRows) { moveSelection(viewRow, set, subset); } } // this.BindingSource.ResetBindings(false); }
private void moveSelection(DataGridViewRow viewRow, DataSourceSet set, int subset) { DataRow row = (viewRow.DataBoundItem as DataRowView).Row; if (row != null) tableSource.MoveRowToSubset(row, set, subset); }