public void Salvar(IConexao conexao, TObjeto model)
{
if ((model == null) || (model.EstadoEntidade == EstadosEntidade.NaoModificado))
{
return;
}
var tabela = (conexao as ContextoFake).ConsultarTabelaDoBancoDeDadosVirtual(typeof(TObjeto));
var registro = DataTableBuilder.ConverterItemEmDataRow(tabela, model, novoRegistro: model.EstadoEntidade == EstadosEntidade.Novo);
switch (model.EstadoEntidade)
{
case EstadosEntidade.Novo:
tabela.Rows.Add(registro);
if (_dicionario.AutoIncremento == OpcoesAutoIncremento.Identity)
{
var autoIncremento = Convert.ToInt32(registro[ConsultarColunaIdentity(tabela)]);
AtribuirValorAutoIncremento(Incremento.Identity, item => item.Propriedade.SetValue(model, autoIncremento, null));
}
break;
case EstadosEntidade.Modificado:
AtualizarRegistro(tabela, registro);
break;
case EstadosEntidade.Excluido:
ExcluirRegistro(tabela, registro);
break;
}
}
/// <summary>
/// Reads all sheets in the excel workbook and returns as a ordered collection of data tables.
/// </summary>
/// <param name="builder">placeholder</param>
/// <param name="filename">excel file to load</param>
/// <returns>Ordered collection of tables corresponding to non-empty sheets. Table name corresponds to sheet name.</returns>
public static IList <MutableDataTable> ReadExcelAllSheets(this DataTableBuilder builder, string filename)
{
using (Stream input = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
return(builder.ReadExcelAllSheets(input));
}
}
public void TestMultinomialLogisticRegression()
{
var dataTable = new DataTableBuilder();
dataTable.AddColumn(ColumnType.Float, "height");
dataTable.AddColumn(ColumnType.Int, "weight").IsContinuous = true;
dataTable.AddColumn(ColumnType.Int, "foot-size").IsContinuous = true;
dataTable.AddColumn(ColumnType.String, "gender", true);
// sample data from: https://en.wikipedia.org/wiki/Naive_Bayes_classifier
dataTable.Add(6f, 180, 12, "male");
dataTable.Add(5.92f, 190, 11, "male");
dataTable.Add(5.58f, 170, 12, "male");
dataTable.Add(5.92f, 165, 10, "male");
dataTable.Add(5f, 100, 6, "female");
dataTable.Add(5.5f, 150, 8, "female");
dataTable.Add(5.42f, 130, 7, "female");
dataTable.Add(5.75f, 150, 9, "female");
var index = dataTable.Build();
var testData = new DataTableBuilder(dataTable.Columns);
var row = testData.Add(6f, 130, 8, "?");
var model = index.TrainMultinomialLogisticRegression(_lap, 100, 0.1f);
var classifier = model.CreateClassifier(_lap);
var classification = classifier.Classify(row);
Assert.IsTrue(classification.First() == "female");
}
/// <summary>
/// Creates random integers added together as feature vectors
/// The input feature contains two features, one for each bit at that position
/// The output feature contains a single feature: 1 or 0 if that bit is set in the result
/// </summary>
/// <param name="sampleCount">How many samples to generate</param>
/// <param name="stochastic">True to generate random integers</param>
/// <returns>A list of sequences</returns>
public static IDataTable Addition(int sampleCount, bool stochastic)
{
Random rand = stochastic ? new Random() : new Random(0);
var builder = DataTableBuilder.CreateTwoColumnMatrix();
for (var i = 0; i < sampleCount; i++)
{
// generate some random numbers (sized to prevent overflow)
var a = rand.Next(int.MaxValue / 2);
var b = rand.Next(int.MaxValue / 2);
var a2 = _GetBitArray(a);
var b2 = _GetBitArray(b);
var r2 = _GetBitArray(a + b);
var inputList = new FloatVector[r2.Length];
var outputList = new FloatVector[r2.Length];
for (int j = 0; j < r2.Length; j++)
{
inputList[j] = new FloatVector {
Data = new[] { a2[j], b2[j] }
};
outputList[j] = new FloatVector {
Data = new[] { r2[j] }
};
}
builder.Add(FloatMatrix.Create(inputList), FloatMatrix.Create(outputList));
}
return(builder.Build());
}
public void ShouldBuildEmptyDataTable()
{
var dt = DataTableBuilder.Create()
.Build();
Assert.That(dt.Rows.Count, Is.EqualTo(0));
}
internal static DtModel InitInstance <T>(DataTableBuilder <T> dtBuilder) where T : class
{
var dtModel = new DtModel
{
TableName = dtBuilder.TableName,
TableClassName = $"{(dtBuilder.TableOptionsFactory.HasClassName ? $" {dtBuilder.TableOptionsFactory.ClassName}" : string.Empty)}",
IdOfHiddenInputHoldingTableData = $"dt_{dtBuilder.TableName}_data",
IsDtAjaxLoadingDefferd = "false",
TableDefaultDatesFormat = dtBuilder.TableOptionsFactory.TableDefaultDatesFormat
};
if (dtBuilder.TableOptionsFactory.DoCreateApiInstance)
{
dtModel.DoCreateDtApiInstance = true;
dtModel.DtApiInstanceName = dtBuilder.TableOptionsFactory.DtApiInstanceName;
}
InitDataTableConfig <T>(dtBuilder, dtModel);
InitFormConfig(dtBuilder, dtModel);
InitNativeDtEventsConfig(dtBuilder, dtModel);
InitOnClickEventsConfig(dtBuilder, dtModel);
return(dtModel);
}
private static void InitFormConfig <T>(DataTableBuilder <T> dtBuilder, DtModel dtModel) where T : class
{
if (!dtBuilder.DataTableDataSourceBuilder.IsUsedInForm)
{
return;
}
dtModel.IsUsedInForm = true;
dtModel.FormId = dtBuilder.DataTableDataSourceBuilder.FormId;
if (dtBuilder.DataTableDataSourceBuilder.SubmitWithMultiHiddenInputs)
{
dtModel.SubmitWithMultiHiddenInputs = true;
dtModel.PrefixForMultiHiddenInputs = dtBuilder.DataTableDataSourceBuilder.PrefixForMultiHiddenInputs;
}
else // A single hiden input that will hold all values as json
{
dtModel.NameAttributeForHiddenInputHoldingTableData = $"name='{dtBuilder.DataTableDataSourceBuilder.StringPropNameForSubmittedValues}'";
}
if (dtBuilder.DataTableDataSourceBuilder.IsSubmitOnlyNewAndModifiedRows)
{
dtModel.IsSubmitOnlyNewAndModifiedRows = true;
}
}
public void TestRegression()
{
var dataTable = new DataTableBuilder();
dataTable.AddColumn(ColumnType.Float, "value");
dataTable.AddColumn(ColumnType.Float, "result", true);
// simple linear relationship: result is twice value
dataTable.AddRow(new object[] { 1f, 2f });
dataTable.AddRow(new object[] { 2f, 4f });
dataTable.AddRow(new object[] { 4f, 8f });
dataTable.AddRow(new object[] { 8f, 16f });
var index = dataTable.Build();
var classifier = index.CreateLinearRegressionTrainer(_lap);
//var theta = classifier.Solve();
//var predictor = theta.CreatePredictor(_lap);
//var prediction = predictor.Predict(3f);
//Assert.IsTrue(Math.Round(prediction) == 6f);
var theta = classifier.GradientDescent(20, 0.01f);
var predictor = theta.CreatePredictor(_lap);
var prediction = predictor.Predict(3f);
Assert.IsTrue(Math.Round(prediction) == 6f);
var prediction3 = predictor.Predict(new[] {
new float[] { 10f },
new float[] { 3f }
});
Assert.IsTrue(Math.Round(prediction3[1]) == 6f);
}
public void TestIndexHydration()
{
var builder = new DataTableBuilder();
builder.AddColumn(ColumnType.Boolean, "target", true);
builder.AddColumn(ColumnType.Int, "val");
builder.AddColumn(ColumnType.String, "label");
for (var i = 0; i < 33000; i++)
{
builder.Add(i % 2 == 0, i, i.ToString());
}
using (var dataStream = new MemoryStream())
using (var indexStream = new MemoryStream()) {
var table = builder.Build(dataStream);
table.WriteIndexTo(indexStream);
dataStream.Seek(0, SeekOrigin.Begin);
indexStream.Seek(0, SeekOrigin.Begin);
var newTable = Provider.CreateDataTable(dataStream, indexStream);
_CompareTables(table, newTable);
dataStream.Seek(0, SeekOrigin.Begin);
var newTable2 = Provider.CreateDataTable(dataStream, null);
_CompareTables(table, newTable2);
}
}
/// <summary>
/// Converts the sparse vectors to a data table
/// </summary>
/// <param name="stream">Optional stream to write the data table to</param>
public IDataTable ConvertToTable(Stream stream = null)
{
var max = GetMaximumIndex();
var dataTable = new DataTableBuilder();
for (var i = 0; i < max; i++)
{
dataTable.AddColumn(ColumnType.Float, "term " + i.ToString());
}
dataTable.AddColumn(ColumnType.String, "classification", true);
foreach (var item in Classification)
{
var data = new object[max + 1];
for (var i = 0; i < max; i++)
{
data[i] = 0f;
}
foreach (var index in item.Data)
{
data[index.Index] = index.Weight;
}
data[max] = item.Name;
dataTable.AddRow(data);
}
return(dataTable.Build(stream));
}
public void SelectColumns()
{
var builder = new DataTableBuilder();
builder.AddColumn(ColumnType.Float, "val1");
builder.AddColumn(ColumnType.Double, "val2");
builder.AddColumn(ColumnType.String, "cls", true);
builder.AddColumn(ColumnType.String, "cls2");
builder.Add(0.5f, 1.1, "a", "a2");
builder.Add(0.2f, 1.5, "b", "b2");
builder.Add(0.7f, 0.5, "c", "c2");
builder.Add(0.2f, 0.6, "d", "d2");
var table = builder.Build();
var table2 = table.SelectColumns(new[] { 1, 2, 3 });
Assert.AreEqual(table2.TargetColumnIndex, 1);
Assert.AreEqual(table2.RowCount, 4);
Assert.AreEqual(table2.ColumnCount, 3);
var column = table2.GetNumericColumns(_lap, new[] { 0 }).First().AsIndexable();
Assert.AreEqual(column[0], 1.1f);
Assert.AreEqual(column[1], 1.5f);
}
public async Task <IActionResult> Index(int?page, string sortColumn, string sortDirection, string filter)
{
var me = await this._userManager.FindByIdAsync(HttpContext.User.FindFirstValue(ClaimTypes.NameIdentifier));
var dataSource = this._notificationReciever.FetchHistory(me);
if (!string.IsNullOrWhiteSpace(filter))
{
dataSource = dataSource.Where(x => x.SubjectId.ToLower() == filter.ToLower());
}
var dataTable = DataTableBuilder
.Create(dataSource)
.Sort(sortColumn ?? "SubjectId", sortDirection ?? "Ascending")
.Page(page)
.RowLink(x => x.URL)
.Columns(columns =>
{
columns.Add(x => x.Id).Title("Id").Width(100);
columns.Add(x => x.SubjectId).Title("Ämne").Width(200);
columns.Add(x => x.ShortDescription).Title("Beskrivning");
});
return(this.ViewOrOk(dataTable));
}
public void DataTableProvider()
{
var builder = new DataTableBuilder();
builder.AddColumn(ColumnType.Float, "val1");
builder.AddColumn(ColumnType.Double, "val2");
builder.AddColumn(ColumnType.String, "val3");
builder.AddColumn(ColumnType.String, "cls", true);
builder.Add(0.5f, 1.1, "d", "a");
builder.Add(0.2f, 1.5, "c", "b");
builder.Add(0.7f, 0.5, "b", "c");
builder.Add(0.2f, 0.6, "a", "d");
var table = builder.Build();
var vectoriser = table.GetVectoriser();
var dataProvider = _lap.NN.CreateTrainingDataProvider(table, vectoriser);
var miniBatch = dataProvider.GetTrainingData(new[] { 1 });
var input = miniBatch.Input.Row(0).AsIndexable();
var expectedOutput = miniBatch.ExpectedOutput.Row(0).AsIndexable();
Assert.AreEqual(input[0], 0.2f);
Assert.AreEqual(input[1], 1.5f);
Assert.AreEqual(expectedOutput.Count, 4);
Assert.AreEqual(vectoriser.GetOutputLabel(2, expectedOutput.MaximumIndex()), "b");
}
public void ShouldFindUnmatchingValuesPerKey()
{
var a = DataTableBuilder.Create("a")
.WithColumns("ID", "Name")
.WithRows(new object[] { 5, "Peter" })
.Build();
var b = DataTableBuilder.Create("b")
.WithColumns("ID", "Name")
.WithRows(new object[] { 5, "Henell" })
.Build();
var keys = new List <string>();
keys.Add("ID");
var result = comparer.Compare(a, b, keys);
Assert.That(result.ComparisonErrors.Where(e => e.ValuesDoesNotMatch).Count(), Is.EqualTo(1));
var diffValue = result.ComparisonErrors.Where(e => e.ValuesDoesNotMatch).FirstOrDefault().ValueDifferences.FirstOrDefault();
Assert.That(diffValue.ColumnName, Is.EqualTo("Name"));
Assert.That(diffValue.ExpectedValue, Is.EqualTo("Peter"));
Assert.That(diffValue.ActualValue, Is.EqualTo("Henell"));
}
public void CanFillDataSet()
{
var users = new DataTableBuilder()
.AddColumn("customerid", typeof(Int32))
.AddColumn("firstname", typeof(String))
.AddColumn("lastname", typeof(String))
.AddRow(1, "joe", "black")
.AddRow(1, "kurt", "vonnegut").DataTable;
var orders = new DataTableBuilder()
.AddColumn("orderid", typeof(Int32))
.AddColumn("userid", typeof(Int32))
.AddColumn("total", typeof(double))
.AddRow(100, 1, 10.10)
.AddRow(101, 1, 10.20)
.AddRow(202, 2, 20.10)
.AddRow(203, 2, 20.20).DataTable;
Factory.AddBehavior(new MockCommandBehavior()
.When(cmd => cmd.CommandText.Contains("from customers"))
.ReturnsData(users));
Factory.AddBehavior(new MockCommandBehavior()
.When(cmd => cmd.CommandText.Contains("from orders"))
.ReturnsData(orders));
var result = SUT.DataAccess.GetAllOrders();
Assert.AreEqual(2, result.Tables.Count);
Assert.AreEqual(2, result.Tables["customers"].Rows.Count);
Assert.AreEqual(4, result.Tables["orders"].Rows.Count);
}
public async Task <IActionResult> Index(int?page, string sortColumn, string sortDirection, string filter)
{
var dataSource = this._context.Designs.Include(x => x.Images).AsQueryable();
if (!string.IsNullOrWhiteSpace(filter))
{
dataSource = dataSource.Where(x => x.Title.ToLower() == filter.ToLower());
}
var dataTable = DataTableBuilder
.Create(dataSource)
.Sort(sortColumn ?? "Title", sortDirection ?? "Ascending")
.Page(page)
.RowLink(x => Url.Action("Display", new { id = x.Id }))
//.CssClass("mox-datatable clean")
.Columns(columns =>
{
columns.Add(x => x.Id).Title("Id").Width(100);
columns.Add(x => x.Title).Title("Titel");
columns.Add(x => x.Images.Count).Title("Bilder").Width(100);
})
.Buttons(buttons =>
{
buttons.Add(x => Url.Action("display", new { Id = x.Id })).CssClass("display");
buttons.Add(x => Url.Action("edit", new { Id = x.Id })).CssClass("edit");
buttons.Add(x => Url.Action("delete", new { Id = x.Id })).CssClass("delete");
});
return(this.ViewOrOk(dataTable));
}
public Table(string name, List <ColumnDescriptor> columns)
{
Merge = new Merge();
Name = name;
Columns = columns;
Data = new DataTableBuilder(Name, Columns).Get();
}
/// <summary>
/// Create an in-memory table where each row is an item in the enumeration.
/// The columns are from the "flattened" properties of the T (not fields).
/// The column names are inferred from T's property names.
/// </summary>
public static MutableDataTable FromEnumerable <T>(this DataTableBuilder builder, IEnumerable <T> a)
{
Debug.Assert(builder != null);
string[] columnNames = Utility.InferColumnNames <T>();
return(Utility.ToTableX <T>(a, columnNames));
}
public void ReadFromStreamWithDefaultColumnsShouldHandleFirstRowAsRowData()
{
DataTableBuilder builder = new DataTableBuilder();
var stream = new MemoryStream();
var sw = new StreamWriter(stream);
var rows = new[] { "first,row,is,data", "second,row,is,johnny", "second,row,was,laura", };
foreach (var row in rows)
{
sw.WriteLine(row);
}
sw.Flush();
stream.Seek(0, SeekOrigin.Begin);
try
{
var lazy = builder.ReadLazy(stream, rows[0].Split(','));
Assert.Equal(rows[0].Split(','), lazy.ColumnNames);
var rowEnumerator = rows.Skip(0).GetEnumerator();
rowEnumerator.MoveNext();
var rowCount = 0;
foreach (var row in lazy.Rows)
{
Assert.Equal(rowEnumerator.Current, string.Join(",", row.Values));
rowEnumerator.MoveNext();
rowCount++;
}
Assert.Equal(rows.Length, rowCount);
}
finally
{
sw.Dispose();
stream.Dispose();
}
}
public void CanFillDataSet()
{
var users = new DataTableBuilder()
.AddColumn("customerid", typeof(Int32))
.AddColumn("firstname", typeof(String))
.AddColumn("lastname", typeof(String))
.AddRow(1, "joe", "black")
.AddRow(1, "kurt", "vonnegut").DataTable;
var orders = new DataTableBuilder()
.AddColumn("orderid", typeof(Int32))
.AddColumn("userid", typeof(Int32))
.AddColumn("total", typeof(double))
.AddRow(100, 1, 10.10)
.AddRow(101, 1, 10.20)
.AddRow(202, 2, 20.10)
.AddRow(203, 2, 20.20).DataTable;
Factory.AddBehavior(new MockCommandBehavior()
.When(cmd=>cmd.CommandText.Contains("from customers"))
.ReturnsData(users));
Factory.AddBehavior(new MockCommandBehavior()
.When(cmd=>cmd.CommandText.Contains("from orders"))
.ReturnsData(orders));
var result = SUT.DataAccess.GetAllOrders();
Assert.AreEqual(2, result.Tables.Count);
Assert.AreEqual(2, result.Tables["customers"].Rows.Count);
Assert.AreEqual(4, result.Tables["orders"].Rows.Count);
}
public void CanSetupResultForSimpleForQuery()
{
DataTable customers;
using (DataTableBuilder builder = new DataTableBuilder())
{
builder.AddColumn("userid", typeof(int))
.AddColumn("email", typeof(string))
.AddRow(1, "*****@*****.**")
.AddRow(10, "*****@*****.**");
customers = builder.DataTable;
}
var behavior = new MockCommandBehavior()
.When(c => c.CommandText.StartsWith("select *"))
.ReturnsData(customers);
Factory.AddBehavior(behavior);
var table = SUT.DataAccess.GetAllUsers();
Assert.AreEqual(2, table.Rows.Count);
Assert.AreEqual(1, table.Rows[0][0]);
Assert.AreEqual("*****@*****.**", table.Rows[0][1]);
Assert.AreEqual(10, table.Rows[1][0]);
Assert.AreEqual("*****@*****.**", table.Rows[1][1]);
}
/// <summary>
/// Creates a new data table with the vectorised weighted classification outputs linked with each row's classification
/// </summary>
/// <param name="output">The output stream to write the table to (optional)</param>
public IDataTable GetTable(Stream output = null)
{
if (_trainingData.Any())
{
var fieldCount = _trainingData.First().Item1.Length;
var builder = new DataTableBuilder();
for (var i = 0; i < fieldCount; i++)
{
builder.AddColumn(ColumnType.Float, "v" + i);
}
builder.AddColumn(ColumnType.String, "target", true);
foreach (var item in _trainingData)
{
var data = new object[fieldCount + 1];
for (var i = 0; i < fieldCount; i++)
{
data[i] = item.Item1[i];
}
data[fieldCount] = item.Item2;
builder.AddRow(data);
}
return(builder.Build(output));
}
return(null);
}
public void TestColumnTypes()
{
var builder = new DataTableBuilder();
builder.AddColumn(ColumnType.Boolean, "boolean");
builder.AddColumn(ColumnType.Byte, "byte");
builder.AddColumn(ColumnType.Date, "date");
builder.AddColumn(ColumnType.Double, "double");
builder.AddColumn(ColumnType.Float, "float");
builder.AddColumn(ColumnType.Int, "int");
builder.AddColumn(ColumnType.Long, "long");
builder.AddColumn(ColumnType.Null, "null");
builder.AddColumn(ColumnType.String, "string");
var now = DateTime.Now;
builder.Add(true, (byte)100, now, 1.0 / 3, 0.5f, int.MaxValue, long.MaxValue, null, "test");
var dataTable = builder.Build();
var firstRow = dataTable.GetRow(0);
Assert.AreEqual(firstRow.GetField <bool>(0), true);
Assert.AreEqual(firstRow.GetField <byte>(1), 100);
Assert.AreEqual(firstRow.GetField <DateTime>(2), now);
Assert.AreEqual(firstRow.GetField <double>(3), 1.0 / 3);
Assert.AreEqual(firstRow.GetField <float>(4), 0.5f);
Assert.AreEqual(firstRow.GetField <int>(5), int.MaxValue);
Assert.AreEqual(firstRow.GetField <long>(6), long.MaxValue);
Assert.AreEqual(firstRow.GetField <object>(7), null);
Assert.AreEqual(firstRow.GetField <string>(8), "test");
}
/// <summary>
/// One hot encodes the REBER strings
/// </summary>
/// <param name="strList">A list of REBER sequences</param>
/// <returns>A data table with matrices to represent the sequences of vectors and their corresponding outputs</returns>
public static IDataTable GetOneHot(IEnumerable <string> strList)
{
var strList2 = strList.ToList();
// build the following item table
var following = new Dictionary <string, HashSet <int> >();
foreach (var str in strList2)
{
var sb = new StringBuilder();
string prev = null;
foreach (var ch in str)
{
sb.Append(ch);
var key = sb.ToString();
if (prev != null)
{
if (!following.TryGetValue(prev, out HashSet <int> temp))
{
following.Add(prev, temp = new HashSet <int>());
}
temp.Add(_ch[ch]);
}
prev = key;
}
}
var builder = DataTableBuilder.CreateTwoColumnMatrix();
foreach (var str in strList2)
{
var inputList = new FloatVector[str.Length];
var outputList = new FloatVector[str.Length];
var sb = new StringBuilder();
for (var i = 0; i < str.Length; i++)
{
var ch = str[i];
sb.Append(ch);
var input = new float[_ch.Count];
var output = new float[_ch.Count];
input[_ch[ch]] = 1f;
if (following.TryGetValue(sb.ToString(), out HashSet <int> temp))
{
foreach (var item in temp)
{
output[item] = 1f;
}
}
inputList[i] = new FloatVector {
Data = input
};
outputList[i] = new FloatVector {
Data = output
};
}
builder.Add(FloatMatrix.Create(inputList), FloatMatrix.Create(outputList));
}
return(builder.Build());
}
public void SeCriarDataTableNaoDevePermitirValoresDuplicados()
{
var dicionario = new Dicionario(typeof(ObjetoDeTestes));
var tabela = DataTableBuilder.CriarDataTable(dicionario);
var registro = tabela.NewRow();
registro["Codigo"] = 1;
registro["Nome"] = "ABC";
registro["Duplo"] = 123.56;
registro["Decimal"] = 234.67M;
registro["Logico"] = true;
registro["DataHora"] = DateTime.Now;
tabela.Rows.Add(registro);
var novoRegistro = tabela.NewRow();
novoRegistro["Codigo"] = 1;
novoRegistro["Nome"] = "ABC";
novoRegistro["Duplo"] = 123.56;
novoRegistro["Decimal"] = 234.67M;
novoRegistro["Logico"] = true;
novoRegistro["DataHora"] = DateTime.Now;
Action incluir = () => tabela.Rows.Add(novoRegistro);
incluir
.Should().Throw <ConstraintException>();
}
public void SeCriarDataTableDoObjetoNaoDeveGerarErro()
{
var dicionario = new Dicionario(typeof(ObjetoDeTestes));
Action criar = () => DataTableBuilder.CriarDataTable(dicionario);
criar
.Should().NotThrow();
}
/// <summary>
/// Read an Azure Table as a CSV. Returned CSV includes columns for the ParitionKey and RowKey.
/// The row order is the same as Azure's natural row ordering (sorted by partition key, rowkey)
/// This is a lazy function, so it reads the table rows at a time and does not read the entire table into memory.
/// </summary>
/// <param name="builder">builder</param>
/// <param name="account">azure storage account</param>
/// <param name="tableName">name of table within account</param>
/// <returns></returns>
public static DataTable ReadAzureTableLazy(this DataTableBuilder builder, CloudStorageAccount account, string tableName)
{
var tableClient = account.CreateCloudTableClient();
return(new AzureStreamingTable {
_tableName = tableName, _tableClient = tableClient, Name = tableName
});
}
private static void InitOnClickEventsConfig <T>(DataTableBuilder <T> dtBuilder, DtModel dtModel) where T : class
{
if (dtBuilder.DtaTableColumnsFactory.Columns.Any(c => !string.IsNullOrEmpty(c.Click)))
{
dtModel.HasClickEvents = true;
dtModel.ClickEvents = string.Join(",", dtBuilder.DtaTableColumnsFactory.Columns.Select(e => e.Click));
}
}
/// <summary>
/// Add a table to the builder
/// </summary>
/// <param name="tableBuilderAction">The table builder action.</param>
/// <returns>The builder for chaining</returns>
public DataSetBuilder HavingTable(Action<DataTableBuilder> tableBuilderAction)
{
var tableBuilder = new DataTableBuilder();
tableBuilderAction(tableBuilder);
_TableBuilders.Add(tableBuilder);
return this;
}
/// <summary>
/// Read an entire CSV file into memory.
/// </summary>
/// <param name="builder">ignored</param>
/// <param name="filename">filename of CSV file to load</param>
/// <returns>a mutable in-memory DataTable for the given CSV file</returns>
public static MutableDataTable ReadCsv(this DataTableBuilder builder, string filename)
{
Debug.Assert(builder != null);
if (filename == null)
{
throw new ArgumentNullException("filename");
}
return(Read(builder, filename));
}
/// <summary>
/// Read a data table from azure blob. This reads streaming.
/// </summary>
/// <param name="builder">builder</param>
/// <param name="container">conatiner</param>
/// <param name="blobName">blob name</param>
/// <returns>in-memory mutable datatable from blob</returns>
public static DataTable ReadAzureBlobLazy(this DataTableBuilder builder, CloudBlobContainer container, string blobName)
{
var blob = GetBlobAndVerify(container, blobName);
var stream = blob.OpenRead();
var dt = DataTable.New.ReadLazy(stream);
dt.Name = container.Name + "." + blobName;
return(dt);
}
public static string ToHtmlString(DataTableBuilder <T> dtBuilder)
{
if (string.IsNullOrEmpty(dtBuilder.TableName))
{
throw new ArgumentException("Table MUST have a name.");
}
return(DtModel.InitInstance(dtBuilder).ToString());
}
/// <summary>
/// Return a streaming data table over a file. This just reads a row at a time and avoids reading the whole
/// table into memory. But it only provides sequential read-only access.
/// </summary>
/// <param name="builder"></param>
/// <param name="filename">filename of CSV to read</param>
/// <returns>a streaming data table for the given filename</returns>
public static DataTable ReadLazy(this DataTableBuilder builder, string filename)
{
Debug.Assert(builder != null);
return(new StreamingDataTable(filename)
{
Name = filename
});
}
public void Can_create_a_table_with_an_integer_column()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.HavingAnIntColumnNamed("SomeNumber");
// assert
Assert.IsTrue(dataTable.Columns.Contains("SomeNumber"));
Assert.IsTrue(dataTable.Columns["SomeNumber"].DataType == typeof(int));
}
public void Can_create_a_table_with_a_decimal_column()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.HavingADecimalColumnNamed("SomeDecimal");
// assert
Assert.IsTrue(dataTable.Columns.Contains("SomeDecimal"));
Assert.IsTrue(dataTable.Columns["SomeDecimal"].DataType == typeof(decimal));
}
public void Can_create_a_table_with_a_string_column()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.HavingAStringColumnNamed("SomeString");
// assert
Assert.IsTrue(dataTable.Columns.Contains("SomeString"));
Assert.IsTrue(dataTable.Columns["SomeString"].DataType == typeof(string));
}
public void Can_create_a_table_with_a_row_with_decimal_data()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.WithThisManyRowsOfData(1)
.HavingADecimalColumnWithSetterNamed("SomeDecimal", () => 3.50M);
// assert
Assert.AreEqual(1, dataTable.Rows.Count);
Assert.AreEqual(3.50M, dataTable.Rows[0]["SomeDecimal"]);
}
public void Can_create_a_table_with_a_row_with_integer_data()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.WithThisManyRowsOfData(1)
.HavingAnIntColumnWithSetterNamed("SomeNumber", () => 1);
// assert
Assert.AreEqual(1, dataTable.Rows.Count);
Assert.AreEqual(1, dataTable.Rows[0]["SomeNumber"]);
}
public void Can_create_a_table_with_a_row_with_string_data()
{
// arrange
// act
DataTable dataTable = new DataTableBuilder()
.WithThisManyRowsOfData(1)
.HavingAStringColumnWithSetterNamed("SomeString", () => "string!");
// assert
Assert.AreEqual(1, dataTable.Rows.Count);
Assert.AreEqual("string!", dataTable.Rows[0]["SomeString"]);
}
public void ReadFromTextReaderWithDefaultColumnsShouldHandleFirstRowAsRowData()
{
// arrange
var tmpFile = Path.GetTempFileName();
var rows = new[] { "first,row,is,data", "second,row,is,johnny", "second,row,was,laura", };
using (var sw = new StreamWriter(tmpFile))
{
foreach (var row in rows)
{
sw.WriteLine(row);
}
sw.Flush();
}
// act
try
{
var builder = new DataTableBuilder();
var lazy = builder.ReadLazy(tmpFile, rows[0].Split(','));
Assert.Equal(rows[0].Split(','), lazy.ColumnNames);
var rowEnumerator = rows.Skip(0).GetEnumerator();
rowEnumerator.MoveNext();
var rowCount = 0;
// assert
foreach (var row in lazy.Rows)
{
Assert.Equal(rowEnumerator.Current, string.Join(",", row.Values));
rowEnumerator.MoveNext();
rowCount++;
}
Assert.Equal(rows.Length, rowCount);
}
finally
{
// cleanup
File.Delete(tmpFile);
}
}
public void CanSetupResultForSimpleForQuery()
{
var customers = new DataTableBuilder()
.AddColumn("userid", typeof(Int32))
.AddColumn("email", typeof(String))
.AddRow(1, "*****@*****.**")
.AddRow(10, "*****@*****.**").DataTable;
var behavior = new MockCommandBehavior()
.When(c=> c.CommandText.StartsWith("select *"))
.ReturnsData(customers);
Factory.AddBehavior(behavior);
var table = SUT.DataAccess.GetAllUsers();
Assert.AreEqual(2, table.Rows.Count);
Assert.AreEqual(1, table.Rows[0][0]);
Assert.AreEqual("*****@*****.**", table.Rows[0][1]);
Assert.AreEqual(10, table.Rows[1][0]);
Assert.AreEqual("*****@*****.**", table.Rows[1][1]);
}
public void SetUp()
{
_builder = new DataTableBuilder();
}
public DataDirectoryTablesBuilder(DataTableBuilder dataTableBuilder, IOffsetConverter offsetConverter)
{
_dataTableBuilder = dataTableBuilder;
_offsetConverter = offsetConverter;
}
