public void CreateDatasetVersionTest()
{
long numberOfTuples = 1000;
var dm = new DatasetManager();
var rsm = new ResearchPlanManager();
var mdm = new MetadataStructureManager();
try
{
var dsHelper = new DatasetHelper();
StructuredDataStructure dataStructure = dsHelper.CreateADataStructure();
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
var rp = dsHelper.CreateResearchPlan();
rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required.");
var mds = mdm.Repo.Query().First();
mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required.");
Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds);
dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad");
dataset.Should().NotBeNull("The dataset tuple generation has failed!");
dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None);
//dm.SyncView(ds.Id, ViewCreationBehavior.Create);
//dm.SyncView(ds.Id, ViewCreationBehavior.Refresh);
dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh);
dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted.");
dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong.");
dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure.");
dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status.");
dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples);
dm.DatasetVersionRepo.Evict();
dm.DataTupleRepo.Evict();
dm.DatasetRepo.Evict();
dm.PurgeDataset(dataset.Id, true);
dsHelper.PurgeAllDataStructures();
}
finally
{
dm.Dispose();
rsm.Dispose();
mdm.Dispose();
}
}
/// <summary>
/// Shows a berif intro about the functions available as well as some warnings that inofrom the user about non recoverability of some of the operations
/// such as purge.
/// </summary>
/// <returns></returns>
public ActionResult Index()
{
ViewBag.Title = PresentationModel.GetViewTitleForTenant("Maintain Datasets", Session.GetTenant());
DatasetManager dm = new DatasetManager();
var entityPermissionManager = new EntityPermissionManager();
List <Dataset> datasets = new List <Dataset>();
List <long> datasetIds = new List <long>();
//if (!string.IsNullOrWhiteSpace(HttpContext.User.Identity.Name))
//{
// datasetIds.AddRange(entityPermissionManager.GetKeys(HttpContext.User.Identity.Name, "Dataset", typeof(Dataset), RightType.Delete));
// if(datasetIds.Count() > 0)
// datasets = dm.DatasetRepo.Query().OrderBy(p => p.Id).ToList();
//}
datasets = dm.DatasetRepo.Query().OrderBy(p => p.Id).ToList();
datasetIds = datasets.Select(p => p.Id).ToList();
// dataset id, dataset status, number of data tuples of the latest version, number of variables in the dataset's structure
List <DatasetStatModel> datasetStat = new List <DatasetStatModel>();
foreach (Dataset ds in datasets)
{
long noColumns = ds.DataStructure.Self is StructuredDataStructure ? (ds.DataStructure.Self as StructuredDataStructure).Variables.Count() : 0L;
long noRows = ds.DataStructure.Self is StructuredDataStructure?dm.GetDatasetLatestVersionEffectiveTupleCount(ds) : 0; // It would save time to calc the row count for all the datasets at once!
bool synced = false;
if (string.Compare(ds.StateInfo?.State, "Synced", true) == 0 &&
ds.StateInfo?.Timestamp != null &&
ds.StateInfo?.Timestamp > DateTime.MinValue &&
ds.StateInfo?.Timestamp < DateTime.MaxValue)
{
synced = ds.StateInfo?.Timestamp >= ds.LastCheckIOTimestamp;
}
datasetStat.Add(new DatasetStatModel {
Id = ds.Id, Status = ds.Status, NoOfRows = noRows, NoOfCols = noColumns, IsSynced = synced
});
}
ViewData["DatasetIds"] = datasetIds;
return(View(datasetStat));
}
public ActionResult CountRows(long id)
{
int number = 0;
using (DatasetManager dm = new DatasetManager())
{
try
{
if (id > 0)
{
Dataset ds = dm.GetDataset(id);
number = ds.DataStructure.Self is StructuredDataStructure?dm.GetDatasetLatestVersionEffectiveTupleCount(ds) : 0;
}
return(Json(number, JsonRequestBehavior.AllowGet));
}
finally
{
dm.Dispose();
}
}
}
public void ProjectExpressionTest()
{
var dsHelper = new DatasetHelper();
StructuredDataStructure dataStructure = dsHelper.CreateADataStructure();
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
string var1Name = "var" + dataStructure.Variables.First().Id;
string var3Name = "var" + dataStructure.Variables.Skip(2).First().Id;
//create prjection expression
ProjectionExpression projectionExpression = new ProjectionExpression();
projectionExpression.Items.Add(new ProjectionItemExpression()
{
FieldName = var1Name
});
projectionExpression.Items.Add(new ProjectionItemExpression()
{
FieldName = var3Name
});
// create a dataset and test the filter, sorting, and projectgion
long numberOfTuples = 10;
var dm = new DatasetManager();
var rsm = new ResearchPlanManager();
var mdm = new MetadataStructureManager();
try
{
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
var rp = dsHelper.CreateResearchPlan();
rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required.");
var mds = mdm.Repo.Query().First();
mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required.");
Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds);
dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad");
dataset.Should().NotBeNull("The dataset tuple generation has failed!");
dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None);
dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh);
dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted.");
dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong.");
dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure.");
dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status.");
dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples);
// pass this filter to get a subset of dataset X
var dst = dm.GetLatestDatasetVersionTuples(dataset.Id, null, null, projectionExpression, 1, 3);
dst.Should().NotBeNull();
dst.Rows.Count.Should().BeLessOrEqualTo(3);
dst.Columns.Count.Should().BeLessOrEqualTo(3, "Projection failed, wrong number of columns");
dm.DatasetVersionRepo.Evict();
dm.DataTupleRepo.Evict();
dm.DatasetRepo.Evict();
dm.PurgeDataset(dataset.Id, true);
dsHelper.PurgeAllDataStructures();
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
rsm.Dispose();
mdm.Dispose();
}
}
public void CreateAndExpressionForQueryingTest()
{
var dsHelper = new DatasetHelper();
StructuredDataStructure dataStructure = dsHelper.CreateADataStructure();
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
string var1Name = "var" + dataStructure.Variables.First().Id;
string var2Name = "var" + dataStructure.Variables.Skip(1).First().Id;
FilterExpression fex = BinaryFilterExpression
.And(
new FilterNumberItemExpression()
{
Field = new Field()
{
DataType = Utils.NH.Querying.DataType.Ineteger, Name = var1Name
}
,
Operator = NumberOperator.Operation.GreaterThan
,
Value = 12
}
,
new FilterStringItemExpression()
{
Field = new Field()
{
DataType = Utils.NH.Querying.DataType.String, Name = var2Name
}
,
Operator = StringOperator.Operation.EndsWith
,
Value = "Test"
}
);
fex.ToSQL().Should().Be($"(({var1Name}) > (12)) AND (({var2Name}) ILIKE ('%Test'))");
// this is to show how to apply a NOT operator on any other expression.
// It can be applied on Numeric, String, Date, and any other type of expression
FilterExpression notFex = UnaryFilterExpression.Not(fex);
notFex.ToSQL().Should().Be($"NOT ((({var1Name}) > (12)) AND (({var2Name}) ILIKE ('%Test')))");
notFex.ToSQL().Should().Be($"NOT ({fex.ToSQL()})");
OrderByExpression orderByExpr = new OrderByExpression(
new List <OrderItemExpression>()
{
new OrderItemExpression(var1Name),
new OrderItemExpression(var2Name, SortDirection.Descending)
});
orderByExpr.ToSQL().Should().Be($"{var1Name} ASC, {var2Name} DESC");
// create a dataset and test the filter, sorting, and projectgion
long numberOfTuples = 100;
var dm = new DatasetManager();
var rsm = new ResearchPlanManager();
var mdm = new MetadataStructureManager();
try
{
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
var rp = dsHelper.CreateResearchPlan();
rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required.");
var mds = mdm.Repo.Query().First();
mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required.");
Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds);
dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad");
dataset.Should().NotBeNull("The dataset tuple generation has failed!");
dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None);
dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh);
dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted.");
dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong.");
dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure.");
dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status.");
dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples);
// pass this filter to get a subset of dataset X
var dst = dm.GetLatestDatasetVersionTuples(dataset.Id, fex, null, null, 1, 10);
dst.Should().NotBeNull();
dst.Rows.Count.Should().BeLessOrEqualTo(10);
dm.DatasetVersionRepo.Evict();
dm.DataTupleRepo.Evict();
dm.DatasetRepo.Evict();
dm.PurgeDataset(dataset.Id, true);
dsHelper.PurgeAllDataStructures();
}
finally
{
dm.Dispose();
rsm.Dispose();
mdm.Dispose();
}
}
