public void CreateDatasetVersionTest() { long numberOfTuples = 1000; var dm = new DatasetManager(); var rsm = new ResearchPlanManager(); var mdm = new MetadataStructureManager(); try { var dsHelper = new DatasetHelper(); StructuredDataStructure dataStructure = dsHelper.CreateADataStructure(); dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required."); var rp = dsHelper.CreateResearchPlan(); rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required."); var mds = mdm.Repo.Query().First(); mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required."); Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds); dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad"); dataset.Should().NotBeNull("The dataset tuple generation has failed!"); dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None); //dm.SyncView(ds.Id, ViewCreationBehavior.Create); //dm.SyncView(ds.Id, ViewCreationBehavior.Refresh); dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh); dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted."); dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong."); dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure."); dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status."); dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples); dm.DatasetVersionRepo.Evict(); dm.DataTupleRepo.Evict(); dm.DatasetRepo.Evict(); dm.PurgeDataset(dataset.Id, true); dsHelper.PurgeAllDataStructures(); } finally { dm.Dispose(); rsm.Dispose(); mdm.Dispose(); } }
/// <summary> /// Shows a berif intro about the functions available as well as some warnings that inofrom the user about non recoverability of some of the operations /// such as purge. /// </summary> /// <returns></returns> public ActionResult Index() { ViewBag.Title = PresentationModel.GetViewTitleForTenant("Maintain Datasets", Session.GetTenant()); DatasetManager dm = new DatasetManager(); var entityPermissionManager = new EntityPermissionManager(); List <Dataset> datasets = new List <Dataset>(); List <long> datasetIds = new List <long>(); //if (!string.IsNullOrWhiteSpace(HttpContext.User.Identity.Name)) //{ // datasetIds.AddRange(entityPermissionManager.GetKeys(HttpContext.User.Identity.Name, "Dataset", typeof(Dataset), RightType.Delete)); // if(datasetIds.Count() > 0) // datasets = dm.DatasetRepo.Query().OrderBy(p => p.Id).ToList(); //} datasets = dm.DatasetRepo.Query().OrderBy(p => p.Id).ToList(); datasetIds = datasets.Select(p => p.Id).ToList(); // dataset id, dataset status, number of data tuples of the latest version, number of variables in the dataset's structure List <DatasetStatModel> datasetStat = new List <DatasetStatModel>(); foreach (Dataset ds in datasets) { long noColumns = ds.DataStructure.Self is StructuredDataStructure ? (ds.DataStructure.Self as StructuredDataStructure).Variables.Count() : 0L; long noRows = ds.DataStructure.Self is StructuredDataStructure?dm.GetDatasetLatestVersionEffectiveTupleCount(ds) : 0; // It would save time to calc the row count for all the datasets at once! bool synced = false; if (string.Compare(ds.StateInfo?.State, "Synced", true) == 0 && ds.StateInfo?.Timestamp != null && ds.StateInfo?.Timestamp > DateTime.MinValue && ds.StateInfo?.Timestamp < DateTime.MaxValue) { synced = ds.StateInfo?.Timestamp >= ds.LastCheckIOTimestamp; } datasetStat.Add(new DatasetStatModel { Id = ds.Id, Status = ds.Status, NoOfRows = noRows, NoOfCols = noColumns, IsSynced = synced }); } ViewData["DatasetIds"] = datasetIds; return(View(datasetStat)); }
public ActionResult CountRows(long id) { int number = 0; using (DatasetManager dm = new DatasetManager()) { try { if (id > 0) { Dataset ds = dm.GetDataset(id); number = ds.DataStructure.Self is StructuredDataStructure?dm.GetDatasetLatestVersionEffectiveTupleCount(ds) : 0; } return(Json(number, JsonRequestBehavior.AllowGet)); } finally { dm.Dispose(); } } }
public void ProjectExpressionTest() { var dsHelper = new DatasetHelper(); StructuredDataStructure dataStructure = dsHelper.CreateADataStructure(); dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required."); string var1Name = "var" + dataStructure.Variables.First().Id; string var3Name = "var" + dataStructure.Variables.Skip(2).First().Id; //create prjection expression ProjectionExpression projectionExpression = new ProjectionExpression(); projectionExpression.Items.Add(new ProjectionItemExpression() { FieldName = var1Name }); projectionExpression.Items.Add(new ProjectionItemExpression() { FieldName = var3Name }); // create a dataset and test the filter, sorting, and projectgion long numberOfTuples = 10; var dm = new DatasetManager(); var rsm = new ResearchPlanManager(); var mdm = new MetadataStructureManager(); try { dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required."); var rp = dsHelper.CreateResearchPlan(); rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required."); var mds = mdm.Repo.Query().First(); mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required."); Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds); dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad"); dataset.Should().NotBeNull("The dataset tuple generation has failed!"); dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None); dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh); dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted."); dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong."); dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure."); dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status."); dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples); // pass this filter to get a subset of dataset X var dst = dm.GetLatestDatasetVersionTuples(dataset.Id, null, null, projectionExpression, 1, 3); dst.Should().NotBeNull(); dst.Rows.Count.Should().BeLessOrEqualTo(3); dst.Columns.Count.Should().BeLessOrEqualTo(3, "Projection failed, wrong number of columns"); dm.DatasetVersionRepo.Evict(); dm.DataTupleRepo.Evict(); dm.DatasetRepo.Evict(); dm.PurgeDataset(dataset.Id, true); dsHelper.PurgeAllDataStructures(); } catch (Exception ex) { throw ex; } finally { dm.Dispose(); rsm.Dispose(); mdm.Dispose(); } }
public void CreateAndExpressionForQueryingTest() { var dsHelper = new DatasetHelper(); StructuredDataStructure dataStructure = dsHelper.CreateADataStructure(); dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required."); string var1Name = "var" + dataStructure.Variables.First().Id; string var2Name = "var" + dataStructure.Variables.Skip(1).First().Id; FilterExpression fex = BinaryFilterExpression .And( new FilterNumberItemExpression() { Field = new Field() { DataType = Utils.NH.Querying.DataType.Ineteger, Name = var1Name } , Operator = NumberOperator.Operation.GreaterThan , Value = 12 } , new FilterStringItemExpression() { Field = new Field() { DataType = Utils.NH.Querying.DataType.String, Name = var2Name } , Operator = StringOperator.Operation.EndsWith , Value = "Test" } ); fex.ToSQL().Should().Be($"(({var1Name}) > (12)) AND (({var2Name}) ILIKE ('%Test'))"); // this is to show how to apply a NOT operator on any other expression. // It can be applied on Numeric, String, Date, and any other type of expression FilterExpression notFex = UnaryFilterExpression.Not(fex); notFex.ToSQL().Should().Be($"NOT ((({var1Name}) > (12)) AND (({var2Name}) ILIKE ('%Test')))"); notFex.ToSQL().Should().Be($"NOT ({fex.ToSQL()})"); OrderByExpression orderByExpr = new OrderByExpression( new List <OrderItemExpression>() { new OrderItemExpression(var1Name), new OrderItemExpression(var2Name, SortDirection.Descending) }); orderByExpr.ToSQL().Should().Be($"{var1Name} ASC, {var2Name} DESC"); // create a dataset and test the filter, sorting, and projectgion long numberOfTuples = 100; var dm = new DatasetManager(); var rsm = new ResearchPlanManager(); var mdm = new MetadataStructureManager(); try { dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required."); var rp = dsHelper.CreateResearchPlan(); rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required."); var mds = mdm.Repo.Query().First(); mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required."); Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds); dataset = dsHelper.GenerateTuplesForDataset(dataset, dataStructure, numberOfTuples, "Javad"); dataset.Should().NotBeNull("The dataset tuple generation has failed!"); dm.CheckInDataset(dataset.Id, "for testing purposes 2", "Javad", ViewCreationBehavior.None); dm.SyncView(dataset.Id, ViewCreationBehavior.Create | ViewCreationBehavior.Refresh); dataset.Id.Should().BeGreaterThan(0, "Dataset was not persisted."); dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong."); dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure."); dataset.Status.Should().Be(DatasetStatus.CheckedIn, "Dataset must be in the CheckedIn status."); dm.GetDatasetLatestVersionEffectiveTupleCount(dataset.Id).Should().Be(numberOfTuples); // pass this filter to get a subset of dataset X var dst = dm.GetLatestDatasetVersionTuples(dataset.Id, fex, null, null, 1, 10); dst.Should().NotBeNull(); dst.Rows.Count.Should().BeLessOrEqualTo(10); dm.DatasetVersionRepo.Evict(); dm.DataTupleRepo.Evict(); dm.DatasetRepo.Evict(); dm.PurgeDataset(dataset.Id, true); dsHelper.PurgeAllDataStructures(); } finally { dm.Dispose(); rsm.Dispose(); mdm.Dispose(); } }