/// <summary> /// If the <paramref name="entity"/> is not associated to any <see cref="Dateset"/>, the method deletes it from the database. /// </summary> /// <param name="entity">The data structure object to be deleted.</param> /// <returns>True if the data structure is deleted, False otherwise.</returns> /// <remarks>Database exceptions are not handled intentionally, so that if the data structure is related to some datasets, a proper exception will be thrown.</remarks> public bool DeleteStructuredDataStructure(StructuredDataStructure entity) { Contract.Requires(entity != null); Contract.Requires(entity.Id >= 0); IReadOnlyRepository <Dataset> datasetRepo = this.GetUnitOfWork().GetReadOnlyRepository <Dataset>(); if (datasetRepo.Query(p => p.DataStructure.Id == entity.Id).Count() > 0) { throw new Exception(string.Format("Data structure {0} is used by datasets. Deletion Failed", entity.Id)); } using (IUnitOfWork uow = this.GetUnitOfWork()) { IRepository <StructuredDataStructure> repo = uow.GetRepository <StructuredDataStructure>(); IRepository <Variable> variableRepo = uow.GetRepository <Variable>(); IRepository <Parameter> paramRepo = uow.GetRepository <Parameter>(); variableRepo.Evict(); paramRepo.Evict(); entity = repo.Reload(entity); // delete associated variables and thier parameters foreach (var usage in entity.Variables) { var localVar = variableRepo.Reload(usage); paramRepo.Delete(localVar.Parameters.ToList()); variableRepo.Delete(localVar); } //uow.Commit(); // should not be needed repo.Delete(entity); uow.Commit(); } return(true); }
public ActionResult _CustomDataStructureBinding(GridCommand command, long datasetID) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); try { long id = datasetID; if (dm.IsDatasetCheckedIn(id)) { DatasetVersion ds = dm.GetDatasetLatestVersion(id); if (ds != null) { StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(ds.Dataset.DataStructure.Id); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); //StructuredDataStructure sds = (StructuredDataStructure)(ds.Dataset.DataStructure.Self); SearchUIHelper suh = new SearchUIHelper(); DataTable table = suh.ConvertStructuredDataStructureToDataTable(sds); return(View(new GridModel(table))); } } else { ModelState.AddModelError(String.Empty, "Dataset is just in processing."); } } finally { dm.Dispose(); dsm.Dispose(); } return(View(new GridModel(new DataTable()))); }
public DataTable ConvertDatasetVersion(DatasetManager datasetManager, DatasetVersion datasetVersion, string tableName = "") { DataTable dt = new DataTable(); if (string.IsNullOrEmpty(tableName)) { dt.TableName = "Primary data table"; } else { dt.TableName = tableName; } DataStructureManager dsm = new DataStructureManager(); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id); var tupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); if (tupleIds != null && tupleIds.Count > 0 && sds != null) { buildTheHeader(sds, dt); buildTheBody(datasetManager, tupleIds, dt, sds); } return(dt); }
/// <summary> /// Creates a structured data structure <seealso cref="StructuredDataStructure"/> and persists the entity in the database. /// </summary> /// <param name="name">The name of the data structure</param> /// <param name="description">A free text describing the purpose, usage, and/or the domain of the data structure usage.</param> /// <param name="xsdFileName">Not in use.</param> /// <param name="xslFileName">Not in use.</param> /// <param name="indexerType">If the data structure is used as a matrix, The indexer type show what kind of column would be represented by the indexer variable. <see cref="DataStructureCategory"/></param> /// <param name="indexer">The variable indicating the first indexing column of the matrix, if the data structure is representing a matrix.</param> /// <returns>The persisted structured data structure instance.</returns> public StructuredDataStructure CreateStructuredDataStructure(string name, string description, string xsdFileName, string xslFileName, DataStructureCategory indexerType, Variable indexer = null) { Contract.Requires(!string.IsNullOrWhiteSpace(name)); Contract.Requires(indexerType != DataStructureCategory.Generic ? (indexer != null) : true); Contract.Ensures(Contract.Result <StructuredDataStructure>() != null && Contract.Result <StructuredDataStructure>().Id >= 0); StructuredDataStructure e = new StructuredDataStructure() { Name = name, Description = description, XsdFileName = xsdFileName, XslFileName = xslFileName, IndexerType = indexerType, // Indexer = indexer, // how its possible to have the indexer before assigning variable to the structure }; using (IUnitOfWork uow = this.GetUnitOfWork()) { IRepository <StructuredDataStructure> repo = uow.GetRepository <StructuredDataStructure>(); repo.Put(e); uow.Commit(); } return(e); }
public List <string> GenerateRowsWithRandomValuesBasedOnDatastructureWithErrors(StructuredDataStructure dataStructure, string seperator, long numberOfTuples, bool withQuotes) { List <string> rows = new List <string>(); numberOfTuples.Should().BeGreaterThan(0); var r = new Random(); try { for (int i = 0; i < numberOfTuples; i++) { string row = r.Next().ToString(); if (withQuotes) { row += seperator.ToString() + "\"Test\""; } else { row += seperator.ToString() + "Test"; } row += seperator.ToString() + "abc";//Convert.ToDouble(r.Next()).ToString(); row += seperator.ToString() + true.ToString(); row += seperator.ToString() + DateTime.Now.ToString(); rows.Add(row); } return(rows); } catch (Exception ex) { return(null); } }
protected override bool AddHeader(StructuredDataStructure header) { return(false); }
// add table header protected abstract bool AddHeader(StructuredDataStructure header);
private void buildTheHeader(StructuredDataStructure sds, DataTable dt) { foreach (var vu in sds.Variables) { DataColumn col = null; string columnName = "var" + vu.Id.ToString(); col = dt.Columns.Add(columnName); // or DisplayName also col.Caption = string.IsNullOrEmpty(vu.Label) ? columnName : vu.Label; switch (vu.DataAttribute.DataType.SystemType) { case "String": { col.DataType = Type.GetType("System.String"); break; } case "Double": { col.DataType = Type.GetType("System.Double"); break; } case "Int16": { col.DataType = Type.GetType("System.Int16"); break; } case "Int32": { col.DataType = Type.GetType("System.Int32"); break; } case "Int64": { col.DataType = Type.GetType("System.Int64"); break; } case "Decimal": { col.DataType = Type.GetType("System.Decimal"); break; } case "DateTime": { col.DataType = Type.GetType("System.DateTime"); break; } default: { col.DataType = Type.GetType("System.String"); break; } } if (vu.Parameters.Count > 0) { foreach (var pu in vu.Parameters) { DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also col2.Caption = pu.Label; } } } }
public DataTable ConvertDatasetVersion(List <AbstractTuple> tuples, DatasetVersion datasetVersion, StructuredDataStructure sds, string tableName = "") { DataTable dt = new DataTable(); if (string.IsNullOrEmpty(tableName)) { dt.TableName = "Primary data table"; } else { dt.TableName = tableName; } if (tuples != null && tuples.Count > 0 && sds != null) { buildTheHeader(sds, dt); buildTheBody(tuples, dt, sds); } return(dt); }
private void buildTheBody(DatasetManager datasetManager, List <long> tupleIds, DataTable dt, StructuredDataStructure sds) { DataTupleIterator tupleIterator = new DataTupleIterator(tupleIds, datasetManager); foreach (var tuple in tupleIterator) { dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds)); } }
public Dataset UpdateOneTupleForDataset(Dataset dataset, StructuredDataStructure dataStructure, long id, int value, DatasetManager datasetManager) { dataset.Status.Should().Be(DatasetStatus.CheckedIn); dataset.Should().NotBeNull(); try { if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, "David") || datasetManager.CheckOutDataset(dataset.Id, "David")) { dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status."); DatasetVersion workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id); DataTuple oldDt = datasetManager.DataTupleRepo.Get(id); DataTuple dt = new DataTuple(); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.First().Id, Value = value }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(1).First().Id, Value = "Test" }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(2).First().Id, Value = 5 }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(3).First().Id, Value = true }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(4).First().Id, Value = DateTime.Now.ToString(new CultureInfo("en-US")) }); dt.Dematerialize(); dt.Should().NotBeNull(); dt.JsonVariableValues.Should().NotBeNull(); List <DataTuple> tuples = new List <DataTuple>(); DataTuple newDt = new DataTuple(); newDt.Id = id; newDt.XmlAmendments = dt.XmlAmendments; newDt.JsonVariableValues = dt.JsonVariableValues; newDt.Materialize(); newDt.OrderNo = oldDt.OrderNo; tuples.Add(newDt); datasetManager.EditDatasetVersion(workingCopy, null, tuples, null); dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status."); } return(dataset); } catch (Exception ex) { return(null); } }
public Dataset GenerateTuplesWithRandomValuesForDataset(Dataset dataset, StructuredDataStructure dataStructure, long numberOfTuples, string username) { dataset.Status.Should().Be(DatasetStatus.CheckedIn); dataset.Should().NotBeNull(); numberOfTuples.Should().BeGreaterThan(0); var r = new Random(); DatasetManager dm = new DatasetManager(); try { if (dm.IsDatasetCheckedOutFor(dataset.Id, username) || dm.CheckOutDataset(dataset.Id, username)) { dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status."); DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(dataset.Id); List <DataTuple> tuples = new List <DataTuple>(); for (int i = 0; i < numberOfTuples; i++) { DataTuple dt = new DataTuple(); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.First().Id, Value = r.Next() }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(1).First().Id, Value = "Test" }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(2).First().Id, Value = Convert.ToDouble(r.Next()) }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(3).First().Id, Value = true }); dt.VariableValues.Add(new VariableValue() { VariableId = dataStructure.Variables.Skip(4).First().Id, Value = "01.01.2017" }); dt.Dematerialize(); dt.Should().NotBeNull(); //dt.XmlVariableValues.Should().NotBeNull(); DataTuple newDt = new DataTuple(); newDt.XmlAmendments = dt.XmlAmendments; //newDt.XmlVariableValues = dt.XmlVariableValues; newDt.JsonVariableValues = dt.JsonVariableValues; newDt.Materialize(); newDt.OrderNo = i; tuples.Add(newDt); } dm.EditDatasetVersion(workingCopy, tuples, null, null); dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status."); } return(dataset); } catch (Exception ex) { return(null); } finally { dm.Dispose(); } }
public List <Variable> getOrderedVariables(StructuredDataStructure structuredDataStructure) { return(getOrderedVariables(structuredDataStructure.Id, structuredDataStructure.Variables.ToList())); }
//[MeasurePerformance] //temporary solution: norman //For original solution, look into Aquadiva Code public List <Error> FinishUpload(EasyUploadTaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); DataContainerManager dam = new DataContainerManager(); //SubjectManager sm = new SubjectManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); List <Error> temp = new List <Error>(); try { using (IUnitOfWork unitOfWork = this.GetUnitOfWork()) { // initialize all necessary manager DataTuple[] rows = null; //First, try to validate - if there are errors, return immediately string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString(); List <Error> ValidationErrors = ValidateRows(JsonArray); if (ValidationErrors.Count != 0) { temp.AddRange(ValidationErrors); return(temp); } string timestamp = DateTime.UtcNow.ToString("r"); string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]); if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE)) { string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); if (!String.IsNullOrWhiteSpace(tmp)) { title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); } } StructuredDataStructure sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp); if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE)) { TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title); TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title); } MetadataStructure metadataStructure = null; if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA)) { long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]); metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Id == metadataStructureId).FirstOrDefault(); } else { //Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault(); } ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault(); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title); #region Progress Information if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0); } #endregion #region DataStructure XmlDocument xmldoc = new XmlDocument(); XmlElement extraElement = xmldoc.CreateElement("extra"); XmlElement orderElement = xmldoc.CreateElement("order"); List <Tuple <int, string, UnitInfo> > MappedHeaders = (List <Tuple <int, string, UnitInfo> >)TaskManager.Bus[EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS]; //Sorting necessary to prevent problems when inserting the tuples MappedHeaders.Sort((head1, head2) => head1.Item1.CompareTo(head2.Item1)); List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>(); var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>(); var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>(); List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList(); foreach (Tuple <int, string, UnitInfo> Entry in MappedHeaders) { int i = MappedHeaders.IndexOf(Entry); DataType dataType = dataTypeRepo.Get(Entry.Item3.SelectedDataTypeId); Unit CurrentSelectedUnit = unitRepo.Get(Entry.Item3.UnitId); DataAttribute CurrentDataAttribute = new DataAttribute(); //If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match) DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(Entry.Item2).ToLower()) && da.DataType.Id == dataType.Id && da.Unit.Id == CurrentSelectedUnit.Id).FirstOrDefault(); if (existingDataAttribute != null) { CurrentDataAttribute = existingDataAttribute; } else { //No matching DataAttribute => Create a new one CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(Entry.Item2), Entry.Item2, "", false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null); } Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, Entry.Item2, "", "", ""); VariableIdentifier vi = new VariableIdentifier { name = newVariable.Label, id = newVariable.Id }; identifiers.Add(vi); XmlElement newVariableXml = xmldoc.CreateElement("variable"); newVariableXml.InnerText = Convert.ToString(newVariable.Id); orderElement.AppendChild(newVariableXml); } extraElement.AppendChild(orderElement); xmldoc.AppendChild(extraElement); sds.Extra = xmldoc; sds.Name = "generated import structure " + timestamp; sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp; #endregion Dataset ds = null; ds = dm.CreateEmptyDataset(sds, rp, metadataStructure); //TODO Should a template be created? /*ExcelTemplateProvider etp = new ExcelTemplateProvider(); * etp.CreateTemplate(sds);*/ long datasetId = ds.Id; long sdsId = sds.Id; if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault())) { DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId); long METADATASTRUCTURE_ID = metadataStructure.Id; XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath); XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID); XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX); dsv.Metadata = metadataXml; try { dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title); } catch (NullReferenceException ex) { //Reference of the title node is missing throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!"); } dm.EditDatasetVersion(dsv, null, null, null); } #region security // add security if (GetUsernameOrDefault() != "DEFAULT") { //PermissionManager pm = new PermissionManager(); //User user = sm.GetUserByName(GetUsernameOrDefault()); //Rights-Management /* * TODO: Use the BExIS Party API for that * * */ /* * UserPiManager upm = new UserPiManager(); * List<long> piList = (new UserSelectListModel(GetUsernameOrDefault())).UserList.Select(x => x.Id).ToList(); */ foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>()) { //The user gets full permissions // add security if (GetUsernameOrDefault() != "DEFAULT") { entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList()); } // adding the rights for the pis /*foreach (long piId in piList) * { * //Each pi gets full permissions * pm.CreateDataPermission(piId, 1, ds.Id, rightType); * * // get all pi members * List<UserPi> currentMembers = upm.GetAllPiMember(piId).ToList(); * * foreach (UserPi currentMapping in currentMembers) * { * switch (rightType) * { * //Each member of each of the pis gets view-rights * case RightType.View: * pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType); * break; * //Each member of each of the pis gets download-rights * case RightType.Download: * pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType); * break; * default: * //No other permissions - is this call necessary? * pm.CreateDataPermission(currentMapping.UserId, 0, ds.Id, rightType); * break; * } * } * }*/ } } #endregion security #region excel reader int packageSize = 10000; //HACK ? TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id); counter++; TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter; //rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray(); List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA]; string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString(); List <int[]> areaDataValuesList = new List <int[]>(); foreach (string area in selectedDataAreaJsonArray) { areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area)); } int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray); Orientation orientation = 0; switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString()) { case "LeftRight": orientation = Orientation.rowwise; break; case "Matrix": //orientation = Orientation.matrix; break; default: orientation = Orientation.columnwise; break; } String worksheetUri = null; //Get the Uri to identify the correct worksheet if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI)) { worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString(); } int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize; int batchnr = 1; foreach (int[] areaDataValues in areaDataValuesList) { //First batch starts at the start of the current data area int currentBatchStartRow = areaDataValues[0] + 1; while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached { //Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work EasyUploadExcelReader reader = new EasyUploadExcelReader(); // open file Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString()); //End row is start row plus batch size int currentBatchEndRow = currentBatchStartRow + batchSize; //Set the indices for the reader EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo { DataStartRow = currentBatchStartRow, //End row is either at the end of the batch or the end of the marked area //DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow, DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1), //Column indices as marked in a previous step DataStartColumn = areaDataValues[1] + 1, DataEndColumn = areaDataValues[3] + 1, //Header area as marked in a previous step VariablesStartRow = areaHeaderValues[0] + 1, VariablesStartColumn = areaHeaderValues[1] + 1, VariablesEndRow = areaHeaderValues[2] + 1, VariablesEndColumn = areaHeaderValues[3] + 1, Offset = areaDataValues[1], Orientation = orientation }; //Set variable identifiers because they might differ from the variable names in the file reader.setSubmittedVariableIdentifiers(identifiers); //Read the rows and convert them to DataTuples rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, sds, (int)datasetId, worksheetUri); //After reading the rows, add them to the dataset if (rows != null) { dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null); } //Close the Stream so the next ExcelReader can open it again Stream.Close(); //Debug information int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1); int batches = lines / batchSize; batchnr++; //Next batch starts after the current one currentBatchStartRow = currentBatchEndRow + 1; } } #endregion dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); //Reindex search if (this.IsAccessibale("DDM", "SearchIndex", "ReIndexSingle")) { this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary() { { "id", datasetId } }); } TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id); return(temp); } } catch (Exception ex) { temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " + "Please try again later. If this problem keeps occuring, please contact your administrator.")); return(temp); } finally { dsm.Dispose(); dm.Dispose(); dam.Dispose(); //sm.Dispose(); entityPermissionManager.Dispose(); } }
public DataReader(StructuredDataStructure structuredDatastructure, FileReaderInfo fileReaderInfo, DatasetManager datasetManager) : this(structuredDatastructure, fileReaderInfo, new IOUtility(), datasetManager) { }
public DataTable ConvertDatasetVersion(IEnumerable <AbstractTuple> tupleIterator, StructuredDataStructure dataStructure, string tableName = "") { return(ConvertPrimaryDataToDatatable(tupleIterator, dataStructure, tableName)); }
private void buildTheBody(IEnumerable <AbstractTuple> tupleIterator, DataTable dt, StructuredDataStructure sds) { foreach (var tuple in tupleIterator) { dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds)); } }
public StructuredDataStructure CreateADataStructure() { var unitManager = new UnitManager(); var dataTypeManager = new DataTypeManager(); var attributeManager = new DataContainerManager(); var dsManager = new DataStructureManager(); try { var dim = unitManager.Create("TestDimnesion", "For Unit Testing", ""); var unit = unitManager.Create("None_UT", "NoneUT", "Use in unit tsting", dim, Dlm.Entities.DataStructure.MeasurementSystem.Metric); var intType = dataTypeManager.Create("Integer", "Integer", TypeCode.Int32); var strType = dataTypeManager.Create("String", "String", TypeCode.String); var doubleType = dataTypeManager.Create("Double", "Double", TypeCode.Double); var boolType = dataTypeManager.Create("Bool", "Bool", TypeCode.Boolean); var dateTimeType = dataTypeManager.Create("DateTime", "DateTime", TypeCode.DateTime); var dataAttribute1 = attributeManager.CreateDataAttribute( "att1UT", "att1UT", "Attribute for Unit testing", false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType, "", intType, unit, null, null, null, null, null, null ); var dataAttribute2 = attributeManager.CreateDataAttribute( "att2UT", "att1UT", "Attribute for Unit testing", false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType, "", strType, unit, null, null, null, null, null, null ); var dataAttribute3 = attributeManager.CreateDataAttribute( "att3UT", "att3UT", "Attribute for Unit testing", false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType, "", doubleType, unit, null, null, null, null, null, null ); var dataAttribute4 = attributeManager.CreateDataAttribute( "att4UT", "att4UT", "Attribute for Unit testing", false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType, "", boolType, unit, null, null, null, null, null, null ); var dataAttribute5 = attributeManager.CreateDataAttribute( "att5UT", "att5UT", "Attribute for Unit testing", false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType, "", dateTimeType, unit, null, null, null, null, null, null ); StructuredDataStructure dataStructure = dsManager.CreateStructuredDataStructure("dsForTesting", "DS for unit testing", "", "", Dlm.Entities.DataStructure.DataStructureCategory.Generic); dsManager.AddVariableUsage(dataStructure, dataAttribute1, true, "var1UT", "", "", "Used for unit testing"); dsManager.AddVariableUsage(dataStructure, dataAttribute2, true, "var2UT", "", "", "Used for unit testing"); dsManager.AddVariableUsage(dataStructure, dataAttribute3, true, "var3UT", "", "", "Used for unit testing"); dsManager.AddVariableUsage(dataStructure, dataAttribute4, true, "var4UT", "", "", "Used for unit testing"); dsManager.AddVariableUsage(dataStructure, dataAttribute5, true, "var5UT", "", "", "Used for unit testing"); return(dataStructure); } catch (Exception ex) { return(null); } finally { unitManager.Dispose(); dataTypeManager.Dispose(); attributeManager.Dispose(); dsManager.Dispose(); } }
private DataRow ConvertTupleIntoDataRow(DataTable dt, AbstractTuple t, StructuredDataStructure sts) { DataRow dr = dt.NewRow(); string columnName = ""; foreach (var vv in t.VariableValues) { columnName = "var" + vv.VariableId.ToString(); if (vv.VariableId > 0) { string valueAsString = ""; if (vv.Value == null) { dr[columnName] = DBNull.Value; } else { valueAsString = vv.Value.ToString(); Dlm.Entities.DataStructure.Variable varr = sts.Variables.Where(p => p.Id == vv.VariableId).SingleOrDefault(); switch (varr.DataAttribute.DataType.SystemType) { case "String": { dr[columnName] = valueAsString; break; } case "Double": { double value; if (double.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = double.MaxValue; } break; } case "Int16": { Int16 value; if (Int16.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = Int16.MaxValue; } break; } case "Int32": { Int32 value; if (Int32.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = Int32.MaxValue; } break; } case "Int64": { Int64 value; if (Int64.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = Int64.MaxValue; } break; } case "Decimal": { decimal value; if (decimal.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = decimal.MaxValue; } break; } case "Float": { decimal value; if (decimal.TryParse(valueAsString, out value)) { dr[columnName] = value; } else { dr[columnName] = decimal.MaxValue; } break; } case "DateTime": { if (!String.IsNullOrEmpty(valueAsString)) { dr[columnName] = Convert.ToDateTime(valueAsString, CultureInfo.InvariantCulture); } else { dr[columnName] = DateTime.MaxValue; } break; } default: { if (!String.IsNullOrEmpty(vv.Value.ToString())) { dr[columnName] = valueAsString; } else { dr[columnName] = DBNull.Value; } break; } } } } } return(dr); }
public DataTable ConvertPrimaryDataToDatatable(DatasetVersion dsv, IEnumerable <AbstractTuple> dsVersionTuples) { DataTable dt = new DataTable(); dt.TableName = "Primary data table"; StructuredDataStructure sds = this.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>().Get(dsv.Dataset.DataStructure.Id); XmlDocument doc = new XmlDocument(); doc = (XmlDocument)sds.Extra; if (dsVersionTuples != null && sds != null && doc != null) { IEnumerable <XElement> orderList = XmlUtility.GetXElementByNodeName("variable", XmlUtility.ToXDocument(doc)); foreach (XElement element in orderList) { var vu = sds.Variables.Where(v => v.Id.Equals(Convert.ToInt64(element.Value))).FirstOrDefault(); // use vu.Label or vu.DataAttribute.Name DataColumn col = dt.Columns.Add("ID" + vu.Id.ToString()); // or DisplayName also col.Caption = vu.Label; switch (vu.DataAttribute.DataType.SystemType) { case "String": { col.DataType = Type.GetType("System.String"); break; } case "Double": { col.DataType = Type.GetType("System.Double"); break; } case "Int16": { col.DataType = Type.GetType("System.Int16"); break; } case "Int32": { col.DataType = Type.GetType("System.Int32"); break; } case "Int64": { col.DataType = Type.GetType("System.Int64"); break; } case "Decimal": { col.DataType = Type.GetType("System.Decimal"); break; } case "DateTime": { col.DataType = Type.GetType("System.DateTime"); break; } default: { col.DataType = Type.GetType("System.String"); break; } } if (vu.Parameters.Count > 0) { foreach (var pu in vu.Parameters) { DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also col2.Caption = pu.Label; } } } foreach (var tuple in dsVersionTuples) { dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds)); } } return(dt); }
private DataTable ConvertPrimaryDataToDatatable(IEnumerable <AbstractTuple> tupleIterator, StructuredDataStructure dataStructure, string tableName = "") { DataTable dt = new DataTable(); if (string.IsNullOrEmpty(tableName)) { dt.TableName = "Primary data table"; } else { dt.TableName = tableName; } if (dataStructure != null) { buildTheHeader(dataStructure, dt); buildTheBody(tupleIterator, dt, dataStructure); } return(dt); }
/// <summary> /// This function convert a datatuple into datarow for a datatable to show on the client side /// the grid in the client side (in client mode) has unknow problem with value 0 and null /// So every empty cell get the max value of the specific Systemtype. /// On the client side this values replaced with "" /// </summary> /// <param name="dt"></param> /// <param name="t"></param> /// <returns></returns> private static DataRow ConvertTupleIntoDataRow(DataTable dt, AbstractTuple t, StructuredDataStructure sts) { DataRow dr = dt.NewRow(); foreach (var vv in t.VariableValues) { if (vv.VariableId > 0) { string valueAsString = ""; if (vv.Value == null) { dr["ID" + vv.VariableId.ToString()] = DBNull.Value; } else { valueAsString = vv.Value.ToString(); Variable varr = sts.Variables.Where(p => p.Id == vv.VariableId).SingleOrDefault(); switch (varr.DataAttribute.DataType.SystemType) { case "String": { dr["ID" + vv.VariableId.ToString()] = valueAsString; break; } case "Double": { double value; if (double.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToDouble(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = -99999; //double.MaxValue; } break; } case "Int16": { Int16 value; if (Int16.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToInt16(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = Int16.MaxValue; } break; } case "Int32": { Int32 value; if (Int32.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToInt32(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = Int32.MaxValue; } break; } case "Int64": { Int64 value; if (Int64.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToInt64(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = Int64.MaxValue; } break; } case "Decimal": { decimal value; if (decimal.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToDecimal(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = -99999; //decimal.MaxValue; } break; } case "Float": { decimal value; if (decimal.TryParse(valueAsString, out value)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToDecimal(valueAsString); } else { dr["ID" + vv.VariableId.ToString()] = -99999; } break; } case "DateTime": { if (!String.IsNullOrEmpty(valueAsString)) { dr["ID" + vv.VariableId.ToString()] = Convert.ToDateTime(valueAsString, CultureInfo.InvariantCulture); } else { dr["ID" + vv.VariableId.ToString()] = DateTime.MaxValue; } break; } default: { if (!String.IsNullOrEmpty(vv.Value.ToString())) { dr["ID" + vv.VariableId.ToString()] = valueAsString; } else { dr["ID" + vv.VariableId.ToString()] = DBNull.Value; } break; } } } /*if (vv.ParameterValues.Count > 0) * { * foreach (var pu in vv.ParameterValues) * { * dr[pu.Parameter.Label.Replace(" ", "")] = pu.Value; * } * }*/ } } return(dr); }
//[MeasurePerformance] //temporary solution: norman //For original solution, look into Aquadiva Code public List <Error> FinishUpload(EasyUploadTaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); DataContainerManager dam = new DataContainerManager(); //SubjectManager sm = new SubjectManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); List <Error> temp = new List <Error>(); try { using (IUnitOfWork unitOfWork = this.GetUnitOfWork()) { // initialize all necessary manager DataTuple[] rows = null; //First, try to validate - if there are errors, return immediately string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString(); List <Error> ValidationErrors = ValidateRows(JsonArray); if (ValidationErrors.Count != 0) { temp.AddRange(ValidationErrors); return(temp); } string timestamp = DateTime.UtcNow.ToString("r"); string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]); if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE)) { string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); if (!String.IsNullOrWhiteSpace(tmp)) { title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); } } StructuredDataStructure sds = null; List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); //Used in Excel reader //Try to find an exact matching datastructure Boolean foundReusableDataStructure = false; List <RowModel> headers = (List <RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS]; //For some reason, MappedHeaders might be in a different order in this list than what's indicated by its IDs - to prevent mismatching, sort the headers headers.Sort((m1, m2) => m1.Index.CompareTo(m2.Index)); List <StructuredDataStructure> allDatastructures = dsm.StructuredDataStructureRepo.Get().ToList(); foreach (StructuredDataStructure existingStructure in allDatastructures) { if (!foundReusableDataStructure) { //For now a datastructure is considered an exact match if it contains variables with //the same names (labels), datatypes and units in the correct order List <Variable> variablesOfExistingStructure = existingStructure.Variables.ToList(); foundReusableDataStructure = true; if (variablesOfExistingStructure.Count != headers.Count) { foundReusableDataStructure = false; } else { for (int i = 0; i < variablesOfExistingStructure.Count; i++) { Variable exVar = variablesOfExistingStructure.ElementAt(i); RowModel currentHeader = headers.ElementAt(i); if (exVar.Label != currentHeader.Name || exVar.Unit.Id != currentHeader.SelectedUnit.UnitId || exVar.DataAttribute.DataType.Id != currentHeader.SelectedDataType.DataTypeId) { foundReusableDataStructure = false; } } } if (foundReusableDataStructure) { sds = existingStructure; foreach (Variable exVar in variablesOfExistingStructure) { VariableIdentifier vi = new VariableIdentifier { name = exVar.Label, id = exVar.Id }; identifiers.Add(vi); } } } } if (!foundReusableDataStructure) { sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic); } TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp); if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE)) { TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title); TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title); } MetadataStructure metadataStructure = null; if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA)) { long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]); metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Id == metadataStructureId).FirstOrDefault(); } else { //Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault(); } ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault(); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title); #region Progress Information if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0); } #endregion Progress Information if (!foundReusableDataStructure) { #region Set Variables and information for new DataStructure XmlDocument xmldoc = new XmlDocument(); XmlElement extraElement = xmldoc.CreateElement("extra"); XmlElement orderElement = xmldoc.CreateElement("order"); //Sorting necessary to prevent problems when inserting the tuples headers.OrderBy(r => r.Index); var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>(); var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>(); var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>(); List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList(); foreach (RowModel header in headers) { int i = headers.IndexOf(header); DataType dataType = dataTypeRepo.Get(header.SelectedDataType.DataTypeId); Unit CurrentSelectedUnit = unitRepo.Get(header.SelectedUnit.UnitId); DataAttribute CurrentDataAttribute = new DataAttribute(); //If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match) DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(header.SelectedDataAttribute.Name).ToLower()) && da.Unit.Dimension == CurrentSelectedUnit.Dimension).FirstOrDefault(); if (existingDataAttribute != null) { CurrentDataAttribute = existingDataAttribute; } else { //No matching DataAttribute => Create a new one CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(header.Name), header.Name, header.SelectedDataAttribute.Description, false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null); } Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, header.Name, "", "", "", CurrentSelectedUnit); VariableIdentifier vi = new VariableIdentifier { name = newVariable.Label, id = newVariable.Id }; identifiers.Add(vi); XmlElement newVariableXml = xmldoc.CreateElement("variable"); newVariableXml.InnerText = Convert.ToString(newVariable.Id); orderElement.AppendChild(newVariableXml); } extraElement.AppendChild(orderElement); xmldoc.AppendChild(extraElement); sds.Extra = xmldoc; sds.Name = "generated import structure " + timestamp; sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp; #endregion Set Variables and information for new DataStructure } Dataset ds = null; ds = dm.CreateEmptyDataset(sds, rp, metadataStructure); long datasetId = ds.Id; long sdsId = sds.Id; if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault())) { DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId); long METADATASTRUCTURE_ID = metadataStructure.Id; XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath); XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID); XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX); dsv.Metadata = metadataXml; try { dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title); dsv.Title = title; } catch (NullReferenceException ex) { //Reference of the title node is missing throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!"); } dm.EditDatasetVersion(dsv, null, null, null); } #region security // add security if (GetUsernameOrDefault() != "DEFAULT") { foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>()) { //The user gets full permissions // add security if (GetUsernameOrDefault() != "DEFAULT") { entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList()); } } } #endregion security #region excel reader int packageSize = 100000; int numberOfRows = 0; //HACK ? TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id); counter++; TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter; //rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray(); List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA]; string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString(); List <int[]> areaDataValuesList = new List <int[]>(); foreach (string area in selectedDataAreaJsonArray) { areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area)); } int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray); Orientation orientation = 0; switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString()) { case "LeftRight": orientation = Orientation.rowwise; break; case "Matrix": //orientation = Orientation.matrix; break; default: orientation = Orientation.columnwise; break; } String worksheetUri = null; //Get the Uri to identify the correct worksheet if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI)) { worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString(); } int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize; int batchnr = 1; foreach (int[] areaDataValues in areaDataValuesList) { //First batch starts at the start of the current data area int currentBatchStartRow = areaDataValues[0] + 1; while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached { //End row is start row plus batch size int currentBatchEndRow = currentBatchStartRow + batchSize; //Set the indices for the reader EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo { DataStartRow = currentBatchStartRow, //End row is either at the end of the batch or the end of the marked area //DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow, DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1), //Column indices as marked in a previous step DataStartColumn = areaDataValues[1] + 1, DataEndColumn = areaDataValues[3] + 1, //Header area as marked in a previous step VariablesStartRow = areaHeaderValues[0] + 1, VariablesStartColumn = areaHeaderValues[1] + 1, VariablesEndRow = areaHeaderValues[2] + 1, VariablesEndColumn = areaHeaderValues[3] + 1, Offset = areaDataValues[1], Orientation = orientation }; //Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work EasyUploadExcelReader reader = new EasyUploadExcelReader(sds, fri); // open file Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString()); //Set variable identifiers because they might differ from the variable names in the file reader.setSubmittedVariableIdentifiers(identifiers); //Read the rows and convert them to DataTuples rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, (int)datasetId, worksheetUri); //After reading the rows, add them to the dataset if (rows != null) { dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null); numberOfRows += rows.Count(); } //Close the Stream so the next ExcelReader can open it again Stream.Close(); //Debug information int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1); int batches = lines / batchSize; batchnr++; //Next batch starts after the current one currentBatchStartRow = currentBatchEndRow + 1; } } #endregion excel reader //set modification workingCopy.ModificationInfo = new EntityAuditInfo() { Performer = GetUsernameOrDefault(), Comment = "Data", ActionType = AuditActionType.Create }; dm.EditDatasetVersion(workingCopy, null, null, null); dm.CheckInDataset(ds.Id, "Import " + numberOfRows + " rows", GetUsernameOrDefault()); //Reindex search if (this.IsAccessible("DDM", "SearchIndex", "ReIndexSingle")) { this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary() { { "id", datasetId } }); } TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id); return(temp); } } catch (Exception ex) { temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " + "Please try again later. If this problem keeps occuring, please contact your administrator.")); return(temp); } finally { dsm.Dispose(); dm.Dispose(); dam.Dispose(); //sm.Dispose(); entityPermissionManager.Dispose(); } }
public DataTable ConvertStructuredDataStructureToDataTable(StructuredDataStructure sds) { DataTable dt = new DataTable(); dt.TableName = "DataStruture"; dt.Columns.Add("VariableName"); dt.Columns.Add("Description"); dt.Columns.Add("Unit"); //dt.Columns.Add("Parameters"); dt.Columns.Add("DataType"); dt.Columns.Add("Category"); dt.Columns.Add("Optional"); dt.Columns.Add("VariableId"); using (var uow = this.GetUnitOfWork()) { StructuredDataStructure datastructure = uow.GetReadOnlyRepository <StructuredDataStructure>().Get(sds.Id); if (datastructure != null) { List <Variable> variables = SortVariablesOnDatastructure(datastructure.Variables.ToList(), datastructure); foreach (Variable var in variables) { Variable sdvu = uow.GetReadOnlyRepository <Variable>().Get(var.Id); DataRow dr = dt.NewRow(); if (sdvu.Label != null) { dr["VariableName"] = sdvu.Label; } else { dr["VariableName"] = "n/a"; } dr["Optional"] = sdvu.IsValueOptional.ToString(); if (sdvu.Label != null) { dr["VariableId"] = sdvu.Id; } else { dr["VariableId"] = "n/a"; } if (sdvu.DataAttribute.DataType != null) { dr["Category"] = sdvu.DataAttribute.ShortName; } else { dr["Category"] = "n/a"; } //if (sdvu.Parameters.Count > 0) dr["Parameters"] = "current not shown"; //else dr["Parameters"] = "n/a"; if (sdvu.Description != null || sdvu.Description != "") { dr["Description"] = sdvu.Description; } else { dr["Description"] = "n/a"; } if (sdvu.Unit != null) { dr["Unit"] = sdvu.Unit.Name; } else { dr["Unit"] = "n/a"; } if (sdvu.DataAttribute.DataType != null) { dr["DataType"] = sdvu.DataAttribute.DataType.Name; } else { dr["DataType"] = "n/a"; } dt.Rows.Add(dr); } } return(dt); } }
//Add Units protected abstract bool AddUnits(StructuredDataStructure structure);
public DataTable ConvertPrimaryDataToDatatable(DatasetVersion dsv, IEnumerable <long> dsVersionTupleIds) { DataTable dt = new DataTable(); dt.TableName = "Primary data table"; DatasetManager datasetManager = new DatasetManager(); try { StructuredDataStructure sds = this.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>().Get(dsv.Dataset.DataStructure.Id); if (dsVersionTupleIds != null && sds != null) { foreach (var vu in sds.Variables) { // use vu.Label or vu.DataAttribute.Name DataColumn col = dt.Columns.Add("ID" + vu.Id.ToString()); // or DisplayName also col.Caption = vu.Label; switch (vu.DataAttribute.DataType.SystemType) { case "String": { col.DataType = Type.GetType("System.String"); break; } case "Double": { col.DataType = Type.GetType("System.Double"); break; } case "Int16": { col.DataType = Type.GetType("System.Int16"); break; } case "Int32": { col.DataType = Type.GetType("System.Int32"); break; } case "Int64": { col.DataType = Type.GetType("System.Int64"); break; } case "Decimal": { col.DataType = Type.GetType("System.Decimal"); break; } case "DateTime": { col.DataType = Type.GetType("System.DateTime"); break; } default: { col.DataType = Type.GetType("System.String"); break; } } if (vu.Parameters.Count > 0) { foreach (var pu in vu.Parameters) { DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also col2.Caption = pu.Label; } } } foreach (var id in dsVersionTupleIds) { DataTuple dataTuple = datasetManager.DataTupleRepo.Query(d => d.Id.Equals(id)).FirstOrDefault(); dataTuple.Materialize(); dt.Rows.Add(ConvertTupleIntoDataRow(dt, dataTuple, sds)); } } return(dt); } finally { datasetManager.Dispose(); } }
public List <List <string> > GenerateRowsWithRandomValuesBasedOnDatastructure(StructuredDataStructure dataStructure, long numberOfTuples) { List <List <string> > rows = new List <List <string> >(); numberOfTuples.Should().BeGreaterThan(0); var r = new Random(); try { for (int i = 0; i < numberOfTuples; i++) { List <string> row = new List <string>(); row.Add(r.Next().ToString()); row.Add("Test"); row.Add(Convert.ToDouble(r.Next()).ToString()); row.Add(true.ToString()); row.Add(DateTime.Now.ToString()); rows.Add(row); } return(rows); } catch (Exception ex) { return(null); } }
public DataStructureResultStruct(long dataStructureId) { using (DataStructureManager dataStructureManager = new DataStructureManager()) { StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); if (structuredDataStructure != null) { this.Id = structuredDataStructure.Id; this.Title = structuredDataStructure.Name; this.Description = structuredDataStructure.Description; this.inUse = false; DatasetManager datasetManager = null; try { datasetManager = new DatasetManager(); foreach (Dataset d in structuredDataStructure.Datasets) { if (datasetManager.RowAny(d.Id)) { this.inUse = true; break; } else { foreach (DatasetVersion dv in d.Versions) { if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any()) { this.inUse = true; break; } } } } } finally { datasetManager.Dispose(); } this.Structured = true; this.Preview = false; } else { UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId); if (unStructuredDataStructure != null) { this.Id = unStructuredDataStructure.Id; this.Title = unStructuredDataStructure.Name; this.Description = unStructuredDataStructure.Description; if (unStructuredDataStructure.Datasets != null && unStructuredDataStructure.Datasets.Any()) { this.inUse = true; } else { this.inUse = false; } this.Structured = false; this.Preview = false; } else { new DataStructureResultStruct(); } } } }
public ActionResult storeVariables(long Id, storeVariableStruct[] variables) { DataStructureManager dataStructureManager = null; DataContainerManager dataContainerManager = null; MissingValueManager missingValueManager = null; UnitManager um = null; try { dataStructureManager = new DataStructureManager(); missingValueManager = new MissingValueManager(); dataContainerManager = new DataContainerManager(); um = new UnitManager(); var structureRepo = dataStructureManager.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>(); StructuredDataStructure dataStructure = structureRepo.Get(Id); MessageModel returnObject = new MessageModel(); MessageModel messageModel = MessageModel.validateDataStructureInUse(dataStructure.Id, dataStructure); if (messageModel.hasMessage) { foreach (Variable v in dataStructure.Variables) { if (variables.Select(svs => svs.Id).ToList().Contains(v.Id)) { v.Description = variables.Where(svs => svs.Id == v.Id).FirstOrDefault().Description; dataStructure = dataStructureManager.UpdateStructuredDataStructure(dataStructure); } } return(PartialView("_messageWindow", messageModel)); } if (variables != null && variables.Any()) { Variable variable = new Variable(); List <long> order = new List <long>(); foreach (Variable v in dataStructure.Variables) { if (!variables.Select(svs => svs.Id).ToList().Contains(v.Id)) { List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(v.Id)).ToList(); foreach (MissingValue mv in missingValues) { missingValueManager.Delete(mv); } dataStructureManager.RemoveVariableUsage(v); } } foreach (storeVariableStruct svs in variables.Where(svs => svs.Id == 0).ToList()) { if (svs.Lable == null) { svs.Lable = ""; } if (svs.Description == null) { svs.Description = ""; } DataAttribute dataAttribute = dataContainerManager.DataAttributeRepo.Get(svs.AttributeId); if (dataAttribute != null) { variable = dataStructureManager.AddVariableUsage(dataStructure, dataAttribute, svs.isOptional, svs.Lable.Trim(), null, null, svs.Description.Trim(), um.Repo.Get(svs.UnitId)); svs.Id = variable.Id; foreach (MissingValueStruct mvs in svs.MissingValues) { if (mvs.Id == 0) { if (String.IsNullOrEmpty(mvs.Placeholder)) { missingValueManager.Create(mvs.DisplayName, mvs.Description, variable); } else { missingValueManager.Create(mvs.DisplayName, mvs.Description, variable, mvs.Placeholder); } } } } else { returnObject = new MessageModel() { hasMessage = true, Message = "Not all Variables are stored.", CssId = "0" }; } } //dataStructure = structureRepo.Get(Id); // Javad: why it is needed? variables = variables.Where(v => v.Id != 0).ToArray(); MissingValue missingValue = new MissingValue(); foreach (storeVariableStruct svs in variables.Where(svs => svs.Id != 0).ToList()) { if (svs.Lable == null) { svs.Lable = ""; } if (svs.Description == null) { svs.Description = ""; } variable = dataStructure.Variables.Where(v => v.Id == svs.Id).FirstOrDefault(); if (variable != null) { variable.Label = svs.Lable.Trim(); variable.Description = svs.Description.Trim(); variable.Unit = um.Repo.Get(svs.UnitId); variable.DataAttribute = dataContainerManager.DataAttributeRepo.Get(svs.AttributeId); variable.IsValueOptional = svs.isOptional; List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(svs.Id)).ToList(); foreach (MissingValue mv in missingValues) { if (!svs.MissingValues.Select(mvs => mvs.Id).Contains(mv.Id)) { missingValueManager.Delete(mv); } } foreach (MissingValueStruct mvs in svs.MissingValues) { if (mvs.Id == 0) { if (String.IsNullOrEmpty(mvs.Placeholder)) { missingValueManager.Create(mvs.DisplayName, mvs.Description, variable); } else { missingValueManager.Create(mvs.DisplayName, mvs.Description, variable, mvs.Placeholder); } } else if (mvs.Id > 0) { missingValue = missingValues.Where(mv => mv.Id.Equals(mvs.Id)).FirstOrDefault(); if (missingValue != null) { missingValue.DisplayName = mvs.DisplayName; missingValue.Description = mvs.Description; missingValue.Placeholder = mvs.Placeholder; missingValueManager.Update(missingValue); } } } } } dataStructure = dataStructureManager.UpdateStructuredDataStructure(dataStructure); DataStructureIO.setVariableOrder(dataStructure, variables.Select(svs => svs.Id).ToList()); } else { foreach (Variable v in dataStructure.Variables) { List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(v.Id)).ToList(); foreach (MissingValue mv in missingValues) { missingValueManager.Delete(mv); } dataStructureManager.RemoveVariableUsage(v); } } LoggerFactory.LogCustom("Variables for Data Structure " + Id + " stored."); return(Json(returnObject, JsonRequestBehavior.AllowGet)); } finally { dataStructureManager.Dispose(); dataContainerManager.Dispose(); um.Dispose(); missingValueManager.Dispose(); } }
private HttpResponseMessage getData(long id, int variableId, string token) { DatasetManager datasetManager = new DatasetManager(); UserManager userManager = new UserManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); EntityManager entityManager = new EntityManager(); DataStructureManager dataStructureManager = null; bool isPublic = false; try { // if a dataset is public, then the api should also return data if there is no token for a user #region is public dataStructureManager = new DataStructureManager(); long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id; entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1; isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id); #endregion is public if (!isPublic && String.IsNullOrEmpty(token)) { var request = Request.CreateResponse(); request.Content = new StringContent("Bearer token not exist."); return(request); } User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault(); if (isPublic || user != null) { if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read)) { XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper(); OutputDataManager ioOutputDataManager = new OutputDataManager(); Dataset dataset = datasetManager.GetDataset(id); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id); string title = datasetVersion.Title; // check the data sturcture type ... if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure) { object stats = new object(); DataTable dt = new DataTable("Varibales"); List <ApiDataStatisticModel> dataStatisticModels = new List <ApiDataStatisticModel>(); if (variableId == -1) { StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id); List <string> varIds = new List <string>(); foreach (Variable vs in structuredDataStructure.Variables) { varIds.Add("var" + vs.Id); } dt = GetDuplicates(id, varIds); } else { } //dt.Strip(); dt.TableName = id + "_data"; DatasetModel model = new DatasetModel(); model.DataTable = dt; var response = Request.CreateResponse(HttpStatusCode.OK); string resp = JsonConvert.SerializeObject(model); response.Content = new StringContent(resp, System.Text.Encoding.UTF8, "application/json"); response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); return(response); } else { return(Request.CreateResponse()); } } else // has rights? { var request = Request.CreateResponse(); request.Content = new StringContent("User has no read right."); return(request); } } else { var request = Request.CreateResponse(); request.Content = new StringContent("User is not available."); return(request); } } catch (Exception e) { throw e; } finally { datasetManager.Dispose(); userManager.Dispose(); entityPermissionManager.Dispose(); entityManager.Dispose(); dataStructureManager.Dispose(); } }