/// <summary> /// generate a text file with JSON from a datastructure of a dataset /// and stored this file on the server /// and store the path in the content discriptor /// </summary> /// <param name="datasetId"></param> /// <returns>dynamic filepath</returns> public static string GenerateDataStructure(long datasetId) { string path = ""; try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); DataStructureManager dataStructureManager = new DataStructureManager(); long dataStructureId = datasetVersion.Dataset.DataStructure.Id; DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); if (dataStructure != null) { } // store in content descriptor path = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "datastructure", ".txt"); } catch (Exception ex) { throw ex; } return path; }
public ActionResult deleteDataStructure(long Id, string cssId = "") { MessageModel DataStructureDeleteValidation = MessageModel.validateDataStructureDelete(Id); if (DataStructureDeleteValidation.hasMessage) { return PartialView("_messageWindow", DataStructureDeleteValidation); } else { DataStructureManager dataStructureManager = new DataStructureManager(); if (dataStructureManager.StructuredDataStructureRepo.Get(Id) != null) { StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id); DataStructureIO.deleteTemplate(dataStructure.Id); foreach (Variable v in dataStructure.Variables) { dataStructureManager.RemoveVariableUsage(v); } dataStructureManager.DeleteStructuredDataStructure(dataStructure); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Deleted); } else { UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id); dataStructureManager.DeleteUnStructuredDataStructure(dataStructure); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Deleted); } return PartialView("_message", new MessageModel() { Message = "DataStructure" + Id + "deleted", hasMessage = false, CssId = "deleted" }); } }
private static XmlDocument createOderNode(StructuredDataStructure structuredDataStructure) { DataStructureManager dsm = new DataStructureManager(); XmlDocument doc = (XmlDocument)structuredDataStructure.Extra; XmlNode order; if (doc == null) { doc = new XmlDocument(); XmlNode root = doc.CreateNode(XmlNodeType.Element, "extra", null); doc.AppendChild(root); } if (doc.GetElementsByTagName("order").Count == 0) { if (structuredDataStructure.Variables.Count > 0) { order = doc.CreateNode(XmlNodeType.Element, "order", null); foreach (Variable v in structuredDataStructure.Variables) { XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null); variable.InnerText = v.Id.ToString(); order.AppendChild(variable); } doc.FirstChild.AppendChild(order); structuredDataStructure.Extra = doc; dsm.UpdateStructuredDataStructure(structuredDataStructure); } } return doc; }
/// <summary> /// wandelt bexis primardaten (rows) in bexis 2 datatuples um /// </summary> /// <param name="data"></param> /// <param name="dataStructureId"></param> /// <param name="observationIndex"></param> /// <returns></returns> public DataTuple XmlRowReader(XmlDocument data, long dataStructureId, int observationIndex) { DataStructureManager dataStructureManager = new DataStructureManager(); DataTuple dataRow = new DataTuple(); StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); this.StructuredDataStructure = dataStructure; List<VariableIdentifier> variableIdentifierList = new List<VariableIdentifier>(); foreach (Variable variable in dataStructure.Variables) { VariableIdentifier vi = new VariableIdentifier(); vi.id = variable.Id; vi.name = variable.Label; vi.systemType = variable.DataAttribute.DataType.SystemType.ToString(); variableIdentifierList.Add(vi); } this.SubmitedVariableIdentifiers = variableIdentifierList; List<string> dataList = XmlRowToStringList(data, variableIdentifierList); int startRow = 0; int indexOfRow = startRow + observationIndex; //List<Error> errors = ValidateRow(dataList, indexOfRow); //if (errors.Count == 0) { dataRow = ReadRow(dataList, indexOfRow); } return dataRow; }
public ActionResult copyDataStructure(long Id, bool isStructured, string Name = "" , string Description = "", string cssId = "") { Name = Server.UrlDecode(Name); Description = Server.UrlDecode(Description); DataStructureManager dataStructureManager = new DataStructureManager(); if (!isStructured) { UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id); if (dataStructure != null) { if (Name == "") { Name = dataStructure.Name + " - Copy"; } if (Description == "" && dataStructure.Description != null) { Description = dataStructure.Description; } LoggerFactory.LogCustom("Copy Data Structure" + Id); return createDataStructure(0, Name.Trim(), isStructured, Description.Trim(), cssId); } } else { StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id); if (dataStructure != null) { if (Name == "") { Name = dataStructure.Name + " - Copy"; } if (Description == "" && dataStructure.Description != null) { Description = dataStructure.Description; } MessageModel messageModel = storeDataStructure(0, Name.Trim(), isStructured, Description.Trim(), cssId); List<long> order = new List<long>(); Variable variable = new Variable(); if (!messageModel.hasMessage) { StructuredDataStructure dataStructureCopy = dataStructureManager.StructuredDataStructureRepo.Get(Convert.ToInt64(messageModel.Message)); foreach (Variable v in DataStructureIO.getOrderedVariables(dataStructure)) { variable = dataStructureManager.AddVariableUsage(dataStructureCopy, v.DataAttribute, v.IsValueOptional, v.Label.Trim(), v.DefaultValue, v.MissingValue, v.Description.Trim(), v.Unit); order.Add(variable.Id); } DataStructureIO.setVariableOrder(dataStructureCopy, order); } LoggerFactory.LogCustom("Copy Data Structure" + Id); return PartialView("_message", messageModel); } } return PartialView("_message", new MessageModel()); }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> public DataStructureTree() { dataStructureTreeList = new List<DataStructureTreeList>(); DataStructureManager dataStructureManager = new DataStructureManager(); DataStructureTreeList treeList = new DataStructureTreeList(); if(dataStructureManager.StructuredDataStructureRepo.Get() != null) treeList = new DataStructureTreeList(dataStructureManager.StructuredDataStructureRepo.Get()); dataStructureTreeList.Add(treeList); if (dataStructureManager.UnStructuredDataStructureRepo.Get() !=null) treeList = new DataStructureTreeList(dataStructureManager.UnStructuredDataStructureRepo.Get()); dataStructureTreeList.Add(treeList); }
public static XmlDocument setVariableOrder(StructuredDataStructure structuredDataStructure, List<long> orderList) { DataStructureManager dsm = new DataStructureManager(); XmlDocument doc = createOderNode(structuredDataStructure); XmlNode order = doc.GetElementsByTagName("order")[0]; doc.FirstChild.RemoveChild(order); order = doc.CreateNode(XmlNodeType.Element, "order", null); foreach (long l in orderList) { XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null); variable.InnerText = l.ToString(); order.AppendChild(variable); } doc.FirstChild.AppendChild(order); structuredDataStructure.Extra = doc; dsm.UpdateStructuredDataStructure(structuredDataStructure); return doc; }
public FileResult downloadTemplate(long id) { if (id != 0) { DataStructureManager dataStructureManager = new DataStructureManager(); StructuredDataStructure dataStructure = new StructuredDataStructure(); dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(id); if (dataStructure != null) { ExcelTemplateProvider provider = new ExcelTemplateProvider("BExISppTemplate_Clean.xlsm"); provider.CreateTemplate(dataStructure); string path = ""; XmlNode resources = dataStructure.TemplatePaths.FirstChild; XmlNodeList resource = resources.ChildNodes; foreach (XmlNode x in resource) { if (x.Attributes.GetNamedItem("Type").Value == "Excel") path = x.Attributes.GetNamedItem("Path").Value; } string rgxPattern = "[<>?\":|\\\\/*]"; string rgxReplace = "-"; Regex rgx = new Regex(rgxPattern); string filename = rgx.Replace(dataStructure.Name, rgxReplace); if (filename.Length > 50) filename = filename.Substring(0, 50); return File(Path.Combine(AppConfiguration.DataPath, path), "application/xlsm", "Template_" + dataStructure.Id + "_" + filename + ".xlsm"); } } return File(Path.Combine(AppConfiguration.GetModuleWorkspacePath("RPM"), "Template", "BExISppTemplate_Clean.xlsm"), "application/xlsm", "Template_" + id + "_No_Data_Structure.xlsm"); }
public long createDataset(string dataSetID, XmlDocument metadataXml, long metadataStructureId, DataStructure dataStructure, string researchPlanName) { Dataset dataset = new Dataset(); DatasetManager datasetManager = new DatasetManager(); DataStructureManager dataStructureManager = new DataStructureManager(); ResearchPlanManager researchPlanManager = new ResearchPlanManager(); MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); PermissionManager permissionManager = new PermissionManager(); SubjectManager subjectManager = new SubjectManager(); // get existing researchPlan ResearchPlan researchPlan = researchPlanManager.Repo.Get(r => researchPlanName.Equals(r.Title)).FirstOrDefault(); // get existing metadataStructure (created manualy by using edited Bexis1_XSD) MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(metadataStructureId); // update verwendet, da bei unstructured structures im dataset constructor dataset(dataStructure) nullexception dataset = datasetManager.CreateEmptyDataset(dataStructure, researchPlan, metadataStructure); long datasetId = dataset.Id; // dataPermission List<string> usersWithDataPerm = new List<string>(); // create dataPermission for originalDatasetManager string originalDatasetManager = metadataXml.SelectSingleNode("Metadata/general/general/originalDatasetManager/originalDatasetManager").InnerText; User userODM = subjectManager.UsersRepo.Get(u => originalDatasetManager.Equals(u.FullName)).FirstOrDefault(); permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Create); permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.View); permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Update); permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Download); usersWithDataPerm.Add(originalDatasetManager); // create dataPermissions for metadataAuthor string metadataAuthor = metadataXml.SelectSingleNode("Metadata/general/general/metadataCreator/metadataCreator").InnerText; if (!usersWithDataPerm.Contains(metadataAuthor)) { User userMA = subjectManager.UsersRepo.Get(u => metadataAuthor.Equals(u.FullName)).FirstOrDefault(); permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Create); permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.View); permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Update); permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Download); usersWithDataPerm.Add(metadataAuthor); } // create dataPermissions for designatedDatasetManager string designatedDatasetManager = metadataXml.SelectSingleNode("Metadata/general/general/designatedDatasetManager/contactType/designatedDatasetManagerName/designatedDatasetManagerName").InnerText; if (!usersWithDataPerm.Contains(designatedDatasetManager)) { User userDDM = subjectManager.UsersRepo.Get(u => designatedDatasetManager.Equals(u.FullName)).FirstOrDefault(); permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Create); permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.View); permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Update); permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Download); usersWithDataPerm.Add(designatedDatasetManager); } // create dataPermissions for owners XmlNodeList ownerNodes = metadataXml.SelectNodes("Metadata/general/general/owners/ownerType/owner/owner"); foreach (XmlNode ownerNode in ownerNodes) { string owner = ownerNode.InnerText; if (!usersWithDataPerm.Contains(owner)) { User userO = subjectManager.UsersRepo.Get(u => owner.Equals(u.FullName)).FirstOrDefault(); if (userO != null) { permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Create); permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.View); permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Update); permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Download); usersWithDataPerm.Add(owner); } } } // integrate metadataXml to dataset // checkOut if (datasetManager.IsDatasetCheckedOutFor(datasetId, userODM.Name) || datasetManager.CheckOutDataset(datasetId, userODM.Name)) { DatasetVersion workingCopy = datasetManager.GetDatasetWorkingCopy(datasetId); // get dataset workingCopy.Metadata = metadataXml; // set metadata to dataset datasetManager.EditDatasetVersion(workingCopy, null, null, null); // edit dataset datasetManager.CheckInDataset(datasetId, "Metadata was submited.", userODM.Name); // check in } return dataset.Id; }
public List<ListViewItem> LoadDataStructureViewList( DataStructureType dataStructureType ) { DataStructureManager dsm = new DataStructureManager(); List<ListViewItem> temp = new List<ListViewItem>(); foreach (DataStructure datasStructure in dsm.StructuredDataStructureRepo.Get()) { string title = datasStructure.Name; temp.Add(new ListViewItem(datasStructure.Id, title)); } return temp.OrderBy(p => p.Title).ToList(); }
public long DatasetExistsByOldDatasetId(string dataSetID, long dataStructureId) { DatasetManager datasetManager = new DatasetManager(); DataStructureManager dataStructureManager = new DataStructureManager(); List<Dataset> datasetsWithCurrentDataStructure = new List<Dataset>(); datasetsWithCurrentDataStructure = datasetManager.DatasetRepo.Get(d => dataStructureId.Equals(d.DataStructure.Id)).ToList(); if (datasetsWithCurrentDataStructure != null && datasetsWithCurrentDataStructure.Count > 0) { foreach (Dataset datasetWCDS in datasetsWithCurrentDataStructure) { string oldDatasetId = ""; try { XmlNode extraID = datasetWCDS.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id"); oldDatasetId = extraID.InnerText; } catch { } if (oldDatasetId == dataSetID) { return datasetWCDS.Id; } } } return -999; }
// create unstructured DataStructures public DataStructure CreateDataStructure(string fileType) { DataStructureManager dataStructureManager = new DataStructureManager(); UnStructuredDataStructure dataStructure = new UnStructuredDataStructure(); // values of DataStructure string name = fileType; string description = "old BExIS unstrctured data structure: " + fileType; UnStructuredDataStructure existDS = dataStructureManager.UnStructuredDataStructureRepo.Get(s => name.Equals(s.Name)).FirstOrDefault(); if (existDS == null) { // create dataStructure return dataStructureManager.CreateUnStructuredDataStructure(name, description); } else { return existDS; } }
public List<ListViewItem> LoadDatasetVersionViewList( DataStructureType dataStructureType) { PermissionManager permissionManager = new PermissionManager(); SubjectManager subjectManager = new SubjectManager(); // add security ICollection<long> datasetIDs = permissionManager.GetAllDataIds(subjectManager.GetUserByName(GetUsernameOrDefault()).Id, 1, RightType.Update).ToList(); DataStructureManager dataStructureManager = new DataStructureManager(); DatasetManager dm = new DatasetManager(); Dictionary<long, XmlDocument> dmtemp = new Dictionary<long, XmlDocument>(); dmtemp = dm.GetDatasetLatestMetadataVersions(); List<ListViewItem> temp = new List<ListViewItem>(); if (dataStructureType.Equals(DataStructureType.Structured)) { List<StructuredDataStructure> list = dataStructureManager.StructuredDataStructureRepo.Get().ToList(); foreach (StructuredDataStructure sds in list) { sds.Materialize(); foreach (Dataset d in sds.Datasets) { if (dm.IsDatasetCheckedIn(d.Id)) { if (datasetIDs.Contains(d.Id)) { temp.Add(new ListViewItem(d.Id, XmlDatasetHelper.GetInformation(dm.GetDatasetLatestVersion(d), NameAttributeValues.title))); } } } } } else { List<UnStructuredDataStructure> list = dataStructureManager.UnStructuredDataStructureRepo.Get().ToList(); foreach (UnStructuredDataStructure sds in list) { foreach (Dataset d in sds.Datasets) { if (datasetIDs.Contains(d.Id)) { if (dm.IsDatasetCheckedIn(d.Id)) { DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(d); temp.Add(new ListViewItem(d.Id, XmlDatasetHelper.GetInformation(datasetVersion, NameAttributeValues.title))); } } } } } return temp.OrderBy(p => p.Title).ToList(); }
public StructuredDataStructure GetDataStructureByID(long ID) { DataStructureManager dsm = new DataStructureManager(); StructuredDataStructure structuredDataStructure = dsm.StructuredDataStructureRepo.Get(ID); this.dataStructure = structuredDataStructure; if (this.dataStructure != null) { this.variableStructs = getOrderedVariableStructs(structuredDataStructure); if (this.dataStructure.Datasets == null) { inUse = false; } else { if (this.dataStructure.Datasets.Count > 0) { inUse = true; } else { inUse = false; } } this.BuildDataTable(); return (structuredDataStructure); } else { this.dataStructure = new StructuredDataStructure(); return (structuredDataStructure); } }
public DataStructure CreateDataStructure(string dataSetID, DataTable mapVariables, List<string> variableNames) { DataStructureManager dataStructureManager = new DataStructureManager(); DataContainerManager attributeManager = new DataContainerManager(); StructuredDataStructure dataStructure = new StructuredDataStructure(); UnitManager unitManager = new UnitManager(); // values of DataStructure ExcelTemplateProvider provider = new ExcelTemplateProvider(); string name = "oldBExIS" + dataSetID; string description = "old BExIS datastructure, created for " + dataSetID + ", possibly used by other"; //metadata.Title; string xsdFileName = "";//"xsdFileName"; string xslFileName = "";//"xslFileName"; DataStructureCategory indexerType = DataStructureCategory.Generic; // if dataStructure not exists StructuredDataStructure existDS = existingDataStructures(mapVariables, dataSetID); if (existDS.Name == null) { // create dataStructure dataStructure = dataStructureManager.CreateStructuredDataStructure(name, description, xsdFileName, xslFileName, indexerType, null); foreach (string varName in variableNames) { // values of variables string attName = ""; string convFactor = ""; string varDescription = ""; int Block = -999; long AttributeId = -999, UnitId = -999, VarUnitId = -999; foreach (DataRow mapRow in mapVariables.Select("DatasetId = '" + dataSetID + "'")) { if (mapRow["Name"].ToString() == varName) { attName = mapRow["Attribute"].ToString(); convFactor = mapRow["ConvFactor"].ToString(); varDescription = mapRow["Description"].ToString(); Block = int.Parse(mapRow["Block"].ToString()); if (attName.Length > 1) // if not mapped yet { AttributeId = Convert.ToInt64(mapRow["AttributeId"].ToString()); UnitId = Convert.ToInt64(mapRow["UnitId"].ToString()); if (mapRow["Unit"].ToString().Length > 0) VarUnitId = Convert.ToInt64(mapRow["VarUnitId"].ToString()); } } } if (AttributeId > 0 && Block == 0) // if not mapped yet AND variable is in block 0 { // find existing attribute for each variable DataAttribute attribute = attributeManager.DataAttributeRepo.Get(AttributeId); Unit varUnit = null; if (VarUnitId > 0) { varUnit = unitManager.Repo.Get(VarUnitId); } // add variables to dataStructure Variable variable = dataStructureManager.AddVariableUsage(dataStructure, attribute, true, varName, null, null, varDescription, varUnit); dataStructure.Variables.Add(variable); } } provider.CreateTemplate(dataStructure); return dataStructure; } else { return existDS; } }
public DataStructurePreviewModel fill(long dataStructureId) { if (dataStructureId > 0) { DataStructureManager dataStructureManager = new DataStructureManager(); if (dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId) != null) { StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); VariablePreviewStruct variablePreview; this.Id = dataStructure.Id; this.Name = dataStructure.Name; this.Description = dataStructure.Description; if(dataStructure.Datasets.Count > 0) { this.inUse = true; } foreach (Variable v in DataStructureIO.getOrderedVariables(dataStructure)) { variablePreview = new VariablePreviewStruct().fill(v); this.VariablePreviews.Add(variablePreview); } } else if (dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId) != null) { UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId); this.Id = dataStructure.Id; this.Name = dataStructure.Name; this.Description = dataStructure.Description; this.VariablePreviews = null; if(dataStructure.Datasets.Count > 0) { this.inUse = true; } } return this; } else { return new DataStructurePreviewModel(); } }
public List<Error> FinishUpload2(TaskManager taskManager) { List<Error> temp = new List<Error>(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); //datatuple list List<DataTuple> rows; DatasetManager dm = new DatasetManager(); Dataset ds = dm.GetDataset(id); DatasetVersion workingCopy = new DatasetVersion(); #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); //Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } } while (rows.Count() > 0); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion // start download generator // filepath //string path = ""; //if (workingCopy != null) //{ // path = GenerateDownloadFile(workingCopy); // dm.EditDatasetVersion(workingCopy, null, null, null); //} // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { workingCopy = dm.GetDatasetLatestVersion(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } return temp; }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="dataStructureId"></param> /// <param name="extention"></param> /// <returns></returns> public string GetDataStructureTemplatePath(long dataStructureId, string extention) { DataStructureManager dataStructureManager = new DataStructureManager(); StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); string dataStructureTitle = dataStructure.Name; // load datastructure from db an get the filepath from this object ExcelTemplateProvider provider = new ExcelTemplateProvider("BExISppTemplate_Clean.xlsm"); string path = ""; if (dataStructure.TemplatePaths != null) { XmlNode resources = dataStructure.TemplatePaths.FirstChild; XmlNodeList resource = resources.ChildNodes; foreach (XmlNode x in resource) { if (x.Attributes.GetNamedItem("Type").Value == "Excel") if (File.Exists(x.Attributes.GetNamedItem("Path").Value)) { path = x.Attributes.GetNamedItem("Path").Value; } else { path = provider.CreateTemplate(dataStructure); } } //string dataPath = AppConfiguration.DataPath; //Path.Combine(AppConfiguration.WorkspaceRootPath, "Data"); return Path.Combine(AppConfiguration.DataPath, path); } path = provider.CreateTemplate(dataStructure); return Path.Combine(AppConfiguration.DataPath, path); }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="ID"></param> /// <param name="structured"></param> /// <returns></returns> public DataStructure GetDataStructureByID(long ID, bool structured) { this.structured = structured; if (structured) { return this.GetDataStructureByID(ID); } else { DataStructureManager dsm = new DataStructureManager(); UnStructuredDataStructure unStructuredDataStructure = dsm.UnStructuredDataStructureRepo.Get(ID); this.dataStructure = unStructuredDataStructure; if (this.dataStructure != null) { this.variableStructs = null; if (this.dataStructure.Datasets == null) { inUse = false; } else { if (this.dataStructure.Datasets.Count > 0) { inUse = true; //DatasetListElement datasetListElement = new DatasetListElement(); //DatasetManager dm = new DatasetManager(); //foreach (Dataset d in unStructuredDataStructure.Datasets) //{ // datasetListElement = new DatasetListElement(d.Id, XmlDatasetHelper.GetInformation(d,AttributeNames.title)); // datasets.Add(datasetListElement); //} } else { inUse = false; } } return (unStructuredDataStructure); } else { this.dataStructure = new StructuredDataStructure(); return (unStructuredDataStructure); } } }
/// <summary> /// Read Row and convert each value into a variableValue /// and each row to a Datatuple /// </summary> /// <param name="row">List of values in one row</param> /// <param name="indexOfRow">Currently row index</param> /// <returns>DataTuple</returns> public DataTuple ReadRow(List<string> row, int indexOfRow) { DataTuple dt = new DataTuple(); DataStructureManager dsm = new DataStructureManager(); string value = ""; // convert row to List<VariableValue> for(int i=0;i< row.Count();i++ ) { VariableIdentifier variableIdentifier = this.SubmitedVariableIdentifiers.ElementAt(i); long variableId = 0; if (variableIdentifier.id > 0) variableId = this.SubmitedVariableIdentifiers.ElementAt(i).id; else variableId = getVariableUsage(variableIdentifier).Id; // if variable from systemtype datatime // maybee needs to convert into the default datetime culture format if (this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("DateTime")) { value = IOUtility.ConvertDateToCulture(row[i]); } else { if (this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Double")|| this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Decimal")|| this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Float")) { value = row[i]; if (Info.Decimal.Equals(DecimalCharacter.comma)) { if(value.Contains(".")) value = value.Replace(".",""); if(value.Contains(",")) value = value.Replace(',', '.'); } if (Info.Decimal.Equals(DecimalCharacter.point)) { if (value.Contains(",")) value = value.Remove(','); } } else { value = row[i]; } } dt.VariableValues.Add(DatasetManager.CreateVariableValue(value,"", DateTime.Now, DateTime.Now, new ObtainingMethod(), variableId, new List<ParameterValue>())); } return dt; }
/// <summary> /// CAUTION !!!!!!!!!!!!!!!!! /// upload bezieht sich derzeit nur auf daten mit einem block (einer tabelle) /// umsetzung von mehreren blöckes noch nicht geklärt. /// (für einen wert gibt es eine weitere tabelle) /// </summary> /// <param name="dataSetID"></param> /// <param name="DataBase"></param> public void uploadData(string dataSetID, string DataBase) { DatasetManager datasetManager = new DatasetManager(); DataStructureManager dataStructureManager = new DataStructureManager(); XmlDataReader xmlDataReader = new XmlDataReader(); User user = new User(); string variableNames = ""; // query metadataAuthor and variable names from explorer.datasets queryAuthorAndVariables(ref user, ref variableNames, dataSetID, DataBase); List<string> varNames = variableNames.Split(',').ToList(); // get all dataStructures with equal variables count List<StructuredDataStructure> dataStructures = dataStructureManager.StructuredDataStructureRepo.Get(s => varNames.Count().Equals(s.Variables.Count)).ToList(); // get all Ids of dataStructures with equal variables List<long> dataStructureIds = new List<long>(); foreach (StructuredDataStructure dataStructure in dataStructures) { bool isSimilarStructure = true; foreach (Variable variable in dataStructure.Variables) { if (!varNames.Contains(variable.Label)) { isSimilarStructure &= false; break; } } if (isSimilarStructure) dataStructureIds.Add(dataStructure.Id); } // get the wanted dataset by comparing the old B1 datasetId out of the datasets with similar dataStructure Dataset dataset = null; List<Dataset> datasets = datasetManager.DatasetRepo.Get(d => dataStructureIds.Contains(d.DataStructure.Id)).ToList(); foreach (Dataset ds in datasets) { string oldDatasetId = ""; try { XmlNode extraID = ds.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id"); oldDatasetId = extraID.InnerText; } catch { } if (oldDatasetId == dataSetID) { dataset = ds; break; } } if (dataset != null) { // get distinct and ascending ordered insertdates from DB List<DB2TimeStamp> distInsertDates = queryDistInsertDates(dataSetID, DataBase); bool checkObsIds = false; foreach (DB2TimeStamp insertDate in distInsertDates) { List<DataTuple> createdDataTuples = new List<DataTuple>(); List<DataTuple> editedDataTuples = new List<DataTuple>(); List<DataTuple> deletedDataTuples = new List<DataTuple>(); DatasetVersion workingCopy = datasetManager.GetDatasetLatestVersion(dataset.Id); // get dataset // get obsid, data, deleted and newest for each observation from DB List<Observation> observations = queryObservation(dataSetID, DataBase, insertDate.ToString()); Dictionary<long, long> obsIdMapsToTupleId = new Dictionary<long, long>(); if (checkObsIds) { // get all EffectiveTupleIds List<long> datasetTupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(workingCopy).ToList(); // id-mapping-list key=obsId, value=TupleId foreach EffectiveTupleId and obsId from Tuple.Extra obsIdMapsToTupleId = idMapping(datasetTupleIds, ref datasetManager); } // observation counter int observationIndex = 0; bool isNewest = true; foreach (Observation observation in observations) //////////////parallel { isNewest = (isNewest && observation.newest != 'Y') ? false : isNewest; // create dataTuple with xmlDataReader // split xml to string list and use DataReader.ReadRow DataTuple dataRow = xmlDataReader.XmlRowReader(observation.data, dataset.DataStructure.Id, observationIndex); // check if observation.obsid is in id-mapping-list long TupleId; if (checkObsIds && obsIdMapsToTupleId.TryGetValue(observation.obsid, out TupleId)) { DataTuple dataTuple = datasetManager.DataTupleRepo.Get(TupleId); if (observation.deleted != 'Y') { dataTuple.VariableValues = dataRow.VariableValues; // edit tuple if observation exists as tuple in EffectiveTuple and observation is not deleted editedDataTuples.Add(dataTuple); } else { // delete tuple if observation exists as tuple in EffectiveTuple and observ. is deleted deletedDataTuples.Add(dataTuple); } } else { // write the obsId and oldBExISdatasetId in Extra: <extra><obsid>[obsid]</obsid><oldBExISdatasetId>[dataSetID]</oldBExISdatasetId></extra> dataRow.Extra = oldIdsIntoExtra(observation.obsid, dataSetID, dataRow.Extra); // create tuple if observation exists not in EffectiveTuple createdDataTuples.Add(dataRow); } observationIndex++; // observation counter } checkObsIds = (!checkObsIds && !isNewest) ? true : checkObsIds; // checkOut if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, user.Name) || datasetManager.CheckOutDataset(dataset.Id, user.Name)) { workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id); // get dataset datasetManager.EditDatasetVersion(workingCopy, createdDataTuples, editedDataTuples, deletedDataTuples); // edit dataset datasetManager.CheckInDataset(dataset.Id, "Primary data row was submited.", user.Name); // checkIn } } } }
public List<Variable> getOrderedVariables(StructuredDataStructure structuredDataStructure) { DataStructureManager dsm = new DataStructureManager(); XmlDocument doc = (XmlDocument)structuredDataStructure.Extra; XmlNode order; if (doc == null) { doc = new XmlDocument(); XmlNode root = doc.CreateNode(XmlNodeType.Element, "extra", null); doc.AppendChild(root); } if (doc.GetElementsByTagName("order").Count == 0) { if (structuredDataStructure.Variables.Count > 0) { order = doc.CreateNode(XmlNodeType.Element, "order", null); foreach (Variable v in structuredDataStructure.Variables) { XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null); variable.InnerText = v.Id.ToString(); order.AppendChild(variable); } doc.FirstChild.AppendChild(order); structuredDataStructure.Extra = doc; dsm.UpdateStructuredDataStructure(structuredDataStructure); } } order = doc.GetElementsByTagName("order")[0]; List<Variable> orderedVariables = new List<Variable>(); if (structuredDataStructure.Variables.Count != 0) { foreach (XmlNode x in order) { foreach (Variable v in structuredDataStructure.Variables) { if (v.Id == Convert.ToInt64(x.InnerText)) orderedVariables.Add(v); } } } return orderedVariables; }
public MessageModel storeDataStructure(long Id, string Name, bool isStructured, string Description ="", string cssId = "", bool inUse = false) { Name = Server.UrlDecode(Name); Description = Server.UrlDecode(Description); MessageModel DataStructureValidation = MessageModel.validateDataStructureInUse(Id); if (DataStructureValidation.hasMessage && inUse == false) { return DataStructureValidation; } else { DataStructureValidation = MessageModel.validateDataStructureName(Id, Name, cssId); if (DataStructureValidation.hasMessage) { return DataStructureValidation; } else { DataStructureManager dataStructureManager = new DataStructureManager(); DataStructure dataStructure; if (isStructured) { if (Id == 0) { dataStructure = dataStructureManager.CreateStructuredDataStructure(Name.Trim(), Description.Trim(), null, null, DataStructureCategory.Generic); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created); return new MessageModel() { Message = dataStructure.Id.ToString(), hasMessage = false, CssId = "redirect" }; } else { StructuredDataStructure StructuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id); StructuredDataStructure.Name = Name; StructuredDataStructure.Description = Description; dataStructure = dataStructureManager.UpdateStructuredDataStructure(StructuredDataStructure); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created); return new MessageModel() { Message = Id.ToString(), hasMessage = false, CssId = "redirect" }; } } else { if (Id == 0) { dataStructure = dataStructureManager.CreateUnStructuredDataStructure(Name.Trim(), Description.Trim()); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created); return new MessageModel() { Message = "refresh DataStructureResultGrid", hasMessage = false, CssId = "refresh" }; } else { UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id); unStructuredDataStructure.Name = Name; unStructuredDataStructure.Description = Description; dataStructure = dataStructureManager.UpdateUnStructuredDataStructure(unStructuredDataStructure); LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created); return new MessageModel() { Message = "refresh DataStructureResultGrid", hasMessage = false, CssId = "refresh" }; } } } } }
private Dataset createDataset() { DataStructureManager dsManager = new DataStructureManager(); ResearchPlanManager rpManager = new ResearchPlanManager(); DatasetManager dm = new DatasetManager(); MetadataStructureManager mdsManager = new MetadataStructureManager(); MDS.MetadataStructure mds = mdsManager.Repo.Query().First(); Dataset ds = dm.CreateEmptyDataset(dsManager.StructuredDataStructureRepo.Get(1), rpManager.Repo.Get(1), mds); return ds; }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="id"></param> /// <returns></returns> protected StructuredDataStructure GetDataStructure(long id) { if (dataStructure == null) { DataStructureManager dataStructureManager = new DataStructureManager(); dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(id); } return dataStructure; }
/// <summary> /// create a new dataset, check it out to create the first version, add a tuple to it. /// </summary> /// <returns></returns> private Int64 createDatasetVersion() { DataStructureManager dsManager = new DataStructureManager(); ResearchPlanManager rpManager = new ResearchPlanManager(); DatasetManager dm = new DatasetManager(); MetadataStructureManager mdsManager = new MetadataStructureManager(); MDS.MetadataStructure mds = mdsManager.Repo.Query().First(); Dataset ds = dm.CreateEmptyDataset(dsManager.StructuredDataStructureRepo.Get(1), rpManager.Repo.Get(1), mds); if (dm.IsDatasetCheckedOutFor(ds.Id, "Javad") || dm.CheckOutDataset(ds.Id, "Javad")) { DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id); //DataTuple changed = dm.GetDatasetVersionEffectiveTuples(workingCopy).First(); //changed.VariableValues.First().Value = (new Random()).Next().ToString(); DataTuple dt = dm.DataTupleRepo.Get(1); // its sample data List<DataTuple> tuples = new List<DataTuple>(); for (int i = 0; i < 10000; i++) { DataTuple newDt = new DataTuple(); newDt.XmlAmendments = dt.XmlAmendments; newDt.XmlVariableValues = dt.XmlVariableValues; // in normal cases, the VariableValues are set and then Dematerialize is called newDt.Materialize(); newDt.OrderNo = i; //newDt.TupleAction = TupleAction.Created;//not required //newDt.Timestamp = DateTime.UtcNow; //required? no, its set in the Edit //newDt.DatasetVersion = workingCopy;//required? no, its set in the Edit tuples.Add(newDt); } dm.EditDatasetVersion(workingCopy, tuples, null, null); dm.CheckInDataset(ds.Id, "for testing purposes", "Javad"); dm.DatasetVersionRepo.Evict(); dm.DataTupleRepo.Evict(); dm.DatasetRepo.Evict(); workingCopy.PriliminaryTuples.Clear(); workingCopy = null; } var dsId = ds.Id; ds = null; return (dsId); }
// // GET: /DIM/Import/ public ActionResult Index() { //xml metadata for import string metadataForImportPath = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), "MetadataIDIV_EXAMPLE.xml"); XmlDocument metadataForImport = new XmlDocument(); metadataForImport.Load(metadataForImportPath); // metadataStructure DI long metadataStructureId = 3; MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); string metadataStructrueName = metadataStructureManager.Repo.Get(metadataStructureId).Name; // loadMapping file string path_mappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), XmlMetadataImportHelper.GetMappingFileName(metadataStructureId, TransmissionType.mappingFileImport, metadataStructrueName)); // XML mapper + mapping file XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern); xmlMapperManager.Load(path_mappingFile, "IDIV"); // generate intern metadata XmlDocument metadataResult = xmlMapperManager.Generate(metadataForImport,1); // generate intern template XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(BExIS.Xml.Helpers.XmlNodeMode.xPath); XDocument metadataXml = xmlMetadatWriter.CreateMetadataXml(metadataStructureId); XmlDocument metadataXmlTemplate = XmlMetadataWriter.ToXmlDocument(metadataXml); XmlDocument completeMetadata = XmlMetadataImportHelper.FillInXmlAttributes(metadataResult, metadataXmlTemplate); // create Dataset //load datastructure DataStructureManager dsm = new DataStructureManager(); ResearchPlanManager rpm = new ResearchPlanManager(); MetadataStructureManager msm = new MetadataStructureManager(); DatasetManager dm = new DatasetManager(); Dataset dataset = dm.CreateEmptyDataset(dsm.UnStructuredDataStructureRepo.Get(1), rpm.Repo.Get(1), msm.Repo.Get(3)); if (dm.IsDatasetCheckedOutFor(dataset.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(dataset.Id, GetUsernameOrDefault())) { DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(dataset.Id); workingCopy.Metadata = completeMetadata; string title = XmlDatasetHelper.GetInformation(workingCopy, NameAttributeValues.title); if (String.IsNullOrEmpty(title)) title = "No Title available."; dm.EditDatasetVersion(workingCopy, null, null, null); dm.CheckInDataset(dataset.Id, "Metadata was submited.", GetUsernameOrDefault()); // add security if (GetUsernameOrDefault() != "DEFAULT") { PermissionManager pm = new PermissionManager(); SubjectManager sm = new SubjectManager(); BExIS.Security.Entities.Subjects.User user = sm.GetUserByName(GetUsernameOrDefault()); foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast<RightType>()) { pm.CreateDataPermission(user.Id, 1, dataset.Id, rightType); } } } return View(); }
private void getDataStructures() { var dsm = new DataStructureManager(); var all = dsm.AllTypesDataStructureRepo.Get(); var uns = all.Where(p => p.Name.Contains("uns")).FirstOrDefault(); }
public List<Error> FinishUpload(TaskManager taskManager) { List<Error> temp = new List<Error>(); DatasetManager dm = new DatasetManager(); DatasetVersion workingCopy = new DatasetVersion(); //datatuple list List<DataTuple> rows = new List<DataTuple>(); Dataset ds = null; bool inputWasAltered = false; if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); ds = dm.GetDataset(id); // Javad: Please check if the dataset does exists!! #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); //workingCopy.ContentDescriptors = new List<ContentDescriptor>(); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); Stopwatch upload = Stopwatch.StartNew(); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); upload.Stop(); Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); upload.Stop(); Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; inputWasAltered = false; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { foreach (var err in reader.ErrorMessages) { temp.Add(new Error(ErrorType.Dataset, err.GetMessage())); } //return temp; } //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>(); var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0 || inputWasAltered == true); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion #region contentdescriptors //remove all contentdescriptors from the old version //generatedTXT if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generatedCSV if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generated if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } #endregion // ToDo: Get Comment from ui and users MoveAndSaveOriginalFileInContentDiscriptor(workingCopy); dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { // checkout the dataset, apply the changes, and check it in. if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } if (temp.Count <= 0) { dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } else { dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault()); } return temp; }
private StructuredDataStructure existingDataStructures(DataTable mapVariables, string dataSetID) { DataStructureManager dataStructureManager = new DataStructureManager(); StructuredDataStructure dataStructure = new StructuredDataStructure(); // get alldatasetId respective variables-rows from mapped-variables-table DataRow[] mappedRows = mapVariables.Select("DatasetId = '" + dataSetID + "' AND Block = 0"); // get all datastrcutures from repo which has the same number of variables List<StructuredDataStructure> existDatastructures = dataStructureManager.StructuredDataStructureRepo.Get(s => mappedRows.Count().Equals(s.Variables.Count)).ToList(); foreach (StructuredDataStructure existDs in existDatastructures) { bool isEqualStructure = true; //foreach (DataRow mapRow in mapVariables.Rows) foreach (DataRow mapRow in mappedRows) { int exvarno = existDs.Variables.Where(v => Convert.ToInt64(mapRow["UnitId"]).Equals(v.Unit.Id) && mapRow["Name"].ToString().Equals(v.Label) && Convert.ToInt64(mapRow["AttributeId"]).Equals(v.DataAttribute.Id) ).Count(); if (exvarno == 0) { isEqualStructure &= false; break; } } if (isEqualStructure) { dataStructure = existDs; break; } } return dataStructure; }