//[MeasurePerformance] //temporary solution: norman //For original solution, look into Aquadiva Code public List <Error> FinishUpload(EasyUploadTaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); DataContainerManager dam = new DataContainerManager(); //SubjectManager sm = new SubjectManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); List <Error> temp = new List <Error>(); try { using (IUnitOfWork unitOfWork = this.GetUnitOfWork()) { // initialize all necessary manager DataTuple[] rows = null; //First, try to validate - if there are errors, return immediately string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString(); List <Error> ValidationErrors = ValidateRows(JsonArray); if (ValidationErrors.Count != 0) { temp.AddRange(ValidationErrors); return(temp); } string timestamp = DateTime.UtcNow.ToString("r"); string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]); if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE)) { string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); if (!String.IsNullOrWhiteSpace(tmp)) { title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); } } StructuredDataStructure sds = null; List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); //Used in Excel reader //Try to find an exact matching datastructure Boolean foundReusableDataStructure = false; List <RowModel> headers = (List <RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS]; //For some reason, MappedHeaders might be in a different order in this list than what's indicated by its IDs - to prevent mismatching, sort the headers headers.Sort((m1, m2) => m1.Index.CompareTo(m2.Index)); List <StructuredDataStructure> allDatastructures = dsm.StructuredDataStructureRepo.Get().ToList(); foreach (StructuredDataStructure existingStructure in allDatastructures) { if (!foundReusableDataStructure) { //For now a datastructure is considered an exact match if it contains variables with //the same names (labels), datatypes and units in the correct order List <Variable> variablesOfExistingStructure = existingStructure.Variables.ToList(); foundReusableDataStructure = true; if (variablesOfExistingStructure.Count != headers.Count) { foundReusableDataStructure = false; } else { for (int i = 0; i < variablesOfExistingStructure.Count; i++) { Variable exVar = variablesOfExistingStructure.ElementAt(i); RowModel currentHeader = headers.ElementAt(i); if (exVar.Label != currentHeader.Name || exVar.Unit.Id != currentHeader.SelectedUnit.UnitId || exVar.DataAttribute.DataType.Id != currentHeader.SelectedDataType.DataTypeId) { foundReusableDataStructure = false; } } } if (foundReusableDataStructure) { sds = existingStructure; foreach (Variable exVar in variablesOfExistingStructure) { VariableIdentifier vi = new VariableIdentifier { name = exVar.Label, id = exVar.Id }; identifiers.Add(vi); } } } } if (!foundReusableDataStructure) { sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic); } TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp); if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE)) { TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title); TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title); } MetadataStructure metadataStructure = null; if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA)) { long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]); metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Id == metadataStructureId).FirstOrDefault(); } else { //Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault(); } ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault(); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title); #region Progress Information if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0); } #endregion Progress Information if (!foundReusableDataStructure) { #region Set Variables and information for new DataStructure XmlDocument xmldoc = new XmlDocument(); XmlElement extraElement = xmldoc.CreateElement("extra"); XmlElement orderElement = xmldoc.CreateElement("order"); //Sorting necessary to prevent problems when inserting the tuples headers.OrderBy(r => r.Index); var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>(); var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>(); var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>(); List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList(); foreach (RowModel header in headers) { int i = headers.IndexOf(header); DataType dataType = dataTypeRepo.Get(header.SelectedDataType.DataTypeId); Unit CurrentSelectedUnit = unitRepo.Get(header.SelectedUnit.UnitId); DataAttribute CurrentDataAttribute = new DataAttribute(); //If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match) DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(header.SelectedDataAttribute.Name).ToLower()) && da.Unit.Dimension == CurrentSelectedUnit.Dimension).FirstOrDefault(); if (existingDataAttribute != null) { CurrentDataAttribute = existingDataAttribute; } else { //No matching DataAttribute => Create a new one CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(header.Name), header.Name, header.SelectedDataAttribute.Description, false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null); } Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, header.Name, "", "", "", CurrentSelectedUnit); VariableIdentifier vi = new VariableIdentifier { name = newVariable.Label, id = newVariable.Id }; identifiers.Add(vi); XmlElement newVariableXml = xmldoc.CreateElement("variable"); newVariableXml.InnerText = Convert.ToString(newVariable.Id); orderElement.AppendChild(newVariableXml); } extraElement.AppendChild(orderElement); xmldoc.AppendChild(extraElement); sds.Extra = xmldoc; sds.Name = "generated import structure " + timestamp; sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp; #endregion Set Variables and information for new DataStructure } Dataset ds = null; ds = dm.CreateEmptyDataset(sds, rp, metadataStructure); long datasetId = ds.Id; long sdsId = sds.Id; if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault())) { DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId); long METADATASTRUCTURE_ID = metadataStructure.Id; XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath); XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID); XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX); dsv.Metadata = metadataXml; try { dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title); dsv.Title = title; } catch (NullReferenceException ex) { //Reference of the title node is missing throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!"); } dm.EditDatasetVersion(dsv, null, null, null); } #region security // add security if (GetUsernameOrDefault() != "DEFAULT") { foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>()) { //The user gets full permissions // add security if (GetUsernameOrDefault() != "DEFAULT") { entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList()); } } } #endregion security #region excel reader int packageSize = 100000; int numberOfRows = 0; //HACK ? TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id); counter++; TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter; //rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray(); List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA]; string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString(); List <int[]> areaDataValuesList = new List <int[]>(); foreach (string area in selectedDataAreaJsonArray) { areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area)); } int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray); Orientation orientation = 0; switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString()) { case "LeftRight": orientation = Orientation.rowwise; break; case "Matrix": //orientation = Orientation.matrix; break; default: orientation = Orientation.columnwise; break; } String worksheetUri = null; //Get the Uri to identify the correct worksheet if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI)) { worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString(); } int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize; int batchnr = 1; foreach (int[] areaDataValues in areaDataValuesList) { //First batch starts at the start of the current data area int currentBatchStartRow = areaDataValues[0] + 1; while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached { //End row is start row plus batch size int currentBatchEndRow = currentBatchStartRow + batchSize; //Set the indices for the reader EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo { DataStartRow = currentBatchStartRow, //End row is either at the end of the batch or the end of the marked area //DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow, DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1), //Column indices as marked in a previous step DataStartColumn = areaDataValues[1] + 1, DataEndColumn = areaDataValues[3] + 1, //Header area as marked in a previous step VariablesStartRow = areaHeaderValues[0] + 1, VariablesStartColumn = areaHeaderValues[1] + 1, VariablesEndRow = areaHeaderValues[2] + 1, VariablesEndColumn = areaHeaderValues[3] + 1, Offset = areaDataValues[1], Orientation = orientation }; //Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work EasyUploadExcelReader reader = new EasyUploadExcelReader(sds, fri); // open file Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString()); //Set variable identifiers because they might differ from the variable names in the file reader.setSubmittedVariableIdentifiers(identifiers); //Read the rows and convert them to DataTuples rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, (int)datasetId, worksheetUri); //After reading the rows, add them to the dataset if (rows != null) { dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null); numberOfRows += rows.Count(); } //Close the Stream so the next ExcelReader can open it again Stream.Close(); //Debug information int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1); int batches = lines / batchSize; batchnr++; //Next batch starts after the current one currentBatchStartRow = currentBatchEndRow + 1; } } #endregion excel reader //set modification workingCopy.ModificationInfo = new EntityAuditInfo() { Performer = GetUsernameOrDefault(), Comment = "Data", ActionType = AuditActionType.Create }; dm.EditDatasetVersion(workingCopy, null, null, null); dm.CheckInDataset(ds.Id, "Import " + numberOfRows + " rows", GetUsernameOrDefault()); //Reindex search if (this.IsAccessible("DDM", "SearchIndex", "ReIndexSingle")) { this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary() { { "id", datasetId } }); } TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id); return(temp); } } catch (Exception ex) { temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " + "Please try again later. If this problem keeps occuring, please contact your administrator.")); return(temp); } finally { dsm.Dispose(); dm.Dispose(); dam.Dispose(); //sm.Dispose(); entityPermissionManager.Dispose(); } }
//[MeasurePerformance] //temporary solution: norman //For original solution, look into Aquadiva Code public List <Error> FinishUpload(EasyUploadTaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); DataContainerManager dam = new DataContainerManager(); //SubjectManager sm = new SubjectManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); List <Error> temp = new List <Error>(); try { using (IUnitOfWork unitOfWork = this.GetUnitOfWork()) { // initialize all necessary manager DataTuple[] rows = null; //First, try to validate - if there are errors, return immediately string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString(); List <Error> ValidationErrors = ValidateRows(JsonArray); if (ValidationErrors.Count != 0) { temp.AddRange(ValidationErrors); return(temp); } string timestamp = DateTime.UtcNow.ToString("r"); string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]); if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE)) { string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); if (!String.IsNullOrWhiteSpace(tmp)) { title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]); } } StructuredDataStructure sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id); TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp); if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE)) { TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title); TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title); } MetadataStructure metadataStructure = null; if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA)) { long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]); metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Id == metadataStructureId).FirstOrDefault(); } else { //Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>() .Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault(); } ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault(); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id); TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title); #region Progress Information if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE)) { TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0); } #endregion #region DataStructure XmlDocument xmldoc = new XmlDocument(); XmlElement extraElement = xmldoc.CreateElement("extra"); XmlElement orderElement = xmldoc.CreateElement("order"); List <Tuple <int, string, UnitInfo> > MappedHeaders = (List <Tuple <int, string, UnitInfo> >)TaskManager.Bus[EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS]; //Sorting necessary to prevent problems when inserting the tuples MappedHeaders.Sort((head1, head2) => head1.Item1.CompareTo(head2.Item1)); List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>(); var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>(); var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>(); List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList(); foreach (Tuple <int, string, UnitInfo> Entry in MappedHeaders) { int i = MappedHeaders.IndexOf(Entry); DataType dataType = dataTypeRepo.Get(Entry.Item3.SelectedDataTypeId); Unit CurrentSelectedUnit = unitRepo.Get(Entry.Item3.UnitId); DataAttribute CurrentDataAttribute = new DataAttribute(); //If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match) DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(Entry.Item2).ToLower()) && da.DataType.Id == dataType.Id && da.Unit.Id == CurrentSelectedUnit.Id).FirstOrDefault(); if (existingDataAttribute != null) { CurrentDataAttribute = existingDataAttribute; } else { //No matching DataAttribute => Create a new one CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(Entry.Item2), Entry.Item2, "", false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null); } Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, Entry.Item2, "", "", ""); VariableIdentifier vi = new VariableIdentifier { name = newVariable.Label, id = newVariable.Id }; identifiers.Add(vi); XmlElement newVariableXml = xmldoc.CreateElement("variable"); newVariableXml.InnerText = Convert.ToString(newVariable.Id); orderElement.AppendChild(newVariableXml); } extraElement.AppendChild(orderElement); xmldoc.AppendChild(extraElement); sds.Extra = xmldoc; sds.Name = "generated import structure " + timestamp; sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp; #endregion Dataset ds = null; ds = dm.CreateEmptyDataset(sds, rp, metadataStructure); //TODO Should a template be created? /*ExcelTemplateProvider etp = new ExcelTemplateProvider(); * etp.CreateTemplate(sds);*/ long datasetId = ds.Id; long sdsId = sds.Id; if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault())) { DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId); long METADATASTRUCTURE_ID = metadataStructure.Id; XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath); XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID); XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX); dsv.Metadata = metadataXml; try { dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title); } catch (NullReferenceException ex) { //Reference of the title node is missing throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!"); } dm.EditDatasetVersion(dsv, null, null, null); } #region security // add security if (GetUsernameOrDefault() != "DEFAULT") { //PermissionManager pm = new PermissionManager(); //User user = sm.GetUserByName(GetUsernameOrDefault()); //Rights-Management /* * TODO: Use the BExIS Party API for that * * */ /* * UserPiManager upm = new UserPiManager(); * List<long> piList = (new UserSelectListModel(GetUsernameOrDefault())).UserList.Select(x => x.Id).ToList(); */ foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>()) { //The user gets full permissions // add security if (GetUsernameOrDefault() != "DEFAULT") { entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList()); } // adding the rights for the pis /*foreach (long piId in piList) * { * //Each pi gets full permissions * pm.CreateDataPermission(piId, 1, ds.Id, rightType); * * // get all pi members * List<UserPi> currentMembers = upm.GetAllPiMember(piId).ToList(); * * foreach (UserPi currentMapping in currentMembers) * { * switch (rightType) * { * //Each member of each of the pis gets view-rights * case RightType.View: * pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType); * break; * //Each member of each of the pis gets download-rights * case RightType.Download: * pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType); * break; * default: * //No other permissions - is this call necessary? * pm.CreateDataPermission(currentMapping.UserId, 0, ds.Id, rightType); * break; * } * } * }*/ } } #endregion security #region excel reader int packageSize = 10000; //HACK ? TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id); counter++; TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter; //rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray(); List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA]; string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString(); List <int[]> areaDataValuesList = new List <int[]>(); foreach (string area in selectedDataAreaJsonArray) { areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area)); } int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray); Orientation orientation = 0; switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString()) { case "LeftRight": orientation = Orientation.rowwise; break; case "Matrix": //orientation = Orientation.matrix; break; default: orientation = Orientation.columnwise; break; } String worksheetUri = null; //Get the Uri to identify the correct worksheet if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI)) { worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString(); } int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize; int batchnr = 1; foreach (int[] areaDataValues in areaDataValuesList) { //First batch starts at the start of the current data area int currentBatchStartRow = areaDataValues[0] + 1; while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached { //Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work EasyUploadExcelReader reader = new EasyUploadExcelReader(); // open file Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString()); //End row is start row plus batch size int currentBatchEndRow = currentBatchStartRow + batchSize; //Set the indices for the reader EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo { DataStartRow = currentBatchStartRow, //End row is either at the end of the batch or the end of the marked area //DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow, DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1), //Column indices as marked in a previous step DataStartColumn = areaDataValues[1] + 1, DataEndColumn = areaDataValues[3] + 1, //Header area as marked in a previous step VariablesStartRow = areaHeaderValues[0] + 1, VariablesStartColumn = areaHeaderValues[1] + 1, VariablesEndRow = areaHeaderValues[2] + 1, VariablesEndColumn = areaHeaderValues[3] + 1, Offset = areaDataValues[1], Orientation = orientation }; //Set variable identifiers because they might differ from the variable names in the file reader.setSubmittedVariableIdentifiers(identifiers); //Read the rows and convert them to DataTuples rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, sds, (int)datasetId, worksheetUri); //After reading the rows, add them to the dataset if (rows != null) { dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null); } //Close the Stream so the next ExcelReader can open it again Stream.Close(); //Debug information int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1); int batches = lines / batchSize; batchnr++; //Next batch starts after the current one currentBatchStartRow = currentBatchEndRow + 1; } } #endregion dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); //Reindex search if (this.IsAccessibale("DDM", "SearchIndex", "ReIndexSingle")) { this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary() { { "id", datasetId } }); } TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id); return(temp); } } catch (Exception ex) { temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " + "Please try again later. If this problem keeps occuring, please contact your administrator.")); return(temp); } finally { dsm.Dispose(); dm.Dispose(); dam.Dispose(); //sm.Dispose(); entityPermissionManager.Dispose(); } }
public List <Error> FinishUpload2(TaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); try { List <Error> temp = new List <Error>(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); //datatuple list List <DataTuple> rows; DatasetManager dm = new DatasetManager(); Dataset ds = dm.GetDataset(id); DatasetVersion workingCopy = new DatasetVersion(); #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List <AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } workingCopy = dm.GetDatasetWorkingCopy(ds.Id); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); //Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } } else { if (rows.Count() > 0) { Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } } while (rows.Count() > 0); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion // start download generator // filepath //string path = ""; //if (workingCopy != null) //{ // path = GenerateDownloadFile(workingCopy); // dm.EditDatasetVersion(workingCopy, null, null, null); //} // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault(), ViewCreationBehavior.None); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { workingCopy = dm.GetDatasetLatestVersion(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault(), ViewCreationBehavior.None); } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } return(temp); } finally { dsm.Dispose(); } }
//temporary solution: norman :FinishUpload2 public List <Error> FinishUpload(TaskManager taskManager) { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); try { List <Error> temp = new List <Error>(); DatasetVersion workingCopy = new DatasetVersion(); //datatuple list List <DataTuple> rows = new List <DataTuple>(); Dataset ds = null; bool inputWasAltered = false; if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); ds = dm.GetDataset(id); // Javad: Please check if the dataset does exists!! //GetValues from the previus version // Status DatasetVersion latestVersion = dm.GetDatasetLatestVersion(ds); string status = DatasetStateInfo.NotValid.ToString(); if (latestVersion.StateInfo != null) { status = latestVersion.StateInfo.State; } #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { string title = ""; long datasetid = ds.Id; XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper(); title = xmlDatasetHelper.GetInformation(ds.Id, NameAttributeValues.title); try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List <long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); } workingCopy = dm.GetDatasetWorkingCopy(ds.Id); //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); Stopwatch upload = Stopwatch.StartNew(); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); upload.Stop(); Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); upload.Stop(); Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } do { counter++; inputWasAltered = false; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { foreach (var err in reader.ErrorMessages) { temp.Add(new Error(ErrorType.Dataset, err.GetMessage())); } //return temp; } //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>(); var splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } } else { if (rows.Count() > 0) { Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0 || inputWasAltered == true); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion #region contentdescriptors //remove all contentdescriptors from the old version //generatedTXT if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generatedCSV if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generated if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } #endregion // ToDo: Get Comment from ui and users MoveAndSaveOriginalFileInContentDiscriptor(workingCopy); dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); //send email var es = new EmailService(); es.Send(MessageHelper.GetUpdateDatasetHeader(), MessageHelper.GetUpdateDatasetMessage(datasetid, title, GetUsernameOrDefault()), ConfigurationManager.AppSettings["SystemEmail"] ); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); var es = new EmailService(); es.Send(MessageHelper.GetErrorHeader(), "Can not upload. : " + e.Message, ConfigurationManager.AppSettings["SystemEmail"] ); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { // checkout the dataset, apply the changes, and check it in. if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { try { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); using (var unitOfWork = this.GetUnitOfWork()) { workingCopy = unitOfWork.GetReadOnlyRepository <DatasetVersion>().Get(workingCopy.Id); //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } unitOfWork.GetReadOnlyRepository <DatasetVersion>().Load(workingCopy.ContentDescriptors); SaveFileInContentDiscriptor(workingCopy); } dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault(), ViewCreationBehavior.None); } catch (Exception ex) { throw ex; } } } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } if (temp.Count <= 0) { dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault(), ViewCreationBehavior.None); } else { dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault()); } return(temp); } finally { dm.Dispose(); dsm.Dispose(); } }
//temporary solution: norman :FinishUpload2 public async Task <List <Error> > FinishUpload() { DataStructureManager dsm = new DataStructureManager(); DatasetManager dm = new DatasetManager(); IOUtility iOUtility = new IOUtility(); List <Error> temp = new List <Error>(); long id = 0; string title = ""; int numberOfRows = 0; int numberOfSkippedRows = 0; try { DatasetVersion workingCopy = new DatasetVersion(); //datatuple list List <DataTuple> rows = new List <DataTuple>(); //Dataset ds = null; bool inputWasAltered = false; if (Bus.ContainsKey(TaskManager.DATASET_ID) && Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { id = Convert.ToInt32(Bus[TaskManager.DATASET_ID]); long iddsd = Convert.ToInt32(Bus[TaskManager.DATASTRUCTURE_ID]); //GetValues from the previus version // Status DatasetVersion latestVersion = dm.GetDatasetLatestVersion(id); title = latestVersion.Title; string status = DatasetStateInfo.NotValid.ToString(); if (latestVersion.StateInfo != null) { status = latestVersion.StateInfo.State; } #region Progress Informations if (Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { Bus[TaskManager.CURRENTPACKAGE] = 0; } else { Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion Progress Informations #region structured data if (Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { long datasetid = id; XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper(); try { // load all data tuple ids from the latest version List <long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(id)); // load structured data structure StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm") || iOUtility.IsSupportedExcelFile(Bus[TaskManager.EXTENTION].ToString())) { int packageSize = 100000; Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; //schleife dm.CheckOutDatasetIfNot(id, User.Name); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(id, User.Name)) { throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", id, User.Name)); } workingCopy = dm.GetDatasetWorkingCopy(id); //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } ExcelReader reader = null; ExcelFileReaderInfo excelFileReaderInfo = null; if (iOUtility.IsSupportedExcelFile(Bus[TaskManager.EXTENTION].ToString())) { excelFileReaderInfo = (ExcelFileReaderInfo)Bus[TaskManager.FILE_READER_INFO]; } reader = new ExcelReader(sds, excelFileReaderInfo); do { counter++; Bus[TaskManager.CURRENTPACKAGE] = counter; //open stream Stream = reader.Open(Bus[TaskManager.FILEPATH].ToString()); rows = new List <DataTuple>(); if (iOUtility.IsSupportedExcelFile(Bus[TaskManager.EXTENTION].ToString())) { if (reader.Position < excelFileReaderInfo.DataEndRow) { rows = reader.ReadFile(Stream, Bus[TaskManager.FILENAME].ToString(), (int)id, packageSize); } } else { rows = reader.ReadTemplateFile(Stream, Bus[TaskManager.FILENAME].ToString(), (int)id, packageSize); } //Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (Bus[TaskManager.DATASET_STATUS].ToString().Equals("new") || ((UploadMethod)Bus[TaskManager.UPLOAD_METHOD]).Equals(UploadMethod.Append)) { dm.EditDatasetVersion(workingCopy, rows, null, null); //Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } else if (Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } } else { } } Stream?.Close(); //count rows numberOfRows += rows.Count(); } while (rows.Count() > 0 && rows.Count() <= packageSize); numberOfSkippedRows = reader.NumberOSkippedfRows; } #endregion excel reader #region ascii reader if (iOUtility.IsSupportedAsciiFile(Bus[TaskManager.EXTENTION].ToString())) { // open file AsciiReader reader = new AsciiReader(sds, (AsciiFileReaderInfo)Bus[TaskManager.FILE_READER_INFO]); if (dm.IsDatasetCheckedOutFor(id, User.Name) || dm.CheckOutDataset(id, User.Name)) { workingCopy = dm.GetDatasetWorkingCopy(id); //set packagsize for one loop int packageSize = 100000; Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } do { counter++; inputWasAltered = false; Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, Bus[TaskManager.FILENAME].ToString(), id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { foreach (var err in reader.ErrorMessages) { temp.Add(new Error(ErrorType.Dataset, err.GetMessage())); } //return temp; } if (Bus.ContainsKey(TaskManager.DATASET_STATUS)) //check wheter there is a dataset status in the upload wizard bus { // based the dataset status and/ or the upload method if (Bus[TaskManager.DATASET_STATUS].ToString().Equals("new") || ((UploadMethod)Bus[TaskManager.UPLOAD_METHOD]).Equals(UploadMethod.Append)) { dm.EditDatasetVersion(workingCopy, rows, null, null); // add all datatuples to the datasetversion } else if (Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) // datatuples allready exist { if (rows.Count() > 0) { //split the incoming datatuples to (new|edit) based on the primary keys var splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } } else // if there is no dataset status in the bus, use dataset status edit { if (rows.Count() > 0) { //split the incoming datatuples to (new|edit) based on the primary keys Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >(); splittedDatatuples = uploadWizardHelper.GetSplitDatatuples(rows, (List <long>)Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } //count rows numberOfRows += rows.Count(); } while ((rows.Count() > 0 && rows.Count() <= packageSize) || inputWasAltered == true); numberOfSkippedRows = reader.NumberOSkippedfRows; } //Stream.Close(); } #endregion ascii reader #region contentdescriptors //remove all contentdescriptors from the old version //generatedTXT if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generatedCSV if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generated if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } #endregion contentdescriptors #region set System value into metadata if (Bus.ContainsKey(TaskManager.DATASET_STATUS)) { bool newdataset = Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"); int v = 1; if (workingCopy.Dataset.Versions != null && workingCopy.Dataset.Versions.Count > 1) { v = workingCopy.Dataset.Versions.Count(); } //set modification workingCopy.ModificationInfo = new EntityAuditInfo() { Performer = User.Name, Comment = "Data", ActionType = newdataset ? AuditActionType.Create : AuditActionType.Edit }; setSystemValuesToMetadata(id, v, workingCopy.Dataset.MetadataStructure.Id, workingCopy.Metadata, newdataset); dm.EditDatasetVersion(workingCopy, null, null, null); } #endregion set System value into metadata // ToDo: Get Comment from ui and users MoveAndSaveOriginalFileInContentDiscriptor(workingCopy); dm.CheckInDataset(id, numberOfRows + " rows", User.Name); //send email var es = new EmailService(); es.Send(MessageHelper.GetUpdateDatasetHeader(datasetid), MessageHelper.GetUpdateDatasetMessage(datasetid, title, User.DisplayName), ConfigurationManager.AppSettings["SystemEmail"] ); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); var es = new EmailService(); es.Send(MessageHelper.GetErrorHeader(), "Can not upload. : " + e.Message, ConfigurationManager.AppSettings["SystemEmail"] ); } finally { } } #endregion structured data #region unstructured data if (Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { // checkout the dataset, apply the changes, and check it in. if (dm.IsDatasetCheckedOutFor(id, User.Name) || dm.CheckOutDataset(id, User.Name)) { try { workingCopy = dm.GetDatasetWorkingCopy(id); using (var unitOfWork = this.GetUnitOfWork()) { workingCopy.VersionNo += 1; //set StateInfo of the previus version if (workingCopy.StateInfo == null) { workingCopy.StateInfo = new Vaiona.Entities.Common.EntityStateInfo() { State = status }; } else { workingCopy.StateInfo.State = status; } unitOfWork.GetReadOnlyRepository <DatasetVersion>().Load(workingCopy.ContentDescriptors); SaveFileInContentDiscriptor(workingCopy); } if (Bus.ContainsKey(TaskManager.DATASET_STATUS)) { bool newdataset = Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"); int v = 1; if (workingCopy.Dataset.Versions != null && workingCopy.Dataset.Versions.Count > 1) { v = workingCopy.Dataset.Versions.Count(); } //set modification workingCopy.ModificationInfo = new EntityAuditInfo() { Performer = User.Name, Comment = "File", ActionType = AuditActionType.Create }; setSystemValuesToMetadata(id, v, workingCopy.Dataset.MetadataStructure.Id, workingCopy.Metadata, newdataset); dm.EditDatasetVersion(workingCopy, null, null, null); } //filename string filename = ""; if (Bus.ContainsKey(TaskManager.FILENAME)) { filename = Bus[TaskManager.FILENAME]?.ToString(); } // ToDo: Get Comment from ui and users dm.CheckInDataset(id, filename, User.Name, ViewCreationBehavior.None); } catch (Exception ex) { throw ex; } } } #endregion unstructured data } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } if (temp.Count <= 0) { dm.CheckInDataset(id, "no update on data tuples", User.Name, ViewCreationBehavior.None); } else { dm.UndoCheckoutDataset(id, User.Name); } } catch (Exception ex) { temp.Add(new Error(ErrorType.Dataset, ex.Message)); dm.CheckInDataset(id, "no update on data tuples", User.Name, ViewCreationBehavior.None); } finally { if (RunningASync) { var user = User; if (temp.Any()) { var es = new EmailService(); es.Send(MessageHelper.GetPushApiUploadFailHeader(id, title), MessageHelper.GetPushApiUploadFailMessage(id, user.Name, temp.Select(e => e.ToString()).ToArray()), new List <string> { user.Email }, null, new List <string> { ConfigurationManager.AppSettings["SystemEmail"] }); } else { var es = new EmailService(); es.Send(MessageHelper.GetASyncFinishUploadHeader(id, title), MessageHelper.GetASyncFinishUploadMessage(id, title, numberOfRows, numberOfSkippedRows), new List <string> { user.Email }, null, new List <string> { ConfigurationManager.AppSettings["SystemEmail"] }); } } dm.Dispose(); dsm.Dispose(); } return(temp); }