/// <summary> /// Show all Datasetids with export functions /// </summary> /// <returns></returns> // GET: api/Export public IEnumerable<long> Get() { DatasetManager dm = new DatasetManager(); var datasetIds = dm.GetDatasetLatestIds(); return datasetIds; }
public ActionResult AddSelectedDatasetToBus(string id) { ChooseDatasetViewModel model = new ChooseDatasetViewModel(); long datasetId = Convert.ToInt64(id); DatasetManager datasetManager = new DatasetManager(); Dataset dataset = datasetManager.GetDataset(datasetId); DatasetVersion datasetVersion; if(datasetManager.IsDatasetCheckedIn(datasetId)) { addSelectedDatasetToBus(datasetId); } else { model.ErrorList.Add(new Error(ErrorType.Dataset, "Dataset is not checked in.")); } Session["TaskManager"] = TaskManager; //create Model model.StepInfo = TaskManager.Current(); if ((List<ListViewItem>)Session["DatasetVersionViewList"] != null) model.DatasetsViewList = (List<ListViewItem>)Session["DatasetVersionViewList"]; if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_TITLE)) model.DatasetTitle = TaskManager.Bus[TaskManager.DATASET_TITLE].ToString(); model.SelectedDatasetId = Convert.ToInt32(id); return PartialView("SpecifyDataset", model); }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> public void Index() { configureBexisIndexing(true); // there is no need for the metadataAccess class anymore. Talked with David and deleted. 30.18.13. Javad/ compare to the previous version to see the deletions DatasetManager dm = new DatasetManager(); IList<long> ids = dm.GetDatasetLatestIds(); //ToDo only enitities from type dataset should be indexed in this index foreach (var id in ids) { //the values in the dictionary are already xml documents or null. Javad writeBexisIndex(id, dm.GetDatasetLatestMetadataVersion(id)); } indexWriter.Optimize(); autoCompleteIndexWriter.Optimize(); if (!reIndex) { indexWriter.Dispose(); autoCompleteIndexWriter.Dispose(); } }
/// <summary> /// generate a text file with JSON from a datastructure of a dataset /// and stored this file on the server /// and store the path in the content discriptor /// </summary> /// <param name="datasetId"></param> /// <returns>dynamic filepath</returns> public static string GenerateDataStructure(long datasetId) { string path = ""; try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); DataStructureManager dataStructureManager = new DataStructureManager(); long dataStructureId = datasetVersion.Dataset.DataStructure.Id; DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); if (dataStructure != null) { } // store in content descriptor path = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "datastructure", ".txt"); } catch (Exception ex) { throw ex; } return path; }
public ActionResult Cancel() { //public ActionResult LoadMetadata(long datasetId, bool locked = false, bool created = false, bool fromEditMode = false, bool resetTaskManager = false, XmlDocument newMetadata = null) TaskManager = (CreateTaskmanager)Session["CreateDatasetTaskmanager"]; if (TaskManager != null) { DatasetManager dm = new DatasetManager(); long datasetid = -1; bool resetTaskManager = true; XmlDocument metadata = null; bool editmode = false; bool created = false; if (TaskManager.Bus.ContainsKey(CreateTaskmanager.ENTITY_ID)) { datasetid = Convert.ToInt64(TaskManager.Bus[CreateTaskmanager.ENTITY_ID]); } if (datasetid > -1 && dm.IsDatasetCheckedIn(datasetid)) { metadata = dm.GetDatasetLatestMetadataVersion(datasetid); editmode = true; created = true; } return RedirectToAction("LoadMetadata", "Form", new { area = "DCM", entityId = datasetid, created = created, locked = true, fromEditMode = editmode, resetTaskManager = resetTaskManager, newMetadata = metadata }); } return RedirectToAction("StartMetadataEditor", "Form"); }
public ActionResult Datasets_Select() { DatasetManager datasetManager = new DatasetManager(); PermissionManager permissionManager = new PermissionManager(); SubjectManager subjectManager = new SubjectManager(); // DATA //var ids = datasetManager.GetDatasetLatestIds(); List<DatasetVersion> data = datasetManager.GetDatasetLatestVersions(); // .GetDatasetLatestVersions(ids); List<DatasetGridRowModel> datasets = new List<DatasetGridRowModel>(); data.ForEach(d => datasets.Add(DatasetGridRowModel.Convert(d, permissionManager.ExistsDataPermission(subjectManager.GetGroupByName("everyone").Id, 1, d.Id, RightType.View)))); return View(new GridModel<DatasetGridRowModel> { Data = datasets }); }
// GET: api/data/5 /// <summary> /// In addition to the id, it is possible to have projection and selection criteria passed to the action via query string parameters /// </summary> /// <param name="id">Dataset Id</param> /// <returns></returns> /// <remarks> The action accepts the following additional parameters via the query string /// 1: projection: is a comman separated list of ids that determines which variables of the dataset version tuples should take part in the result set /// 2: selection: is a logical expression that filters the tuples of the chosen dataset. The expression should have been written against the variables of the dataset only. /// logical operators, nesting, precedence, and SOME functions should be supported. /// </remarks> public HttpResponseMessage Get(int id) { string projection = this.Request.GetQueryNameValuePairs().FirstOrDefault(p => "header".Equals(p.Key, StringComparison.InvariantCultureIgnoreCase)).Value; string selection = this.Request.GetQueryNameValuePairs().FirstOrDefault(p => "filter" .Equals(p.Key, StringComparison.InvariantCultureIgnoreCase)).Value; OutputDataManager ioOutputDataManager = new OutputDataManager(); DatasetManager dm = new DatasetManager(); DatasetVersion version = dm.GetDatasetLatestVersion(id); string title = XmlDatasetHelper.GetInformation(version, NameAttributeValues.title); // check the data sturcture type ... if (version.Dataset.DataStructure.Self is StructuredDataStructure) { // apply selection and projection //var tuples = dm.GetDatasetVersionEffectiveTuples(version); DataTable dt = OutputDataManager.ConvertPrimaryDataToDatatable(dm, version, title, true); if (!string.IsNullOrEmpty(selection)) { dt = OutputDataManager.SelectionOnDataTable(dt, selection); } if (!string.IsNullOrEmpty(projection)) { // make the header names upper case to make them case insensitive dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(',')); } DatasetModel model = new DatasetModel(); model.DataTable = dt; var response = Request.CreateResponse(); response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName)); //set headers on the "response" return response; //return model; } else { return Request.CreateResponse(); } }
private static string storeGeneratedFilePathToContentDiscriptor(long datasetId, DatasetVersion datasetVersion, string title, string ext) { string name = ""; string mimeType = ""; if (ext.Contains("csv")) { name = "datastructure"; mimeType = "text/comma-separated-values"; } // create the generated FileStream and determine its location string dynamicPath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId, datasetVersion.Id, title, ext); //Register the generated data FileStream as a resource of the current dataset version //ContentDescriptor generatedDescriptor = new ContentDescriptor() //{ // OrderNo = 1, // Name = name, // MimeType = mimeType, // URI = dynamicPath, // DatasetVersion = datasetVersion, //}; DatasetManager dm = new DatasetManager(); if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(name)) > 0) { // remove the one contentdesciptor foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors) { if (cd.Name == name) { cd.URI = dynamicPath; dm.UpdateContentDescriptor(cd); } } } else { // add current contentdesciptor to list //datasetVersion.ContentDescriptors.Add(generatedDescriptor); dm.CreateContentDescriptor(name, mimeType, dynamicPath, 1, datasetVersion); } //dm.EditDatasetVersion(datasetVersion, null, null, null); return dynamicPath; }
/// <summary> /// Deletes a dataset, which means the dataset is marked as deleted, but is not physically removed from the database. /// </summary> /// <param name="id">the identifier of the dataset to be purged.</param> /// <remarks>When a dataset is deleted, it is consodered as non-exisiting, but for the sake or provenance, citation, history, etc, it is not removed froom the database. /// The function to recover a deleted dataset, will not be provided.</remarks> /// <returns></returns> public ActionResult Delete(long id) { ViewBag.Title = PresentationModel.GetViewTitleForTenant("Delete", this.Session.GetTenant()); try { DatasetManager dm = new DatasetManager(); if (dm.DeleteDataset(id, this.ControllerContext.HttpContext.User.Identity.Name, true)) { // during the delete permissions are not removed, to allow purging the dataset later on. // it is a safe operation, because deleted datasets are not returned by the DLM anyway. // Javad and Sven decided on 22.11.2016. //try //{ // PermissionManager pm = new PermissionManager(); // pm.DeleteDataPermissionsByEntity(1, id); //} //catch //{ // ViewData.ModelState.AddModelError("", string.Format("Dataset {0} was deleted, but its permissions were not altered. You need to remove them manually from the data permission management.", id)); //} try { ISearchProvider provider = IoCFactory.Container.ResolveForSession<ISearchProvider>() as ISearchProvider; provider?.UpdateSingleDatasetIndex(id, IndexingAction.DELETE); } catch { ViewData.ModelState.AddModelError("", string.Format("Dataset {0} was deleted, but it is still indexed for searching. You need to reindex the search via the managemnet console.", id)); } } else { ViewData.ModelState.AddModelError("", string.Format("Dataset {0} could not be deleted. Details: Internal system error!", id)); } } catch (Exception ex) { ViewData.ModelState.AddModelError("", string.Format("Dataset {0} could not be deleted. Details: {1}", id, ex.Message)); } return View(); //return RedirectToAction("List"); }
// GET: api/Metadata public IEnumerable<MetadataViewObject> Get() { DatasetManager dm = new DatasetManager(); var datasetIds = dm.GetDatasetLatestIds(); List<MetadataViewObject> tmp = new List<MetadataViewObject>(); foreach (var id in datasetIds) { MetadataViewObject mvo = new MetadataViewObject(); mvo.DatasetId = id; List<string> t = XmlDatasetHelper.GetAllTransmissionInformation(id, TransmissionType.mappingFileExport, AttributeNames.name).ToList(); mvo.Format = t.ToArray(); tmp.Add(mvo); } return tmp; }
public ActionResult ConvertSelectedDatasetVersion(string Id, string SelectedDatasetIds) { #region load Model DatasetManager datasetManager = new DatasetManager(); datasetVersionIds = datasetManager.GetDatasetVersionLatestIds(); MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(Convert.ToInt64(Id)); MetadataStructureModel model = new MetadataStructureModel( metadataStructure.Id, metadataStructure.Name, metadataStructure.Description, getDatasetVersionsDic(metadataStructure,datasetVersionIds), IsExportAvailable(metadataStructure) ); #endregion #region convert if (SelectedDatasetIds != null && SelectedDatasetIds!="") { string[] ids = SelectedDatasetIds.Split(','); foreach (string id in ids) { string path = Export(Convert.ToInt64(id)); model.AddMetadataPath(Convert.ToInt64(id), path); } } #endregion return PartialView("_metadataStructureView",model); }
// GET: api/Metadata/5 // HttpResponseMessage response = new HttpResponseMessage { Content = new StringContent(doc.innerXml, Encoding.UTF8,"application/xml") }; public HttpResponseMessage Get(int id) { string convertTo = ""; try { convertTo = this.Request.GetQueryNameValuePairs().FirstOrDefault(p => "format".Equals(p.Key, StringComparison.InvariantCultureIgnoreCase)).Value; } catch (Exception ex) { } DatasetManager dm = new DatasetManager(); DatasetVersion dsv = dm.GetDatasetLatestVersion(id); XmlDocument xmldoc = dsv.Metadata; if (string.IsNullOrEmpty(convertTo)) { //return xmldoc; HttpResponseMessage response = new HttpResponseMessage { Content = new StringContent(xmldoc.InnerXml, Encoding.UTF8, "application/xml") }; return response; } else { try { XmlDocument newXmlDoc = OutputMetadataManager.GetConvertedMetadata(id, TransmissionType.mappingFileExport, convertTo); HttpResponseMessage response = new HttpResponseMessage { Content = new StringContent(newXmlDoc.InnerXml, Encoding.UTF8, "application/xml") }; return response; } catch (Exception ex) { return null; } } }
public static string GetSchemaDirectoryPath(long datasetId) { try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(datasetVersion.Dataset.MetadataStructure.Id); string path = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DCM"), "Metadata", metadataStructure.Name); if(!String.IsNullOrEmpty(path) && Directory.Exists(path)) return path; } catch (Exception ex) { throw ex; } return String.Empty; }
public static XmlDocument GetConvertedMetadata(long datasetId, TransmissionType type, string mappingName, bool storing = true) { XmlDocument newXml; try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); string mappingFileName = XmlDatasetHelper.GetTransmissionInformation(datasetVersion, type, mappingName); string pathMappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), mappingFileName); XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern); xmlMapperManager.Load(pathMappingFile, "exporttest"); newXml = xmlMapperManager.Export(datasetVersion.Metadata, datasetVersion.Id, mappingName, true); string title = XmlDatasetHelper.GetInformation(datasetVersion, NameAttributeValues.title); // store in content descriptor if (storing) { if(String.IsNullOrEmpty(mappingName) || mappingName.ToLower() == "generic") storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "metadata", ".xml"); else storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "metadata_"+ mappingName, ".xml"); } } catch (Exception ex) { throw ex; } return newXml; }
public List<Error> FinishUpload2(TaskManager taskManager) { List<Error> temp = new List<Error>(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); //datatuple list List<DataTuple> rows; DatasetManager dm = new DatasetManager(); Dataset ds = dm.GetDataset(id); DatasetVersion workingCopy = new DatasetVersion(); #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); //Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } } while (rows.Count() > 0); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion // start download generator // filepath //string path = ""; //if (workingCopy != null) //{ // path = GenerateDownloadFile(workingCopy); // dm.EditDatasetVersion(workingCopy, null, null, null); //} // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { workingCopy = dm.GetDatasetLatestVersion(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } return temp; }
/// <summary> /// Shows the content of a specific dataset version /// </summary> /// <param name="id"></param> /// <returns></returns> public ActionResult Version(int id) { ViewBag.Title = PresentationModel.GetViewTitleForTenant("Version", this.Session.GetTenant()); DatasetManager dm = new DatasetManager(); DatasetVersion version = dm.DatasetVersionRepo.Get(p => p.Id == id).First(); var tuples = dm.GetDatasetVersionEffectiveTuples(version); ViewBag.VersionId = id; ViewBag.DatasetId = version.Dataset.Id; if (version.Dataset.DataStructure is StructuredDataStructure) { ViewBag.Variables = ((StructuredDataStructure) version.Dataset.DataStructure.Self).Variables.ToList(); } else { ViewBag.Variables = new List<Variable>(); } return View(tuples); }
public static string IsValideAgainstSchema(long datasetId, TransmissionType type, string mappingName) { try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); string mappingFileName = XmlDatasetHelper.GetTransmissionInformation(datasetVersion, type, mappingName); string pathMappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), mappingFileName); XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern); xmlMapperManager.Load(pathMappingFile, "exporttest"); XmlDocument tmp = GetConvertedMetadata(datasetId, type, mappingName, false); string path = Path.Combine(AppConfiguration.DataPath, "Temp", "System", "convertedMetadata.xml"); if (FileHelper.FileExist(path)) FileHelper.Delete(path); FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(path)); tmp.Save(path); XmlDocument metadataForImport = new XmlDocument(); metadataForImport.Load(path); return xmlMapperManager.Validate(metadataForImport); } catch (Exception ex) { return ex.Message; } }
public ActionResult SpecifyDataset(object[] data) { TaskManager = (TaskManager)Session["TaskManager"]; ChooseDatasetViewModel model = new ChooseDatasetViewModel(); model.StepInfo = TaskManager.Current(); if (TaskManager != null) { TaskManager.Current().SetValid(false); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID)) { DatasetManager dm = new DatasetManager(); Dataset ds = new Dataset(); try { dm = new DatasetManager(); ds = dm.GetDataset((long)Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID])); TaskManager.AddToBus(TaskManager.DATASTRUCTURE_ID, ((DataStructure)(ds.DataStructure.Self)).Id); TaskManager.AddToBus(TaskManager.DATASTRUCTURE_TITLE, ((DataStructure)(ds.DataStructure.Self)).Name); TaskManager.Current().SetValid(true); } catch { model.ErrorList.Add(new Error(ErrorType.Other, "Dataset not exist.")); } } else { model.ErrorList.Add(new Error(ErrorType.Other, "Dataset not exist.")); } if (TaskManager.Current().valid == true) { TaskManager.AddExecutedStep(TaskManager.Current()); TaskManager.GoToNext(); Session["TaskManager"] = TaskManager; ActionInfo actionInfo = TaskManager.Current().GetActionInfo; return RedirectToAction(actionInfo.ActionName, actionInfo.ControllerName, new RouteValueDictionary { { "area", actionInfo.AreaName }, { "index", TaskManager.GetCurrentStepInfoIndex() } }); } else { TaskManager.Current().SetStatus(StepStatus.error); //reload model model.StepInfo = TaskManager.Current(); if ((List<ListViewItem>)Session["DatasetVersionViewList"] != null) model.DatasetsViewList = (List<ListViewItem>)Session["DatasetVersionViewList"]; } } return PartialView(model); }
private void addSelectedDatasetToBus(long datasetId) { TaskManager = (TaskManager)Session["TaskManager"]; DatasetManager datasetManager = new DatasetManager(); if (datasetManager.GetDatasetVersionEffectiveTupleCount(datasetManager.GetDatasetLatestVersion(datasetId)) > 0) { TaskManager.AddToBus("DatasetStatus", "edit"); } else TaskManager.AddToBus("DatasetStatus", "new"); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); TaskManager.AddToBus(TaskManager.DATASET_ID, datasetId); //Add Metadata to Bus //TITLE TaskManager.AddToBus(TaskManager.DATASET_TITLE, XmlDatasetHelper.GetInformation(datasetVersion, NameAttributeValues.title)); ResearchPlanManager rpm = new ResearchPlanManager(); ResearchPlan rp = rpm.Repo.Get(datasetVersion.Dataset.ResearchPlan.Id); TaskManager.AddToBus(TaskManager.RESEARCHPLAN_ID, rp.Id); TaskManager.AddToBus(TaskManager.RESEARCHPLAN_TITLE, rp.Title); }
public ActionResult LoadMetadataFromExternal(long entityId, string title, long metadatastructureId, long datastructureId=-1,long researchplanId=-1, string sessionKeyForMetadata="", bool resetTaskManager = false) { bool loadFromExternal = true; long metadataStructureId = -1; //load metadata from session if exist XmlDocument metadata = Session[sessionKeyForMetadata] != null ? (XmlDocument) Session[sessionKeyForMetadata] : new XmlDocument(); ViewBag.Title = PresentationModel.GetViewTitleForTenant("Create Dataset", this.Session.GetTenant()); ; ViewData["Locked"] = true; ViewData["ShowOptional"] = false; TaskManager = (CreateTaskmanager)Session["CreateDatasetTaskmanager"]; if (TaskManager == null || resetTaskManager) { TaskManager = new CreateTaskmanager(); } List<StepModelHelper> stepInfoModelHelpers = new List<StepModelHelper>(); MetadataEditorModel Model = new MetadataEditorModel(); if (loadFromExternal) { string entityClassPath = ""; //TaskManager = new CreateTaskmanager(); Session["CreateDatasetTaskmanager"] = TaskManager; TaskManager.AddToBus(CreateTaskmanager.ENTITY_ID, entityId); if (TaskManager.Bus.ContainsKey(CreateTaskmanager.ENTITY_CLASS_PATH)) entityClassPath = TaskManager.Bus[CreateTaskmanager.ENTITY_CLASS_PATH].ToString(); bool ready = true; // todo i case of entity "BExIS.Dlm.Entities.Data.Dataset" we need to have a check if the dataset is checked in later all enitities should support such functions over webapis if (entityClassPath.Equals("BExIS.Dlm.Entities.Data.Dataset")) { DatasetManager dm = new DatasetManager(); //todo need a check if entity is in use if (!dm.IsDatasetCheckedIn(entityId)) { ready = false; } } if (ready) { TaskManager.AddToBus(CreateTaskmanager.METADATASTRUCTURE_ID, metadatastructureId); if(researchplanId!=-1)TaskManager.AddToBus(CreateTaskmanager.RESEARCHPLAN_ID, researchplanId); if(datastructureId!=-1) TaskManager.AddToBus(CreateTaskmanager.DATASTRUCTURE_ID, datastructureId); if (metadata != null && metadata.DocumentElement != null) TaskManager.AddToBus(CreateTaskmanager.METADATA_XML, XmlUtility.ToXDocument(metadata)); TaskManager.AddToBus(CreateTaskmanager.ENTITY_TITLE,title); ResearchPlanManager rpm = new ResearchPlanManager(); TaskManager.AddToBus(CreateTaskmanager.RESEARCHPLAN_ID, rpm.Repo.Get().First().Id); AdvanceTaskManagerBasedOnExistingMetadata(metadatastructureId); //AdvanceTaskManager(dsv.Dataset.MetadataStructure.Id); foreach (var stepInfo in TaskManager.StepInfos) { StepModelHelper stepModelHelper = GetStepModelhelper(stepInfo.Id); if (stepModelHelper.Model == null) { if (stepModelHelper.Usage is MetadataPackageUsage) { stepModelHelper.Model = CreatePackageModel(stepInfo.Id, false); if (stepModelHelper.Model.StepInfo.IsInstanze) LoadSimpleAttributesForModelFromXml(stepModelHelper); } if (stepModelHelper.Usage is MetadataNestedAttributeUsage) { stepModelHelper.Model = CreateCompoundModel(stepInfo.Id, false); if (stepModelHelper.Model.StepInfo.IsInstanze) LoadSimpleAttributesForModelFromXml(stepModelHelper); } getChildModelsHelper(stepModelHelper); } stepInfoModelHelpers.Add(stepModelHelper); } if (TaskManager.Bus.ContainsKey(CreateTaskmanager.METADATA_XML)) { XDocument xMetadata = (XDocument) TaskManager.Bus[CreateTaskmanager.METADATA_XML]; if (String.IsNullOrEmpty(title)) title = "No Title available."; if (TaskManager.Bus.ContainsKey(CreateTaskmanager.ENTITY_TITLE)) { if(TaskManager.Bus[CreateTaskmanager.ENTITY_TITLE]!=null) Model.DatasetTitle = TaskManager.Bus[CreateTaskmanager.ENTITY_TITLE].ToString(); } else Model.DatasetTitle = "No Title available."; } } else { ModelState.AddModelError(String.Empty, "Dataset is just in processing."); } } Model.DatasetId = entityId; Model.StepModelHelpers = stepInfoModelHelpers; Model.Created = false; //check if a metadatastructure has a import mapping if (TaskManager.Bus.ContainsKey(CreateTaskmanager.METADATASTRUCTURE_ID)) metadataStructureId = Convert.ToInt64(TaskManager.Bus[CreateTaskmanager.METADATASTRUCTURE_ID]); if (metadataStructureId != -1) Model.Import = IsImportAvavilable(metadataStructureId); //FromCreateOrEditMode TaskManager.AddToBus(CreateTaskmanager.EDIT_MODE, false); Model.FromEditMode = (bool)TaskManager.Bus[CreateTaskmanager.EDIT_MODE]; // set edit rights Model.EditRight = hasUserEditRights(entityId); Model.EditAccessRight = hasUserEditAccessRights(entityId); //set addtionaly functions Model.Actions = getAddtionalActions(); return PartialView("MetadataEditor", Model); }
public ActionResult Reset() { TaskManager = (CreateTaskmanager)Session["CreateDatasetTaskmanager"]; if (TaskManager != null) { DatasetManager dm = new DatasetManager(); long datasetid = -1; long metadataStructureid = -1; bool resetTaskManager = true; XmlDocument metadata = null; bool edit = true; bool created = false; if (TaskManager.Bus.ContainsKey(CreateTaskmanager.ENTITY_ID)) { datasetid = Convert.ToInt64(TaskManager.Bus[CreateTaskmanager.ENTITY_ID]); } if (datasetid > -1 && dm.IsDatasetCheckedIn(datasetid)) { Dataset dataset = dm.GetDataset(datasetid); metadataStructureid = dataset.MetadataStructure.Id; metadata = dm.GetDatasetLatestMetadataVersion(datasetid); TaskManager.UpdateBus(CreateTaskmanager.METADATA_XML, metadata); } return RedirectToAction("ImportMetadata", "Form", new { area = "DCM", metadataStructureId = metadataStructureid, edit, created }); } return RedirectToAction("StartMetadataEditor", "Form"); }
private List<long> GetDataTuples(long datasetId) { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); return datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); }
private string MoveAndSaveOriginalFileInContentDiscriptor(DatasetVersion datasetVersion) { TaskManager TaskManager = (TaskManager)Session["TaskManager"]; //dataset id and data structure id are available via datasetVersion properties,why you are passing them via the BUS? Javad long datasetId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASET_ID]); long dataStructureId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); DatasetManager datasetManager = new DatasetManager(); string title = TaskManager.Bus[TaskManager.DATASET_TITLE].ToString(); string ext = ".xlsm";// TaskManager.Bus[TaskManager.EXTENTION].ToString(); ExcelWriter excelWriter = new ExcelWriter(); // Move Original File to its permanent location String tempPath = TaskManager.Bus[TaskManager.FILEPATH].ToString(); string originalFileName = TaskManager.Bus[TaskManager.FILENAME].ToString(); string storePath = excelWriter.GetFullStorePathOriginalFile(datasetId, datasetVersion.Id, originalFileName); string dynamicStorePath = excelWriter.GetDynamicStorePathOriginalFile(datasetId, datasetVersion.VersionNo, originalFileName); //Why using the excel writer, isn't any function available in System.IO.File/ Directory, etc. Javad FileHelper.MoveFile(tempPath, storePath); //Register the original data as a resource of the current dataset version ContentDescriptor originalDescriptor = new ContentDescriptor() { OrderNo = 1, Name = "original", MimeType = "application/xlsm", URI = dynamicStorePath, DatasetVersion = datasetVersion, }; if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(originalDescriptor.Name)) > 0) { // remove the one contentdesciptor foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors) { if (cd.Name == originalDescriptor.Name) { cd.URI = originalDescriptor.URI; } } } else { // add current contentdesciptor to list datasetVersion.ContentDescriptors.Add(originalDescriptor); } return storePath; }
private string GenerateDownloadFile(DatasetVersion datasetVersion) { TaskManager TaskManager = (TaskManager)Session["TaskManager"]; //dataset id and data structure id are available via datasetVersion properties,why you are passing them via the BUS? Javad long datasetId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASET_ID]); long dataStructureId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); DatasetManager datasetManager = new DatasetManager(); string title = TaskManager.Bus[TaskManager.DATASET_TITLE].ToString(); string ext = ".xlsm";// TaskManager.Bus[TaskManager.EXTENTION].ToString(); ExcelWriter excelWriter = new ExcelWriter(); // create the generated file and determine its location string path = excelWriter.CreateFile(datasetId, datasetVersion.VersionNo, dataStructureId, title, ext); string dynamicPath = excelWriter.GetDynamicStorePath(datasetId, datasetVersion.VersionNo, title, ext); //Register the generated data file as a resource of the current dataset version ContentDescriptor generatedDescriptor = new ContentDescriptor() { OrderNo = 1, Name = "generated", MimeType = "application/xlsm", URI = dynamicPath, DatasetVersion = datasetVersion, }; if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(generatedDescriptor.Name)) > 0) { // remove the one contentdesciptor foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors) { if (cd.Name == generatedDescriptor.Name) { cd.URI = generatedDescriptor.URI; } } } else { // add current contentdesciptor to list datasetVersion.ContentDescriptors.Add(generatedDescriptor); } // note: the descriptors are not persisted yet, they will be persisted if the caller of this method persists the datasetVersion object. return path; }
private void AddDatatuplesToFile(long datasetId, long dataStructureId, string path) { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); List<long> tempDataTuplesIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); ExcelWriter excelWriter = new ExcelWriter(); excelWriter.AddDataTuplesToTemplate(datasetManager, tempDataTuplesIds, path, dataStructureId); }
/// <summary> /// Having the identifier of a dataset, lists all its versions. /// </summary> /// <param name="id">the identifier of the dataset.</param> /// <returns></returns> public ActionResult Versions(int id) { ViewBag.Title = PresentationModel.GetViewTitleForTenant("Versions", this.Session.GetTenant()); DatasetManager dm = new DatasetManager(); List<DatasetVersion> versions = dm.DatasetVersionRepo.Query(p => p.Dataset.Id == id).OrderBy(p => p.Id).ToList(); ViewBag.VersionId = id; return View(versions); }
/// <summary> /// Add Rows to a WorksheetPart /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="worksheetPart"></param> /// <param name="startRow"></param> /// <param name="endRow"></param> /// <param name="dataTuplesIds"></param> protected void AddRows(WorksheetPart worksheetPart, int startRow, int endRow, List<long> dataTuplesIds, DatasetManager datasetManager) { Worksheet worksheet = worksheetPart.Worksheet; SheetData sheetData = worksheet.GetFirstChild<SheetData>(); int rowIndex = endRow; DataTupleIterator tupleIterator = new DataTupleIterator(dataTuplesIds, datasetManager); foreach (var tuple in tupleIterator) { // convert datatuple to row and add it to sheetdata Row row = DatatupleToRow(tuple, rowIndex); bool empty = true; foreach (Cell c in row.Elements<Cell>().ToList()) { if (!String.IsNullOrEmpty(c.InnerText)) { empty = false; break; } } if (!empty) { sheetData.Append(row); if (!tuple.Id.Equals(dataTuplesIds.Last())) rowIndex++; } } numOfDataRows = rowIndex; }
// // GET: /DIM/Import/ public ActionResult Index() { //xml metadata for import string metadataForImportPath = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), "MetadataIDIV_EXAMPLE.xml"); XmlDocument metadataForImport = new XmlDocument(); metadataForImport.Load(metadataForImportPath); // metadataStructure DI long metadataStructureId = 3; MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); string metadataStructrueName = metadataStructureManager.Repo.Get(metadataStructureId).Name; // loadMapping file string path_mappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), XmlMetadataImportHelper.GetMappingFileName(metadataStructureId, TransmissionType.mappingFileImport, metadataStructrueName)); // XML mapper + mapping file XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern); xmlMapperManager.Load(path_mappingFile, "IDIV"); // generate intern metadata XmlDocument metadataResult = xmlMapperManager.Generate(metadataForImport,1); // generate intern template XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(BExIS.Xml.Helpers.XmlNodeMode.xPath); XDocument metadataXml = xmlMetadatWriter.CreateMetadataXml(metadataStructureId); XmlDocument metadataXmlTemplate = XmlMetadataWriter.ToXmlDocument(metadataXml); XmlDocument completeMetadata = XmlMetadataImportHelper.FillInXmlAttributes(metadataResult, metadataXmlTemplate); // create Dataset //load datastructure DataStructureManager dsm = new DataStructureManager(); ResearchPlanManager rpm = new ResearchPlanManager(); MetadataStructureManager msm = new MetadataStructureManager(); DatasetManager dm = new DatasetManager(); Dataset dataset = dm.CreateEmptyDataset(dsm.UnStructuredDataStructureRepo.Get(1), rpm.Repo.Get(1), msm.Repo.Get(3)); if (dm.IsDatasetCheckedOutFor(dataset.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(dataset.Id, GetUsernameOrDefault())) { DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(dataset.Id); workingCopy.Metadata = completeMetadata; string title = XmlDatasetHelper.GetInformation(workingCopy, NameAttributeValues.title); if (String.IsNullOrEmpty(title)) title = "No Title available."; dm.EditDatasetVersion(workingCopy, null, null, null); dm.CheckInDataset(dataset.Id, "Metadata was submited.", GetUsernameOrDefault()); // add security if (GetUsernameOrDefault() != "DEFAULT") { PermissionManager pm = new PermissionManager(); SubjectManager sm = new SubjectManager(); BExIS.Security.Entities.Subjects.User user = sm.GetUserByName(GetUsernameOrDefault()); foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast<RightType>()) { pm.CreateDataPermission(user.Id, 1, dataset.Id, rightType); } } } return View(); }
/// <summary> /// Add Datatuples to a Excel Template file /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="dataTuples"> Datatuples to add</param> /// <param name="filePath">Path of the excel template file</param> /// <param name="dataStructureId">Id of datastructure</param> /// <returns>List of Errors or null</returns> public List<Error> AddDataTuplesToTemplate(DatasetManager datasetManager, List<long> dataTuplesIds, string filePath, long dataStructureId ) { if (File.Exists(filePath)) { //Stream file = Open(filePath); //_dataTuples = dataTuples; // loading datastructure dataStructure = GetDataStructure(dataStructureId); // open excel file spreadsheetDocument = SpreadsheetDocument.Open(filePath, true); // get workbookpart WorkbookPart workbookPart = spreadsheetDocument.WorkbookPart; // get all the defined area List<DefinedNameVal> namesTable = buildDefinedNamesTable(workbookPart); // select data area this.areaOfData = namesTable.Where(p => p.Key.Equals("Data")).FirstOrDefault(); // Select variable area this.areaOfVariables = namesTable.Where(p => p.Key.Equals("VariableIdentifiers")).FirstOrDefault(); // Get intergers for reading data startColumn = getColumnNumber(this.areaOfData.StartColumn); endColumn = getColumnNumber(this.areaOfData.EndColumn); numOfColumns = (endColumn - startColumn) + 1; offset = getColumnNumber(getColumnName(this.areaOfData.StartColumn)) - 1; // gerneat Style for cell types generateStyle(spreadsheetDocument); // get styleSheet stylesheet = workbookPart.WorkbookStylesPart.Stylesheet; // Get shared strings sharedStrings = workbookPart.SharedStringTablePart.SharedStringTable.Elements<SharedStringItem>().ToArray(); // select worksheetpart by selected defined name area like data in sheet // sheet where data area is inside WorksheetPart worksheetPart = getWorkSheetPart(workbookPart, this.areaOfData); // Get VarioableIndentifiers this.VariableIdentifiers = getVariableIdentifiers(worksheetPart, this.areaOfVariables.StartRow, this.areaOfVariables.EndRow); AddRows(worksheetPart, this.areaOfData.StartRow, this.areaOfData.EndRow, dataTuplesIds, datasetManager); // set data area foreach (DefinedName name in workbookPart.Workbook.GetFirstChild<DefinedNames>()) { if (name.Name == "Data") { string[] tempArr = name.InnerText.Split('$'); string temp = ""; //$A$10:$C$15 tempArr[tempArr.Count() - 1] = numOfDataRows.ToString(); foreach (string t in tempArr) { if (t == tempArr.First()) { temp = temp + t; } else { temp = temp + "$" + t; } } name.Text = temp; } } spreadsheetDocument.WorkbookPart.Workbook.Save(); spreadsheetDocument.Close(); } return ErrorMessages; }
public List<Error> FinishUpload(TaskManager taskManager) { List<Error> temp = new List<Error>(); DatasetManager dm = new DatasetManager(); DatasetVersion workingCopy = new DatasetVersion(); //datatuple list List<DataTuple> rows = new List<DataTuple>(); Dataset ds = null; bool inputWasAltered = false; if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); ds = dm.GetDataset(id); // Javad: Please check if the dataset does exists!! #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); //workingCopy.ContentDescriptors = new List<ContentDescriptor>(); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); Stopwatch upload = Stopwatch.StartNew(); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); upload.Stop(); Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); upload.Stop(); Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; inputWasAltered = false; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { foreach (var err in reader.ErrorMessages) { temp.Add(new Error(ErrorType.Dataset, err.GetMessage())); } //return temp; } //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>(); var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0 || inputWasAltered == true); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion #region contentdescriptors //remove all contentdescriptors from the old version //generatedTXT if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generatedCSV if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generated if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } #endregion // ToDo: Get Comment from ui and users MoveAndSaveOriginalFileInContentDiscriptor(workingCopy); dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { // checkout the dataset, apply the changes, and check it in. if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } if (temp.Count <= 0) { dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } else { dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault()); } return temp; }