public string Convert(long datasetVersionId) { MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); DatasetManager datasetManager = new DatasetManager(); SubmissionManager submissionManager = new SubmissionManager(); PublicationManager publicationManager = new PublicationManager(); try { DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); _broker = publicationManager.GetBroker(_broker.Id); _dataRepo = publicationManager.GetRepository(_dataRepo.Id); long datasetId = datasetVersion.Dataset.Id; string name = datasetManager.GetDatasetVersion(datasetVersionId).Dataset.MetadataStructure.Name; XmlDocument metadata = OutputMetadataManager.GetConvertedMetadata(datasetId, TransmissionType.mappingFileExport, name, false); // create links to the api calls of the primary data? //add all to a zip //save document and return filepath for download string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name); string filename = submissionManager.GetFileNameForDataRepo(datasetId, datasetVersionId, _dataRepo.Name, "xml"); string filepath = Path.Combine(path, filename); FileHelper.CreateDicrectoriesIfNotExist(path); metadata.Save(filepath); return(filepath); } finally { datasetManager.Dispose(); metadataStructureManager.Dispose(); publicationManager.Dispose(); } }
private string getStorePath(long datasetVersionId, string exportTo) { using (DatasetManager datasetManager = new DatasetManager()) using (MetadataStructureManager metadataStructureManager = new MetadataStructureManager()) { DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); Dataset dataset = datasetManager.GetDataset(datasetVersionId); string md_title = metadataStructureManager.Repo.Get(datasetVersion.Dataset.MetadataStructure.Id).Name; string path; int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion); if (string.IsNullOrEmpty(exportTo) || exportTo.ToLower().Equals("generic")) { path = IOHelper.GetDynamicStorePath(datasetVersion.Dataset.Id, versionNr, "metadata", ".xml"); } else { path = IOHelper.GetDynamicStorePath(datasetVersion.Dataset.Id, versionNr, "metadata_" + exportTo, ".xml"); } return(path); } }
private string generatePrimaryData(long datasetVersionId) { try { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure) { OutputDataManager outputDataManager = new OutputDataManager(); SubmissionManager submissionManager = new SubmissionManager(); long datasetId = datasetVersion.Dataset.Id; string fileName = submissionManager.GetFileNameForDataRepo(datasetId, datasetVersionId, _dataRepo.Name, "csv"); string filepath = outputDataManager.GenerateAsciiFile( datasetId, datasetVersionId, fileName, "text/txt"); return(filepath); } } catch (Exception) { throw; } return(""); }
public ActionResult Save(MetadataStructureModel metadataStructureModel) { MetadataStructureManager metadataStructureManager = new MetadataStructureManager(); DatasetManager datasetManager = new DatasetManager(); try { if (ModelState.IsValid) { MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(metadataStructureModel.Id); metadataStructure = updateMetadataStructure(metadataStructure, metadataStructureModel); metadataStructureManager.Update(metadataStructure); //update dsv title and description if there is a change //ToDo check if there is a change in the xpaths // update datasetversion // get all datasetIds which using the metadata structure var datasetIds = datasetManager.DatasetRepo.Query().Where(d => d.MetadataStructure.Id.Equals(metadataStructure.Id)).Select(d => d.Id); if (datasetIds.Any()) { //get all datasetversions of the dataset ids var datasetVersionIds = datasetManager.DatasetVersionRepo.Query().Where(dsv => datasetIds.Contains(dsv.Dataset.Id)).Select(dsv => dsv.Id).ToList(); //load all titles & descriptions from versions var allTitles = xmlDatasetHelper.GetInformationFromVersions(datasetVersionIds, metadataStructure.Id, NameAttributeValues.title); var allDescriptions = xmlDatasetHelper.GetInformationFromVersions(datasetVersionIds, metadataStructure.Id, NameAttributeValues.description); // update each datasetversion foreach (var datasetVersionId in datasetVersionIds) { // load dataset version var datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); datasetVersion.Title = allTitles.ContainsKey(datasetVersion.Id) ? allTitles[datasetVersion.Id] : string.Empty; datasetVersion.Description = allDescriptions.ContainsKey(datasetVersion.Id) ? allDescriptions[datasetVersion.Id] : string.Empty; datasetManager.UpdateDatasetVersion(datasetVersion); } } return(Json(true)); } return(Json(false)); } catch (Exception ex) { return(Json(ex.Message)); } finally { metadataStructureManager.Dispose(); datasetManager.Dispose(); } }
private static void storeGeneratedFilePathToContentDiscriptor(long datasetId, DatasetVersion datasetVersion, string title, string ext) { string name = ""; string mimeType = ""; if (ext.Contains("xml")) { name = "metadata"; mimeType = "text/xml"; } DatasetManager dm = new DatasetManager(); int versionNr = dm.GetDatasetVersionNr(datasetVersion); // create the generated FileStream and determine its location string dynamicPath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId, versionNr, title, ext); //Register the generated data FileStream as a resource of the current dataset version //ContentDescriptor generatedDescriptor = new ContentDescriptor() //{ // OrderNo = 1, // Name = name, // MimeType = mimeType, // URI = dynamicPath, // DatasetVersion = datasetVersion, //}; try { datasetVersion = dm.GetDatasetVersion(datasetVersion.Id); if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(name)) > 0) { // remove the one contentdesciptor foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors) { if (cd.Name == name) { cd.URI = dynamicPath; dm.UpdateContentDescriptor(cd); } } } else { // add current contentdesciptor to list //datasetVersion.ContentDescriptors.Add(generatedDescriptor); dm.CreateContentDescriptor(name, mimeType, dynamicPath, 1, datasetVersion); } //dm.EditDatasetVersion(datasetVersion, null, null, null); } finally { dm.Dispose(); } }
private DataTable getHistoryData(long versionId) { DatasetManager dm = new DatasetManager(); try { DatasetVersion dsv = dm.GetDatasetVersion(versionId); DataTable table = null; long rowCount = dm.GetDatasetVersionEffectiveTuples(dsv).Count; table = dm.GetDatasetVersionTuples(versionId, 0, (int)rowCount); return(table); } finally { dm.Dispose(); } }
public List <FileInformation> getFilesByDataset(Dataset dataset, DatasetManager datasetManager, string entityType, long versionId = 0) { EntityPermissionManager entityPermissionManager = null; try { List <FileInformation> fileInfos = new List <FileInformation>(); entityPermissionManager = new EntityPermissionManager(); bool access = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), dataset.Id, RightType.Read); if (dataset != null && access) { DatasetVersion datasetVersion = new DatasetVersion(); if (versionId > 0) { datasetVersion = datasetManager.GetDatasetVersion(versionId); } else { datasetVersion = datasetManager.GetDatasetLatestVersion(dataset); } if (datasetVersion != null) { List <ContentDescriptor> contentDescriptors = datasetVersion.ContentDescriptors.ToList(); if (contentDescriptors.Count > 0) { foreach (ContentDescriptor cd in contentDescriptors) { if (cd.Name.ToLower().Equals("unstructureddata")) { fileInfos.Add(getFileInfo(cd)); } } } } } return(fileInfos); } finally { entityPermissionManager.Dispose(); } }
private DatasetFilesModel LoadDatasetModel(long versionId) { using (EntityPermissionManager entityPermissionManager = new EntityPermissionManager()) using (EntityManager entityManager = new EntityManager()) using (UserManager userManager = new UserManager()) using (DatasetManager dm = new DatasetManager()) { var datasetVersion = dm.GetDatasetVersion(versionId); var model = new DatasetFilesModel { ServerFileList = GetDatasetFileList(datasetVersion), FileSize = this.Session.GetTenant().MaximumUploadSize }; //Parse user right var entity = entityManager.EntityRepository.Query(e => e.Name.ToUpperInvariant() == "Dataset".ToUpperInvariant() && e.EntityType == typeof(Dataset)).FirstOrDefault(); var userTask = userManager.FindByNameAsync(HttpContext.User.Identity.Name); userTask.Wait(); var user = userTask.Result; int rights = 0; if (user == null) { rights = entityPermissionManager.GetEffectiveRights(subjectId: null, entity.Id, datasetVersion.Dataset.Id); } else { rights = entityPermissionManager.GetEffectiveRights(user.Id, entity.Id, datasetVersion.Dataset.Id); } model.UploadAccess = (((rights & (int)RightType.Write) > 0) || ((rights & (int)RightType.Grant) > 0)); model.DeleteAccess = (((rights & (int)RightType.Delete) > 0) || ((rights & (int)RightType.Grant) > 0)); model.DownloadAccess = ((rights & (int)RightType.Read) > 0 || ((rights & (int)RightType.Grant) > 0)); model.ViewAccess = ((rights & (int)RightType.Read) > 0 || ((rights & (int)RightType.Grant) > 0)); return(model); } }
// GET: DQ public ActionResult ShowDQ(long datasetId, long versionId) { DQModels dqModel = new DQModels(); Dictionary <string, string> datasetInfo = new Dictionary <string, string>(); List <performer> performers = new List <performer>(); List <varVariable> varVariables = new List <varVariable>(); Dictionary <string, double> datasetSize = new Dictionary <string, double>(); DatasetManager dm = new DatasetManager(); DataStructureManager dsm = new DataStructureManager(); EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); PartyManager pm = new PartyManager(); UserManager um = new UserManager(); DatasetVersion dsv = new DatasetVersion(); UserManager userManager = new UserManager(); // data quality files try { string pathPerformerDataset = @"C:\Data\DatasetQualities\PerformerDataset.csv"; StreamReader readerPerformerDataset = new StreamReader(pathPerformerDataset); } catch (Exception ex) { } ////////////////////////////////////////////////////////////////////////// DatasetVersion currentDatasetVersion = dm.GetDatasetVersion(versionId); //Current dataset version DataStructure currentDataStr = dsm.AllTypesDataStructureRepo.Get(currentDatasetVersion.Dataset.DataStructure.Id); //current data structure var currentUser = userManager.FindByNameAsync(GetUsernameOrDefault()).Result; //Find current user //Find the dataset Type string currentDatasetType = "file"; if (currentDataStr.Self.GetType() == typeof(StructuredDataStructure)) { currentDatasetType = "tabular"; } dqModel.type = currentDatasetType; #region performers #region dataset's performers try { string pathPerformerDataset = @"C:\Data\DatasetQualities\PerformerDataset.csv"; StreamReader readerPerformerDataset = new StreamReader(pathPerformerDataset); string pathPerformers = @"C:\Data\DatasetQualities\Performers.csv"; StreamReader readerPerformers = new StreamReader(pathPerformers); string performerLine; List <string> pfs = new List <string>(); List <performer> ps = new List <performer>(); while ((performerLine = readerPerformerDataset.ReadLine()) != null) { string[] s = performerLine.Split(','); if (long.Parse(s[1]) == datasetId) { pfs.Add(s[0]); } } while ((performerLine = readerPerformers.ReadLine()) != null) { string[] s = performerLine.Split(','); if (pfs.Contains(s[0])) { performer p = new performer(); p.performerName = FindPerformerNameFromUsername(um, s[0]); //find performer name p.performerRate = int.Parse(s[1]); List <long> pfIds = FindDatasetsFromPerformerUsername(dm, um, s[0]); //Find all datasets in wich the username is involved. p.DatasetIds = pfIds; ps.Add(p); } } dqModel.performers = ps; readerPerformerDataset.Close(); readerPerformers.Close(); } catch (Exception ex) { } #endregion #endregion //performers //dqModel.isPublic = entityPermissionManager.GetRights(null, 1, datasetId); //check if dataset is public //check the read permission for current dataset bool rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), datasetId, Security.Entities.Authorization.RightType.Read); //find if user has read permission if (rPermission == true) //has read permission or public = readable { dqModel.readable = 1; } else { dqModel.readable = 0; } //cannot read //Check if the current metadata is valid if (currentDatasetVersion.StateInfo != null) { dqModel.isValid = DatasetStateInfo.Valid.ToString().Equals(currentDatasetVersion.StateInfo.State) ? 1 : 0; //1:valid; 0:invalid. } else { dqModel.isValid = 0; } List <long> datasetIds = dm.GetDatasetLatestIds(); dqModel.allDatasets = datasetIds.Count; List <int> metadataRates = new List <int>(); List <int> dsDescLength = new List <int>(); List <int> dstrDescLength = new List <int>(); List <int> dstrUsage = new List <int>(); List <int> datasetSizeTabular = new List <int>(); List <int> datasetRows = new List <int>(); List <int> datasetCols = new List <int>(); List <double> datasetSizeFiles = new List <double>(); double datasetSizeFile = new double(); List <int> datasetFileNumber = new List <int>(); List <int> restrictions = new List <int>(); int fileNumber = 0; List <int> sizeTabular = new List <int>(); //collect size, column number, and row number for one dataset int publicDatasets = 0; int restrictedDatasets = 0; int fileDatasets = 0; int tabularDatasets = 0; int rpTrue = 0; int rp; int validMetadata = 0; int allValidMetadas = 0; foreach (long Id in datasetIds) //for each dataset { if (dm.IsDatasetCheckedIn(Id)) { DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(Id); //get last dataset versions //If user has read permission rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), Id, Security.Entities.Authorization.RightType.Read); if (rPermission == true) //has read permission or public = readable { rp = 1; rpTrue += 1; } else { rp = 0; } //cannot read } } dqModel.allReadables = rpTrue; string pathDatasetInfo = @"C:\Data\DatasetQualities\datasetInfo.csv"; StreamReader readerDatasetInfo = new StreamReader(pathDatasetInfo); List <datasetInformation> datasetsInformation = new List <datasetInformation>(); try { string lineDatasetInfo; while ((lineDatasetInfo = readerDatasetInfo.ReadLine()) != null) { string[] dsInf = lineDatasetInfo.Split(';'); datasetInformation datasetInformation = new datasetInformation(); long id = long.Parse(dsInf[0]); datasetInformation.datasetId = id; datasetInformation.title = dm.GetDatasetLatestVersion(id).Title; rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), id, Security.Entities.Authorization.RightType.Read); if (rPermission == true) { datasetInformation.readable = 1; } if (rPermission == false) { datasetInformation.readable = 0; } datasetInformation.type = dsInf[1]; datasetInformation.metadataValidation = int.Parse(dsInf[2]); datasetInformation.metadataComplition = int.Parse(dsInf[3]); datasetInformation.descriptionLength = int.Parse(dsInf[4]); datasetInformation.structureDescriptionLength = int.Parse(dsInf[5]); datasetInformation.structureUsage = int.Parse(dsInf[6]); datasetInformation.columnNumber = int.Parse(dsInf[7]); datasetInformation.rowNumber = int.Parse(dsInf[8]); datasetInformation.fileNumber = int.Parse(dsInf[9]); datasetInformation.datasetSizeFile = double.Parse(dsInf[10]); string[] pfrms = dsInf[11].Split(','); List <string> performerNames = new List <string>(); foreach (string p in pfrms) { performerNames.Add(p); } datasetInformation.performerNames = performerNames; datasetsInformation.Add(datasetInformation); if (datasetId == id) { dqModel.metadataComplition.requiredFields = int.Parse(dsInf[2]); dqModel.metadataComplition.totalFields = int.Parse(dsInf[3]); dqModel.datasetDescriptionLength.currentDescriptionLength = int.Parse(dsInf[4]); dqModel.dataStrDescriptionLength.currentDescriptionLength = int.Parse(dsInf[5]); dqModel.dataStrUsage.currentDataStrUsage = int.Parse(dsInf[6]); dqModel.columnNumber = datasetInformation.columnNumber; dqModel.rowNumber = datasetInformation.rowNumber; dqModel.fileNumber = datasetInformation.fileNumber; dqModel.datasetTotalSize.currentTotalSize = datasetInformation.datasetSizeFile; } } } catch { } dqModel.datasetsInformation = datasetsInformation; readerDatasetInfo.Close(); //CURRENT DATASET VERSION //dqModel.metadataComplition.totalFields = GetMetadataRate(currentDatasetVersion); //current dataset version: metadata rate //dqModel.metadataComplition.requiredFields = 100; //Need to calculate: metadataStructureId = dsv.Dataset.MetadataStructure.Id; //dqModel.datasetDescriptionLength.currentDescriptionLength = currentDatasetVersion.Description.Length; // Current dataset vesion: dataset description length //dqModel.dataStrDescriptionLength.currentDescriptionLength = currentDatasetVersion.Dataset.DataStructure.Description.Length; // Current dataset version: data structure description length //dqModel.dataStrUsage.currentDataStrUsage = currentDataStr.Datasets.Count() - 1; // Current dataset version: how many times the data structure is used in other datasets #region comparision try { string pathComparison = @"C:\Data\DatasetQualities\Comparison.csv"; StreamReader readerComparison = new StreamReader(pathComparison); string infoline; List <int> infos = new List <int>(); while ((infoline = readerComparison.ReadLine()) != null) { string[] s = infoline.Split(','); if (s[0] == "performersActivity") { dqModel.performersActivity.minActivity = int.Parse(s[1]); dqModel.performersActivity.medianActivity = int.Parse(s[2]); dqModel.performersActivity.maxActivity = int.Parse(s[3]); } else if (s[0] == "type") { dqModel.allDatasets = int.Parse(s[1]); dqModel.tabularDatasets = int.Parse(s[2]); dqModel.fileDatasets = int.Parse(s[3]); } else if (s[0] == "metadataRates") { dqModel.metadataComplition.minRate = int.Parse(s[1]); dqModel.metadataComplition.medianRate = int.Parse(s[2]); dqModel.metadataComplition.maxRate = int.Parse(s[3]); } else if (s[0] == "allValidMetadas") { dqModel.allValids = int.Parse(s[1]); } else if (s[0] == "datasetDescriptionLength") { dqModel.datasetDescriptionLength.minDescriptionLength = int.Parse(s[1]); dqModel.datasetDescriptionLength.medianDescriptionLength = int.Parse(s[2]); dqModel.datasetDescriptionLength.maxDescriptionLength = int.Parse(s[3]); } else if (s[0] == "dataStrDescriptionLength") { dqModel.dataStrDescriptionLength.minDescriptionLength = int.Parse(s[1]); dqModel.dataStrDescriptionLength.medianDescriptionLength = int.Parse(s[2]); dqModel.dataStrDescriptionLength.maxDescriptionLength = int.Parse(s[3]); } else if (s[0] == "dataStrUsage") { dqModel.dataStrUsage.minDataStrUsage = int.Parse(s[1]); dqModel.dataStrUsage.medianDataStrUsage = int.Parse(s[2]); dqModel.dataStrUsage.maxDataStrUsage = int.Parse(s[3]); } else if (s[0] == "datasetColNumber") { dqModel.datasetColNumber.minColNumber = int.Parse(s[1]); dqModel.datasetColNumber.medianColNumber = int.Parse(s[2]); dqModel.datasetColNumber.maxColNumber = int.Parse(s[3]); } else if (s[0] == "datasetRowNumber") { dqModel.datasetRowNumber.minRowNumber = int.Parse(s[1]); dqModel.datasetRowNumber.medianRowNumber = int.Parse(s[2]); dqModel.datasetRowNumber.maxRowNumber = int.Parse(s[3]); } else if (s[0] == "datasetFileNumber") { dqModel.datasetFileNumber.minFileNumber = int.Parse(s[1]); dqModel.datasetFileNumber.medianFileNumber = int.Parse(s[2]); dqModel.datasetFileNumber.maxFileNumber = int.Parse(s[3]); } else if (s[0] == "datasetTotalSizeFiles") { dqModel.datasetTotalSize.minSizeFile = double.Parse(s[1]); dqModel.datasetTotalSize.medianSizeFile = double.Parse(s[2]); dqModel.datasetTotalSize.maxSizeFile = double.Parse(s[3]); } } readerComparison.Close(); } catch (Exception ex) { } #endregion /////////////////////////////////////////////////////////////////////// #region TABULAR FORMAT DATASET //If it is a tabular format dataset if (currentDatasetType == "tabular") { string pathVariables = @"C:\Data\DatasetQualities\Variables.csv"; StreamReader readerVariables = new StreamReader(pathVariables); string varLine; while ((varLine = readerVariables.ReadLine()) != null) { string[] varDetail = varLine.Split(','); if (varDetail[0] == datasetId.ToString()) { varVariable v = new varVariable(); v.varLabel = varDetail[1]; v.varType = varDetail[2]; v.varDescription = varDetail[3]; v.varUsage = int.Parse(varDetail[4]); v.missing = int.Parse(varDetail[5]); varVariables.Add(v); } } dqModel.varVariables = varVariables; readerVariables.Close(); } // string serverName = "http://*****:*****@"C:\Data\DatasetQualities\Files.csv"; try { StreamReader readerFiles = new StreamReader(pathFiles); while ((fileLine = readerFiles.ReadLine()) != null) { string[] fileDetail = fileLine.Split(','); if (fileDetail[0] == datasetId.ToString()) { fileInformation f = new fileInformation(); f.fileName = fileDetail[1]; f.fileFormat = fileDetail[2]; double d = Convert.ToDouble(fileDetail[3]); f.fileSize = d; filesInformations.Add(f); } } readerFiles.Close(); } catch { } dqModel.filesInformation = filesInformations; } //if (currentDatasetVersion != null) //{ // List<ContentDescriptor> contentDescriptors = currentDatasetVersion.ContentDescriptors.ToList(); // double totalSize = 0; // if (contentDescriptors.Count > 0) // { // foreach (ContentDescriptor cd in contentDescriptors) // { // if (cd.Name.ToLower().Equals("unstructureddata")) // { // fileInformation fileInformation = new fileInformation(); // string uri = cd.URI; // //get the file path // try // { // String path = Server.UrlDecode(uri); // path = Path.Combine(AppConfiguration.DataPath, path); // Stream fileStream = System.IO.File.OpenRead(path); // if (fileStream != null) // { // FileStream fs = fileStream as FileStream; // if (fs != null) // { // //get file information // FileInformation fileInfo = new FileInformation(fs.Name.Split('\\').LastOrDefault(), MimeMapping.GetMimeMapping(fs.Name), (uint)fs.Length, uri); // fileInformation.fileName = fileInfo.Name.Split('.')[0]; //file name // fileInformation.fileFormat = fileInfo.Name.Split('.')[1].ToLower(); //file extension // fileInformation.fileSize = fileInfo.Size; //file size // totalSize += fileInfo.Size; // } // } // } // catch // { // } // filesInformation.Add(fileInformation); // } //} //} //dqModel.fileNumber = contentDescriptors.Count; //dqModel.datasetTotalSize.currentTotalSize = totalSize; //} //dqModel.filesInformation = filesInformation; //} #endregion return(PartialView(dqModel)); }
//ToDO -> David <- do not store the files on the server public string Convert(long datasetVersionId) { string datarepo = _dataRepo.Name; SubmissionManager publishingManager = new SubmissionManager(); using (DatasetManager datasetManager = new DatasetManager()) using (DataStructureManager dataStructureManager = new DataStructureManager()) using (PublicationManager publicationManager = new PublicationManager()) using (ZipFile zip = new ZipFile()) { DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); long datasetId = datasetVersion.Dataset.Id; Publication publication = publicationManager.GetPublication() .Where( p => p.DatasetVersion.Id.Equals(datasetVersion.Id) && p.Broker.Name.ToLower().Equals(_broker.Name)) .FirstOrDefault(); // if(broker exist) if (publication == null && _broker != null) { Broker broker = _broker; if (broker != null) { OutputMetadataManager.GetConvertedMetadata(datasetId, TransmissionType.mappingFileExport, broker.MetadataFormat); // get primary data // check the data sturcture type ... if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure) { OutputDataManager odm = new OutputDataManager(); // apply selection and projection odm.GenerateAsciiFile(datasetId, datasetVersion.Id, broker.PrimaryDataFormat, false); } int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion); string zipName = publishingManager.GetZipFileName(datasetId, versionNr); string zipPath = publishingManager.GetDirectoryPath(datasetId, broker.Name); string dynamicZipPath = publishingManager.GetDynamicDirectoryPath(datasetId, broker.Name); string zipFilePath = Path.Combine(zipPath, zipName); string dynamicFilePath = Path.Combine(dynamicZipPath, zipName); FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(zipFilePath)); if (FileHelper.FileExist(zipFilePath)) { if (FileHelper.WaitForFile(zipFilePath)) { FileHelper.Delete(zipFilePath); } } // add datastructure //ToDo put that functiom to the outputDatatructureManager #region datatructure long dataStructureId = datasetVersion.Dataset.DataStructure.Id; DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId); if (dataStructure != null) { try { string dynamicPathOfDS = ""; dynamicPathOfDS = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "datastructure", ".txt"); string datastructureFilePath = AsciiWriter.CreateFile(dynamicPathOfDS); string json = OutputDataStructureManager.GetVariableListAsJson(dataStructureId); AsciiWriter.AllTextToFile(datastructureFilePath, json); } catch (Exception ex) { throw ex; } } #endregion datatructure foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors) { string path = Path.Combine(AppConfiguration.DataPath, cd.URI); string name = cd.URI.Split('\\').Last(); if (FileHelper.FileExist(path)) { zip.AddFile(path, ""); } } // add xsd of the metadata schema string xsdDirectoryPath = OutputMetadataManager.GetSchemaDirectoryPath(datasetId); if (Directory.Exists(xsdDirectoryPath)) { zip.AddDirectory(xsdDirectoryPath, "Schema"); } XmlDocument manifest = OutputDatasetManager.GenerateManifest(datasetId, datasetVersion.Id); if (manifest != null) { string dynamicManifestFilePath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId, versionNr, "manifest", ".xml"); string fullFilePath = Path.Combine(AppConfiguration.DataPath, dynamicManifestFilePath); manifest.Save(fullFilePath); zip.AddFile(fullFilePath, ""); } string message = string.Format("dataset {0} version {1} was published for repository {2}", datasetId, datasetVersion.Id, broker.Name); LoggerFactory.LogCustom(message); //Session["ZipFilePath"] = dynamicFilePath; zip.Save(zipFilePath); return(zipFilePath); } } return(""); } }
public string GenerateAsciiFile(long id, long versionId, string mimeType, bool withUnits) { DatasetManager datasetManager = new DatasetManager(); DataStructureManager datasetStructureManager = new DataStructureManager(); try { DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(versionId); int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion); string contentDescriptorTitle = ""; string ext = ""; string nameExt = ""; TextSeperator textSeperator = TextSeperator.semicolon; if (withUnits) { nameExt = "_withunits"; } switch (mimeType) { case "text/csv": case "text/comma-separated-values": case "application/octet-stream": /* of course this is a wrong mimetype for csv. * but the c# class MimeMapping.GetMimeMapping(ext) currently returns this as a result for .csv. * since we don't use the datatype at the moment, * it will be rebuilt into the case here*/ { contentDescriptorTitle = "generatedCSV" + nameExt; ext = ".csv"; textSeperator = TextSeperator.semicolon; break; } case "text/tsv": case "text/tab-separated-values": { contentDescriptorTitle = "generatedTSV" + nameExt; ext = ".tsv"; textSeperator = TextSeperator.tab; break; } default: { contentDescriptorTitle = "generatedTXT" + nameExt; ext = ".txt"; textSeperator = TextSeperator.tab; break; } } AsciiWriter writer = new AsciiWriter(textSeperator); string path = ""; //ascii allready exist if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(contentDescriptorTitle) && p.URI.Contains(datasetVersion.Id.ToString())) > 0 && !withUnits) { #region FileStream exist ContentDescriptor contentdescriptor = datasetVersion.ContentDescriptors.Where(p => p.Name.Equals(contentDescriptorTitle)).FirstOrDefault(); path = Path.Combine(AppConfiguration.DataPath, contentdescriptor.URI); if (FileHelper.FileExist(path)) { return(path); } #endregion FileStream exist } // not exist, needs to generated - get data first as datatable DataTable data = getData(id, versionId); long datastuctureId = datasetVersion.Dataset.DataStructure.Id; path = createDownloadFile(id, versionNr, datastuctureId, "data", ext, writer, null, withUnits); storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext, withUnits); //add units if want string[] units = null; if (withUnits) { units = getUnits(datastuctureId, null); } writer.AddData(data, path, datastuctureId, units); return(path); } finally { datasetManager.Dispose(); datasetStructureManager.Dispose(); } }
//ToDO -> David <- do not store the files on the server public string Convert(long datasetVersionId) { DatasetManager datasetManager = new DatasetManager(); PublicationManager publicationManager = new PublicationManager(); try { _broker = publicationManager.GetBroker(_broker.Id); _dataRepo = publicationManager.GetRepository(_dataRepo.Id); /*** * Generate a txt file for pangaea * 1. json metadata * 2. tabseperated primary Data * * * if data only unstructred, then only metadata */ string primaryDataFilePath = ""; #region create primary Data primaryDataFilePath = generatePrimaryData(datasetVersionId); #endregion DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId); long datasetId = datasetVersion.Dataset.Id; long metadataStructureId = datasetVersion.Dataset.MetadataStructure.Id; DataStructureDataList datastructureDataList = OutputDataStructureManager.GetVariableList(datasetId); PanageaMetadata metadata = getMetadata(datasetVersion.Metadata, metadataStructureId); metadata.ParameterIDs = datastructureDataList.Variables; string json = JsonConvert.SerializeObject(metadata, Formatting.Indented); SubmissionManager submissionManager = new SubmissionManager(); string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name); string filename = submissionManager.GetFileNameForDataRepo(datasetVersionId, datasetId, _dataRepo.Name, "txt"); string filepath = Path.Combine(path, filename); try { if (File.Exists(filepath)) { File.Delete(filepath); } FileHelper.Create(filepath).Close(); File.WriteAllText(filepath, json + Environment.NewLine + Environment.NewLine); if (!string.IsNullOrEmpty(primaryDataFilePath)) { File.AppendAllText(filepath, File.ReadAllText(primaryDataFilePath)); } return(filepath); } catch (Exception exception) { throw exception; } return(""); } finally { datasetManager.Dispose(); publicationManager.Dispose(); } }
public ActionResult ShowPrimaryData(long datasetID, int versionId) { Session["Filter"] = null; Session["Columns"] = null; Session["DownloadFullDataset"] = false; ViewData["DownloadOptions"] = null; IOUtility iOUtility = new IOUtility(); DatasetManager dm = new DatasetManager(); DataStructureManager dsm = new DataStructureManager(); //permission download EntityPermissionManager entityPermissionManager = new EntityPermissionManager(); try { if (dm.IsDatasetCheckedIn(datasetID)) { // get latest or other datasetversion DatasetVersion dsv = dm.GetDatasetVersion(versionId); bool latestVersion = versionId == dm.GetDatasetLatestVersionId(datasetID); DataStructure ds = dsm.AllTypesDataStructureRepo.Get(dsv.Dataset.DataStructure.Id); // TODO: refactor Download Right not existing, so i set it to read bool downloadAccess = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), datasetID, RightType.Read); bool editRights = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), datasetID, RightType.Write); //TITLE string title = dsv.Title; if (ds.Self.GetType() == typeof(UnStructuredDataStructure)) { if (this.IsAccessible("MMM", "ShowMultimediaData", "multimediaData") && ConfigurationManager.AppSettings["useMultimediaModule"].ToLower().Equals("true")) { return(RedirectToAction("multimediaData", "ShowMultimediaData", new RouteValueDictionary { { "area", "MMM" }, { "datasetID", datasetID }, { "versionId", versionId }, { "entityType", "Publication" } })); } else { return (PartialView(ShowPrimaryDataModel.Convert(datasetID, versionId, title, ds, SearchUIHelper.GetContantDescriptorFromKey(dsv, "unstructuredData"), downloadAccess, iOUtility.GetSupportedAsciiFiles(), latestVersion, editRights))); } } } else { ModelState.AddModelError(string.Empty, "Dataset is just in processing."); } return(PartialView(null)); } catch (Exception ex) { throw ex; } finally { dm.Dispose(); dsm.Dispose(); entityPermissionManager.Dispose(); } }
public string GenerateAsciiFile(long id, long versionId, string title, string mimeType) { DatasetManager datasetManager = new DatasetManager(); try { DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(versionId); string contentDescriptorTitle = ""; string ext = ""; TextSeperator textSeperator = TextSeperator.semicolon; switch (mimeType) { case "text/csv": { contentDescriptorTitle = "generatedCSV"; ext = ".csv"; textSeperator = TextSeperator.semicolon; break; } default: { contentDescriptorTitle = "generatedTXT"; ext = ".txt"; textSeperator = TextSeperator.tab; break; } } AsciiWriter writer = new AsciiWriter(textSeperator); string path = ""; //ascii allready exist if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(contentDescriptorTitle) && p.URI.Contains(datasetVersion.Id.ToString())) > 0) { #region FileStream exist ContentDescriptor contentdescriptor = datasetVersion.ContentDescriptors.Where(p => p.Name.Equals(contentDescriptorTitle)).FirstOrDefault(); path = Path.Combine(AppConfiguration.DataPath, contentdescriptor.URI); if (FileHelper.FileExist(path)) { return(path); } else { List <long> datatupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); long datastuctureId = datasetVersion.Dataset.DataStructure.Id; path = generateDownloadFile(id, datasetVersion.Id, datastuctureId, "Data", ext, writer); storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext); writer.AddDataTuples(datasetManager, datatupleIds, path, datastuctureId); return(path); } #endregion } // not exist needs to generated else { #region FileStream not exist List <long> datatupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); long datastuctureId = datasetVersion.Dataset.DataStructure.Id; path = generateDownloadFile(id, datasetVersion.Id, datastuctureId, "data", ext, writer); storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext); writer.AddDataTuples(datasetManager, datatupleIds, path, datastuctureId); return(path); #endregion } } finally { datasetManager.Dispose(); } }