public string Convert(long datasetVersionId)
{
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
DatasetManager datasetManager = new DatasetManager();
SubmissionManager submissionManager = new SubmissionManager();
PublicationManager publicationManager = new PublicationManager();
try
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
_broker = publicationManager.GetBroker(_broker.Id);
_dataRepo = publicationManager.GetRepository(_dataRepo.Id);
long datasetId = datasetVersion.Dataset.Id;
string name = datasetManager.GetDatasetVersion(datasetVersionId).Dataset.MetadataStructure.Name;
XmlDocument metadata = OutputMetadataManager.GetConvertedMetadata(datasetId,
TransmissionType.mappingFileExport, name, false);
// create links to the api calls of the primary data?
//add all to a zip
//save document and return filepath for download
string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name);
string filename = submissionManager.GetFileNameForDataRepo(datasetId, datasetVersionId, _dataRepo.Name, "xml");
string filepath = Path.Combine(path, filename);
FileHelper.CreateDicrectoriesIfNotExist(path);
metadata.Save(filepath);
return(filepath);
}
finally
{
datasetManager.Dispose();
metadataStructureManager.Dispose();
publicationManager.Dispose();
}
}
private string getStorePath(long datasetVersionId, string exportTo)
{
using (DatasetManager datasetManager = new DatasetManager())
using (MetadataStructureManager metadataStructureManager = new MetadataStructureManager())
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
Dataset dataset = datasetManager.GetDataset(datasetVersionId);
string md_title = metadataStructureManager.Repo.Get(datasetVersion.Dataset.MetadataStructure.Id).Name;
string path;
int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion);
if (string.IsNullOrEmpty(exportTo) || exportTo.ToLower().Equals("generic"))
{
path = IOHelper.GetDynamicStorePath(datasetVersion.Dataset.Id, versionNr, "metadata", ".xml");
}
else
{
path = IOHelper.GetDynamicStorePath(datasetVersion.Dataset.Id, versionNr, "metadata_" + exportTo, ".xml");
}
return(path);
}
}
private string generatePrimaryData(long datasetVersionId)
{
try
{
DatasetManager datasetManager = new DatasetManager();
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
OutputDataManager outputDataManager = new OutputDataManager();
SubmissionManager submissionManager = new SubmissionManager();
long datasetId = datasetVersion.Dataset.Id;
string fileName = submissionManager.GetFileNameForDataRepo(datasetId, datasetVersionId,
_dataRepo.Name,
"csv");
string filepath = outputDataManager.GenerateAsciiFile(
datasetId,
datasetVersionId,
fileName,
"text/txt");
return(filepath);
}
}
catch (Exception)
{
throw;
}
return("");
}
public ActionResult Save(MetadataStructureModel metadataStructureModel)
{
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
DatasetManager datasetManager = new DatasetManager();
try
{
if (ModelState.IsValid)
{
MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(metadataStructureModel.Id);
metadataStructure = updateMetadataStructure(metadataStructure, metadataStructureModel);
metadataStructureManager.Update(metadataStructure);
//update dsv title and description if there is a change
//ToDo check if there is a change in the xpaths
// update datasetversion
// get all datasetIds which using the metadata structure
var datasetIds = datasetManager.DatasetRepo.Query().Where(d => d.MetadataStructure.Id.Equals(metadataStructure.Id)).Select(d => d.Id);
if (datasetIds.Any())
{
//get all datasetversions of the dataset ids
var datasetVersionIds = datasetManager.DatasetVersionRepo.Query().Where(dsv => datasetIds.Contains(dsv.Dataset.Id)).Select(dsv => dsv.Id).ToList();
//load all titles & descriptions from versions
var allTitles = xmlDatasetHelper.GetInformationFromVersions(datasetVersionIds, metadataStructure.Id, NameAttributeValues.title);
var allDescriptions = xmlDatasetHelper.GetInformationFromVersions(datasetVersionIds, metadataStructure.Id, NameAttributeValues.description);
// update each datasetversion
foreach (var datasetVersionId in datasetVersionIds)
{
// load dataset version
var datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
datasetVersion.Title = allTitles.ContainsKey(datasetVersion.Id) ? allTitles[datasetVersion.Id] : string.Empty;
datasetVersion.Description = allDescriptions.ContainsKey(datasetVersion.Id) ? allDescriptions[datasetVersion.Id] : string.Empty;
datasetManager.UpdateDatasetVersion(datasetVersion);
}
}
return(Json(true));
}
return(Json(false));
}
catch (Exception ex)
{
return(Json(ex.Message));
}
finally
{
metadataStructureManager.Dispose();
datasetManager.Dispose();
}
}
private static void storeGeneratedFilePathToContentDiscriptor(long datasetId, DatasetVersion datasetVersion, string title, string ext)
{
string name = "";
string mimeType = "";
if (ext.Contains("xml"))
{
name = "metadata";
mimeType = "text/xml";
}
DatasetManager dm = new DatasetManager();
int versionNr = dm.GetDatasetVersionNr(datasetVersion);
// create the generated FileStream and determine its location
string dynamicPath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId, versionNr, title, ext);
//Register the generated data FileStream as a resource of the current dataset version
//ContentDescriptor generatedDescriptor = new ContentDescriptor()
//{
// OrderNo = 1,
// Name = name,
// MimeType = mimeType,
// URI = dynamicPath,
// DatasetVersion = datasetVersion,
//};
try
{
datasetVersion = dm.GetDatasetVersion(datasetVersion.Id);
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(name)) > 0)
{ // remove the one contentdesciptor
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
if (cd.Name == name)
{
cd.URI = dynamicPath;
dm.UpdateContentDescriptor(cd);
}
}
}
else
{
// add current contentdesciptor to list
//datasetVersion.ContentDescriptors.Add(generatedDescriptor);
dm.CreateContentDescriptor(name, mimeType, dynamicPath, 1, datasetVersion);
}
//dm.EditDatasetVersion(datasetVersion, null, null, null);
}
finally
{
dm.Dispose();
}
}
private DataTable getHistoryData(long versionId)
{
DatasetManager dm = new DatasetManager();
try
{
DatasetVersion dsv = dm.GetDatasetVersion(versionId);
DataTable table = null;
long rowCount = dm.GetDatasetVersionEffectiveTuples(dsv).Count;
table = dm.GetDatasetVersionTuples(versionId, 0, (int)rowCount);
return(table);
}
finally
{
dm.Dispose();
}
}
public List <FileInformation> getFilesByDataset(Dataset dataset, DatasetManager datasetManager, string entityType, long versionId = 0)
{
EntityPermissionManager entityPermissionManager = null;
try
{
List <FileInformation> fileInfos = new List <FileInformation>();
entityPermissionManager = new EntityPermissionManager();
bool access = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), dataset.Id, RightType.Read);
if (dataset != null && access)
{
DatasetVersion datasetVersion = new DatasetVersion();
if (versionId > 0)
{
datasetVersion = datasetManager.GetDatasetVersion(versionId);
}
else
{
datasetVersion = datasetManager.GetDatasetLatestVersion(dataset);
}
if (datasetVersion != null)
{
List <ContentDescriptor> contentDescriptors = datasetVersion.ContentDescriptors.ToList();
if (contentDescriptors.Count > 0)
{
foreach (ContentDescriptor cd in contentDescriptors)
{
if (cd.Name.ToLower().Equals("unstructureddata"))
{
fileInfos.Add(getFileInfo(cd));
}
}
}
}
}
return(fileInfos);
}
finally
{
entityPermissionManager.Dispose();
}
}
private DatasetFilesModel LoadDatasetModel(long versionId)
{
using (EntityPermissionManager entityPermissionManager = new EntityPermissionManager())
using (EntityManager entityManager = new EntityManager())
using (UserManager userManager = new UserManager())
using (DatasetManager dm = new DatasetManager())
{
var datasetVersion = dm.GetDatasetVersion(versionId);
var model = new DatasetFilesModel
{
ServerFileList = GetDatasetFileList(datasetVersion),
FileSize = this.Session.GetTenant().MaximumUploadSize
};
//Parse user right
var entity = entityManager.EntityRepository.Query(e => e.Name.ToUpperInvariant() == "Dataset".ToUpperInvariant() && e.EntityType == typeof(Dataset)).FirstOrDefault();
var userTask = userManager.FindByNameAsync(HttpContext.User.Identity.Name);
userTask.Wait();
var user = userTask.Result;
int rights = 0;
if (user == null)
{
rights = entityPermissionManager.GetEffectiveRights(subjectId: null, entity.Id, datasetVersion.Dataset.Id);
}
else
{
rights = entityPermissionManager.GetEffectiveRights(user.Id, entity.Id, datasetVersion.Dataset.Id);
}
model.UploadAccess = (((rights & (int)RightType.Write) > 0) || ((rights & (int)RightType.Grant) > 0));
model.DeleteAccess = (((rights & (int)RightType.Delete) > 0) || ((rights & (int)RightType.Grant) > 0));
model.DownloadAccess = ((rights & (int)RightType.Read) > 0 || ((rights & (int)RightType.Grant) > 0));
model.ViewAccess = ((rights & (int)RightType.Read) > 0 || ((rights & (int)RightType.Grant) > 0));
return(model);
}
}
// GET: DQ
public ActionResult ShowDQ(long datasetId, long versionId)
{
DQModels dqModel = new DQModels();
Dictionary <string, string> datasetInfo = new Dictionary <string, string>();
List <performer> performers = new List <performer>();
List <varVariable> varVariables = new List <varVariable>();
Dictionary <string, double> datasetSize = new Dictionary <string, double>();
DatasetManager dm = new DatasetManager();
DataStructureManager dsm = new DataStructureManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
PartyManager pm = new PartyManager();
UserManager um = new UserManager();
DatasetVersion dsv = new DatasetVersion();
UserManager userManager = new UserManager();
// data quality files
try
{
string pathPerformerDataset = @"C:\Data\DatasetQualities\PerformerDataset.csv";
StreamReader readerPerformerDataset = new StreamReader(pathPerformerDataset);
}
catch (Exception ex)
{ }
//////////////////////////////////////////////////////////////////////////
DatasetVersion currentDatasetVersion = dm.GetDatasetVersion(versionId); //Current dataset version
DataStructure currentDataStr = dsm.AllTypesDataStructureRepo.Get(currentDatasetVersion.Dataset.DataStructure.Id); //current data structure
var currentUser = userManager.FindByNameAsync(GetUsernameOrDefault()).Result; //Find current user
//Find the dataset Type
string currentDatasetType = "file";
if (currentDataStr.Self.GetType() == typeof(StructuredDataStructure))
{
currentDatasetType = "tabular";
}
dqModel.type = currentDatasetType;
#region performers
#region dataset's performers
try
{
string pathPerformerDataset = @"C:\Data\DatasetQualities\PerformerDataset.csv";
StreamReader readerPerformerDataset = new StreamReader(pathPerformerDataset);
string pathPerformers = @"C:\Data\DatasetQualities\Performers.csv";
StreamReader readerPerformers = new StreamReader(pathPerformers);
string performerLine;
List <string> pfs = new List <string>();
List <performer> ps = new List <performer>();
while ((performerLine = readerPerformerDataset.ReadLine()) != null)
{
string[] s = performerLine.Split(',');
if (long.Parse(s[1]) == datasetId)
{
pfs.Add(s[0]);
}
}
while ((performerLine = readerPerformers.ReadLine()) != null)
{
string[] s = performerLine.Split(',');
if (pfs.Contains(s[0]))
{
performer p = new performer();
p.performerName = FindPerformerNameFromUsername(um, s[0]); //find performer name
p.performerRate = int.Parse(s[1]);
List <long> pfIds = FindDatasetsFromPerformerUsername(dm, um, s[0]); //Find all datasets in wich the username is involved.
p.DatasetIds = pfIds;
ps.Add(p);
}
}
dqModel.performers = ps;
readerPerformerDataset.Close();
readerPerformers.Close();
}
catch (Exception ex)
{
}
#endregion
#endregion //performers
//dqModel.isPublic = entityPermissionManager.GetRights(null, 1, datasetId); //check if dataset is public
//check the read permission for current dataset
bool rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), datasetId, Security.Entities.Authorization.RightType.Read); //find if user has read permission
if (rPermission == true) //has read permission or public = readable
{
dqModel.readable = 1;
}
else
{
dqModel.readable = 0;
} //cannot read
//Check if the current metadata is valid
if (currentDatasetVersion.StateInfo != null)
{
dqModel.isValid = DatasetStateInfo.Valid.ToString().Equals(currentDatasetVersion.StateInfo.State) ? 1 : 0; //1:valid; 0:invalid.
}
else
{
dqModel.isValid = 0;
}
List <long> datasetIds = dm.GetDatasetLatestIds();
dqModel.allDatasets = datasetIds.Count;
List <int> metadataRates = new List <int>();
List <int> dsDescLength = new List <int>();
List <int> dstrDescLength = new List <int>();
List <int> dstrUsage = new List <int>();
List <int> datasetSizeTabular = new List <int>();
List <int> datasetRows = new List <int>();
List <int> datasetCols = new List <int>();
List <double> datasetSizeFiles = new List <double>();
double datasetSizeFile = new double();
List <int> datasetFileNumber = new List <int>();
List <int> restrictions = new List <int>();
int fileNumber = 0;
List <int> sizeTabular = new List <int>(); //collect size, column number, and row number for one dataset
int publicDatasets = 0;
int restrictedDatasets = 0;
int fileDatasets = 0;
int tabularDatasets = 0;
int rpTrue = 0;
int rp;
int validMetadata = 0;
int allValidMetadas = 0;
foreach (long Id in datasetIds) //for each dataset
{
if (dm.IsDatasetCheckedIn(Id))
{
DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(Id); //get last dataset versions
//If user has read permission
rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), Id, Security.Entities.Authorization.RightType.Read);
if (rPermission == true) //has read permission or public = readable
{
rp = 1;
rpTrue += 1;
}
else
{
rp = 0;
} //cannot read
}
}
dqModel.allReadables = rpTrue;
string pathDatasetInfo = @"C:\Data\DatasetQualities\datasetInfo.csv";
StreamReader readerDatasetInfo = new StreamReader(pathDatasetInfo);
List <datasetInformation> datasetsInformation = new List <datasetInformation>();
try
{
string lineDatasetInfo;
while ((lineDatasetInfo = readerDatasetInfo.ReadLine()) != null)
{
string[] dsInf = lineDatasetInfo.Split(';');
datasetInformation datasetInformation = new datasetInformation();
long id = long.Parse(dsInf[0]);
datasetInformation.datasetId = id;
datasetInformation.title = dm.GetDatasetLatestVersion(id).Title;
rPermission = entityPermissionManager.HasEffectiveRight(currentUser.UserName, typeof(Dataset), id, Security.Entities.Authorization.RightType.Read);
if (rPermission == true)
{
datasetInformation.readable = 1;
}
if (rPermission == false)
{
datasetInformation.readable = 0;
}
datasetInformation.type = dsInf[1];
datasetInformation.metadataValidation = int.Parse(dsInf[2]);
datasetInformation.metadataComplition = int.Parse(dsInf[3]);
datasetInformation.descriptionLength = int.Parse(dsInf[4]);
datasetInformation.structureDescriptionLength = int.Parse(dsInf[5]);
datasetInformation.structureUsage = int.Parse(dsInf[6]);
datasetInformation.columnNumber = int.Parse(dsInf[7]);
datasetInformation.rowNumber = int.Parse(dsInf[8]);
datasetInformation.fileNumber = int.Parse(dsInf[9]);
datasetInformation.datasetSizeFile = double.Parse(dsInf[10]);
string[] pfrms = dsInf[11].Split(',');
List <string> performerNames = new List <string>();
foreach (string p in pfrms)
{
performerNames.Add(p);
}
datasetInformation.performerNames = performerNames;
datasetsInformation.Add(datasetInformation);
if (datasetId == id)
{
dqModel.metadataComplition.requiredFields = int.Parse(dsInf[2]);
dqModel.metadataComplition.totalFields = int.Parse(dsInf[3]);
dqModel.datasetDescriptionLength.currentDescriptionLength = int.Parse(dsInf[4]);
dqModel.dataStrDescriptionLength.currentDescriptionLength = int.Parse(dsInf[5]);
dqModel.dataStrUsage.currentDataStrUsage = int.Parse(dsInf[6]);
dqModel.columnNumber = datasetInformation.columnNumber;
dqModel.rowNumber = datasetInformation.rowNumber;
dqModel.fileNumber = datasetInformation.fileNumber;
dqModel.datasetTotalSize.currentTotalSize = datasetInformation.datasetSizeFile;
}
}
}
catch
{
}
dqModel.datasetsInformation = datasetsInformation;
readerDatasetInfo.Close();
//CURRENT DATASET VERSION
//dqModel.metadataComplition.totalFields = GetMetadataRate(currentDatasetVersion); //current dataset version: metadata rate
//dqModel.metadataComplition.requiredFields = 100; //Need to calculate: metadataStructureId = dsv.Dataset.MetadataStructure.Id;
//dqModel.datasetDescriptionLength.currentDescriptionLength = currentDatasetVersion.Description.Length; // Current dataset vesion: dataset description length
//dqModel.dataStrDescriptionLength.currentDescriptionLength = currentDatasetVersion.Dataset.DataStructure.Description.Length; // Current dataset version: data structure description length
//dqModel.dataStrUsage.currentDataStrUsage = currentDataStr.Datasets.Count() - 1; // Current dataset version: how many times the data structure is used in other datasets
#region comparision
try
{
string pathComparison = @"C:\Data\DatasetQualities\Comparison.csv";
StreamReader readerComparison = new StreamReader(pathComparison);
string infoline;
List <int> infos = new List <int>();
while ((infoline = readerComparison.ReadLine()) != null)
{
string[] s = infoline.Split(',');
if (s[0] == "performersActivity")
{
dqModel.performersActivity.minActivity = int.Parse(s[1]);
dqModel.performersActivity.medianActivity = int.Parse(s[2]);
dqModel.performersActivity.maxActivity = int.Parse(s[3]);
}
else if (s[0] == "type")
{
dqModel.allDatasets = int.Parse(s[1]);
dqModel.tabularDatasets = int.Parse(s[2]);
dqModel.fileDatasets = int.Parse(s[3]);
}
else if (s[0] == "metadataRates")
{
dqModel.metadataComplition.minRate = int.Parse(s[1]);
dqModel.metadataComplition.medianRate = int.Parse(s[2]);
dqModel.metadataComplition.maxRate = int.Parse(s[3]);
}
else if (s[0] == "allValidMetadas")
{
dqModel.allValids = int.Parse(s[1]);
}
else if (s[0] == "datasetDescriptionLength")
{
dqModel.datasetDescriptionLength.minDescriptionLength = int.Parse(s[1]);
dqModel.datasetDescriptionLength.medianDescriptionLength = int.Parse(s[2]);
dqModel.datasetDescriptionLength.maxDescriptionLength = int.Parse(s[3]);
}
else if (s[0] == "dataStrDescriptionLength")
{
dqModel.dataStrDescriptionLength.minDescriptionLength = int.Parse(s[1]);
dqModel.dataStrDescriptionLength.medianDescriptionLength = int.Parse(s[2]);
dqModel.dataStrDescriptionLength.maxDescriptionLength = int.Parse(s[3]);
}
else if (s[0] == "dataStrUsage")
{
dqModel.dataStrUsage.minDataStrUsage = int.Parse(s[1]);
dqModel.dataStrUsage.medianDataStrUsage = int.Parse(s[2]);
dqModel.dataStrUsage.maxDataStrUsage = int.Parse(s[3]);
}
else if (s[0] == "datasetColNumber")
{
dqModel.datasetColNumber.minColNumber = int.Parse(s[1]);
dqModel.datasetColNumber.medianColNumber = int.Parse(s[2]);
dqModel.datasetColNumber.maxColNumber = int.Parse(s[3]);
}
else if (s[0] == "datasetRowNumber")
{
dqModel.datasetRowNumber.minRowNumber = int.Parse(s[1]);
dqModel.datasetRowNumber.medianRowNumber = int.Parse(s[2]);
dqModel.datasetRowNumber.maxRowNumber = int.Parse(s[3]);
}
else if (s[0] == "datasetFileNumber")
{
dqModel.datasetFileNumber.minFileNumber = int.Parse(s[1]);
dqModel.datasetFileNumber.medianFileNumber = int.Parse(s[2]);
dqModel.datasetFileNumber.maxFileNumber = int.Parse(s[3]);
}
else if (s[0] == "datasetTotalSizeFiles")
{
dqModel.datasetTotalSize.minSizeFile = double.Parse(s[1]);
dqModel.datasetTotalSize.medianSizeFile = double.Parse(s[2]);
dqModel.datasetTotalSize.maxSizeFile = double.Parse(s[3]);
}
}
readerComparison.Close();
}
catch (Exception ex)
{
}
#endregion
///////////////////////////////////////////////////////////////////////
#region TABULAR FORMAT DATASET
//If it is a tabular format dataset
if (currentDatasetType == "tabular")
{
string pathVariables = @"C:\Data\DatasetQualities\Variables.csv";
StreamReader readerVariables = new StreamReader(pathVariables);
string varLine;
while ((varLine = readerVariables.ReadLine()) != null)
{
string[] varDetail = varLine.Split(',');
if (varDetail[0] == datasetId.ToString())
{
varVariable v = new varVariable();
v.varLabel = varDetail[1];
v.varType = varDetail[2];
v.varDescription = varDetail[3];
v.varUsage = int.Parse(varDetail[4]);
v.missing = int.Parse(varDetail[5]);
varVariables.Add(v);
}
}
dqModel.varVariables = varVariables;
readerVariables.Close();
}
// string serverName = "http://*****:*****@"C:\Data\DatasetQualities\Files.csv";
try
{
StreamReader readerFiles = new StreamReader(pathFiles);
while ((fileLine = readerFiles.ReadLine()) != null)
{
string[] fileDetail = fileLine.Split(',');
if (fileDetail[0] == datasetId.ToString())
{
fileInformation f = new fileInformation();
f.fileName = fileDetail[1];
f.fileFormat = fileDetail[2];
double d = Convert.ToDouble(fileDetail[3]);
f.fileSize = d;
filesInformations.Add(f);
}
}
readerFiles.Close();
}
catch
{
}
dqModel.filesInformation = filesInformations;
}
//if (currentDatasetVersion != null)
//{
// List<ContentDescriptor> contentDescriptors = currentDatasetVersion.ContentDescriptors.ToList();
// double totalSize = 0;
// if (contentDescriptors.Count > 0)
// {
// foreach (ContentDescriptor cd in contentDescriptors)
// {
// if (cd.Name.ToLower().Equals("unstructureddata"))
// {
// fileInformation fileInformation = new fileInformation();
// string uri = cd.URI;
// //get the file path
// try
// {
// String path = Server.UrlDecode(uri);
// path = Path.Combine(AppConfiguration.DataPath, path);
// Stream fileStream = System.IO.File.OpenRead(path);
// if (fileStream != null)
// {
// FileStream fs = fileStream as FileStream;
// if (fs != null)
// {
// //get file information
// FileInformation fileInfo = new FileInformation(fs.Name.Split('\\').LastOrDefault(), MimeMapping.GetMimeMapping(fs.Name), (uint)fs.Length, uri);
// fileInformation.fileName = fileInfo.Name.Split('.')[0]; //file name
// fileInformation.fileFormat = fileInfo.Name.Split('.')[1].ToLower(); //file extension
// fileInformation.fileSize = fileInfo.Size; //file size
// totalSize += fileInfo.Size;
// }
// }
// }
// catch
// {
// }
// filesInformation.Add(fileInformation);
// }
//}
//}
//dqModel.fileNumber = contentDescriptors.Count;
//dqModel.datasetTotalSize.currentTotalSize = totalSize;
//}
//dqModel.filesInformation = filesInformation;
//}
#endregion
return(PartialView(dqModel));
}
//ToDO -> David <- do not store the files on the server
public string Convert(long datasetVersionId)
{
string datarepo = _dataRepo.Name;
SubmissionManager publishingManager = new SubmissionManager();
using (DatasetManager datasetManager = new DatasetManager())
using (DataStructureManager dataStructureManager = new DataStructureManager())
using (PublicationManager publicationManager = new PublicationManager())
using (ZipFile zip = new ZipFile())
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
long datasetId = datasetVersion.Dataset.Id;
Publication publication =
publicationManager.GetPublication()
.Where(
p =>
p.DatasetVersion.Id.Equals(datasetVersion.Id) &&
p.Broker.Name.ToLower().Equals(_broker.Name))
.FirstOrDefault();
// if(broker exist)
if (publication == null && _broker != null)
{
Broker broker = _broker;
if (broker != null)
{
OutputMetadataManager.GetConvertedMetadata(datasetId, TransmissionType.mappingFileExport,
broker.MetadataFormat);
// get primary data
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
OutputDataManager odm = new OutputDataManager();
// apply selection and projection
odm.GenerateAsciiFile(datasetId, datasetVersion.Id, broker.PrimaryDataFormat, false);
}
int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion);
string zipName = publishingManager.GetZipFileName(datasetId, versionNr);
string zipPath = publishingManager.GetDirectoryPath(datasetId, broker.Name);
string dynamicZipPath = publishingManager.GetDynamicDirectoryPath(datasetId, broker.Name);
string zipFilePath = Path.Combine(zipPath, zipName);
string dynamicFilePath = Path.Combine(dynamicZipPath, zipName);
FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(zipFilePath));
if (FileHelper.FileExist(zipFilePath))
{
if (FileHelper.WaitForFile(zipFilePath))
{
FileHelper.Delete(zipFilePath);
}
}
// add datastructure
//ToDo put that functiom to the outputDatatructureManager
#region datatructure
long dataStructureId = datasetVersion.Dataset.DataStructure.Id;
DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (dataStructure != null)
{
try
{
string dynamicPathOfDS = "";
dynamicPathOfDS = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion,
"datastructure", ".txt");
string datastructureFilePath = AsciiWriter.CreateFile(dynamicPathOfDS);
string json = OutputDataStructureManager.GetVariableListAsJson(dataStructureId);
AsciiWriter.AllTextToFile(datastructureFilePath, json);
}
catch (Exception ex)
{
throw ex;
}
}
#endregion datatructure
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
string path = Path.Combine(AppConfiguration.DataPath, cd.URI);
string name = cd.URI.Split('\\').Last();
if (FileHelper.FileExist(path))
{
zip.AddFile(path, "");
}
}
// add xsd of the metadata schema
string xsdDirectoryPath = OutputMetadataManager.GetSchemaDirectoryPath(datasetId);
if (Directory.Exists(xsdDirectoryPath))
{
zip.AddDirectory(xsdDirectoryPath, "Schema");
}
XmlDocument manifest = OutputDatasetManager.GenerateManifest(datasetId, datasetVersion.Id);
if (manifest != null)
{
string dynamicManifestFilePath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId,
versionNr, "manifest", ".xml");
string fullFilePath = Path.Combine(AppConfiguration.DataPath, dynamicManifestFilePath);
manifest.Save(fullFilePath);
zip.AddFile(fullFilePath, "");
}
string message = string.Format("dataset {0} version {1} was published for repository {2}", datasetId,
datasetVersion.Id, broker.Name);
LoggerFactory.LogCustom(message);
//Session["ZipFilePath"] = dynamicFilePath;
zip.Save(zipFilePath);
return(zipFilePath);
}
}
return("");
}
}
public string GenerateAsciiFile(long id, long versionId, string mimeType, bool withUnits)
{
DatasetManager datasetManager = new DatasetManager();
DataStructureManager datasetStructureManager = new DataStructureManager();
try
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(versionId);
int versionNr = datasetManager.GetDatasetVersionNr(datasetVersion);
string contentDescriptorTitle = "";
string ext = "";
string nameExt = "";
TextSeperator textSeperator = TextSeperator.semicolon;
if (withUnits)
{
nameExt = "_withunits";
}
switch (mimeType)
{
case "text/csv":
case "text/comma-separated-values":
case "application/octet-stream":
/* of course this is a wrong mimetype for csv.
* but the c# class MimeMapping.GetMimeMapping(ext) currently returns this as a result for .csv.
* since we don't use the datatype at the moment,
* it will be rebuilt into the case here*/
{
contentDescriptorTitle = "generatedCSV" + nameExt;
ext = ".csv";
textSeperator = TextSeperator.semicolon;
break;
}
case "text/tsv":
case "text/tab-separated-values":
{
contentDescriptorTitle = "generatedTSV" + nameExt;
ext = ".tsv";
textSeperator = TextSeperator.tab;
break;
}
default:
{
contentDescriptorTitle = "generatedTXT" + nameExt;
ext = ".txt";
textSeperator = TextSeperator.tab;
break;
}
}
AsciiWriter writer = new AsciiWriter(textSeperator);
string path = "";
//ascii allready exist
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(contentDescriptorTitle) &&
p.URI.Contains(datasetVersion.Id.ToString())) > 0 &&
!withUnits)
{
#region FileStream exist
ContentDescriptor contentdescriptor = datasetVersion.ContentDescriptors.Where(p => p.Name.Equals(contentDescriptorTitle)).FirstOrDefault();
path = Path.Combine(AppConfiguration.DataPath, contentdescriptor.URI);
if (FileHelper.FileExist(path))
{
return(path);
}
#endregion FileStream exist
}
// not exist, needs to generated - get data first as datatable
DataTable data = getData(id, versionId);
long datastuctureId = datasetVersion.Dataset.DataStructure.Id;
path = createDownloadFile(id, versionNr, datastuctureId, "data", ext, writer, null, withUnits);
storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext, withUnits);
//add units if want
string[] units = null;
if (withUnits)
{
units = getUnits(datastuctureId, null);
}
writer.AddData(data, path, datastuctureId, units);
return(path);
}
finally
{
datasetManager.Dispose();
datasetStructureManager.Dispose();
}
}
//ToDO -> David <- do not store the files on the server
public string Convert(long datasetVersionId)
{
DatasetManager datasetManager = new DatasetManager();
PublicationManager publicationManager = new PublicationManager();
try
{
_broker = publicationManager.GetBroker(_broker.Id);
_dataRepo = publicationManager.GetRepository(_dataRepo.Id);
/***
* Generate a txt file for pangaea
* 1. json metadata
* 2. tabseperated primary Data
*
*
* if data only unstructred, then only metadata
*/
string primaryDataFilePath = "";
#region create primary Data
primaryDataFilePath = generatePrimaryData(datasetVersionId);
#endregion
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
long datasetId = datasetVersion.Dataset.Id;
long metadataStructureId = datasetVersion.Dataset.MetadataStructure.Id;
DataStructureDataList datastructureDataList = OutputDataStructureManager.GetVariableList(datasetId);
PanageaMetadata metadata = getMetadata(datasetVersion.Metadata, metadataStructureId);
metadata.ParameterIDs = datastructureDataList.Variables;
string json = JsonConvert.SerializeObject(metadata, Formatting.Indented);
SubmissionManager submissionManager = new SubmissionManager();
string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name);
string filename = submissionManager.GetFileNameForDataRepo(datasetVersionId, datasetId, _dataRepo.Name, "txt");
string filepath = Path.Combine(path, filename);
try
{
if (File.Exists(filepath))
{
File.Delete(filepath);
}
FileHelper.Create(filepath).Close();
File.WriteAllText(filepath, json + Environment.NewLine + Environment.NewLine);
if (!string.IsNullOrEmpty(primaryDataFilePath))
{
File.AppendAllText(filepath, File.ReadAllText(primaryDataFilePath));
}
return(filepath);
}
catch (Exception exception)
{
throw exception;
}
return("");
}
finally
{
datasetManager.Dispose();
publicationManager.Dispose();
}
}
public ActionResult ShowPrimaryData(long datasetID, int versionId)
{
Session["Filter"] = null;
Session["Columns"] = null;
Session["DownloadFullDataset"] = false;
ViewData["DownloadOptions"] = null;
IOUtility iOUtility = new IOUtility();
DatasetManager dm = new DatasetManager();
DataStructureManager dsm = new DataStructureManager();
//permission download
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
try
{
if (dm.IsDatasetCheckedIn(datasetID))
{
// get latest or other datasetversion
DatasetVersion dsv = dm.GetDatasetVersion(versionId);
bool latestVersion = versionId == dm.GetDatasetLatestVersionId(datasetID);
DataStructure ds = dsm.AllTypesDataStructureRepo.Get(dsv.Dataset.DataStructure.Id);
// TODO: refactor Download Right not existing, so i set it to read
bool downloadAccess = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), datasetID, RightType.Read);
bool editRights = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, typeof(Dataset), datasetID, RightType.Write);
//TITLE
string title = dsv.Title;
if (ds.Self.GetType() == typeof(UnStructuredDataStructure))
{
if (this.IsAccessible("MMM", "ShowMultimediaData", "multimediaData") && ConfigurationManager.AppSettings["useMultimediaModule"].ToLower().Equals("true"))
{
return(RedirectToAction("multimediaData", "ShowMultimediaData", new RouteValueDictionary {
{ "area", "MMM" }, { "datasetID", datasetID }, { "versionId", versionId }, { "entityType", "Publication" }
}));
}
else
{
return
(PartialView(ShowPrimaryDataModel.Convert(datasetID,
versionId,
title,
ds,
SearchUIHelper.GetContantDescriptorFromKey(dsv, "unstructuredData"),
downloadAccess,
iOUtility.GetSupportedAsciiFiles(),
latestVersion,
editRights)));
}
}
}
else
{
ModelState.AddModelError(string.Empty, "Dataset is just in processing.");
}
return(PartialView(null));
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
dsm.Dispose();
entityPermissionManager.Dispose();
}
}
public string GenerateAsciiFile(long id, long versionId, string title, string mimeType)
{
DatasetManager datasetManager = new DatasetManager();
try
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(versionId);
string contentDescriptorTitle = "";
string ext = "";
TextSeperator textSeperator = TextSeperator.semicolon;
switch (mimeType)
{
case "text/csv":
{
contentDescriptorTitle = "generatedCSV";
ext = ".csv";
textSeperator = TextSeperator.semicolon;
break;
}
default:
{
contentDescriptorTitle = "generatedTXT";
ext = ".txt";
textSeperator = TextSeperator.tab;
break;
}
}
AsciiWriter writer = new AsciiWriter(textSeperator);
string path = "";
//ascii allready exist
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(contentDescriptorTitle) && p.URI.Contains(datasetVersion.Id.ToString())) > 0)
{
#region FileStream exist
ContentDescriptor contentdescriptor = datasetVersion.ContentDescriptors.Where(p => p.Name.Equals(contentDescriptorTitle)).FirstOrDefault();
path = Path.Combine(AppConfiguration.DataPath, contentdescriptor.URI);
if (FileHelper.FileExist(path))
{
return(path);
}
else
{
List <long> datatupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
long datastuctureId = datasetVersion.Dataset.DataStructure.Id;
path = generateDownloadFile(id, datasetVersion.Id, datastuctureId, "Data", ext, writer);
storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext);
writer.AddDataTuples(datasetManager, datatupleIds, path, datastuctureId);
return(path);
}
#endregion
}
// not exist needs to generated
else
{
#region FileStream not exist
List <long> datatupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
long datastuctureId = datasetVersion.Dataset.DataStructure.Id;
path = generateDownloadFile(id, datasetVersion.Id, datastuctureId, "data", ext, writer);
storeGeneratedFilePathToContentDiscriptor(id, datasetVersion, ext);
writer.AddDataTuples(datasetManager, datatupleIds, path, datastuctureId);
return(path);
#endregion
}
}
finally
{
datasetManager.Dispose();
}
}