public void Load()
{
PublicationManager publicationManager = new PublicationManager();
try
{
string filepath = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), sourceFile);
if (FileHelper.FileExist(filepath))
{
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load(filepath);
requirementXmlDocument = xmlDoc;
XmlNodeList brokerNodes = requirementXmlDocument.GetElementsByTagName("broker");
foreach (XmlNode child in brokerNodes)
{
Brokers.Add(createBroker(child, publicationManager));
}
}
}
finally
{
publicationManager.Dispose();
}
}
public string Convert(long datasetVersionId)
{
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
DatasetManager datasetManager = new DatasetManager();
SubmissionManager submissionManager = new SubmissionManager();
PublicationManager publicationManager = new PublicationManager();
try
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
_broker = publicationManager.GetBroker(_broker.Id);
_dataRepo = publicationManager.GetRepository(_dataRepo.Id);
long datasetId = datasetVersion.Dataset.Id;
string name = datasetManager.GetDatasetVersion(datasetVersionId).Dataset.MetadataStructure.Name;
XmlDocument metadata = OutputMetadataManager.GetConvertedMetadata(datasetId,
TransmissionType.mappingFileExport, name, false);
// create links to the api calls of the primary data?
//add all to a zip
//save document and return filepath for download
string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name);
string filename = submissionManager.GetFileNameForDataRepo(datasetId, datasetVersionId, _dataRepo.Name, "xml");
string filepath = Path.Combine(path, filename);
FileHelper.CreateDicrectoriesIfNotExist(path);
metadata.Save(filepath);
return(filepath);
}
finally
{
datasetManager.Dispose();
metadataStructureManager.Dispose();
publicationManager.Dispose();
}
}
private void createMetadataStructureRepoMaps()
{
PublicationManager publicationManager = new PublicationManager();
try
{
//set MetadataStructureToRepository for gbif and pensoft
long metadataStrutcureId = 0;
long repositoryId = 0;
//get id of metadatstructure
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
string metadatStrutcureName = "gbif";
if (metadataStructureManager.Repo.Get().Any(m => m.Name.ToLower().Equals(metadatStrutcureName)))
{
MetadataStructure metadataStructure =
metadataStructureManager.Repo.Get()
.FirstOrDefault(m => m.Name.ToLower().Equals(metadatStrutcureName));
if (metadataStructure != null)
{
metadataStrutcureId = metadataStructure.Id;
}
}
//get id of metadatstructure
string repoName = "pensoft";
if (publicationManager.RepositoryRepo.Get().Any(m => m.Name.ToLower().Equals(repoName)))
{
Repository repository =
publicationManager.RepositoryRepo.Get().FirstOrDefault(m => m.Name.ToLower().Equals(repoName));
if (repository != null)
{
repositoryId = repository.Id;
}
}
if (metadataStrutcureId > 0 && repositoryId > 0)
{
publicationManager.CreateMetadataStructureToRepository(metadataStrutcureId, repositoryId);
}
}
finally
{
publicationManager.Dispose();
}
}
//ToDO -> David <- do not store the files on the server
public string Convert(long datasetVersionId)
{
DatasetManager datasetManager = new DatasetManager();
PublicationManager publicationManager = new PublicationManager();
try
{
_broker = publicationManager.GetBroker(_broker.Id);
_dataRepo = publicationManager.GetRepository(_dataRepo.Id);
/***
* Generate a txt file for pangaea
* 1. json metadata
* 2. tabseperated primary Data
*
*
* if data only unstructred, then only metadata
*/
string primaryDataFilePath = "";
#region create primary Data
primaryDataFilePath = generatePrimaryData(datasetVersionId);
#endregion
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
long datasetId = datasetVersion.Dataset.Id;
long metadataStructureId = datasetVersion.Dataset.MetadataStructure.Id;
DataStructureDataList datastructureDataList = OutputDataStructureManager.GetVariableList(datasetId);
PanageaMetadata metadata = getMetadata(datasetVersion.Metadata, metadataStructureId);
metadata.ParameterIDs = datastructureDataList.Variables;
string json = JsonConvert.SerializeObject(metadata, Formatting.Indented);
SubmissionManager submissionManager = new SubmissionManager();
string path = submissionManager.GetDirectoryPath(datasetId, _broker.Name);
string filename = submissionManager.GetFileNameForDataRepo(datasetVersionId, datasetId, _dataRepo.Name, "txt");
string filepath = Path.Combine(path, filename);
try
{
if (File.Exists(filepath))
{
File.Delete(filepath);
}
FileHelper.Create(filepath).Close();
File.WriteAllText(filepath, json + Environment.NewLine + Environment.NewLine);
if (!string.IsNullOrEmpty(primaryDataFilePath))
{
File.AppendAllText(filepath, File.ReadAllText(primaryDataFilePath));
}
return(filepath);
}
catch (Exception exception)
{
throw exception;
}
return("");
}
finally
{
datasetManager.Dispose();
publicationManager.Dispose();
}
}
public ActionResult GenerateZip(long id, long versionid, string format)
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
DatasetManager dm = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
PublicationManager publicationManager = new PublicationManager();
SubmissionManager publishingManager = new SubmissionManager();
string brokerName = "generic";
try
{
using (var uow = this.GetUnitOfWork())
{
LoggerFactory.LogCustom("Generate Zip Start");
long dsvId = versionid;
if (dsvId <= 0)
{
dsvId = dm.GetDatasetLatestVersion(id).Id;
}
DatasetVersion datasetVersion = uow.GetUnitOfWork().GetReadOnlyRepository <DatasetVersion>().Get(dsvId);
int versionNr = dm.GetDatasetVersionNr(datasetVersion);
#region metadata
LoggerFactory.LogCustom("Metadata Start");
//metadata as xml output
XmlDocument document = OutputMetadataManager.GetConvertedMetadata(id, TransmissionType.mappingFileExport,
datasetVersion.Dataset.MetadataStructure.Name);
//metadata as html
generateMetadataHtml(datasetVersion);
#endregion metadata
#region primary data
LoggerFactory.LogCustom("Primary Data Start");
// check the data sturcture type ...
if (format != null && datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
OutputDataManager odm = new OutputDataManager();
// apply selection and projection
//check wheter title is empty or not
string title = String.IsNullOrEmpty(datasetVersion.Title) ? "no title available" : datasetVersion.Title;
switch (format)
{
case "application/xlsx":
odm.GenerateExcelFile(id, datasetVersion.Id, false);
break;
case "application/xlsm":
odm.GenerateExcelFile(id, datasetVersion.Id, true);
break;
default:
odm.GenerateAsciiFile(id, datasetVersion.Id, format, false);
break;
}
}
#endregion primary data
LoggerFactory.LogCustom("check zip on server Start");
string zipName = publishingManager.GetZipFileName(id, versionNr);
string zipPath = publishingManager.GetDirectoryPath(id, brokerName);
string dynamicZipPath = publishingManager.GetDynamicDirectoryPath(id, brokerName);
string zipFilePath = Path.Combine(zipPath, zipName);
string dynamicFilePath = Path.Combine(dynamicZipPath, zipName);
FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(zipFilePath));
if (FileHelper.FileExist(zipFilePath))
{
if (FileHelper.WaitForFile(zipFilePath))
{
FileHelper.Delete(zipFilePath);
}
}
// add datastructure
//ToDo put that functiom to the outputDatatructureManager
#region datatructure
LoggerFactory.LogCustom("Datastructure Start");
long dataStructureId = datasetVersion.Dataset.DataStructure.Id;
DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (dataStructure != null)
{
try
{
string dynamicPathOfDS = "";
dynamicPathOfDS = storeGeneratedFilePathToContentDiscriptor(id, datasetVersion,
"datastructure", ".txt");
string datastructureFilePath = AsciiWriter.CreateFile(dynamicPathOfDS);
string json = OutputDataStructureManager.GetVariableListAsJson(dataStructureId);
AsciiWriter.AllTextToFile(datastructureFilePath, json);
//generate datastructure as html
DatasetVersion ds = uow.GetUnitOfWork().GetReadOnlyRepository <DatasetVersion>().Get(dsvId);
generateDataStructureHtml(ds);
}
catch (Exception ex)
{
throw ex;
}
}
#endregion datatructure
LoggerFactory.LogCustom("Zip Start");
using (ZipFile zip = new ZipFile())
{
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
bool addFile = true;
if (cd.Name.ToLower().Contains("generated"))
{
if (!cd.MimeType.ToLower().Equals(format))
{
addFile = false;
}
}
if (addFile)
{
string path = Path.Combine(AppConfiguration.DataPath, cd.URI);
string name = cd.URI.Split('\\').Last();
if (FileHelper.FileExist(path))
{
zip.AddFile(path, "");
}
}
}
// add xsd of the metadata schema
LoggerFactory.LogCustom("Schema Start");
string xsdDirectoryPath = OutputMetadataManager.GetSchemaDirectoryPath(id);
if (Directory.Exists(xsdDirectoryPath))
{
zip.AddDirectory(xsdDirectoryPath, "Schema");
}
LoggerFactory.LogCustom("Manifest Start");
XmlDocument manifest = OutputDatasetManager.GenerateManifest(id, datasetVersion.Id);
if (manifest != null)
{
string dynamicManifestFilePath = OutputDatasetManager.GetDynamicDatasetStorePath(id,
versionNr, "manifest", ".xml");
string fullFilePath = Path.Combine(AppConfiguration.DataPath, dynamicManifestFilePath);
manifest.Save(fullFilePath);
zip.AddFile(fullFilePath, "");
}
LoggerFactory.LogCustom("Save zip Start");
zip.Save(zipFilePath);
LoggerFactory.LogCustom("Return");
string title = datasetVersion.Title;
title = String.IsNullOrEmpty(title) ? "unknown" : title;
string message = string.Format("dataset {0} version {1} was downloaded as zip - {2}.", id,
versionNr, format);
LoggerFactory.LogCustom(message);
var es = new EmailService();
es.Send(MessageHelper.GetDownloadDatasetHeader(id, versionNr),
MessageHelper.GetDownloadDatasetMessage(id, title, getPartyNameOrDefault(), "zip - " + format, versionNr),
ConfigurationManager.AppSettings["SystemEmail"]
);
return(File(zipFilePath, "application/zip", Path.GetFileName(zipFilePath)));
}
}
}
catch (Exception ex)
{
LoggerFactory.LogCustom("Error: " + ex.Message);
return(null);
}
finally
{
dm.Dispose();
dataStructureManager.Dispose();
publicationManager.Dispose();
}
}
//ToDO -> David <- do not store the files on the server
public string Convert(long datasetVersionId)
{
string datarepo = _dataRepo.Name;
DatasetManager datasetManager = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
PublicationManager publicationManager = new PublicationManager();
SubmissionManager publishingManager = new SubmissionManager();
try
{
DatasetVersion datasetVersion = datasetManager.GetDatasetVersion(datasetVersionId);
long datasetId = datasetVersion.Dataset.Id;
Publication publication =
publicationManager.GetPublication()
.Where(
p =>
p.DatasetVersion.Id.Equals(datasetVersion.Id) &&
p.Broker.Name.ToLower().Equals(_broker.Name))
.FirstOrDefault();
// if(broker exist)
if (publication == null && _broker != null)
{
Broker broker = _broker;
if (broker != null)
{
OutputMetadataManager.GetConvertedMetadata(datasetId, TransmissionType.mappingFileExport,
broker.MetadataFormat);
// get primary data
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
OutputDataManager odm = new OutputDataManager();
// apply selection and projection
string title = xmlDatasetHelper.GetInformation(datasetId, NameAttributeValues.title);
odm.GenerateAsciiFile(datasetId, title, broker.PrimaryDataFormat);
}
string zipName = publishingManager.GetZipFileName(datasetId, datasetVersion.Id);
string zipPath = publishingManager.GetDirectoryPath(datasetId, broker.Name);
string dynamicZipPath = publishingManager.GetDynamicDirectoryPath(datasetId, broker.Name);
string zipFilePath = Path.Combine(zipPath, zipName);
string dynamicFilePath = Path.Combine(dynamicZipPath, zipName);
FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(zipFilePath));
if (FileHelper.FileExist(zipFilePath))
{
if (FileHelper.WaitForFile(zipFilePath))
{
FileHelper.Delete(zipFilePath);
}
}
// add datastructure
//ToDo put that functiom to the outputDatatructureManager
#region datatructure
long dataStructureId = datasetVersion.Dataset.DataStructure.Id;
DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (dataStructure != null)
{
try
{
string dynamicPathOfDS = "";
dynamicPathOfDS = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion,
"datastructure", ".txt");
string datastructureFilePath = AsciiWriter.CreateFile(dynamicPathOfDS);
string json = OutputDataStructureManager.GetVariableListAsJson(datasetId);
AsciiWriter.AllTextToFile(datastructureFilePath, json);
}
catch (Exception ex)
{
throw ex;
}
}
#endregion
ZipFile zip = new ZipFile();
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
string path = Path.Combine(AppConfiguration.DataPath, cd.URI);
string name = cd.URI.Split('\\').Last();
if (FileHelper.FileExist(path))
{
zip.AddFile(path, "");
}
}
// add xsd of the metadata schema
string xsdDirectoryPath = OutputMetadataManager.GetSchemaDirectoryPath(datasetId);
if (Directory.Exists(xsdDirectoryPath))
{
zip.AddDirectory(xsdDirectoryPath, "Schema");
}
XmlDocument manifest = OutputDatasetManager.GenerateManifest(datasetId, datasetVersion.Id);
if (manifest != null)
{
string dynamicManifestFilePath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId,
datasetVersion.Id, "manifest", ".xml");
string fullFilePath = Path.Combine(AppConfiguration.DataPath, dynamicManifestFilePath);
manifest.Save(fullFilePath);
zip.AddFile(fullFilePath, "");
}
string message = string.Format("dataset {0} version {1} was published for repository {2}", datasetId,
datasetVersion.Id, broker.Name);
LoggerFactory.LogCustom(message);
//Session["ZipFilePath"] = dynamicFilePath;
zip.Save(zipFilePath);
return(zipFilePath);
}
}
return("");
}
finally
{
datasetManager.Dispose();
dataStructureManager.Dispose();
publicationManager.Dispose();
}
}
private ShowPublishDataModel getShowPublishDataModel(long datasetId, long datasetVersionId = -1)
{
PublicationManager publicationManager = new PublicationManager();
DatasetManager datasetManager = new DatasetManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
ShowPublishDataModel model = new ShowPublishDataModel();
try
{
Dataset dataset = datasetManager.GetDataset(datasetId);
List <Broker> Brokers = GetBrokers(dataset.MetadataStructure.Id, publicationManager);
model.Brokers = Brokers.Select(b => b.Name).ToList();
model.DatasetId = datasetId;
//Todo Download Rigths -> currently set read rigths for this case
model.DownloadRights = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, "Dataset",
typeof(Dataset), datasetId, RightType.Read);
model.EditRights = entityPermissionManager.HasEffectiveRight(HttpContext.User.Identity.Name, "Dataset",
typeof(Dataset), datasetId, RightType.Write);
List <long> versions = new List <long>();
if (datasetVersionId == -1)
{
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
datasetVersionId = datasetVersion.Id;
versions = datasetManager.GetDatasetVersions(datasetId).Select(d => d.Id).ToList();
if (datasetVersion.StateInfo != null)
{
model.MetadataIsValid = DatasetStateInfo.Valid.ToString().Equals(datasetVersion.StateInfo.State) ? true : false;
}
}
//todo check if datasetversion id is correct
List <Publication> publications =
publicationManager.PublicationRepo.Get().Where(p => versions.Contains(p.DatasetVersion.Id)).ToList();
foreach (var pub in publications)
{
Broker broker = publicationManager.BrokerRepo.Get(pub.Broker.Id);
Repository repo = null;
if (pub.Repository != null)
{
repo = publicationManager.RepositoryRepo.Get(pub.Repository.Id);
}
string dataRepoName = repo == null ? "" : repo.Name;
List <string> repos =
GetRepos(dataset.MetadataStructure.Id, broker.Id, publicationManager).Select(r => r.Name).ToList();
model.Publications.Add(new PublicationModel()
{
Broker = new BrokerModel(broker.Name, repos, broker.Link),
DataRepo = dataRepoName,
DatasetVersionId = pub.DatasetVersion.Id,
CreationDate = pub.Timestamp,
ExternalLink = pub.ExternalLink,
FilePath = pub.FilePath,
Status = pub.Status,
});
}
return(model);
}
finally
{
publicationManager.Dispose();
datasetManager.Dispose();
entityPermissionManager.Dispose();
}
}
public async Task <ActionResult> SendDataToDataRepo(long datasetId, string datarepo)
{
PublicationManager publicationManager = new PublicationManager();
DatasetManager datasetManager = new DatasetManager();
try
{
string zipfilepath = "";
if (Session["ZipFilePath"] != null)
{
zipfilepath = Session["ZipFilePath"].ToString();
}
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
Publication publication =
publicationManager.GetPublication()
.Where(
p =>
p.DatasetVersion.Id.Equals(datasetVersion.Id) &&
p.Broker.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
if (publication == null)
{
//ToDo [SUBMISSION] -> create broker specfic function
// check case for gfbio
if (datarepo.ToLower().Contains("gfbio"))
{
#region GFBIO
//SubmissionManager publishingManager = new SubmissionManager();
//publishingManager.Load();
//DataRepository dataRepository = publishingManager.DataRepositories.Where(d => d.Name.Equals(datarepo)).FirstOrDefault();
Broker broker =
publicationManager.GetBroker()
.Where(b => b.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
if (broker != null)
{
//Store ro in db
string title = xmlDatasetHelper.GetInformationFromVersion(datasetVersion.Id,
NameAttributeValues.title);
publicationManager.CreatePublication(datasetVersion, broker, title, 0, zipfilepath, "", "no status available");
//sendToGFBIO(broker, datasetId, datasetVersion, zipfilepath);
}
#endregion
}
if (datarepo.ToLower().Contains("pensoft"))
{
#region pensoft
Broker broker =
publicationManager.BrokerRepo.Get()
.Where(b => b.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
Repository repository =
publicationManager.RepositoryRepo.Get()
.Where(b => b.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
string title = xmlDatasetHelper.GetInformationFromVersion(datasetVersion.Id, NameAttributeValues.title);
publicationManager.CreatePublication(datasetVersion, broker, repository, title, 0, zipfilepath, "",
"no status available");
#endregion
}
if (datarepo.ToLower().Equals("generic"))
{
#region GENERIC
Broker broker =
publicationManager.BrokerRepo.Get()
.Where(b => b.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
string title = xmlDatasetHelper.GetInformationFromVersion(datasetVersion.Id, NameAttributeValues.title);
publicationManager.CreatePublication(datasetVersion, broker, title, 0, zipfilepath, "",
"created");
#endregion
}
}
else
{
Json("Publication exist.");
}
}
catch (Exception ex)
{
return(Json(ex.Message));
}
finally
{
publicationManager.Dispose();
datasetManager.Dispose();
}
return(Json(true));
}
public ActionResult LoadDataRepoRequirementView(string datarepo, long datasetid)
{
DataRepoRequirentModel model = new DataRepoRequirentModel();
model.DatasetId = datasetid;
//get broker
PublicationManager publicationManager = new PublicationManager();
// datasetversion
DatasetManager dm = new DatasetManager();
try
{
long version = dm.GetDatasetLatestVersionId(datasetid);
model.DatasetVersionId = version;
if (publicationManager.BrokerRepo.Get().Any(d => d.Name.ToLower().Equals(datarepo.ToLower())))
{
Broker broker =
publicationManager.BrokerRepo.Get()
.Where(d => d.Name.ToLower().Equals(datarepo.ToLower()))
.FirstOrDefault();
Publication publication =
publicationManager.PublicationRepo.Get()
.Where(p => p.Broker.Id.Equals(broker.Id) && p.DatasetVersion.Id.Equals(version))
.FirstOrDefault();
if (publication != null && !String.IsNullOrEmpty(publication.FilePath) &&
FileHelper.FileExist(Path.Combine(AppConfiguration.DataPath, publication.FilePath)))
{
model.Exist = true;
}
else
{
//if convertion check ist needed
//get all export attr from metadata structure
List <string> exportNames =
xmlDatasetHelper.GetAllTransmissionInformation(datasetid,
TransmissionType.mappingFileExport, AttributeNames.name).ToList();
if (string.IsNullOrEmpty(broker.MetadataFormat) || exportNames.Contains(broker.MetadataFormat))
{
model.IsMetadataConvertable = true;
}
// Validate
model.metadataValidMessage = OutputMetadataManager.IsValideAgainstSchema(datasetid,
TransmissionType.mappingFileExport, datarepo);
#region primary Data
if (broker.PrimaryDataFormat.ToLower().Contains("text/plain") ||
broker.PrimaryDataFormat.ToLower().Contains("text/csv") ||
broker.PrimaryDataFormat.ToLower().Contains("application/excel") ||
String.IsNullOrEmpty(broker.PrimaryDataFormat))
{
model.IsDataConvertable = true;
}
#endregion
}
}
return(PartialView("_dataRepositoryRequirementsView", model));
}
catch (Exception ex)
{
throw ex;
}
finally
{
publicationManager.Dispose();
dm.Dispose();
}
}
