public List <ListViewItem> LoadDatasetVersionViewList(DataStructureType dataStructureType)
{
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
DataStructureManager dataStructureManager = new DataStructureManager();
DatasetManager dm = new DatasetManager();
try
{
List <long> datasetIds = entityPermissionManager.GetKeys(GetUsernameOrDefault(), "Dataset", typeof(Dataset), RightType.Write).ToList();
List <ListViewItem> tempStructured = new List <ListViewItem>();
List <ListViewItem> tempUnStructured = new List <ListViewItem>();
var DatasetVersions = dm.GetDatasetLatestVersions(datasetIds, false);
foreach (var dsv in DatasetVersions)
{
if (dsv.Dataset.DataStructure.Self.GetType().Equals(typeof(StructuredDataStructure)))
{
tempStructured.Add(new ListViewItem(dsv.Dataset.Id, dsv.Title));
}
else
{
tempUnStructured.Add(new ListViewItem(dsv.Dataset.Id, dsv.Title));
}
}
if (dataStructureType.Equals(DataStructureType.Structured))
{
return(tempStructured.OrderBy(p => p.Title).ToList());
}
else
{
return(tempUnStructured.OrderBy(p => p.Title).ToList());
}
}
finally
{
entityPermissionManager.Dispose();
dataStructureManager.Dispose();
dm.Dispose();
}
}
public ActionResult FlipDateTime(long id, long variableid)
{
DatasetManager datasetManager = new DatasetManager();
try
{
DatasetVersion dsv = datasetManager.GetDatasetLatestVersion(id);
IEnumerable <long> datatupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(dsv);
foreach (var tid in datatupleIds)
{
DataTuple dataTuple = datasetManager.DataTupleRepo.Get(tid);
dataTuple.Materialize();
bool needUpdate = false;
foreach (var vv in dataTuple.VariableValues)
{
string systemType = vv.DataAttribute.DataType.SystemType;
if (systemType.Equals(typeof(DateTime).Name) && vv.VariableId.Equals(variableid))
{
string value = vv.Value.ToString();
vv.Value = flip(value, out needUpdate);
}
}
if (needUpdate)
{
dataTuple.Dematerialize();
datasetManager.UpdateDataTuple(dataTuple);
}
}
}
catch (Exception ex)
{
}
finally
{
datasetManager.Dispose();
}
return(RedirectToAction("Index"));
}
public static string IsValideAgainstSchema(long datasetId, TransmissionType type, string mappingName)
{
DatasetManager datasetManager = new DatasetManager();
try
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
string mappingFileName = xmlDatasetHelper.GetTransmissionInformation(datasetVersion.Id, type, mappingName);
string pathMappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), mappingFileName);
XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern);
xmlMapperManager.Load(pathMappingFile, "exporttest");
XmlDocument tmp = GetConvertedMetadata(datasetId, type, mappingName, false);
string path = Path.Combine(AppConfiguration.DataPath, "Temp", "System", "convertedMetadata.xml");
if (FileHelper.FileExist(path))
{
FileHelper.Delete(path);
}
FileHelper.CreateDicrectoriesIfNotExist(Path.GetDirectoryName(path));
tmp.Save(path);
XmlDocument metadataForImport = new XmlDocument();
metadataForImport.Load(path);
return(xmlMapperManager.Validate(metadataForImport));
}
catch (Exception ex)
{
return(ex.Message);
}
finally
{
datasetManager.Dispose();
}
}
public void GetDataTuples_WhenCalledForLatestVersionComplete_ReturnIQueryable()
{
//Arrange
DatasetManager datasetManager = new DatasetManager();
try
{
//Act
DatasetVersion datasetversion = datasetManager.GetDatasetLatestVersion(datasetId);
var result = datasetManager.GetDataTuples(datasetversion.Id);
int c = datasetManager.GetDataTuplesCount(datasetversion.Id);
//Assert
Assert.That(result.Count(), Is.EqualTo(numberOfTuples));
Assert.That(c, Is.EqualTo(result.Count()));
}
finally
{
datasetManager.Dispose();
}
}
public string EarliestDatestamp()
{
DatasetManager datasetManager = new DatasetManager();
try
{
var firstdsv = datasetManager.DatasetVersionRepo.Get().OrderBy(dsv => dsv.Timestamp).FirstOrDefault();
string format = API.Common.Enums.Granularity.DateTime.Replace("'", "");
if (firstdsv != null)
{
return(firstdsv.Timestamp.ToString(format));
}
return("");
}
finally
{
datasetManager.Dispose();
}
}
// GET: Test
public ActionResult Index(long id = -1)
{
DatasetManager datasetManager = new DatasetManager();
try
{
//get all
var x = MappingUtils.GetAllMatchesInSystem(1, LinkElementType.MetadataNestedAttributeUsage);
// get all where value = david
x = MappingUtils.GetAllMatchesInSystem(1, LinkElementType.MetadataNestedAttributeUsage, "David");
// get value from metadata over the system
// partytpe person - attr firstname
long partyCustomtAttr = 1;
LinkElementType type = LinkElementType.PartyCustomType;
long datasetId = id;
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
List <string> tmp = MappingUtils.GetValuesFromMetadata(partyCustomtAttr, type,
datasetVersion.Dataset.MetadataStructure.Id, XmlUtility.ToXDocument(datasetVersion.Metadata));
tmp = MappingUtils.GetValuesFromMetadata(Convert.ToInt64(Key.Keyword), LinkElementType.Key,
datasetVersion.Dataset.MetadataStructure.Id, XmlUtility.ToXDocument(datasetVersion.Metadata));
tmp = MappingUtils.GetValuesFromMetadata(Convert.ToInt64(Key.Author), LinkElementType.Key,
datasetVersion.Dataset.MetadataStructure.Id, XmlUtility.ToXDocument(datasetVersion.Metadata));
tmp = MappingUtils.GetValuesFromMetadata(Convert.ToInt64(Key.Title), LinkElementType.Key,
datasetVersion.Dataset.MetadataStructure.Id, XmlUtility.ToXDocument(datasetVersion.Metadata));
return(View("Index"));
}
finally
{
datasetManager.Dispose();
}
}
/// <summary>
/// Get a html file of the data structure from a dataset
/// </summary>
/// <param name="id">dataset id</param>
/// <returns>html file</returns>
public ActionResult GetDataStructureAsHtmlFile(long id)
{
DatasetManager dm = new DatasetManager();
DataStructureManager dsm = new DataStructureManager();
try
{
using (var uow = this.GetUnitOfWork())
{
long dsId = dm.GetDatasetLatestVersion(id).Id;
DatasetVersion ds = uow.GetUnitOfWork().GetReadOnlyRepository <DatasetVersion>().Get(dsId);
DataStructure dataStructure = null;
if (ds != null)
{
dataStructure = uow.GetReadOnlyRepository <DataStructure>().Get(ds.Dataset.DataStructure.Id);
}
if (dataStructure != null && dataStructure.Self is StructuredDataStructure)
{
SimpleDataStructureModel model = new SimpleDataStructureModel((StructuredDataStructure)dataStructure.Self);
string htmlPage = PartialView("SimpleDataStructure", model).RenderToString();
byte[] content = Encoding.ASCII.GetBytes(htmlPage);
return(File(content, "text/html", "dataStructure.html"));
}
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
dsm.Dispose();
}
return(null);
}
public int CountEntities()
{
using (var uow = this.GetUnitOfWork())
{
DatasetManager dm = new DatasetManager();
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
var entities = new List <EntityStoreItem>();
int count = 0;
try
{
List <long> metadataStructureIds = metadataStructureManager.Repo.Query().Select(m => m.Id).ToList();
List <long> metadataSturctureIdsForDatasets = new List <long>();
metadataStructureIds.ForEach(m => xmlDatasetHelper.HasEntity(m, _entityName));
foreach (var msid in metadataStructureIds)
{
var datasetIds = new List <long>();
// get all datasets based on metadata data structure id
datasetIds = dm.DatasetRepo.Query(d => d.MetadataStructure.Id.Equals(msid)).Select(d => d.Id).ToList();
count += datasetIds.Count;
}
return(count);
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
metadataStructureManager.Dispose();
}
}
}
public string GetTitleById(long id)
{
using (var uow = this.GetUnitOfWork())
{
var dm = new DatasetManager();
try
{
var dsv = dm.GetDatasetLatestVersion(id);
return(dsv.Title);
}
catch
{
return(String.Empty);
}
finally
{
dm.Dispose();
}
}
}
public int CountVersions(long id)
{
DatasetManager dm = new DatasetManager();
try
{
var datasetIds = dm.GetDatasetLatestIds();
var datasetHelper = new XmlDatasetHelper();
int version = dm.GetDataset(id).Versions.Count;
return(version);
}
catch (Exception ex)
{
return(0);
}
finally
{
dm.Dispose();
}
}
public void GetDatasetVersionEffectiveDataTuples_PageOfDataTuplesFromLatestVersion_ReturnListOfAbstractTuples()
{
//Arrange
DatasetManager datasetManager = new DatasetManager();
int pagesize = (int)(numberOfTuples / 4);
int page = 2;
try
{
//Act
DatasetVersion datasetversion = datasetManager.GetDatasetLatestVersion(datasetId);
var result = datasetManager.GetDatasetVersionEffectiveTuples(datasetversion, page, pagesize);
//Assert
Assert.That(result.Count(), Is.EqualTo(pagesize));
}
finally
{
datasetManager.Dispose();
}
}
public string GetEntityName(long datasetid)
{
DatasetManager datasetManager = new DatasetManager();
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
try
{
Dataset dataset = datasetManager.GetDataset(datasetid);
// get MetadataStructure
if (dataset != null)
{
return(GetEntityNameFromMetadatStructure(dataset.MetadataStructure.Id));
}
return(string.Empty);
}
finally
{
datasetManager.Dispose();
metadataStructureManager.Dispose();
}
}
public IEnumerable <ApiSimpleDatasetModel> Get()
{
DatasetManager datasetManager = new DatasetManager();
List <ApiSimpleDatasetModel> datasetModels = new List <ApiSimpleDatasetModel>();
try
{
IEnumerable <long> datasetIds = datasetManager.DatasetRepo.Get().Select(d => d.Id);
foreach (long id in datasetIds)
{
Dataset tmpDataset = datasetManager.GetDataset(id);
ApiSimpleDatasetModel datasetModel = new ApiSimpleDatasetModel();
datasetModel.Id = tmpDataset.Id;
DatasetVersion[] tmpVersions = tmpDataset.Versions.OrderBy(ds => ds.Timestamp).ToArray();
for (int i = 0; i < tmpVersions.Length; i++)
{
DatasetVersion dsv = tmpVersions.ElementAt(i);
ApiSimpleDatasetVersionModel datasetVersionModel = new ApiSimpleDatasetVersionModel()
{
Id = dsv.Id,
Number = i + 1
};
datasetModel.Versions.Add(datasetVersionModel);
}
datasetModels.Add(datasetModel);
}
return(datasetModels);
}
finally
{
datasetManager.Dispose();
}
}
public ActionResult CountRows(long id)
{
int number = 0;
using (DatasetManager dm = new DatasetManager())
{
try
{
if (id > 0)
{
Dataset ds = dm.GetDataset(id);
number = ds.DataStructure.Self is StructuredDataStructure?dm.GetDatasetLatestVersionEffectiveTupleCount(ds) : 0;
}
return(Json(number, JsonRequestBehavior.AllowGet));
}
finally
{
dm.Dispose();
}
}
}
public List <EntityStoreItem> GetEntities()
{
using (var uow = this.GetUnitOfWork())
{
DatasetManager dm = new DatasetManager();
try
{
var datasetIds = dm.GetDatasetLatestIds();
var datasetHelper = new XmlDatasetHelper();
var entities = datasetIds.Select(id => new EntityStoreItem()
{
Id = id, Title = datasetHelper.GetInformation(id, NameAttributeValues.title)
});
return(entities.ToList());
}
finally
{
dm.Dispose();
}
}
}
/// <summary>
///
/// </summary>
/// <param name="datasetid"></param>
/// <param name="name"></param>
/// <returns></returns>
public string GetInformation(long datasetid, NameAttributeValues name)
{
DatasetManager dm = new DatasetManager();
try
{
//DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(datasetid);
var dataset = dm.GetDataset(datasetid);
if (!dm.IsDatasetCheckedIn(datasetid))
{
return(string.Empty);
}
var versionId = dm.GetDatasetLatestVersionId(datasetid);
return(GetInformationFromVersion(versionId, dataset.MetadataStructure.Id, name));
}
finally
{
dm.Dispose();
}
}
public void DeleteDatasetTest()
{
var dm = new DatasetManager();
var rsm = new ResearchPlanManager();
var mdm = new MetadataStructureManager();
try
{
var dsHelper = new DatasetHelper();
StructuredDataStructure dataStructure = dsHelper.CreateADataStructure();
dataStructure.Should().NotBeNull("Failed to meet a precondition: a data strcuture is required.");
var rp = dsHelper.CreateResearchPlan();
rp.Should().NotBeNull("Failed to meet a precondition: a research plan is required.");
var mds = mdm.Repo.Query().First();
mds.Should().NotBeNull("Failed to meet a precondition: a metadata strcuture is required.");
Dataset dataset = dm.CreateEmptyDataset(dataStructure, rp, mds);
dm.DeleteDataset(dataset.Id, "Javad", false);
dataset.Should().NotBeNull();
dataset.Id.Should().BeGreaterThan(0, "Dataset is not persisted.");
dataset.LastCheckIOTimestamp.Should().NotBeAfter(DateTime.UtcNow, "The dataset's timestamp is wrong.");
dataset.DataStructure.Should().NotBeNull("Dataset must have a data structure.");
dataset.Status.Should().Be(DatasetStatus.Deleted, "Dataset must be in Deleted status.");
dm.PurgeDataset(dataset.Id);
dsHelper.PurgeAllDataStructures();
}
finally
{
dm.Dispose();
rsm.Dispose();
mdm.Dispose();
}
}
public ActionResult SimpleDataStructure(long id)
{
DatasetManager dm = new DatasetManager();
DataStructureManager dsm = new DataStructureManager();
try
{
using (var uow = this.GetUnitOfWork())
{
long dsId = dm.GetDatasetLatestVersion(id).Id;
DatasetVersion ds = uow.GetUnitOfWork().GetReadOnlyRepository <DatasetVersion>().Get(dsId);
DataStructure dataStructure = null;
if (ds != null)
{
dataStructure = uow.GetReadOnlyRepository <DataStructure>().Get(ds.Dataset.DataStructure.Id);
}
if (dataStructure != null && dataStructure.Self is StructuredDataStructure)
{
SimpleDataStructureModel model = new SimpleDataStructureModel((StructuredDataStructure)dataStructure.Self);
return(PartialView("SimpleDataStructure", model));
}
}
return(null);
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
dsm.Dispose();
}
}
//[MeasurePerformance]
public ActionResult _CustomPrimaryDataBinding(GridCommand command, int datasetID)
{
GridModel model = new GridModel();
Session["Filter"] = command;
DatasetManager dm = new DatasetManager();
try
{
if (dm.IsDatasetCheckedIn(datasetID))
{
DatasetVersion dsv = dm.GetDatasetLatestVersion(datasetID);
// commented by Javad. Now the new API is called
//List<AbstractTuple> dataTuples = dm.GetDatasetVersionEffectiveTuples(dsv, command.Page - 1,
// command.PageSize);
//DataTable table = SearchUIHelper.ConvertPrimaryDataToDatatable(dsv, dataTuples);
DataTable table = dm.GetLatestDatasetVersionTuples(dsv.Dataset.Id, command.Page - 1, command.PageSize);
Session["gridTotal"] = dm.GetDatasetVersionEffectiveTupleCount(dsv);
model = new GridModel(table);
model.Total = Convert.ToInt32(Session["gridTotal"]); // (int)Session["gridTotal"];
}
else
{
ModelState.AddModelError(String.Empty, "Dataset is just in processing.");
}
return(View(model));
}
finally
{
dm.Dispose();
}
}
public List <ListViewItem> LoadDatasetViewList()
{
List <ListViewItem> temp = new List <ListViewItem>();
DatasetManager datasetManager = new DatasetManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
//get all datasetsid where the current userer has access to
UserManager userManager = new UserManager();
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
try
{
List <long> datasetIds = entityPermissionManager.GetKeys(GetUsernameOrDefault(), "Dataset",
typeof(Dataset), RightType.Write);
foreach (long id in datasetIds)
{
if (datasetManager.IsDatasetCheckedIn(id))
{
string title = xmlDatasetHelper.GetInformation(id, NameAttributeValues.title);
string description = xmlDatasetHelper.GetInformation(id, NameAttributeValues.description);
temp.Add(new ListViewItem(id, title, description));
}
}
return(temp.OrderBy(p => p.Title).ToList());
}
finally
{
datasetManager.Dispose();
entityPermissionManager.Dispose();
userManager.Dispose();
}
}
public ActionResult _CustomDataStructureBinding(GridCommand command, long datasetID)
{
DataStructureManager dsm = new DataStructureManager();
DatasetManager dm = new DatasetManager();
try
{
long id = datasetID;
if (dm.IsDatasetCheckedIn(id))
{
DatasetVersion ds = dm.GetDatasetLatestVersion(id);
if (ds != null)
{
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(ds.Dataset.DataStructure.Id);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
//StructuredDataStructure sds = (StructuredDataStructure)(ds.Dataset.DataStructure.Self);
SearchUIHelper suh = new SearchUIHelper();
DataTable table = suh.ConvertStructuredDataStructureToDataTable(sds);
return(View(new GridModel(table)));
}
}
else
{
ModelState.AddModelError(String.Empty, "Dataset is just in processing.");
}
}
finally
{
dm.Dispose();
dsm.Dispose();
}
return(View(new GridModel(new DataTable())));
}
public List <int> Get()
{
List <int> structuredIds = new List <int>();
DatasetManager dm = new DatasetManager();
try
{
var datasetIds = dm.GetDatasetLatestIds();
foreach (int id in datasetIds)
{
DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(id);
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
structuredIds.Add(id);
}
}
return(structuredIds);
}
finally
{
dm.Dispose();
}
}
public void GetDataTuples_WhenCalledOneOlderVersion_ReturnIQueryableOfAbstractTuples()
{
//Arrange
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
//get latest datatupleid before create a new dataset and data
using (var uow = this.GetUnitOfWork())
{
var latestDataTuple = uow.GetReadOnlyRepository <DataTuple>().Get().LastOrDefault();
if (latestDataTuple != null)
{
latestDataTupleId = latestDataTuple.Id;
}
}
var dataset = datasetManager.GetDataset(datasetId);
dataset = dsHelper.UpdateOneTupleForDataset(dataset, (StructuredDataStructure)dataset.DataStructure, latestDataTupleId, 1, datasetManager);
datasetManager.CheckInDataset(dataset.Id, "for testing datatuples with versions", username, ViewCreationBehavior.None);
//Act
List <DatasetVersion> datasetversions = datasetManager.GetDatasetVersions(datasetId).OrderBy(d => d.Timestamp).ToList();
var result = datasetManager.GetDataTuples(datasetversions.ElementAt(datasetversions.Count - 2).Id); // get datatuples from the one before the latest
//Assert
Assert.That(result.Count(), Is.EqualTo(10));
}
finally
{
datasetManager.Dispose();
}
}
private List <DatasetVersionModel> getDatasetVersionsDic(MetadataStructure metadataStructure, List <long> datasetVersionIds)
{
List <DatasetVersionModel> datasetVersions = new List <DatasetVersionModel>();
DatasetManager datasetManager = new DatasetManager();
try
{
using (var uow = this.GetUnitOfWork())
{
// gets all the dataset versions that their Id is in the datasetVersionIds and they are using a specific metadata structure as indicated by metadataStructure parameter
var q = uow.GetReadOnlyRepository <DatasetVersion>().Get(p => datasetVersionIds.Contains(p.Id) &&
p.Dataset.MetadataStructure.Id.Equals(metadataStructure.Id)).Distinct();
foreach (DatasetVersion datasetVersion in q)
{
if (datasetManager.IsDatasetCheckedIn(datasetVersion.Dataset.Id))
{
uow.GetReadOnlyRepository <DatasetVersion>().Load(datasetVersion.ContentDescriptors);
datasetVersions.Add(
new DatasetVersionModel
{
DatasetVersionId = datasetVersion.Id,
DatasetId = datasetVersion.Dataset.Id,
Title = datasetVersion.Title,
MetadataDownloadPath = OutputMetadataManager.GetMetadataPath(datasetVersion.ContentDescriptors)
});
}
}
return(datasetVersions);
}
}
finally
{
datasetManager.Dispose();
}
}
public void fillDatasetList()
{
DatasetListElement datasetListElement = new DatasetListElement();
DatasetManager dm = null;
datasets = new List <DatasetListElement>();
try
{
dm = new DatasetManager();
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
foreach (var item in dm.GetDatasetLatestVersions(dataStructure.Id, true))
{
datasetListElement = new DatasetListElement(item.Key, xmlDatasetHelper.GetInformation(item.Value.Id, NameAttributeValues.title));
datasets.Add(datasetListElement);
}
}
finally
{
dm.Dispose();
}
}
//[MeasurePerformance]
//temporary solution: norman
//For original solution, look into Aquadiva Code
public List <Error> FinishUpload(EasyUploadTaskManager taskManager)
{
DataStructureManager dsm = new DataStructureManager();
DatasetManager dm = new DatasetManager();
DataContainerManager dam = new DataContainerManager();
//SubjectManager sm = new SubjectManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
List <Error> temp = new List <Error>();
try
{
using (IUnitOfWork unitOfWork = this.GetUnitOfWork())
{
// initialize all necessary manager
DataTuple[] rows = null;
//First, try to validate - if there are errors, return immediately
string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString();
List <Error> ValidationErrors = ValidateRows(JsonArray);
if (ValidationErrors.Count != 0)
{
temp.AddRange(ValidationErrors);
return(temp);
}
string timestamp = DateTime.UtcNow.ToString("r");
string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]);
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE))
{
string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
if (!String.IsNullOrWhiteSpace(tmp))
{
title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
}
}
StructuredDataStructure sds = null;
List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); //Used in Excel reader
//Try to find an exact matching datastructure
Boolean foundReusableDataStructure = false;
List <RowModel> headers = (List <RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS];
//For some reason, MappedHeaders might be in a different order in this list than what's indicated by its IDs - to prevent mismatching, sort the headers
headers.Sort((m1, m2) => m1.Index.CompareTo(m2.Index));
List <StructuredDataStructure> allDatastructures = dsm.StructuredDataStructureRepo.Get().ToList();
foreach (StructuredDataStructure existingStructure in allDatastructures)
{
if (!foundReusableDataStructure)
{
//For now a datastructure is considered an exact match if it contains variables with
//the same names (labels), datatypes and units in the correct order
List <Variable> variablesOfExistingStructure = existingStructure.Variables.ToList();
foundReusableDataStructure = true;
if (variablesOfExistingStructure.Count != headers.Count)
{
foundReusableDataStructure = false;
}
else
{
for (int i = 0; i < variablesOfExistingStructure.Count; i++)
{
Variable exVar = variablesOfExistingStructure.ElementAt(i);
RowModel currentHeader = headers.ElementAt(i);
if (exVar.Label != currentHeader.Name ||
exVar.Unit.Id != currentHeader.SelectedUnit.UnitId ||
exVar.DataAttribute.DataType.Id != currentHeader.SelectedDataType.DataTypeId)
{
foundReusableDataStructure = false;
}
}
}
if (foundReusableDataStructure)
{
sds = existingStructure;
foreach (Variable exVar in variablesOfExistingStructure)
{
VariableIdentifier vi = new VariableIdentifier
{
name = exVar.Label,
id = exVar.Id
};
identifiers.Add(vi);
}
}
}
}
if (!foundReusableDataStructure)
{
sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic);
}
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id);
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp);
if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE))
{
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title);
TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title);
}
MetadataStructure metadataStructure = null;
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA))
{
long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]);
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Id == metadataStructureId).FirstOrDefault();
}
else
{
//Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault();
}
ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault();
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id);
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title);
#region Progress Information
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0);
}
#endregion Progress Information
if (!foundReusableDataStructure)
{
#region Set Variables and information for new DataStructure
XmlDocument xmldoc = new XmlDocument();
XmlElement extraElement = xmldoc.CreateElement("extra");
XmlElement orderElement = xmldoc.CreateElement("order");
//Sorting necessary to prevent problems when inserting the tuples
headers.OrderBy(r => r.Index);
var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>();
var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>();
var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>();
List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList();
foreach (RowModel header in headers)
{
int i = headers.IndexOf(header);
DataType dataType = dataTypeRepo.Get(header.SelectedDataType.DataTypeId);
Unit CurrentSelectedUnit = unitRepo.Get(header.SelectedUnit.UnitId);
DataAttribute CurrentDataAttribute = new DataAttribute();
//If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match)
DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(header.SelectedDataAttribute.Name).ToLower()) &&
da.Unit.Dimension == CurrentSelectedUnit.Dimension).FirstOrDefault();
if (existingDataAttribute != null)
{
CurrentDataAttribute = existingDataAttribute;
}
else
{
//No matching DataAttribute => Create a new one
CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(header.Name), header.Name, header.SelectedDataAttribute.Description, false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null);
}
Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, header.Name, "", "", "", CurrentSelectedUnit);
VariableIdentifier vi = new VariableIdentifier
{
name = newVariable.Label,
id = newVariable.Id
};
identifiers.Add(vi);
XmlElement newVariableXml = xmldoc.CreateElement("variable");
newVariableXml.InnerText = Convert.ToString(newVariable.Id);
orderElement.AppendChild(newVariableXml);
}
extraElement.AppendChild(orderElement);
xmldoc.AppendChild(extraElement);
sds.Extra = xmldoc;
sds.Name = "generated import structure " + timestamp;
sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp;
#endregion Set Variables and information for new DataStructure
}
Dataset ds = null;
ds = dm.CreateEmptyDataset(sds, rp, metadataStructure);
long datasetId = ds.Id;
long sdsId = sds.Id;
if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault()))
{
DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId);
long METADATASTRUCTURE_ID = metadataStructure.Id;
XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath);
XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID);
XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX);
dsv.Metadata = metadataXml;
try
{
dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title);
dsv.Title = title;
}
catch (NullReferenceException ex)
{
//Reference of the title node is missing
throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!");
}
dm.EditDatasetVersion(dsv, null, null, null);
}
#region security
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>())
{
//The user gets full permissions
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList());
}
}
}
#endregion security
#region excel reader
int packageSize = 100000;
int numberOfRows = 0;
//HACK ?
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
{
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
}
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
counter++;
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter;
//rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray();
List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA];
string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString();
List <int[]> areaDataValuesList = new List <int[]>();
foreach (string area in selectedDataAreaJsonArray)
{
areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area));
}
int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray);
Orientation orientation = 0;
switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString())
{
case "LeftRight":
orientation = Orientation.rowwise;
break;
case "Matrix":
//orientation = Orientation.matrix;
break;
default:
orientation = Orientation.columnwise;
break;
}
String worksheetUri = null;
//Get the Uri to identify the correct worksheet
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI))
{
worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString();
}
int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize;
int batchnr = 1;
foreach (int[] areaDataValues in areaDataValuesList)
{
//First batch starts at the start of the current data area
int currentBatchStartRow = areaDataValues[0] + 1;
while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached
{
//End row is start row plus batch size
int currentBatchEndRow = currentBatchStartRow + batchSize;
//Set the indices for the reader
EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo
{
DataStartRow = currentBatchStartRow,
//End row is either at the end of the batch or the end of the marked area
//DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow,
DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1),
//Column indices as marked in a previous step
DataStartColumn = areaDataValues[1] + 1,
DataEndColumn = areaDataValues[3] + 1,
//Header area as marked in a previous step
VariablesStartRow = areaHeaderValues[0] + 1,
VariablesStartColumn = areaHeaderValues[1] + 1,
VariablesEndRow = areaHeaderValues[2] + 1,
VariablesEndColumn = areaHeaderValues[3] + 1,
Offset = areaDataValues[1],
Orientation = orientation
};
//Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work
EasyUploadExcelReader reader = new EasyUploadExcelReader(sds, fri);
// open file
Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString());
//Set variable identifiers because they might differ from the variable names in the file
reader.setSubmittedVariableIdentifiers(identifiers);
//Read the rows and convert them to DataTuples
rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, (int)datasetId, worksheetUri);
//After reading the rows, add them to the dataset
if (rows != null)
{
dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null);
numberOfRows += rows.Count();
}
//Close the Stream so the next ExcelReader can open it again
Stream.Close();
//Debug information
int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1);
int batches = lines / batchSize;
batchnr++;
//Next batch starts after the current one
currentBatchStartRow = currentBatchEndRow + 1;
}
}
#endregion excel reader
//set modification
workingCopy.ModificationInfo = new EntityAuditInfo()
{
Performer = GetUsernameOrDefault(),
Comment = "Data",
ActionType = AuditActionType.Create
};
dm.EditDatasetVersion(workingCopy, null, null, null);
dm.CheckInDataset(ds.Id, "Import " + numberOfRows + " rows", GetUsernameOrDefault());
//Reindex search
if (this.IsAccessible("DDM", "SearchIndex", "ReIndexSingle"))
{
this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary()
{
{ "id", datasetId }
});
}
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id);
return(temp);
}
}
catch (Exception ex)
{
temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " +
"Please try again later. If this problem keeps occuring, please contact your administrator."));
return(temp);
}
finally
{
dsm.Dispose();
dm.Dispose();
dam.Dispose();
//sm.Dispose();
entityPermissionManager.Dispose();
}
}
private HttpResponseMessage getData(long id, int variableId, string token)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
DataStructureManager dataStructureManager = null;
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
dataStructureManager = new DataStructureManager();
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
object stats = new object();
DataTable dt = new DataTable("Varibales");
List <ApiDataStatisticModel> dataStatisticModels = new List <ApiDataStatisticModel>();
if (variableId == -1)
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id);
List <string> varIds = new List <string>();
foreach (Variable vs in structuredDataStructure.Variables)
{
varIds.Add("var" + vs.Id);
}
dt = GetDuplicates(id, varIds);
}
else
{
}
//dt.Strip();
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse(HttpStatusCode.OK);
string resp = JsonConvert.SerializeObject(model);
response.Content = new StringContent(resp, System.Text.Encoding.UTF8, "application/json");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return(response);
}
else
{
return(Request.CreateResponse());
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
dataStructureManager.Dispose();
}
}
public List <EntityStoreItem> GetEntities(int skip, int take)
{
bool withPaging = (take > 0);
using (var uow = this.GetUnitOfWork())
{
DatasetManager dm = new DatasetManager();
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
var entities = new List <EntityStoreItem>();
try
{
List <long> metadataStructureIds = metadataStructureManager.Repo.Query().Select(m => m.Id).ToList();
List <long> metadataSturctureIdsForDatasets = new List <long>();
metadataSturctureIdsForDatasets = metadataStructureIds.Where(m => xmlDatasetHelper.HasEntity(m, _entityName)).ToList();
foreach (var msid in metadataSturctureIdsForDatasets)
{
var datasetIds = new List <long>();
// get all datasets based on metadata data structure id
if (withPaging)
{
datasetIds = dm.DatasetRepo
.Query(d => d.MetadataStructure.Id.Equals(msid))
.Skip(skip)
.Take(take)
.Select(d => d.Id).ToList();
}
else
{
datasetIds = dm.DatasetRepo.Query(d => d.MetadataStructure.Id.Equals(msid)).Select(d => d.Id).ToList();
}
if (!datasetIds.Any())
{
continue;
}
// create tuples based on dataset id list, and get latest version of each dataset
List <DatasetVersion> datasetVersions = dm.GetDatasetLatestVersions(datasetIds, false);
foreach (var dsv in datasetVersions)
{
var e = new EntityStoreItem()
{
Id = dsv.Dataset.Id,
Title = dsv.Title,
Version = dm.GetDatasetVersionCount(dsv.Dataset.Id)
};
entities.Add(e);
}
}
return(entities.ToList());
}
catch (Exception ex)
{
throw ex;
}
finally
{
dm.Dispose();
metadataStructureManager.Dispose();
}
}
}
public DataStructureResultsModel fill(long[] previewIds, string saerchTerms, bool structured = true, bool unstructured = true)
{
DataStructureResultStruct dataStructureResult = new DataStructureResultStruct();
DataStructureManager dataStructureManager = null;
DatasetManager datasetManager = null;
try
{
dataStructureManager = new DataStructureManager();
datasetManager = new DatasetManager();
using (IUnitOfWork uow = this.GetBulkUnitOfWork())
{
if (structured)
{
foreach (DataStructure ds in getStucturedDataStructures(saerchTerms, dataStructureManager))
{
dataStructureResult = new DataStructureResultStruct();
dataStructureResult.Id = ds.Id;
dataStructureResult.Title = ds.Name;
dataStructureResult.Description = ds.Description;
foreach (Dataset d in ds.Datasets)
{
if (datasetManager.RowAny(d.Id, uow))
{
dataStructureResult.inUse = true;
break;
}
// currently not working
/* else
* {
* foreach (DatasetVersion dv in d.Versions)
* {
* if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any())
* {
* dataStructureResult.inUse = true;
* break;
* }
* }
* }*/
}
dataStructureResult.Structured = true;
if (previewIds != null && previewIds.Contains(ds.Id))
{
dataStructureResult.Preview = true;
}
this.dataStructureResults.Add(dataStructureResult);
}
}
}
if (unstructured)
{
foreach (DataStructure ds in getUnStucturedDataStructures(saerchTerms, dataStructureManager))
{
dataStructureResult = new DataStructureResultStruct();
dataStructureResult.Id = ds.Id;
dataStructureResult.Title = ds.Name;
dataStructureResult.Description = ds.Description;
if (ds.Datasets.Count > 1) // Allow to edit, if only one file is linked to it
{
dataStructureResult.inUse = true;
}
if (previewIds != null && previewIds.Contains(ds.Id))
{
dataStructureResult.Preview = true;
}
this.dataStructureResults.Add(dataStructureResult);
}
}
return(this);
}
finally
{
dataStructureManager.Dispose();
datasetManager.Dispose();
}
}
public DataStructureResultStruct(long dataStructureId)
{
using (DataStructureManager dataStructureManager = new DataStructureManager())
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (structuredDataStructure != null)
{
this.Id = structuredDataStructure.Id;
this.Title = structuredDataStructure.Name;
this.Description = structuredDataStructure.Description;
this.inUse = false;
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
foreach (Dataset d in structuredDataStructure.Datasets)
{
if (datasetManager.RowAny(d.Id))
{
this.inUse = true;
break;
}
else
{
foreach (DatasetVersion dv in d.Versions)
{
if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any())
{
this.inUse = true;
break;
}
}
}
}
}
finally
{
datasetManager.Dispose();
}
this.Structured = true;
this.Preview = false;
}
else
{
UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId);
if (unStructuredDataStructure != null)
{
this.Id = unStructuredDataStructure.Id;
this.Title = unStructuredDataStructure.Name;
this.Description = unStructuredDataStructure.Description;
if (unStructuredDataStructure.Datasets != null && unStructuredDataStructure.Datasets.Any())
{
this.inUse = true;
}
else
{
this.inUse = false;
}
this.Structured = false;
this.Preview = false;
}
else
{
new DataStructureResultStruct();
}
}
}
}
