/// <summary>
/// If the <paramref name="entity"/> is not associated to any <see cref="Dateset"/>, the method deletes it from the database.
/// </summary>
/// <param name="entity">The data structure object to be deleted.</param>
/// <returns>True if the data structure is deleted, False otherwise.</returns>
/// <remarks>Database exceptions are not handled intentionally, so that if the data structure is related to some datasets, a proper exception will be thrown.</remarks>
public bool DeleteStructuredDataStructure(StructuredDataStructure entity)
{
Contract.Requires(entity != null);
Contract.Requires(entity.Id >= 0);
IReadOnlyRepository <Dataset> datasetRepo = this.GetUnitOfWork().GetReadOnlyRepository <Dataset>();
if (datasetRepo.Query(p => p.DataStructure.Id == entity.Id).Count() > 0)
{
throw new Exception(string.Format("Data structure {0} is used by datasets. Deletion Failed", entity.Id));
}
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository <StructuredDataStructure> repo = uow.GetRepository <StructuredDataStructure>();
IRepository <Variable> variableRepo = uow.GetRepository <Variable>();
IRepository <Parameter> paramRepo = uow.GetRepository <Parameter>();
variableRepo.Evict();
paramRepo.Evict();
entity = repo.Reload(entity);
// delete associated variables and thier parameters
foreach (var usage in entity.Variables)
{
var localVar = variableRepo.Reload(usage);
paramRepo.Delete(localVar.Parameters.ToList());
variableRepo.Delete(localVar);
}
//uow.Commit(); // should not be needed
repo.Delete(entity);
uow.Commit();
}
return(true);
}
public ActionResult _CustomDataStructureBinding(GridCommand command, long datasetID)
{
DataStructureManager dsm = new DataStructureManager();
DatasetManager dm = new DatasetManager();
try
{
long id = datasetID;
if (dm.IsDatasetCheckedIn(id))
{
DatasetVersion ds = dm.GetDatasetLatestVersion(id);
if (ds != null)
{
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(ds.Dataset.DataStructure.Id);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
//StructuredDataStructure sds = (StructuredDataStructure)(ds.Dataset.DataStructure.Self);
SearchUIHelper suh = new SearchUIHelper();
DataTable table = suh.ConvertStructuredDataStructureToDataTable(sds);
return(View(new GridModel(table)));
}
}
else
{
ModelState.AddModelError(String.Empty, "Dataset is just in processing.");
}
}
finally
{
dm.Dispose();
dsm.Dispose();
}
return(View(new GridModel(new DataTable())));
}
public DataTable ConvertDatasetVersion(DatasetManager datasetManager, DatasetVersion datasetVersion, string tableName = "")
{
DataTable dt = new DataTable();
if (string.IsNullOrEmpty(tableName))
{
dt.TableName = "Primary data table";
}
else
{
dt.TableName = tableName;
}
DataStructureManager dsm = new DataStructureManager();
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id);
var tupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
if (tupleIds != null && tupleIds.Count > 0 && sds != null)
{
buildTheHeader(sds, dt);
buildTheBody(datasetManager, tupleIds, dt, sds);
}
return(dt);
}
/// <summary>
/// Creates a structured data structure <seealso cref="StructuredDataStructure"/> and persists the entity in the database.
/// </summary>
/// <param name="name">The name of the data structure</param>
/// <param name="description">A free text describing the purpose, usage, and/or the domain of the data structure usage.</param>
/// <param name="xsdFileName">Not in use.</param>
/// <param name="xslFileName">Not in use.</param>
/// <param name="indexerType">If the data structure is used as a matrix, The indexer type show what kind of column would be represented by the indexer variable. <see cref="DataStructureCategory"/></param>
/// <param name="indexer">The variable indicating the first indexing column of the matrix, if the data structure is representing a matrix.</param>
/// <returns>The persisted structured data structure instance.</returns>
public StructuredDataStructure CreateStructuredDataStructure(string name, string description, string xsdFileName, string xslFileName, DataStructureCategory indexerType, Variable indexer = null)
{
Contract.Requires(!string.IsNullOrWhiteSpace(name));
Contract.Requires(indexerType != DataStructureCategory.Generic ? (indexer != null) : true);
Contract.Ensures(Contract.Result <StructuredDataStructure>() != null && Contract.Result <StructuredDataStructure>().Id >= 0);
StructuredDataStructure e = new StructuredDataStructure()
{
Name = name,
Description = description,
XsdFileName = xsdFileName,
XslFileName = xslFileName,
IndexerType = indexerType,
// Indexer = indexer, // how its possible to have the indexer before assigning variable to the structure
};
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository <StructuredDataStructure> repo = uow.GetRepository <StructuredDataStructure>();
repo.Put(e);
uow.Commit();
}
return(e);
}
public List <string> GenerateRowsWithRandomValuesBasedOnDatastructureWithErrors(StructuredDataStructure dataStructure, string seperator, long numberOfTuples, bool withQuotes)
{
List <string> rows = new List <string>();
numberOfTuples.Should().BeGreaterThan(0);
var r = new Random();
try
{
for (int i = 0; i < numberOfTuples; i++)
{
string row = r.Next().ToString();
if (withQuotes)
{
row += seperator.ToString() + "\"Test\"";
}
else
{
row += seperator.ToString() + "Test";
}
row += seperator.ToString() + "abc";//Convert.ToDouble(r.Next()).ToString();
row += seperator.ToString() + true.ToString();
row += seperator.ToString() + DateTime.Now.ToString();
rows.Add(row);
}
return(rows);
}
catch (Exception ex)
{
return(null);
}
}
protected override bool AddHeader(StructuredDataStructure header)
{
return(false);
}
// add table header protected abstract bool AddHeader(StructuredDataStructure header);
private void buildTheHeader(StructuredDataStructure sds, DataTable dt)
{
foreach (var vu in sds.Variables)
{
DataColumn col = null;
string columnName = "var" + vu.Id.ToString();
col = dt.Columns.Add(columnName); // or DisplayName also
col.Caption = string.IsNullOrEmpty(vu.Label) ? columnName : vu.Label;
switch (vu.DataAttribute.DataType.SystemType)
{
case "String":
{
col.DataType = Type.GetType("System.String");
break;
}
case "Double":
{
col.DataType = Type.GetType("System.Double");
break;
}
case "Int16":
{
col.DataType = Type.GetType("System.Int16");
break;
}
case "Int32":
{
col.DataType = Type.GetType("System.Int32");
break;
}
case "Int64":
{
col.DataType = Type.GetType("System.Int64");
break;
}
case "Decimal":
{
col.DataType = Type.GetType("System.Decimal");
break;
}
case "DateTime":
{
col.DataType = Type.GetType("System.DateTime");
break;
}
default:
{
col.DataType = Type.GetType("System.String");
break;
}
}
if (vu.Parameters.Count > 0)
{
foreach (var pu in vu.Parameters)
{
DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also
col2.Caption = pu.Label;
}
}
}
}
public DataTable ConvertDatasetVersion(List <AbstractTuple> tuples, DatasetVersion datasetVersion, StructuredDataStructure sds, string tableName = "")
{
DataTable dt = new DataTable();
if (string.IsNullOrEmpty(tableName))
{
dt.TableName = "Primary data table";
}
else
{
dt.TableName = tableName;
}
if (tuples != null && tuples.Count > 0 && sds != null)
{
buildTheHeader(sds, dt);
buildTheBody(tuples, dt, sds);
}
return(dt);
}
private void buildTheBody(DatasetManager datasetManager, List <long> tupleIds, DataTable dt, StructuredDataStructure sds)
{
DataTupleIterator tupleIterator = new DataTupleIterator(tupleIds, datasetManager);
foreach (var tuple in tupleIterator)
{
dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds));
}
}
public Dataset UpdateOneTupleForDataset(Dataset dataset, StructuredDataStructure dataStructure, long id, int value, DatasetManager datasetManager)
{
dataset.Status.Should().Be(DatasetStatus.CheckedIn);
dataset.Should().NotBeNull();
try
{
if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, "David") || datasetManager.CheckOutDataset(dataset.Id, "David"))
{
dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status.");
DatasetVersion workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id);
DataTuple oldDt = datasetManager.DataTupleRepo.Get(id);
DataTuple dt = new DataTuple();
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.First().Id, Value = value
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(1).First().Id, Value = "Test"
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(2).First().Id, Value = 5
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(3).First().Id, Value = true
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(4).First().Id, Value = DateTime.Now.ToString(new CultureInfo("en-US"))
});
dt.Dematerialize();
dt.Should().NotBeNull();
dt.JsonVariableValues.Should().NotBeNull();
List <DataTuple> tuples = new List <DataTuple>();
DataTuple newDt = new DataTuple();
newDt.Id = id;
newDt.XmlAmendments = dt.XmlAmendments;
newDt.JsonVariableValues = dt.JsonVariableValues;
newDt.Materialize();
newDt.OrderNo = oldDt.OrderNo;
tuples.Add(newDt);
datasetManager.EditDatasetVersion(workingCopy, null, tuples, null);
dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status.");
}
return(dataset);
}
catch (Exception ex)
{
return(null);
}
}
public Dataset GenerateTuplesWithRandomValuesForDataset(Dataset dataset, StructuredDataStructure dataStructure, long numberOfTuples, string username)
{
dataset.Status.Should().Be(DatasetStatus.CheckedIn);
dataset.Should().NotBeNull();
numberOfTuples.Should().BeGreaterThan(0);
var r = new Random();
DatasetManager dm = new DatasetManager();
try
{
if (dm.IsDatasetCheckedOutFor(dataset.Id, username) || dm.CheckOutDataset(dataset.Id, username))
{
dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status.");
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(dataset.Id);
List <DataTuple> tuples = new List <DataTuple>();
for (int i = 0; i < numberOfTuples; i++)
{
DataTuple dt = new DataTuple();
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.First().Id, Value = r.Next()
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(1).First().Id, Value = "Test"
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(2).First().Id, Value = Convert.ToDouble(r.Next())
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(3).First().Id, Value = true
});
dt.VariableValues.Add(new VariableValue()
{
VariableId = dataStructure.Variables.Skip(4).First().Id, Value = "01.01.2017"
});
dt.Dematerialize();
dt.Should().NotBeNull();
//dt.XmlVariableValues.Should().NotBeNull();
DataTuple newDt = new DataTuple();
newDt.XmlAmendments = dt.XmlAmendments;
//newDt.XmlVariableValues = dt.XmlVariableValues;
newDt.JsonVariableValues = dt.JsonVariableValues;
newDt.Materialize();
newDt.OrderNo = i;
tuples.Add(newDt);
}
dm.EditDatasetVersion(workingCopy, tuples, null, null);
dataset.Status.Should().Be(DatasetStatus.CheckedOut, "Dataset must be in Checkedout status.");
}
return(dataset);
}
catch (Exception ex)
{
return(null);
}
finally
{
dm.Dispose();
}
}
public List <Variable> getOrderedVariables(StructuredDataStructure structuredDataStructure)
{
return(getOrderedVariables(structuredDataStructure.Id, structuredDataStructure.Variables.ToList()));
}
//[MeasurePerformance]
//temporary solution: norman
//For original solution, look into Aquadiva Code
public List <Error> FinishUpload(EasyUploadTaskManager taskManager)
{
DataStructureManager dsm = new DataStructureManager();
DatasetManager dm = new DatasetManager();
DataContainerManager dam = new DataContainerManager();
//SubjectManager sm = new SubjectManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
List <Error> temp = new List <Error>();
try
{
using (IUnitOfWork unitOfWork = this.GetUnitOfWork())
{
// initialize all necessary manager
DataTuple[] rows = null;
//First, try to validate - if there are errors, return immediately
string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString();
List <Error> ValidationErrors = ValidateRows(JsonArray);
if (ValidationErrors.Count != 0)
{
temp.AddRange(ValidationErrors);
return(temp);
}
string timestamp = DateTime.UtcNow.ToString("r");
string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]);
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE))
{
string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
if (!String.IsNullOrWhiteSpace(tmp))
{
title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
}
}
StructuredDataStructure sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic);
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id);
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp);
if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE))
{
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title);
TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title);
}
MetadataStructure metadataStructure = null;
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA))
{
long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]);
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Id == metadataStructureId).FirstOrDefault();
}
else
{
//Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault();
}
ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault();
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id);
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title);
#region Progress Information
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region DataStructure
XmlDocument xmldoc = new XmlDocument();
XmlElement extraElement = xmldoc.CreateElement("extra");
XmlElement orderElement = xmldoc.CreateElement("order");
List <Tuple <int, string, UnitInfo> > MappedHeaders = (List <Tuple <int, string, UnitInfo> >)TaskManager.Bus[EasyUploadTaskManager.VERIFICATION_MAPPEDHEADERUNITS];
//Sorting necessary to prevent problems when inserting the tuples
MappedHeaders.Sort((head1, head2) => head1.Item1.CompareTo(head2.Item1));
List <VariableIdentifier> identifiers = new List <VariableIdentifier>();
var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>();
var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>();
var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>();
List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList();
foreach (Tuple <int, string, UnitInfo> Entry in MappedHeaders)
{
int i = MappedHeaders.IndexOf(Entry);
DataType dataType = dataTypeRepo.Get(Entry.Item3.SelectedDataTypeId);
Unit CurrentSelectedUnit = unitRepo.Get(Entry.Item3.UnitId);
DataAttribute CurrentDataAttribute = new DataAttribute();
//If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match)
DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(Entry.Item2).ToLower()) &&
da.DataType.Id == dataType.Id &&
da.Unit.Id == CurrentSelectedUnit.Id).FirstOrDefault();
if (existingDataAttribute != null)
{
CurrentDataAttribute = existingDataAttribute;
}
else
{
//No matching DataAttribute => Create a new one
CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(Entry.Item2), Entry.Item2, "", false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null);
}
Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, Entry.Item2, "", "", "");
VariableIdentifier vi = new VariableIdentifier
{
name = newVariable.Label,
id = newVariable.Id
};
identifiers.Add(vi);
XmlElement newVariableXml = xmldoc.CreateElement("variable");
newVariableXml.InnerText = Convert.ToString(newVariable.Id);
orderElement.AppendChild(newVariableXml);
}
extraElement.AppendChild(orderElement);
xmldoc.AppendChild(extraElement);
sds.Extra = xmldoc;
sds.Name = "generated import structure " + timestamp;
sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp;
#endregion
Dataset ds = null;
ds = dm.CreateEmptyDataset(sds, rp, metadataStructure);
//TODO Should a template be created?
/*ExcelTemplateProvider etp = new ExcelTemplateProvider();
* etp.CreateTemplate(sds);*/
long datasetId = ds.Id;
long sdsId = sds.Id;
if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault()))
{
DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId);
long METADATASTRUCTURE_ID = metadataStructure.Id;
XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath);
XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID);
XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX);
dsv.Metadata = metadataXml;
try
{
dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title);
}
catch (NullReferenceException ex)
{
//Reference of the title node is missing
throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!");
}
dm.EditDatasetVersion(dsv, null, null, null);
}
#region security
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
//PermissionManager pm = new PermissionManager();
//User user = sm.GetUserByName(GetUsernameOrDefault());
//Rights-Management
/*
* TODO: Use the BExIS Party API for that
*
* */
/*
* UserPiManager upm = new UserPiManager();
* List<long> piList = (new UserSelectListModel(GetUsernameOrDefault())).UserList.Select(x => x.Id).ToList();
*/
foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>())
{
//The user gets full permissions
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList());
}
// adding the rights for the pis
/*foreach (long piId in piList)
* {
* //Each pi gets full permissions
* pm.CreateDataPermission(piId, 1, ds.Id, rightType);
*
* // get all pi members
* List<UserPi> currentMembers = upm.GetAllPiMember(piId).ToList();
*
* foreach (UserPi currentMapping in currentMembers)
* {
* switch (rightType)
* {
* //Each member of each of the pis gets view-rights
* case RightType.View:
* pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType);
* break;
* //Each member of each of the pis gets download-rights
* case RightType.Download:
* pm.CreateDataPermission(currentMapping.UserId, 1, ds.Id, rightType);
* break;
* default:
* //No other permissions - is this call necessary?
* pm.CreateDataPermission(currentMapping.UserId, 0, ds.Id, rightType);
* break;
* }
* }
* }*/
}
}
#endregion security
#region excel reader
int packageSize = 10000;
//HACK ?
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
{
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
}
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
counter++;
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter;
//rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray();
List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA];
string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString();
List <int[]> areaDataValuesList = new List <int[]>();
foreach (string area in selectedDataAreaJsonArray)
{
areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area));
}
int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray);
Orientation orientation = 0;
switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString())
{
case "LeftRight":
orientation = Orientation.rowwise;
break;
case "Matrix":
//orientation = Orientation.matrix;
break;
default:
orientation = Orientation.columnwise;
break;
}
String worksheetUri = null;
//Get the Uri to identify the correct worksheet
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI))
{
worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString();
}
int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize;
int batchnr = 1;
foreach (int[] areaDataValues in areaDataValuesList)
{
//First batch starts at the start of the current data area
int currentBatchStartRow = areaDataValues[0] + 1;
while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached
{
//Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work
EasyUploadExcelReader reader = new EasyUploadExcelReader();
// open file
Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString());
//End row is start row plus batch size
int currentBatchEndRow = currentBatchStartRow + batchSize;
//Set the indices for the reader
EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo
{
DataStartRow = currentBatchStartRow,
//End row is either at the end of the batch or the end of the marked area
//DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow,
DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1),
//Column indices as marked in a previous step
DataStartColumn = areaDataValues[1] + 1,
DataEndColumn = areaDataValues[3] + 1,
//Header area as marked in a previous step
VariablesStartRow = areaHeaderValues[0] + 1,
VariablesStartColumn = areaHeaderValues[1] + 1,
VariablesEndRow = areaHeaderValues[2] + 1,
VariablesEndColumn = areaHeaderValues[3] + 1,
Offset = areaDataValues[1],
Orientation = orientation
};
//Set variable identifiers because they might differ from the variable names in the file
reader.setSubmittedVariableIdentifiers(identifiers);
//Read the rows and convert them to DataTuples
rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, sds, (int)datasetId, worksheetUri);
//After reading the rows, add them to the dataset
if (rows != null)
{
dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null);
}
//Close the Stream so the next ExcelReader can open it again
Stream.Close();
//Debug information
int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1);
int batches = lines / batchSize;
batchnr++;
//Next batch starts after the current one
currentBatchStartRow = currentBatchEndRow + 1;
}
}
#endregion
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
//Reindex search
if (this.IsAccessibale("DDM", "SearchIndex", "ReIndexSingle"))
{
this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary()
{
{ "id", datasetId }
});
}
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id);
return(temp);
}
}
catch (Exception ex)
{
temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " +
"Please try again later. If this problem keeps occuring, please contact your administrator."));
return(temp);
}
finally
{
dsm.Dispose();
dm.Dispose();
dam.Dispose();
//sm.Dispose();
entityPermissionManager.Dispose();
}
}
public DataReader(StructuredDataStructure structuredDatastructure, FileReaderInfo fileReaderInfo, DatasetManager datasetManager) : this(structuredDatastructure, fileReaderInfo, new IOUtility(), datasetManager)
{
}
public DataTable ConvertDatasetVersion(IEnumerable <AbstractTuple> tupleIterator, StructuredDataStructure dataStructure, string tableName = "")
{
return(ConvertPrimaryDataToDatatable(tupleIterator, dataStructure, tableName));
}
private void buildTheBody(IEnumerable <AbstractTuple> tupleIterator, DataTable dt, StructuredDataStructure sds)
{
foreach (var tuple in tupleIterator)
{
dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds));
}
}
public StructuredDataStructure CreateADataStructure()
{
var unitManager = new UnitManager();
var dataTypeManager = new DataTypeManager();
var attributeManager = new DataContainerManager();
var dsManager = new DataStructureManager();
try
{
var dim = unitManager.Create("TestDimnesion", "For Unit Testing", "");
var unit = unitManager.Create("None_UT", "NoneUT", "Use in unit tsting", dim, Dlm.Entities.DataStructure.MeasurementSystem.Metric);
var intType = dataTypeManager.Create("Integer", "Integer", TypeCode.Int32);
var strType = dataTypeManager.Create("String", "String", TypeCode.String);
var doubleType = dataTypeManager.Create("Double", "Double", TypeCode.Double);
var boolType = dataTypeManager.Create("Bool", "Bool", TypeCode.Boolean);
var dateTimeType = dataTypeManager.Create("DateTime", "DateTime", TypeCode.DateTime);
var dataAttribute1 = attributeManager.CreateDataAttribute(
"att1UT", "att1UT", "Attribute for Unit testing",
false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType,
"", intType, unit,
null, null, null, null, null, null
);
var dataAttribute2 = attributeManager.CreateDataAttribute(
"att2UT", "att1UT", "Attribute for Unit testing",
false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType,
"", strType, unit,
null, null, null, null, null, null
);
var dataAttribute3 = attributeManager.CreateDataAttribute(
"att3UT", "att3UT", "Attribute for Unit testing",
false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType,
"", doubleType, unit,
null, null, null, null, null, null
);
var dataAttribute4 = attributeManager.CreateDataAttribute(
"att4UT", "att4UT", "Attribute for Unit testing",
false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType,
"", boolType, unit,
null, null, null, null, null, null
);
var dataAttribute5 = attributeManager.CreateDataAttribute(
"att5UT", "att5UT", "Attribute for Unit testing",
false, false, "", Dlm.Entities.DataStructure.MeasurementScale.Nominal, Dlm.Entities.DataStructure.DataContainerType.ValueType,
"", dateTimeType, unit,
null, null, null, null, null, null
);
StructuredDataStructure dataStructure = dsManager.CreateStructuredDataStructure("dsForTesting", "DS for unit testing", "", "", Dlm.Entities.DataStructure.DataStructureCategory.Generic);
dsManager.AddVariableUsage(dataStructure, dataAttribute1, true, "var1UT", "", "", "Used for unit testing");
dsManager.AddVariableUsage(dataStructure, dataAttribute2, true, "var2UT", "", "", "Used for unit testing");
dsManager.AddVariableUsage(dataStructure, dataAttribute3, true, "var3UT", "", "", "Used for unit testing");
dsManager.AddVariableUsage(dataStructure, dataAttribute4, true, "var4UT", "", "", "Used for unit testing");
dsManager.AddVariableUsage(dataStructure, dataAttribute5, true, "var5UT", "", "", "Used for unit testing");
return(dataStructure);
}
catch (Exception ex) { return(null); }
finally
{
unitManager.Dispose();
dataTypeManager.Dispose();
attributeManager.Dispose();
dsManager.Dispose();
}
}
private DataRow ConvertTupleIntoDataRow(DataTable dt, AbstractTuple t, StructuredDataStructure sts)
{
DataRow dr = dt.NewRow();
string columnName = "";
foreach (var vv in t.VariableValues)
{
columnName = "var" + vv.VariableId.ToString();
if (vv.VariableId > 0)
{
string valueAsString = "";
if (vv.Value == null)
{
dr[columnName] = DBNull.Value;
}
else
{
valueAsString = vv.Value.ToString();
Dlm.Entities.DataStructure.Variable varr = sts.Variables.Where(p => p.Id == vv.VariableId).SingleOrDefault();
switch (varr.DataAttribute.DataType.SystemType)
{
case "String":
{
dr[columnName] = valueAsString;
break;
}
case "Double":
{
double value;
if (double.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = double.MaxValue;
}
break;
}
case "Int16":
{
Int16 value;
if (Int16.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = Int16.MaxValue;
}
break;
}
case "Int32":
{
Int32 value;
if (Int32.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = Int32.MaxValue;
}
break;
}
case "Int64":
{
Int64 value;
if (Int64.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = Int64.MaxValue;
}
break;
}
case "Decimal":
{
decimal value;
if (decimal.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = decimal.MaxValue;
}
break;
}
case "Float":
{
decimal value;
if (decimal.TryParse(valueAsString, out value))
{
dr[columnName] = value;
}
else
{
dr[columnName] = decimal.MaxValue;
}
break;
}
case "DateTime":
{
if (!String.IsNullOrEmpty(valueAsString))
{
dr[columnName] = Convert.ToDateTime(valueAsString, CultureInfo.InvariantCulture);
}
else
{
dr[columnName] = DateTime.MaxValue;
}
break;
}
default:
{
if (!String.IsNullOrEmpty(vv.Value.ToString()))
{
dr[columnName] = valueAsString;
}
else
{
dr[columnName] = DBNull.Value;
}
break;
}
}
}
}
}
return(dr);
}
public DataTable ConvertPrimaryDataToDatatable(DatasetVersion dsv, IEnumerable <AbstractTuple> dsVersionTuples)
{
DataTable dt = new DataTable();
dt.TableName = "Primary data table";
StructuredDataStructure sds = this.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>().Get(dsv.Dataset.DataStructure.Id);
XmlDocument doc = new XmlDocument();
doc = (XmlDocument)sds.Extra;
if (dsVersionTuples != null && sds != null && doc != null)
{
IEnumerable <XElement> orderList = XmlUtility.GetXElementByNodeName("variable", XmlUtility.ToXDocument(doc));
foreach (XElement element in orderList)
{
var vu = sds.Variables.Where(v => v.Id.Equals(Convert.ToInt64(element.Value))).FirstOrDefault();
// use vu.Label or vu.DataAttribute.Name
DataColumn col = dt.Columns.Add("ID" + vu.Id.ToString()); // or DisplayName also
col.Caption = vu.Label;
switch (vu.DataAttribute.DataType.SystemType)
{
case "String":
{
col.DataType = Type.GetType("System.String");
break;
}
case "Double":
{
col.DataType = Type.GetType("System.Double");
break;
}
case "Int16":
{
col.DataType = Type.GetType("System.Int16");
break;
}
case "Int32":
{
col.DataType = Type.GetType("System.Int32");
break;
}
case "Int64":
{
col.DataType = Type.GetType("System.Int64");
break;
}
case "Decimal":
{
col.DataType = Type.GetType("System.Decimal");
break;
}
case "DateTime":
{
col.DataType = Type.GetType("System.DateTime");
break;
}
default:
{
col.DataType = Type.GetType("System.String");
break;
}
}
if (vu.Parameters.Count > 0)
{
foreach (var pu in vu.Parameters)
{
DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also
col2.Caption = pu.Label;
}
}
}
foreach (var tuple in dsVersionTuples)
{
dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds));
}
}
return(dt);
}
private DataTable ConvertPrimaryDataToDatatable(IEnumerable <AbstractTuple> tupleIterator, StructuredDataStructure dataStructure, string tableName = "")
{
DataTable dt = new DataTable();
if (string.IsNullOrEmpty(tableName))
{
dt.TableName = "Primary data table";
}
else
{
dt.TableName = tableName;
}
if (dataStructure != null)
{
buildTheHeader(dataStructure, dt);
buildTheBody(tupleIterator, dt, dataStructure);
}
return(dt);
}
/// <summary>
/// This function convert a datatuple into datarow for a datatable to show on the client side
/// the grid in the client side (in client mode) has unknow problem with value 0 and null
/// So every empty cell get the max value of the specific Systemtype.
/// On the client side this values replaced with ""
/// </summary>
/// <param name="dt"></param>
/// <param name="t"></param>
/// <returns></returns>
private static DataRow ConvertTupleIntoDataRow(DataTable dt, AbstractTuple t, StructuredDataStructure sts)
{
DataRow dr = dt.NewRow();
foreach (var vv in t.VariableValues)
{
if (vv.VariableId > 0)
{
string valueAsString = "";
if (vv.Value == null)
{
dr["ID" + vv.VariableId.ToString()] = DBNull.Value;
}
else
{
valueAsString = vv.Value.ToString();
Variable varr = sts.Variables.Where(p => p.Id == vv.VariableId).SingleOrDefault();
switch (varr.DataAttribute.DataType.SystemType)
{
case "String":
{
dr["ID" + vv.VariableId.ToString()] = valueAsString;
break;
}
case "Double":
{
double value;
if (double.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToDouble(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = -99999; //double.MaxValue;
}
break;
}
case "Int16":
{
Int16 value;
if (Int16.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToInt16(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = Int16.MaxValue;
}
break;
}
case "Int32":
{
Int32 value;
if (Int32.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToInt32(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = Int32.MaxValue;
}
break;
}
case "Int64":
{
Int64 value;
if (Int64.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToInt64(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = Int64.MaxValue;
}
break;
}
case "Decimal":
{
decimal value;
if (decimal.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToDecimal(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = -99999; //decimal.MaxValue;
}
break;
}
case "Float":
{
decimal value;
if (decimal.TryParse(valueAsString, out value))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToDecimal(valueAsString);
}
else
{
dr["ID" + vv.VariableId.ToString()] = -99999;
}
break;
}
case "DateTime":
{
if (!String.IsNullOrEmpty(valueAsString))
{
dr["ID" + vv.VariableId.ToString()] = Convert.ToDateTime(valueAsString, CultureInfo.InvariantCulture);
}
else
{
dr["ID" + vv.VariableId.ToString()] = DateTime.MaxValue;
}
break;
}
default:
{
if (!String.IsNullOrEmpty(vv.Value.ToString()))
{
dr["ID" + vv.VariableId.ToString()] = valueAsString;
}
else
{
dr["ID" + vv.VariableId.ToString()] = DBNull.Value;
}
break;
}
}
}
/*if (vv.ParameterValues.Count > 0)
* {
* foreach (var pu in vv.ParameterValues)
* {
* dr[pu.Parameter.Label.Replace(" ", "")] = pu.Value;
* }
* }*/
}
}
return(dr);
}
//[MeasurePerformance]
//temporary solution: norman
//For original solution, look into Aquadiva Code
public List <Error> FinishUpload(EasyUploadTaskManager taskManager)
{
DataStructureManager dsm = new DataStructureManager();
DatasetManager dm = new DatasetManager();
DataContainerManager dam = new DataContainerManager();
//SubjectManager sm = new SubjectManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
List <Error> temp = new List <Error>();
try
{
using (IUnitOfWork unitOfWork = this.GetUnitOfWork())
{
// initialize all necessary manager
DataTuple[] rows = null;
//First, try to validate - if there are errors, return immediately
string JsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_JSON_DATA].ToString();
List <Error> ValidationErrors = ValidateRows(JsonArray);
if (ValidationErrors.Count != 0)
{
temp.AddRange(ValidationErrors);
return(temp);
}
string timestamp = DateTime.UtcNow.ToString("r");
string title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.FILENAME]);
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DESCRIPTIONTITLE))
{
string tmp = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
if (!String.IsNullOrWhiteSpace(tmp))
{
title = Convert.ToString(TaskManager.Bus[EasyUploadTaskManager.DESCRIPTIONTITLE]);
}
}
StructuredDataStructure sds = null;
List <VariableIdentifier> identifiers = new List <VariableIdentifier>(); //Used in Excel reader
//Try to find an exact matching datastructure
Boolean foundReusableDataStructure = false;
List <RowModel> headers = (List <RowModel>)TaskManager.Bus[EasyUploadTaskManager.ROWS];
//For some reason, MappedHeaders might be in a different order in this list than what's indicated by its IDs - to prevent mismatching, sort the headers
headers.Sort((m1, m2) => m1.Index.CompareTo(m2.Index));
List <StructuredDataStructure> allDatastructures = dsm.StructuredDataStructureRepo.Get().ToList();
foreach (StructuredDataStructure existingStructure in allDatastructures)
{
if (!foundReusableDataStructure)
{
//For now a datastructure is considered an exact match if it contains variables with
//the same names (labels), datatypes and units in the correct order
List <Variable> variablesOfExistingStructure = existingStructure.Variables.ToList();
foundReusableDataStructure = true;
if (variablesOfExistingStructure.Count != headers.Count)
{
foundReusableDataStructure = false;
}
else
{
for (int i = 0; i < variablesOfExistingStructure.Count; i++)
{
Variable exVar = variablesOfExistingStructure.ElementAt(i);
RowModel currentHeader = headers.ElementAt(i);
if (exVar.Label != currentHeader.Name ||
exVar.Unit.Id != currentHeader.SelectedUnit.UnitId ||
exVar.DataAttribute.DataType.Id != currentHeader.SelectedDataType.DataTypeId)
{
foundReusableDataStructure = false;
}
}
}
if (foundReusableDataStructure)
{
sds = existingStructure;
foreach (Variable exVar in variablesOfExistingStructure)
{
VariableIdentifier vi = new VariableIdentifier
{
name = exVar.Label,
id = exVar.Id
};
identifiers.Add(vi);
}
}
}
}
if (!foundReusableDataStructure)
{
sds = dsm.CreateStructuredDataStructure(title, title + " " + timestamp, "", "", DataStructureCategory.Generic);
}
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_ID, sds.Id);
TaskManager.AddToBus(EasyUploadTaskManager.DATASTRUCTURE_TITLE, title + " " + timestamp);
if (!TaskManager.Bus.ContainsKey(EasyUploadTaskManager.DATASET_TITLE))
{
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_TITLE, title);
TaskManager.AddToBus(EasyUploadTaskManager.TITLE, title);
}
MetadataStructure metadataStructure = null;
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.SCHEMA))
{
long metadataStructureId = Convert.ToInt64(TaskManager.Bus[EasyUploadTaskManager.SCHEMA]);
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Id == metadataStructureId).FirstOrDefault();
}
else
{
//Default option but shouldn't happen because previous steps can't be finished without selecting the metadata-structure
metadataStructure = unitOfWork.GetReadOnlyRepository <MetadataStructure>()
.Get(m => m.Name.ToLower().Contains("eml")).FirstOrDefault();
}
ResearchPlan rp = unitOfWork.GetReadOnlyRepository <ResearchPlan>().Get().FirstOrDefault();
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_ID, rp.Id);
TaskManager.AddToBus(EasyUploadTaskManager.RESEARCHPLAN_TITLE, rp.Title);
#region Progress Information
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(EasyUploadTaskManager.CURRENTPACKAGE, 0);
}
#endregion Progress Information
if (!foundReusableDataStructure)
{
#region Set Variables and information for new DataStructure
XmlDocument xmldoc = new XmlDocument();
XmlElement extraElement = xmldoc.CreateElement("extra");
XmlElement orderElement = xmldoc.CreateElement("order");
//Sorting necessary to prevent problems when inserting the tuples
headers.OrderBy(r => r.Index);
var dataTypeRepo = unitOfWork.GetReadOnlyRepository <DataType>();
var unitRepo = unitOfWork.GetReadOnlyRepository <Unit>();
var dataAttributeRepo = unitOfWork.GetReadOnlyRepository <DataAttribute>();
List <DataAttribute> allDataAttributes = dataAttributeRepo.Get().ToList();
foreach (RowModel header in headers)
{
int i = headers.IndexOf(header);
DataType dataType = dataTypeRepo.Get(header.SelectedDataType.DataTypeId);
Unit CurrentSelectedUnit = unitRepo.Get(header.SelectedUnit.UnitId);
DataAttribute CurrentDataAttribute = new DataAttribute();
//If possible, map the chosen variable name, unit and datatype to an existing DataAttribute (Exact match)
DataAttribute existingDataAttribute = allDataAttributes.Where(da => da.Name.ToLower().Equals(TrimAndLimitString(header.SelectedDataAttribute.Name).ToLower()) &&
da.Unit.Dimension == CurrentSelectedUnit.Dimension).FirstOrDefault();
if (existingDataAttribute != null)
{
CurrentDataAttribute = existingDataAttribute;
}
else
{
//No matching DataAttribute => Create a new one
CurrentDataAttribute = dam.CreateDataAttribute(TrimAndLimitString(header.Name), header.Name, header.SelectedDataAttribute.Description, false, false, "", MeasurementScale.Categorial, DataContainerType.ReferenceType, "", dataType, CurrentSelectedUnit, null, null, null, null, null, null);
}
Variable newVariable = dsm.AddVariableUsage(sds, CurrentDataAttribute, true, header.Name, "", "", "", CurrentSelectedUnit);
VariableIdentifier vi = new VariableIdentifier
{
name = newVariable.Label,
id = newVariable.Id
};
identifiers.Add(vi);
XmlElement newVariableXml = xmldoc.CreateElement("variable");
newVariableXml.InnerText = Convert.ToString(newVariable.Id);
orderElement.AppendChild(newVariableXml);
}
extraElement.AppendChild(orderElement);
xmldoc.AppendChild(extraElement);
sds.Extra = xmldoc;
sds.Name = "generated import structure " + timestamp;
sds.Description = "automatically generated structured data structure by user " + GetUsernameOrDefault() + " for file " + title + " on " + timestamp;
#endregion Set Variables and information for new DataStructure
}
Dataset ds = null;
ds = dm.CreateEmptyDataset(sds, rp, metadataStructure);
long datasetId = ds.Id;
long sdsId = sds.Id;
if (dm.IsDatasetCheckedOutFor(datasetId, GetUsernameOrDefault()) || dm.CheckOutDataset(datasetId, GetUsernameOrDefault()))
{
DatasetVersion dsv = dm.GetDatasetWorkingCopy(datasetId);
long METADATASTRUCTURE_ID = metadataStructure.Id;
XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(XmlNodeMode.xPath);
XDocument metadataX = xmlMetadatWriter.CreateMetadataXml(METADATASTRUCTURE_ID);
XmlDocument metadataXml = XmlMetadataWriter.ToXmlDocument(metadataX);
dsv.Metadata = metadataXml;
try
{
dsv.Metadata = xmlDatasetHelper.SetInformation(dsv, metadataXml, NameAttributeValues.title, title);
dsv.Title = title;
}
catch (NullReferenceException ex)
{
//Reference of the title node is missing
throw new NullReferenceException("The extra-field of this metadata-structure is missing the title-node-reference!");
}
dm.EditDatasetVersion(dsv, null, null, null);
}
#region security
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast <RightType>())
{
//The user gets full permissions
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
entityPermissionManager.Create <User>(GetUsernameOrDefault(), "Dataset", typeof(Dataset), ds.Id, Enum.GetValues(typeof(RightType)).Cast <RightType>().ToList());
}
}
}
#endregion security
#region excel reader
int packageSize = 100000;
int numberOfRows = 0;
//HACK ?
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
{
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
}
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
counter++;
TaskManager.Bus[EasyUploadTaskManager.CURRENTPACKAGE] = counter;
//rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), oldSds, (int)id, packageSize).ToArray();
List <string> selectedDataAreaJsonArray = (List <string>)TaskManager.Bus[EasyUploadTaskManager.SHEET_DATA_AREA];
string selectedHeaderAreaJsonArray = TaskManager.Bus[EasyUploadTaskManager.SHEET_HEADER_AREA].ToString();
List <int[]> areaDataValuesList = new List <int[]>();
foreach (string area in selectedDataAreaJsonArray)
{
areaDataValuesList.Add(JsonConvert.DeserializeObject <int[]>(area));
}
int[] areaHeaderValues = JsonConvert.DeserializeObject <int[]>(selectedHeaderAreaJsonArray);
Orientation orientation = 0;
switch (TaskManager.Bus[EasyUploadTaskManager.SHEET_FORMAT].ToString())
{
case "LeftRight":
orientation = Orientation.rowwise;
break;
case "Matrix":
//orientation = Orientation.matrix;
break;
default:
orientation = Orientation.columnwise;
break;
}
String worksheetUri = null;
//Get the Uri to identify the correct worksheet
if (TaskManager.Bus.ContainsKey(EasyUploadTaskManager.ACTIVE_WOKSHEET_URI))
{
worksheetUri = TaskManager.Bus[EasyUploadTaskManager.ACTIVE_WOKSHEET_URI].ToString();
}
int batchSize = (new Object()).GetUnitOfWork().PersistenceManager.PreferredPushSize;
int batchnr = 1;
foreach (int[] areaDataValues in areaDataValuesList)
{
//First batch starts at the start of the current data area
int currentBatchStartRow = areaDataValues[0] + 1;
while (currentBatchStartRow <= areaDataValues[2] + 1) //While the end of the current data area has not yet been reached
{
//End row is start row plus batch size
int currentBatchEndRow = currentBatchStartRow + batchSize;
//Set the indices for the reader
EasyUploadFileReaderInfo fri = new EasyUploadFileReaderInfo
{
DataStartRow = currentBatchStartRow,
//End row is either at the end of the batch or the end of the marked area
//DataEndRow = (currentBatchEndRow > areaDataValues[2] + 1) ? areaDataValues[2] + 1 : currentBatchEndRow,
DataEndRow = Math.Min(currentBatchEndRow, areaDataValues[2] + 1),
//Column indices as marked in a previous step
DataStartColumn = areaDataValues[1] + 1,
DataEndColumn = areaDataValues[3] + 1,
//Header area as marked in a previous step
VariablesStartRow = areaHeaderValues[0] + 1,
VariablesStartColumn = areaHeaderValues[1] + 1,
VariablesEndRow = areaHeaderValues[2] + 1,
VariablesEndColumn = areaHeaderValues[3] + 1,
Offset = areaDataValues[1],
Orientation = orientation
};
//Create a new reader each time because the reader saves ALL tuples it read and therefore the batch processing wouldn't work
EasyUploadExcelReader reader = new EasyUploadExcelReader(sds, fri);
// open file
Stream = reader.Open(TaskManager.Bus[EasyUploadTaskManager.FILEPATH].ToString());
//Set variable identifiers because they might differ from the variable names in the file
reader.setSubmittedVariableIdentifiers(identifiers);
//Read the rows and convert them to DataTuples
rows = reader.ReadFile(Stream, TaskManager.Bus[EasyUploadTaskManager.FILENAME].ToString(), fri, (int)datasetId, worksheetUri);
//After reading the rows, add them to the dataset
if (rows != null)
{
dm.EditDatasetVersion(workingCopy, rows.ToList(), null, null);
numberOfRows += rows.Count();
}
//Close the Stream so the next ExcelReader can open it again
Stream.Close();
//Debug information
int lines = (areaDataValues[2] + 1) - (areaDataValues[0] + 1);
int batches = lines / batchSize;
batchnr++;
//Next batch starts after the current one
currentBatchStartRow = currentBatchEndRow + 1;
}
}
#endregion excel reader
//set modification
workingCopy.ModificationInfo = new EntityAuditInfo()
{
Performer = GetUsernameOrDefault(),
Comment = "Data",
ActionType = AuditActionType.Create
};
dm.EditDatasetVersion(workingCopy, null, null, null);
dm.CheckInDataset(ds.Id, "Import " + numberOfRows + " rows", GetUsernameOrDefault());
//Reindex search
if (this.IsAccessible("DDM", "SearchIndex", "ReIndexSingle"))
{
this.Run("DDM", "SearchIndex", "ReIndexSingle", new RouteValueDictionary()
{
{ "id", datasetId }
});
}
TaskManager.AddToBus(EasyUploadTaskManager.DATASET_ID, ds.Id);
return(temp);
}
}
catch (Exception ex)
{
temp.Add(new Error(ErrorType.Other, "An error occured during the upload. " +
"Please try again later. If this problem keeps occuring, please contact your administrator."));
return(temp);
}
finally
{
dsm.Dispose();
dm.Dispose();
dam.Dispose();
//sm.Dispose();
entityPermissionManager.Dispose();
}
}
public DataTable ConvertStructuredDataStructureToDataTable(StructuredDataStructure sds)
{
DataTable dt = new DataTable();
dt.TableName = "DataStruture";
dt.Columns.Add("VariableName");
dt.Columns.Add("Description");
dt.Columns.Add("Unit");
//dt.Columns.Add("Parameters");
dt.Columns.Add("DataType");
dt.Columns.Add("Category");
dt.Columns.Add("Optional");
dt.Columns.Add("VariableId");
using (var uow = this.GetUnitOfWork())
{
StructuredDataStructure datastructure = uow.GetReadOnlyRepository <StructuredDataStructure>().Get(sds.Id);
if (datastructure != null)
{
List <Variable> variables = SortVariablesOnDatastructure(datastructure.Variables.ToList(), datastructure);
foreach (Variable var in variables)
{
Variable sdvu = uow.GetReadOnlyRepository <Variable>().Get(var.Id);
DataRow dr = dt.NewRow();
if (sdvu.Label != null)
{
dr["VariableName"] = sdvu.Label;
}
else
{
dr["VariableName"] = "n/a";
}
dr["Optional"] = sdvu.IsValueOptional.ToString();
if (sdvu.Label != null)
{
dr["VariableId"] = sdvu.Id;
}
else
{
dr["VariableId"] = "n/a";
}
if (sdvu.DataAttribute.DataType != null)
{
dr["Category"] = sdvu.DataAttribute.ShortName;
}
else
{
dr["Category"] = "n/a";
}
//if (sdvu.Parameters.Count > 0) dr["Parameters"] = "current not shown";
//else dr["Parameters"] = "n/a";
if (sdvu.Description != null || sdvu.Description != "")
{
dr["Description"] = sdvu.Description;
}
else
{
dr["Description"] = "n/a";
}
if (sdvu.Unit != null)
{
dr["Unit"] = sdvu.Unit.Name;
}
else
{
dr["Unit"] = "n/a";
}
if (sdvu.DataAttribute.DataType != null)
{
dr["DataType"] = sdvu.DataAttribute.DataType.Name;
}
else
{
dr["DataType"] = "n/a";
}
dt.Rows.Add(dr);
}
}
return(dt);
}
}
//Add Units protected abstract bool AddUnits(StructuredDataStructure structure);
public DataTable ConvertPrimaryDataToDatatable(DatasetVersion dsv, IEnumerable <long> dsVersionTupleIds)
{
DataTable dt = new DataTable();
dt.TableName = "Primary data table";
DatasetManager datasetManager = new DatasetManager();
try
{
StructuredDataStructure sds = this.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>().Get(dsv.Dataset.DataStructure.Id);
if (dsVersionTupleIds != null && sds != null)
{
foreach (var vu in sds.Variables)
{
// use vu.Label or vu.DataAttribute.Name
DataColumn col = dt.Columns.Add("ID" + vu.Id.ToString()); // or DisplayName also
col.Caption = vu.Label;
switch (vu.DataAttribute.DataType.SystemType)
{
case "String":
{
col.DataType = Type.GetType("System.String");
break;
}
case "Double":
{
col.DataType = Type.GetType("System.Double");
break;
}
case "Int16":
{
col.DataType = Type.GetType("System.Int16");
break;
}
case "Int32":
{
col.DataType = Type.GetType("System.Int32");
break;
}
case "Int64":
{
col.DataType = Type.GetType("System.Int64");
break;
}
case "Decimal":
{
col.DataType = Type.GetType("System.Decimal");
break;
}
case "DateTime":
{
col.DataType = Type.GetType("System.DateTime");
break;
}
default:
{
col.DataType = Type.GetType("System.String");
break;
}
}
if (vu.Parameters.Count > 0)
{
foreach (var pu in vu.Parameters)
{
DataColumn col2 = dt.Columns.Add(pu.Label.Replace(" ", "")); // or DisplayName also
col2.Caption = pu.Label;
}
}
}
foreach (var id in dsVersionTupleIds)
{
DataTuple dataTuple = datasetManager.DataTupleRepo.Query(d => d.Id.Equals(id)).FirstOrDefault();
dataTuple.Materialize();
dt.Rows.Add(ConvertTupleIntoDataRow(dt, dataTuple, sds));
}
}
return(dt);
}
finally
{
datasetManager.Dispose();
}
}
public List <List <string> > GenerateRowsWithRandomValuesBasedOnDatastructure(StructuredDataStructure dataStructure, long numberOfTuples)
{
List <List <string> > rows = new List <List <string> >();
numberOfTuples.Should().BeGreaterThan(0);
var r = new Random();
try
{
for (int i = 0; i < numberOfTuples; i++)
{
List <string> row = new List <string>();
row.Add(r.Next().ToString());
row.Add("Test");
row.Add(Convert.ToDouble(r.Next()).ToString());
row.Add(true.ToString());
row.Add(DateTime.Now.ToString());
rows.Add(row);
}
return(rows);
}
catch (Exception ex)
{
return(null);
}
}
public DataStructureResultStruct(long dataStructureId)
{
using (DataStructureManager dataStructureManager = new DataStructureManager())
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (structuredDataStructure != null)
{
this.Id = structuredDataStructure.Id;
this.Title = structuredDataStructure.Name;
this.Description = structuredDataStructure.Description;
this.inUse = false;
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
foreach (Dataset d in structuredDataStructure.Datasets)
{
if (datasetManager.RowAny(d.Id))
{
this.inUse = true;
break;
}
else
{
foreach (DatasetVersion dv in d.Versions)
{
if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any())
{
this.inUse = true;
break;
}
}
}
}
}
finally
{
datasetManager.Dispose();
}
this.Structured = true;
this.Preview = false;
}
else
{
UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId);
if (unStructuredDataStructure != null)
{
this.Id = unStructuredDataStructure.Id;
this.Title = unStructuredDataStructure.Name;
this.Description = unStructuredDataStructure.Description;
if (unStructuredDataStructure.Datasets != null && unStructuredDataStructure.Datasets.Any())
{
this.inUse = true;
}
else
{
this.inUse = false;
}
this.Structured = false;
this.Preview = false;
}
else
{
new DataStructureResultStruct();
}
}
}
}
public ActionResult storeVariables(long Id, storeVariableStruct[] variables)
{
DataStructureManager dataStructureManager = null;
DataContainerManager dataContainerManager = null;
MissingValueManager missingValueManager = null;
UnitManager um = null;
try
{
dataStructureManager = new DataStructureManager();
missingValueManager = new MissingValueManager();
dataContainerManager = new DataContainerManager();
um = new UnitManager();
var structureRepo = dataStructureManager.GetUnitOfWork().GetReadOnlyRepository <StructuredDataStructure>();
StructuredDataStructure dataStructure = structureRepo.Get(Id);
MessageModel returnObject = new MessageModel();
MessageModel messageModel = MessageModel.validateDataStructureInUse(dataStructure.Id, dataStructure);
if (messageModel.hasMessage)
{
foreach (Variable v in dataStructure.Variables)
{
if (variables.Select(svs => svs.Id).ToList().Contains(v.Id))
{
v.Description = variables.Where(svs => svs.Id == v.Id).FirstOrDefault().Description;
dataStructure = dataStructureManager.UpdateStructuredDataStructure(dataStructure);
}
}
return(PartialView("_messageWindow", messageModel));
}
if (variables != null && variables.Any())
{
Variable variable = new Variable();
List <long> order = new List <long>();
foreach (Variable v in dataStructure.Variables)
{
if (!variables.Select(svs => svs.Id).ToList().Contains(v.Id))
{
List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(v.Id)).ToList();
foreach (MissingValue mv in missingValues)
{
missingValueManager.Delete(mv);
}
dataStructureManager.RemoveVariableUsage(v);
}
}
foreach (storeVariableStruct svs in variables.Where(svs => svs.Id == 0).ToList())
{
if (svs.Lable == null)
{
svs.Lable = "";
}
if (svs.Description == null)
{
svs.Description = "";
}
DataAttribute dataAttribute = dataContainerManager.DataAttributeRepo.Get(svs.AttributeId);
if (dataAttribute != null)
{
variable = dataStructureManager.AddVariableUsage(dataStructure, dataAttribute, svs.isOptional, svs.Lable.Trim(), null, null, svs.Description.Trim(), um.Repo.Get(svs.UnitId));
svs.Id = variable.Id;
foreach (MissingValueStruct mvs in svs.MissingValues)
{
if (mvs.Id == 0)
{
if (String.IsNullOrEmpty(mvs.Placeholder))
{
missingValueManager.Create(mvs.DisplayName, mvs.Description, variable);
}
else
{
missingValueManager.Create(mvs.DisplayName, mvs.Description, variable, mvs.Placeholder);
}
}
}
}
else
{
returnObject = new MessageModel()
{
hasMessage = true,
Message = "Not all Variables are stored.",
CssId = "0"
};
}
}
//dataStructure = structureRepo.Get(Id); // Javad: why it is needed?
variables = variables.Where(v => v.Id != 0).ToArray();
MissingValue missingValue = new MissingValue();
foreach (storeVariableStruct svs in variables.Where(svs => svs.Id != 0).ToList())
{
if (svs.Lable == null)
{
svs.Lable = "";
}
if (svs.Description == null)
{
svs.Description = "";
}
variable = dataStructure.Variables.Where(v => v.Id == svs.Id).FirstOrDefault();
if (variable != null)
{
variable.Label = svs.Lable.Trim();
variable.Description = svs.Description.Trim();
variable.Unit = um.Repo.Get(svs.UnitId);
variable.DataAttribute = dataContainerManager.DataAttributeRepo.Get(svs.AttributeId);
variable.IsValueOptional = svs.isOptional;
List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(svs.Id)).ToList();
foreach (MissingValue mv in missingValues)
{
if (!svs.MissingValues.Select(mvs => mvs.Id).Contains(mv.Id))
{
missingValueManager.Delete(mv);
}
}
foreach (MissingValueStruct mvs in svs.MissingValues)
{
if (mvs.Id == 0)
{
if (String.IsNullOrEmpty(mvs.Placeholder))
{
missingValueManager.Create(mvs.DisplayName, mvs.Description, variable);
}
else
{
missingValueManager.Create(mvs.DisplayName, mvs.Description, variable, mvs.Placeholder);
}
}
else if (mvs.Id > 0)
{
missingValue = missingValues.Where(mv => mv.Id.Equals(mvs.Id)).FirstOrDefault();
if (missingValue != null)
{
missingValue.DisplayName = mvs.DisplayName;
missingValue.Description = mvs.Description;
missingValue.Placeholder = mvs.Placeholder;
missingValueManager.Update(missingValue);
}
}
}
}
}
dataStructure = dataStructureManager.UpdateStructuredDataStructure(dataStructure);
DataStructureIO.setVariableOrder(dataStructure, variables.Select(svs => svs.Id).ToList());
}
else
{
foreach (Variable v in dataStructure.Variables)
{
List <MissingValue> missingValues = missingValueManager.Repo.Query(mv => mv.Variable.Id.Equals(v.Id)).ToList();
foreach (MissingValue mv in missingValues)
{
missingValueManager.Delete(mv);
}
dataStructureManager.RemoveVariableUsage(v);
}
}
LoggerFactory.LogCustom("Variables for Data Structure " + Id + " stored.");
return(Json(returnObject, JsonRequestBehavior.AllowGet));
}
finally
{
dataStructureManager.Dispose();
dataContainerManager.Dispose();
um.Dispose();
missingValueManager.Dispose();
}
}
private HttpResponseMessage getData(long id, int variableId, string token)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
DataStructureManager dataStructureManager = null;
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
dataStructureManager = new DataStructureManager();
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
object stats = new object();
DataTable dt = new DataTable("Varibales");
List <ApiDataStatisticModel> dataStatisticModels = new List <ApiDataStatisticModel>();
if (variableId == -1)
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id);
List <string> varIds = new List <string>();
foreach (Variable vs in structuredDataStructure.Variables)
{
varIds.Add("var" + vs.Id);
}
dt = GetDuplicates(id, varIds);
}
else
{
}
//dt.Strip();
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse(HttpStatusCode.OK);
string resp = JsonConvert.SerializeObject(model);
response.Content = new StringContent(resp, System.Text.Encoding.UTF8, "application/json");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return(response);
}
else
{
return(Request.CreateResponse());
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
dataStructureManager.Dispose();
}
}