/// <summary>
/// generate a text file with JSON from a datastructure of a dataset
/// and stored this file on the server
/// and store the path in the content discriptor
/// </summary>
/// <param name="datasetId"></param>
/// <returns>dynamic filepath</returns>
public static string GenerateDataStructure(long datasetId)
{
string path = "";
try
{
DatasetManager datasetManager = new DatasetManager();
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
DataStructureManager dataStructureManager = new DataStructureManager();
long dataStructureId = datasetVersion.Dataset.DataStructure.Id;
DataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (dataStructure != null)
{
}
// store in content descriptor
path = storeGeneratedFilePathToContentDiscriptor(datasetId, datasetVersion, "datastructure", ".txt");
}
catch (Exception ex)
{
throw ex;
}
return path;
}
public ActionResult deleteDataStructure(long Id, string cssId = "")
{
MessageModel DataStructureDeleteValidation = MessageModel.validateDataStructureDelete(Id);
if (DataStructureDeleteValidation.hasMessage)
{
return PartialView("_messageWindow", DataStructureDeleteValidation);
}
else
{
DataStructureManager dataStructureManager = new DataStructureManager();
if (dataStructureManager.StructuredDataStructureRepo.Get(Id) != null)
{
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id);
DataStructureIO.deleteTemplate(dataStructure.Id);
foreach (Variable v in dataStructure.Variables)
{
dataStructureManager.RemoveVariableUsage(v);
}
dataStructureManager.DeleteStructuredDataStructure(dataStructure);
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Deleted);
}
else
{
UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id);
dataStructureManager.DeleteUnStructuredDataStructure(dataStructure);
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Deleted);
}
return PartialView("_message", new MessageModel()
{
Message = "DataStructure" + Id + "deleted",
hasMessage = false,
CssId = "deleted"
});
}
}
private static XmlDocument createOderNode(StructuredDataStructure structuredDataStructure)
{
DataStructureManager dsm = new DataStructureManager();
XmlDocument doc = (XmlDocument)structuredDataStructure.Extra;
XmlNode order;
if (doc == null)
{
doc = new XmlDocument();
XmlNode root = doc.CreateNode(XmlNodeType.Element, "extra", null);
doc.AppendChild(root);
}
if (doc.GetElementsByTagName("order").Count == 0)
{
if (structuredDataStructure.Variables.Count > 0)
{
order = doc.CreateNode(XmlNodeType.Element, "order", null);
foreach (Variable v in structuredDataStructure.Variables)
{
XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null);
variable.InnerText = v.Id.ToString();
order.AppendChild(variable);
}
doc.FirstChild.AppendChild(order);
structuredDataStructure.Extra = doc;
dsm.UpdateStructuredDataStructure(structuredDataStructure);
}
}
return doc;
}
/// <summary>
/// wandelt bexis primardaten (rows) in bexis 2 datatuples um
/// </summary>
/// <param name="data"></param>
/// <param name="dataStructureId"></param>
/// <param name="observationIndex"></param>
/// <returns></returns>
public DataTuple XmlRowReader(XmlDocument data, long dataStructureId, int observationIndex)
{
DataStructureManager dataStructureManager = new DataStructureManager();
DataTuple dataRow = new DataTuple();
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
this.StructuredDataStructure = dataStructure;
List<VariableIdentifier> variableIdentifierList = new List<VariableIdentifier>();
foreach (Variable variable in dataStructure.Variables)
{
VariableIdentifier vi = new VariableIdentifier();
vi.id = variable.Id;
vi.name = variable.Label;
vi.systemType = variable.DataAttribute.DataType.SystemType.ToString();
variableIdentifierList.Add(vi);
}
this.SubmitedVariableIdentifiers = variableIdentifierList;
List<string> dataList = XmlRowToStringList(data, variableIdentifierList);
int startRow = 0;
int indexOfRow = startRow + observationIndex;
//List<Error> errors = ValidateRow(dataList, indexOfRow);
//if (errors.Count == 0)
{
dataRow = ReadRow(dataList, indexOfRow);
}
return dataRow;
}
public ActionResult copyDataStructure(long Id, bool isStructured, string Name = "" , string Description = "", string cssId = "")
{
Name = Server.UrlDecode(Name);
Description = Server.UrlDecode(Description);
DataStructureManager dataStructureManager = new DataStructureManager();
if (!isStructured)
{
UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id);
if (dataStructure != null)
{
if (Name == "")
{
Name = dataStructure.Name + " - Copy";
}
if (Description == "" && dataStructure.Description != null)
{
Description = dataStructure.Description;
}
LoggerFactory.LogCustom("Copy Data Structure" + Id);
return createDataStructure(0, Name.Trim(), isStructured, Description.Trim(), cssId);
}
}
else
{
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id);
if (dataStructure != null)
{
if (Name == "")
{
Name = dataStructure.Name + " - Copy";
}
if (Description == "" && dataStructure.Description != null)
{
Description = dataStructure.Description;
}
MessageModel messageModel = storeDataStructure(0, Name.Trim(), isStructured, Description.Trim(), cssId);
List<long> order = new List<long>();
Variable variable = new Variable();
if (!messageModel.hasMessage)
{
StructuredDataStructure dataStructureCopy = dataStructureManager.StructuredDataStructureRepo.Get(Convert.ToInt64(messageModel.Message));
foreach (Variable v in DataStructureIO.getOrderedVariables(dataStructure))
{
variable = dataStructureManager.AddVariableUsage(dataStructureCopy, v.DataAttribute, v.IsValueOptional, v.Label.Trim(), v.DefaultValue, v.MissingValue, v.Description.Trim(), v.Unit);
order.Add(variable.Id);
}
DataStructureIO.setVariableOrder(dataStructureCopy, order);
}
LoggerFactory.LogCustom("Copy Data Structure" + Id);
return PartialView("_message", messageModel);
}
}
return PartialView("_message", new MessageModel());
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
public DataStructureTree()
{
dataStructureTreeList = new List<DataStructureTreeList>();
DataStructureManager dataStructureManager = new DataStructureManager();
DataStructureTreeList treeList = new DataStructureTreeList();
if(dataStructureManager.StructuredDataStructureRepo.Get() != null)
treeList = new DataStructureTreeList(dataStructureManager.StructuredDataStructureRepo.Get());
dataStructureTreeList.Add(treeList);
if (dataStructureManager.UnStructuredDataStructureRepo.Get() !=null)
treeList = new DataStructureTreeList(dataStructureManager.UnStructuredDataStructureRepo.Get());
dataStructureTreeList.Add(treeList);
}
public static XmlDocument setVariableOrder(StructuredDataStructure structuredDataStructure, List<long> orderList)
{
DataStructureManager dsm = new DataStructureManager();
XmlDocument doc = createOderNode(structuredDataStructure);
XmlNode order = doc.GetElementsByTagName("order")[0];
doc.FirstChild.RemoveChild(order);
order = doc.CreateNode(XmlNodeType.Element, "order", null);
foreach (long l in orderList)
{
XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null);
variable.InnerText = l.ToString();
order.AppendChild(variable);
}
doc.FirstChild.AppendChild(order);
structuredDataStructure.Extra = doc;
dsm.UpdateStructuredDataStructure(structuredDataStructure);
return doc;
}
public FileResult downloadTemplate(long id)
{
if (id != 0)
{
DataStructureManager dataStructureManager = new DataStructureManager();
StructuredDataStructure dataStructure = new StructuredDataStructure();
dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(id);
if (dataStructure != null)
{
ExcelTemplateProvider provider = new ExcelTemplateProvider("BExISppTemplate_Clean.xlsm");
provider.CreateTemplate(dataStructure);
string path = "";
XmlNode resources = dataStructure.TemplatePaths.FirstChild;
XmlNodeList resource = resources.ChildNodes;
foreach (XmlNode x in resource)
{
if (x.Attributes.GetNamedItem("Type").Value == "Excel")
path = x.Attributes.GetNamedItem("Path").Value;
}
string rgxPattern = "[<>?\":|\\\\/*]";
string rgxReplace = "-";
Regex rgx = new Regex(rgxPattern);
string filename = rgx.Replace(dataStructure.Name, rgxReplace);
if (filename.Length > 50)
filename = filename.Substring(0, 50);
return File(Path.Combine(AppConfiguration.DataPath, path), "application/xlsm", "Template_" + dataStructure.Id + "_" + filename + ".xlsm");
}
}
return File(Path.Combine(AppConfiguration.GetModuleWorkspacePath("RPM"), "Template", "BExISppTemplate_Clean.xlsm"), "application/xlsm", "Template_" + id + "_No_Data_Structure.xlsm");
}
public long createDataset(string dataSetID, XmlDocument metadataXml, long metadataStructureId, DataStructure dataStructure, string researchPlanName)
{
Dataset dataset = new Dataset();
DatasetManager datasetManager = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
ResearchPlanManager researchPlanManager = new ResearchPlanManager();
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
PermissionManager permissionManager = new PermissionManager();
SubjectManager subjectManager = new SubjectManager();
// get existing researchPlan
ResearchPlan researchPlan = researchPlanManager.Repo.Get(r => researchPlanName.Equals(r.Title)).FirstOrDefault();
// get existing metadataStructure (created manualy by using edited Bexis1_XSD)
MetadataStructure metadataStructure = metadataStructureManager.Repo.Get(metadataStructureId);
// update verwendet, da bei unstructured structures im dataset constructor dataset(dataStructure) nullexception
dataset = datasetManager.CreateEmptyDataset(dataStructure, researchPlan, metadataStructure);
long datasetId = dataset.Id;
// dataPermission
List<string> usersWithDataPerm = new List<string>();
// create dataPermission for originalDatasetManager
string originalDatasetManager = metadataXml.SelectSingleNode("Metadata/general/general/originalDatasetManager/originalDatasetManager").InnerText;
User userODM = subjectManager.UsersRepo.Get(u => originalDatasetManager.Equals(u.FullName)).FirstOrDefault();
permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Create);
permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.View);
permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Update);
permissionManager.CreateDataPermission(userODM.Id, 1, dataset.Id, RightType.Download);
usersWithDataPerm.Add(originalDatasetManager);
// create dataPermissions for metadataAuthor
string metadataAuthor = metadataXml.SelectSingleNode("Metadata/general/general/metadataCreator/metadataCreator").InnerText;
if (!usersWithDataPerm.Contains(metadataAuthor))
{
User userMA = subjectManager.UsersRepo.Get(u => metadataAuthor.Equals(u.FullName)).FirstOrDefault();
permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Create);
permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.View);
permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Update);
permissionManager.CreateDataPermission(userMA.Id, 1, dataset.Id, RightType.Download);
usersWithDataPerm.Add(metadataAuthor);
}
// create dataPermissions for designatedDatasetManager
string designatedDatasetManager = metadataXml.SelectSingleNode("Metadata/general/general/designatedDatasetManager/contactType/designatedDatasetManagerName/designatedDatasetManagerName").InnerText;
if (!usersWithDataPerm.Contains(designatedDatasetManager))
{
User userDDM = subjectManager.UsersRepo.Get(u => designatedDatasetManager.Equals(u.FullName)).FirstOrDefault();
permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Create);
permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.View);
permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Update);
permissionManager.CreateDataPermission(userDDM.Id, 1, dataset.Id, RightType.Download);
usersWithDataPerm.Add(designatedDatasetManager);
}
// create dataPermissions for owners
XmlNodeList ownerNodes = metadataXml.SelectNodes("Metadata/general/general/owners/ownerType/owner/owner");
foreach (XmlNode ownerNode in ownerNodes)
{
string owner = ownerNode.InnerText;
if (!usersWithDataPerm.Contains(owner))
{
User userO = subjectManager.UsersRepo.Get(u => owner.Equals(u.FullName)).FirstOrDefault();
if (userO != null)
{
permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Create);
permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.View);
permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Update);
permissionManager.CreateDataPermission(userO.Id, 1, dataset.Id, RightType.Download);
usersWithDataPerm.Add(owner);
}
}
}
// integrate metadataXml to dataset
// checkOut
if (datasetManager.IsDatasetCheckedOutFor(datasetId, userODM.Name) || datasetManager.CheckOutDataset(datasetId, userODM.Name))
{
DatasetVersion workingCopy = datasetManager.GetDatasetWorkingCopy(datasetId); // get dataset
workingCopy.Metadata = metadataXml; // set metadata to dataset
datasetManager.EditDatasetVersion(workingCopy, null, null, null); // edit dataset
datasetManager.CheckInDataset(datasetId, "Metadata was submited.", userODM.Name); // check in
}
return dataset.Id;
}
public List<ListViewItem> LoadDataStructureViewList( DataStructureType dataStructureType )
{
DataStructureManager dsm = new DataStructureManager();
List<ListViewItem> temp = new List<ListViewItem>();
foreach (DataStructure datasStructure in dsm.StructuredDataStructureRepo.Get())
{
string title = datasStructure.Name;
temp.Add(new ListViewItem(datasStructure.Id, title));
}
return temp.OrderBy(p => p.Title).ToList();
}
public long DatasetExistsByOldDatasetId(string dataSetID, long dataStructureId)
{
DatasetManager datasetManager = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
List<Dataset> datasetsWithCurrentDataStructure = new List<Dataset>();
datasetsWithCurrentDataStructure = datasetManager.DatasetRepo.Get(d => dataStructureId.Equals(d.DataStructure.Id)).ToList();
if (datasetsWithCurrentDataStructure != null && datasetsWithCurrentDataStructure.Count > 0)
{
foreach (Dataset datasetWCDS in datasetsWithCurrentDataStructure)
{
string oldDatasetId = "";
try
{
XmlNode extraID = datasetWCDS.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id");
oldDatasetId = extraID.InnerText;
}
catch
{
}
if (oldDatasetId == dataSetID)
{
return datasetWCDS.Id;
}
}
}
return -999;
}
// create unstructured DataStructures
public DataStructure CreateDataStructure(string fileType)
{
DataStructureManager dataStructureManager = new DataStructureManager();
UnStructuredDataStructure dataStructure = new UnStructuredDataStructure();
// values of DataStructure
string name = fileType;
string description = "old BExIS unstrctured data structure: " + fileType;
UnStructuredDataStructure existDS = dataStructureManager.UnStructuredDataStructureRepo.Get(s => name.Equals(s.Name)).FirstOrDefault();
if (existDS == null)
{
// create dataStructure
return dataStructureManager.CreateUnStructuredDataStructure(name, description);
}
else
{
return existDS;
}
}
public List<ListViewItem> LoadDatasetVersionViewList( DataStructureType dataStructureType)
{
PermissionManager permissionManager = new PermissionManager();
SubjectManager subjectManager = new SubjectManager();
// add security
ICollection<long> datasetIDs = permissionManager.GetAllDataIds(subjectManager.GetUserByName(GetUsernameOrDefault()).Id, 1, RightType.Update).ToList();
DataStructureManager dataStructureManager = new DataStructureManager();
DatasetManager dm = new DatasetManager();
Dictionary<long, XmlDocument> dmtemp = new Dictionary<long, XmlDocument>();
dmtemp = dm.GetDatasetLatestMetadataVersions();
List<ListViewItem> temp = new List<ListViewItem>();
if (dataStructureType.Equals(DataStructureType.Structured))
{
List<StructuredDataStructure> list = dataStructureManager.StructuredDataStructureRepo.Get().ToList();
foreach (StructuredDataStructure sds in list)
{
sds.Materialize();
foreach (Dataset d in sds.Datasets)
{
if (dm.IsDatasetCheckedIn(d.Id))
{
if (datasetIDs.Contains(d.Id))
{
temp.Add(new ListViewItem(d.Id,
XmlDatasetHelper.GetInformation(dm.GetDatasetLatestVersion(d),
NameAttributeValues.title)));
}
}
}
}
}
else
{
List<UnStructuredDataStructure> list = dataStructureManager.UnStructuredDataStructureRepo.Get().ToList();
foreach (UnStructuredDataStructure sds in list)
{
foreach (Dataset d in sds.Datasets)
{
if (datasetIDs.Contains(d.Id))
{
if (dm.IsDatasetCheckedIn(d.Id))
{
DatasetVersion datasetVersion = dm.GetDatasetLatestVersion(d);
temp.Add(new ListViewItem(d.Id,
XmlDatasetHelper.GetInformation(datasetVersion, NameAttributeValues.title)));
}
}
}
}
}
return temp.OrderBy(p => p.Title).ToList();
}
public StructuredDataStructure GetDataStructureByID(long ID)
{
DataStructureManager dsm = new DataStructureManager();
StructuredDataStructure structuredDataStructure = dsm.StructuredDataStructureRepo.Get(ID);
this.dataStructure = structuredDataStructure;
if (this.dataStructure != null)
{
this.variableStructs = getOrderedVariableStructs(structuredDataStructure);
if (this.dataStructure.Datasets == null)
{
inUse = false;
}
else
{
if (this.dataStructure.Datasets.Count > 0)
{
inUse = true;
}
else
{
inUse = false;
}
}
this.BuildDataTable();
return (structuredDataStructure);
}
else
{
this.dataStructure = new StructuredDataStructure();
return (structuredDataStructure);
}
}
public DataStructure CreateDataStructure(string dataSetID, DataTable mapVariables, List<string> variableNames)
{
DataStructureManager dataStructureManager = new DataStructureManager();
DataContainerManager attributeManager = new DataContainerManager();
StructuredDataStructure dataStructure = new StructuredDataStructure();
UnitManager unitManager = new UnitManager();
// values of DataStructure
ExcelTemplateProvider provider = new ExcelTemplateProvider();
string name = "oldBExIS" + dataSetID;
string description = "old BExIS datastructure, created for " + dataSetID + ", possibly used by other"; //metadata.Title;
string xsdFileName = "";//"xsdFileName";
string xslFileName = "";//"xslFileName";
DataStructureCategory indexerType = DataStructureCategory.Generic;
// if dataStructure not exists
StructuredDataStructure existDS = existingDataStructures(mapVariables, dataSetID);
if (existDS.Name == null)
{
// create dataStructure
dataStructure = dataStructureManager.CreateStructuredDataStructure(name, description, xsdFileName, xslFileName, indexerType, null);
foreach (string varName in variableNames)
{
// values of variables
string attName = "";
string convFactor = "";
string varDescription = "";
int Block = -999;
long AttributeId = -999, UnitId = -999, VarUnitId = -999;
foreach (DataRow mapRow in mapVariables.Select("DatasetId = '" + dataSetID + "'"))
{
if (mapRow["Name"].ToString() == varName)
{
attName = mapRow["Attribute"].ToString();
convFactor = mapRow["ConvFactor"].ToString();
varDescription = mapRow["Description"].ToString();
Block = int.Parse(mapRow["Block"].ToString());
if (attName.Length > 1) // if not mapped yet
{
AttributeId = Convert.ToInt64(mapRow["AttributeId"].ToString());
UnitId = Convert.ToInt64(mapRow["UnitId"].ToString());
if (mapRow["Unit"].ToString().Length > 0)
VarUnitId = Convert.ToInt64(mapRow["VarUnitId"].ToString());
}
}
}
if (AttributeId > 0 && Block == 0) // if not mapped yet AND variable is in block 0
{
// find existing attribute for each variable
DataAttribute attribute = attributeManager.DataAttributeRepo.Get(AttributeId);
Unit varUnit = null;
if (VarUnitId > 0)
{
varUnit = unitManager.Repo.Get(VarUnitId);
}
// add variables to dataStructure
Variable variable = dataStructureManager.AddVariableUsage(dataStructure, attribute, true, varName, null, null, varDescription, varUnit);
dataStructure.Variables.Add(variable);
}
}
provider.CreateTemplate(dataStructure);
return dataStructure;
}
else
{
return existDS;
}
}
public DataStructurePreviewModel fill(long dataStructureId)
{
if (dataStructureId > 0)
{
DataStructureManager dataStructureManager = new DataStructureManager();
if (dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId) != null)
{
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
VariablePreviewStruct variablePreview;
this.Id = dataStructure.Id;
this.Name = dataStructure.Name;
this.Description = dataStructure.Description;
if(dataStructure.Datasets.Count > 0)
{
this.inUse = true;
}
foreach (Variable v in DataStructureIO.getOrderedVariables(dataStructure))
{
variablePreview = new VariablePreviewStruct().fill(v);
this.VariablePreviews.Add(variablePreview);
}
}
else if (dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId) != null)
{
UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId);
this.Id = dataStructure.Id;
this.Name = dataStructure.Name;
this.Description = dataStructure.Description;
this.VariablePreviews = null;
if(dataStructure.Datasets.Count > 0)
{
this.inUse = true;
}
}
return this;
}
else
{
return new DataStructurePreviewModel();
}
}
public List<Error> FinishUpload2(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
//datatuple list
List<DataTuple> rows;
DatasetManager dm = new DatasetManager();
Dataset ds = dm.GetDataset(id);
DatasetVersion workingCopy = new DatasetVersion();
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
//Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
}
} while (rows.Count() > 0);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
// start download generator
// filepath
//string path = "";
//if (workingCopy != null)
//{
// path = GenerateDownloadFile(workingCopy);
// dm.EditDatasetVersion(workingCopy, null, null, null);
//}
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated);
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
workingCopy = dm.GetDatasetLatestVersion(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
return temp;
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="dataStructureId"></param>
/// <param name="extention"></param>
/// <returns></returns>
public string GetDataStructureTemplatePath(long dataStructureId, string extention)
{
DataStructureManager dataStructureManager = new DataStructureManager();
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
string dataStructureTitle = dataStructure.Name;
// load datastructure from db an get the filepath from this object
ExcelTemplateProvider provider = new ExcelTemplateProvider("BExISppTemplate_Clean.xlsm");
string path = "";
if (dataStructure.TemplatePaths != null)
{
XmlNode resources = dataStructure.TemplatePaths.FirstChild;
XmlNodeList resource = resources.ChildNodes;
foreach (XmlNode x in resource)
{
if (x.Attributes.GetNamedItem("Type").Value == "Excel")
if (File.Exists(x.Attributes.GetNamedItem("Path").Value))
{
path = x.Attributes.GetNamedItem("Path").Value;
}
else
{
path = provider.CreateTemplate(dataStructure);
}
}
//string dataPath = AppConfiguration.DataPath; //Path.Combine(AppConfiguration.WorkspaceRootPath, "Data");
return Path.Combine(AppConfiguration.DataPath, path);
}
path = provider.CreateTemplate(dataStructure);
return Path.Combine(AppConfiguration.DataPath, path);
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="ID"></param>
/// <param name="structured"></param>
/// <returns></returns>
public DataStructure GetDataStructureByID(long ID, bool structured)
{
this.structured = structured;
if (structured)
{
return this.GetDataStructureByID(ID);
}
else
{
DataStructureManager dsm = new DataStructureManager();
UnStructuredDataStructure unStructuredDataStructure = dsm.UnStructuredDataStructureRepo.Get(ID);
this.dataStructure = unStructuredDataStructure;
if (this.dataStructure != null)
{
this.variableStructs = null;
if (this.dataStructure.Datasets == null)
{
inUse = false;
}
else
{
if (this.dataStructure.Datasets.Count > 0)
{
inUse = true;
//DatasetListElement datasetListElement = new DatasetListElement();
//DatasetManager dm = new DatasetManager();
//foreach (Dataset d in unStructuredDataStructure.Datasets)
//{
// datasetListElement = new DatasetListElement(d.Id, XmlDatasetHelper.GetInformation(d,AttributeNames.title));
// datasets.Add(datasetListElement);
//}
}
else
{
inUse = false;
}
}
return (unStructuredDataStructure);
}
else
{
this.dataStructure = new StructuredDataStructure();
return (unStructuredDataStructure);
}
}
}
/// <summary>
/// Read Row and convert each value into a variableValue
/// and each row to a Datatuple
/// </summary>
/// <param name="row">List of values in one row</param>
/// <param name="indexOfRow">Currently row index</param>
/// <returns>DataTuple</returns>
public DataTuple ReadRow(List<string> row, int indexOfRow)
{
DataTuple dt = new DataTuple();
DataStructureManager dsm = new DataStructureManager();
string value = "";
// convert row to List<VariableValue>
for(int i=0;i< row.Count();i++ )
{
VariableIdentifier variableIdentifier = this.SubmitedVariableIdentifiers.ElementAt(i);
long variableId = 0;
if (variableIdentifier.id > 0)
variableId = this.SubmitedVariableIdentifiers.ElementAt(i).id;
else
variableId = getVariableUsage(variableIdentifier).Id;
// if variable from systemtype datatime
// maybee needs to convert into the default datetime culture format
if (this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("DateTime"))
{
value = IOUtility.ConvertDateToCulture(row[i]);
}
else
{
if (this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Double")||
this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Decimal")||
this.StructuredDataStructure.Variables.Where(p => p.Id.Equals(variableId)).FirstOrDefault().DataAttribute.DataType.SystemType.Equals("Float"))
{
value = row[i];
if (Info.Decimal.Equals(DecimalCharacter.comma))
{
if(value.Contains(".")) value = value.Replace(".","");
if(value.Contains(",")) value = value.Replace(',', '.');
}
if (Info.Decimal.Equals(DecimalCharacter.point))
{
if (value.Contains(",")) value = value.Remove(',');
}
}
else
{
value = row[i];
}
}
dt.VariableValues.Add(DatasetManager.CreateVariableValue(value,"", DateTime.Now, DateTime.Now, new ObtainingMethod(), variableId, new List<ParameterValue>()));
}
return dt;
}
/// <summary>
/// CAUTION !!!!!!!!!!!!!!!!!
/// upload bezieht sich derzeit nur auf daten mit einem block (einer tabelle)
/// umsetzung von mehreren blöckes noch nicht geklärt.
/// (für einen wert gibt es eine weitere tabelle)
/// </summary>
/// <param name="dataSetID"></param>
/// <param name="DataBase"></param>
public void uploadData(string dataSetID, string DataBase)
{
DatasetManager datasetManager = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
XmlDataReader xmlDataReader = new XmlDataReader();
User user = new User();
string variableNames = "";
// query metadataAuthor and variable names from explorer.datasets
queryAuthorAndVariables(ref user, ref variableNames, dataSetID, DataBase);
List<string> varNames = variableNames.Split(',').ToList();
// get all dataStructures with equal variables count
List<StructuredDataStructure> dataStructures = dataStructureManager.StructuredDataStructureRepo.Get(s =>
varNames.Count().Equals(s.Variables.Count)).ToList();
// get all Ids of dataStructures with equal variables
List<long> dataStructureIds = new List<long>();
foreach (StructuredDataStructure dataStructure in dataStructures)
{
bool isSimilarStructure = true;
foreach (Variable variable in dataStructure.Variables)
{
if (!varNames.Contains(variable.Label))
{
isSimilarStructure &= false;
break;
}
}
if (isSimilarStructure)
dataStructureIds.Add(dataStructure.Id);
}
// get the wanted dataset by comparing the old B1 datasetId out of the datasets with similar dataStructure
Dataset dataset = null;
List<Dataset> datasets = datasetManager.DatasetRepo.Get(d => dataStructureIds.Contains(d.DataStructure.Id)).ToList();
foreach (Dataset ds in datasets)
{
string oldDatasetId = "";
try
{
XmlNode extraID = ds.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id");
oldDatasetId = extraID.InnerText;
}
catch
{
}
if (oldDatasetId == dataSetID)
{
dataset = ds;
break;
}
}
if (dataset != null)
{
// get distinct and ascending ordered insertdates from DB
List<DB2TimeStamp> distInsertDates = queryDistInsertDates(dataSetID, DataBase);
bool checkObsIds = false;
foreach (DB2TimeStamp insertDate in distInsertDates)
{
List<DataTuple> createdDataTuples = new List<DataTuple>();
List<DataTuple> editedDataTuples = new List<DataTuple>();
List<DataTuple> deletedDataTuples = new List<DataTuple>();
DatasetVersion workingCopy = datasetManager.GetDatasetLatestVersion(dataset.Id); // get dataset
// get obsid, data, deleted and newest for each observation from DB
List<Observation> observations = queryObservation(dataSetID, DataBase, insertDate.ToString());
Dictionary<long, long> obsIdMapsToTupleId = new Dictionary<long, long>();
if (checkObsIds)
{
// get all EffectiveTupleIds
List<long> datasetTupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(workingCopy).ToList();
// id-mapping-list key=obsId, value=TupleId foreach EffectiveTupleId and obsId from Tuple.Extra
obsIdMapsToTupleId = idMapping(datasetTupleIds, ref datasetManager);
}
// observation counter
int observationIndex = 0;
bool isNewest = true;
foreach (Observation observation in observations) //////////////parallel
{
isNewest = (isNewest && observation.newest != 'Y') ? false : isNewest;
// create dataTuple with xmlDataReader
// split xml to string list and use DataReader.ReadRow
DataTuple dataRow = xmlDataReader.XmlRowReader(observation.data, dataset.DataStructure.Id, observationIndex);
// check if observation.obsid is in id-mapping-list
long TupleId;
if (checkObsIds && obsIdMapsToTupleId.TryGetValue(observation.obsid, out TupleId))
{
DataTuple dataTuple = datasetManager.DataTupleRepo.Get(TupleId);
if (observation.deleted != 'Y')
{
dataTuple.VariableValues = dataRow.VariableValues;
// edit tuple if observation exists as tuple in EffectiveTuple and observation is not deleted
editedDataTuples.Add(dataTuple);
}
else
{
// delete tuple if observation exists as tuple in EffectiveTuple and observ. is deleted
deletedDataTuples.Add(dataTuple);
}
}
else
{
// write the obsId and oldBExISdatasetId in Extra: <extra><obsid>[obsid]</obsid><oldBExISdatasetId>[dataSetID]</oldBExISdatasetId></extra>
dataRow.Extra = oldIdsIntoExtra(observation.obsid, dataSetID, dataRow.Extra);
// create tuple if observation exists not in EffectiveTuple
createdDataTuples.Add(dataRow);
}
observationIndex++; // observation counter
}
checkObsIds = (!checkObsIds && !isNewest) ? true : checkObsIds;
// checkOut
if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, user.Name) || datasetManager.CheckOutDataset(dataset.Id, user.Name))
{
workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id); // get dataset
datasetManager.EditDatasetVersion(workingCopy, createdDataTuples, editedDataTuples, deletedDataTuples); // edit dataset
datasetManager.CheckInDataset(dataset.Id, "Primary data row was submited.", user.Name); // checkIn
}
}
}
}
public List<Variable> getOrderedVariables(StructuredDataStructure structuredDataStructure)
{
DataStructureManager dsm = new DataStructureManager();
XmlDocument doc = (XmlDocument)structuredDataStructure.Extra;
XmlNode order;
if (doc == null)
{
doc = new XmlDocument();
XmlNode root = doc.CreateNode(XmlNodeType.Element, "extra", null);
doc.AppendChild(root);
}
if (doc.GetElementsByTagName("order").Count == 0)
{
if (structuredDataStructure.Variables.Count > 0)
{
order = doc.CreateNode(XmlNodeType.Element, "order", null);
foreach (Variable v in structuredDataStructure.Variables)
{
XmlNode variable = doc.CreateNode(XmlNodeType.Element, "variable", null);
variable.InnerText = v.Id.ToString();
order.AppendChild(variable);
}
doc.FirstChild.AppendChild(order);
structuredDataStructure.Extra = doc;
dsm.UpdateStructuredDataStructure(structuredDataStructure);
}
}
order = doc.GetElementsByTagName("order")[0];
List<Variable> orderedVariables = new List<Variable>();
if (structuredDataStructure.Variables.Count != 0)
{
foreach (XmlNode x in order)
{
foreach (Variable v in structuredDataStructure.Variables)
{
if (v.Id == Convert.ToInt64(x.InnerText))
orderedVariables.Add(v);
}
}
}
return orderedVariables;
}
public MessageModel storeDataStructure(long Id, string Name, bool isStructured, string Description ="", string cssId = "", bool inUse = false)
{
Name = Server.UrlDecode(Name);
Description = Server.UrlDecode(Description);
MessageModel DataStructureValidation = MessageModel.validateDataStructureInUse(Id);
if (DataStructureValidation.hasMessage && inUse == false)
{
return DataStructureValidation;
}
else
{
DataStructureValidation = MessageModel.validateDataStructureName(Id, Name, cssId);
if (DataStructureValidation.hasMessage)
{
return DataStructureValidation;
}
else
{
DataStructureManager dataStructureManager = new DataStructureManager();
DataStructure dataStructure;
if (isStructured)
{
if (Id == 0)
{
dataStructure = dataStructureManager.CreateStructuredDataStructure(Name.Trim(), Description.Trim(), null, null, DataStructureCategory.Generic);
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created);
return new MessageModel()
{
Message = dataStructure.Id.ToString(),
hasMessage = false,
CssId = "redirect"
};
}
else
{
StructuredDataStructure StructuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(Id);
StructuredDataStructure.Name = Name;
StructuredDataStructure.Description = Description;
dataStructure = dataStructureManager.UpdateStructuredDataStructure(StructuredDataStructure);
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created);
return new MessageModel()
{
Message = Id.ToString(),
hasMessage = false,
CssId = "redirect"
};
}
}
else
{
if (Id == 0)
{
dataStructure = dataStructureManager.CreateUnStructuredDataStructure(Name.Trim(), Description.Trim());
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created);
return new MessageModel()
{
Message = "refresh DataStructureResultGrid",
hasMessage = false,
CssId = "refresh"
};
}
else
{
UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(Id);
unStructuredDataStructure.Name = Name;
unStructuredDataStructure.Description = Description;
dataStructure = dataStructureManager.UpdateUnStructuredDataStructure(unStructuredDataStructure);
LoggerFactory.LogData(dataStructure.Id.ToString(), typeof(DataStructure).Name, Vaiona.Entities.Logging.CrudState.Created);
return new MessageModel()
{
Message = "refresh DataStructureResultGrid",
hasMessage = false,
CssId = "refresh"
};
}
}
}
}
}
private Dataset createDataset()
{
DataStructureManager dsManager = new DataStructureManager();
ResearchPlanManager rpManager = new ResearchPlanManager();
DatasetManager dm = new DatasetManager();
MetadataStructureManager mdsManager = new MetadataStructureManager();
MDS.MetadataStructure mds = mdsManager.Repo.Query().First();
Dataset ds = dm.CreateEmptyDataset(dsManager.StructuredDataStructureRepo.Get(1), rpManager.Repo.Get(1), mds);
return ds;
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="id"></param>
/// <returns></returns>
protected StructuredDataStructure GetDataStructure(long id)
{
if (dataStructure == null)
{
DataStructureManager dataStructureManager = new DataStructureManager();
dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(id);
}
return dataStructure;
}
/// <summary>
/// create a new dataset, check it out to create the first version, add a tuple to it.
/// </summary>
/// <returns></returns>
private Int64 createDatasetVersion()
{
DataStructureManager dsManager = new DataStructureManager();
ResearchPlanManager rpManager = new ResearchPlanManager();
DatasetManager dm = new DatasetManager();
MetadataStructureManager mdsManager = new MetadataStructureManager();
MDS.MetadataStructure mds = mdsManager.Repo.Query().First();
Dataset ds = dm.CreateEmptyDataset(dsManager.StructuredDataStructureRepo.Get(1), rpManager.Repo.Get(1), mds);
if (dm.IsDatasetCheckedOutFor(ds.Id, "Javad") || dm.CheckOutDataset(ds.Id, "Javad"))
{
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
//DataTuple changed = dm.GetDatasetVersionEffectiveTuples(workingCopy).First();
//changed.VariableValues.First().Value = (new Random()).Next().ToString();
DataTuple dt = dm.DataTupleRepo.Get(1); // its sample data
List<DataTuple> tuples = new List<DataTuple>();
for (int i = 0; i < 10000; i++)
{
DataTuple newDt = new DataTuple();
newDt.XmlAmendments = dt.XmlAmendments;
newDt.XmlVariableValues = dt.XmlVariableValues; // in normal cases, the VariableValues are set and then Dematerialize is called
newDt.Materialize();
newDt.OrderNo = i;
//newDt.TupleAction = TupleAction.Created;//not required
//newDt.Timestamp = DateTime.UtcNow; //required? no, its set in the Edit
//newDt.DatasetVersion = workingCopy;//required? no, its set in the Edit
tuples.Add(newDt);
}
dm.EditDatasetVersion(workingCopy, tuples, null, null);
dm.CheckInDataset(ds.Id, "for testing purposes", "Javad");
dm.DatasetVersionRepo.Evict();
dm.DataTupleRepo.Evict();
dm.DatasetRepo.Evict();
workingCopy.PriliminaryTuples.Clear();
workingCopy = null;
}
var dsId = ds.Id;
ds = null;
return (dsId);
}
//
// GET: /DIM/Import/
public ActionResult Index()
{
//xml metadata for import
string metadataForImportPath = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), "MetadataIDIV_EXAMPLE.xml");
XmlDocument metadataForImport = new XmlDocument();
metadataForImport.Load(metadataForImportPath);
// metadataStructure DI
long metadataStructureId = 3;
MetadataStructureManager metadataStructureManager = new MetadataStructureManager();
string metadataStructrueName = metadataStructureManager.Repo.Get(metadataStructureId).Name;
// loadMapping file
string path_mappingFile = Path.Combine(AppConfiguration.GetModuleWorkspacePath("DIM"), XmlMetadataImportHelper.GetMappingFileName(metadataStructureId, TransmissionType.mappingFileImport, metadataStructrueName));
// XML mapper + mapping file
XmlMapperManager xmlMapperManager = new XmlMapperManager(TransactionDirection.InternToExtern);
xmlMapperManager.Load(path_mappingFile, "IDIV");
// generate intern metadata
XmlDocument metadataResult = xmlMapperManager.Generate(metadataForImport,1);
// generate intern template
XmlMetadataWriter xmlMetadatWriter = new XmlMetadataWriter(BExIS.Xml.Helpers.XmlNodeMode.xPath);
XDocument metadataXml = xmlMetadatWriter.CreateMetadataXml(metadataStructureId);
XmlDocument metadataXmlTemplate = XmlMetadataWriter.ToXmlDocument(metadataXml);
XmlDocument completeMetadata = XmlMetadataImportHelper.FillInXmlAttributes(metadataResult, metadataXmlTemplate);
// create Dataset
//load datastructure
DataStructureManager dsm = new DataStructureManager();
ResearchPlanManager rpm = new ResearchPlanManager();
MetadataStructureManager msm = new MetadataStructureManager();
DatasetManager dm = new DatasetManager();
Dataset dataset = dm.CreateEmptyDataset(dsm.UnStructuredDataStructureRepo.Get(1), rpm.Repo.Get(1), msm.Repo.Get(3));
if (dm.IsDatasetCheckedOutFor(dataset.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(dataset.Id, GetUsernameOrDefault()))
{
DatasetVersion workingCopy = dm.GetDatasetWorkingCopy(dataset.Id);
workingCopy.Metadata = completeMetadata;
string title = XmlDatasetHelper.GetInformation(workingCopy, NameAttributeValues.title);
if (String.IsNullOrEmpty(title)) title = "No Title available.";
dm.EditDatasetVersion(workingCopy, null, null, null);
dm.CheckInDataset(dataset.Id, "Metadata was submited.", GetUsernameOrDefault());
// add security
if (GetUsernameOrDefault() != "DEFAULT")
{
PermissionManager pm = new PermissionManager();
SubjectManager sm = new SubjectManager();
BExIS.Security.Entities.Subjects.User user = sm.GetUserByName(GetUsernameOrDefault());
foreach (RightType rightType in Enum.GetValues(typeof(RightType)).Cast<RightType>())
{
pm.CreateDataPermission(user.Id, 1, dataset.Id, rightType);
}
}
}
return View();
}
private void getDataStructures()
{
var dsm = new DataStructureManager();
var all = dsm.AllTypesDataStructureRepo.Get();
var uns = all.Where(p => p.Name.Contains("uns")).FirstOrDefault();
}
public List<Error> FinishUpload(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
DatasetManager dm = new DatasetManager();
DatasetVersion workingCopy = new DatasetVersion();
//datatuple list
List<DataTuple> rows = new List<DataTuple>();
Dataset ds = null;
bool inputWasAltered = false;
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
ds = dm.GetDataset(id);
// Javad: Please check if the dataset does exists!!
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
//workingCopy.ContentDescriptors = new List<ContentDescriptor>();
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
Stopwatch upload = Stopwatch.StartNew();
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
upload.Stop();
Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
upload.Stop();
Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
inputWasAltered = false;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
foreach (var err in reader.ErrorMessages)
{
temp.Add(new Error(ErrorType.Dataset, err.GetMessage()));
}
//return temp;
}
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>();
var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0 || inputWasAltered == true);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
#region contentdescriptors
//remove all contentdescriptors from the old version
//generatedTXT
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generatedCSV
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generated
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
#endregion
// ToDo: Get Comment from ui and users
MoveAndSaveOriginalFileInContentDiscriptor(workingCopy);
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
// checkout the dataset, apply the changes, and check it in.
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
if (temp.Count <= 0)
{
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
else
{
dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault());
}
return temp;
}
private StructuredDataStructure existingDataStructures(DataTable mapVariables, string dataSetID)
{
DataStructureManager dataStructureManager = new DataStructureManager();
StructuredDataStructure dataStructure = new StructuredDataStructure();
// get alldatasetId respective variables-rows from mapped-variables-table
DataRow[] mappedRows = mapVariables.Select("DatasetId = '" + dataSetID + "' AND Block = 0");
// get all datastrcutures from repo which has the same number of variables
List<StructuredDataStructure> existDatastructures = dataStructureManager.StructuredDataStructureRepo.Get(s =>
mappedRows.Count().Equals(s.Variables.Count)).ToList();
foreach (StructuredDataStructure existDs in existDatastructures)
{
bool isEqualStructure = true;
//foreach (DataRow mapRow in mapVariables.Rows)
foreach (DataRow mapRow in mappedRows)
{
int exvarno = existDs.Variables.Where(v =>
Convert.ToInt64(mapRow["UnitId"]).Equals(v.Unit.Id) &&
mapRow["Name"].ToString().Equals(v.Label) &&
Convert.ToInt64(mapRow["AttributeId"]).Equals(v.DataAttribute.Id)
).Count();
if (exvarno == 0)
{
isEqualStructure &= false;
break;
}
}
if (isEqualStructure)
{
dataStructure = existDs;
break;
}
}
return dataStructure;
}