public GetDatasetVersionEffectiveTupleIds ( |
||
datasetVersion | The object representing the data set version requested | |
return | List |
private List<long> GetDataTuples(long datasetId) { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); return datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); }
private void AddDatatuplesToFile(long datasetId, long dataStructureId, string path) { DatasetManager datasetManager = new DatasetManager(); DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); List<long> tempDataTuplesIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); ExcelWriter excelWriter = new ExcelWriter(); excelWriter.AddDataTuplesToTemplate(datasetManager, tempDataTuplesIds, path, dataStructureId); }
/// <summary> /// CAUTION !!!!!!!!!!!!!!!!! /// upload bezieht sich derzeit nur auf daten mit einem block (einer tabelle) /// umsetzung von mehreren blöckes noch nicht geklärt. /// (für einen wert gibt es eine weitere tabelle) /// </summary> /// <param name="dataSetID"></param> /// <param name="DataBase"></param> public void uploadData(string dataSetID, string DataBase) { DatasetManager datasetManager = new DatasetManager(); DataStructureManager dataStructureManager = new DataStructureManager(); XmlDataReader xmlDataReader = new XmlDataReader(); User user = new User(); string variableNames = ""; // query metadataAuthor and variable names from explorer.datasets queryAuthorAndVariables(ref user, ref variableNames, dataSetID, DataBase); List<string> varNames = variableNames.Split(',').ToList(); // get all dataStructures with equal variables count List<StructuredDataStructure> dataStructures = dataStructureManager.StructuredDataStructureRepo.Get(s => varNames.Count().Equals(s.Variables.Count)).ToList(); // get all Ids of dataStructures with equal variables List<long> dataStructureIds = new List<long>(); foreach (StructuredDataStructure dataStructure in dataStructures) { bool isSimilarStructure = true; foreach (Variable variable in dataStructure.Variables) { if (!varNames.Contains(variable.Label)) { isSimilarStructure &= false; break; } } if (isSimilarStructure) dataStructureIds.Add(dataStructure.Id); } // get the wanted dataset by comparing the old B1 datasetId out of the datasets with similar dataStructure Dataset dataset = null; List<Dataset> datasets = datasetManager.DatasetRepo.Get(d => dataStructureIds.Contains(d.DataStructure.Id)).ToList(); foreach (Dataset ds in datasets) { string oldDatasetId = ""; try { XmlNode extraID = ds.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id"); oldDatasetId = extraID.InnerText; } catch { } if (oldDatasetId == dataSetID) { dataset = ds; break; } } if (dataset != null) { // get distinct and ascending ordered insertdates from DB List<DB2TimeStamp> distInsertDates = queryDistInsertDates(dataSetID, DataBase); bool checkObsIds = false; foreach (DB2TimeStamp insertDate in distInsertDates) { List<DataTuple> createdDataTuples = new List<DataTuple>(); List<DataTuple> editedDataTuples = new List<DataTuple>(); List<DataTuple> deletedDataTuples = new List<DataTuple>(); DatasetVersion workingCopy = datasetManager.GetDatasetLatestVersion(dataset.Id); // get dataset // get obsid, data, deleted and newest for each observation from DB List<Observation> observations = queryObservation(dataSetID, DataBase, insertDate.ToString()); Dictionary<long, long> obsIdMapsToTupleId = new Dictionary<long, long>(); if (checkObsIds) { // get all EffectiveTupleIds List<long> datasetTupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(workingCopy).ToList(); // id-mapping-list key=obsId, value=TupleId foreach EffectiveTupleId and obsId from Tuple.Extra obsIdMapsToTupleId = idMapping(datasetTupleIds, ref datasetManager); } // observation counter int observationIndex = 0; bool isNewest = true; foreach (Observation observation in observations) //////////////parallel { isNewest = (isNewest && observation.newest != 'Y') ? false : isNewest; // create dataTuple with xmlDataReader // split xml to string list and use DataReader.ReadRow DataTuple dataRow = xmlDataReader.XmlRowReader(observation.data, dataset.DataStructure.Id, observationIndex); // check if observation.obsid is in id-mapping-list long TupleId; if (checkObsIds && obsIdMapsToTupleId.TryGetValue(observation.obsid, out TupleId)) { DataTuple dataTuple = datasetManager.DataTupleRepo.Get(TupleId); if (observation.deleted != 'Y') { dataTuple.VariableValues = dataRow.VariableValues; // edit tuple if observation exists as tuple in EffectiveTuple and observation is not deleted editedDataTuples.Add(dataTuple); } else { // delete tuple if observation exists as tuple in EffectiveTuple and observ. is deleted deletedDataTuples.Add(dataTuple); } } else { // write the obsId and oldBExISdatasetId in Extra: <extra><obsid>[obsid]</obsid><oldBExISdatasetId>[dataSetID]</oldBExISdatasetId></extra> dataRow.Extra = oldIdsIntoExtra(observation.obsid, dataSetID, dataRow.Extra); // create tuple if observation exists not in EffectiveTuple createdDataTuples.Add(dataRow); } observationIndex++; // observation counter } checkObsIds = (!checkObsIds && !isNewest) ? true : checkObsIds; // checkOut if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, user.Name) || datasetManager.CheckOutDataset(dataset.Id, user.Name)) { workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id); // get dataset datasetManager.EditDatasetVersion(workingCopy, createdDataTuples, editedDataTuples, deletedDataTuples); // edit dataset datasetManager.CheckInDataset(dataset.Id, "Primary data row was submited.", user.Name); // checkIn } } } }
public List<Error> FinishUpload(TaskManager taskManager) { List<Error> temp = new List<Error>(); DatasetManager dm = new DatasetManager(); DatasetVersion workingCopy = new DatasetVersion(); //datatuple list List<DataTuple> rows = new List<DataTuple>(); Dataset ds = null; bool inputWasAltered = false; if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); ds = dm.GetDataset(id); // Javad: Please check if the dataset does exists!! #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); //workingCopy.ContentDescriptors = new List<ContentDescriptor>(); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); Stopwatch upload = Stopwatch.StartNew(); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); upload.Stop(); Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); upload.Stop(); Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; inputWasAltered = false; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { foreach (var err in reader.ErrorMessages) { temp.Add(new Error(ErrorType.Dataset, err.GetMessage())); } //return temp; } //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>(); var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); inputWasAltered = true; } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0 || inputWasAltered == true); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion #region contentdescriptors //remove all contentdescriptors from the old version //generatedTXT if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generatedCSV if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } //generated if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated"))) { ContentDescriptor tmp = workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated")) .FirstOrDefault(); dm.DeleteContentDescriptor(tmp); } #endregion // ToDo: Get Comment from ui and users MoveAndSaveOriginalFileInContentDiscriptor(workingCopy); dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { // checkout the dataset, apply the changes, and check it in. if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } if (temp.Count <= 0) { dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } else { dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault()); } return temp; }
/// <summary> /// test unique of primary keys on a dataset /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="datasetId"></param> /// <param name="primaryKeys"></param> /// <returns></returns> ////[MeasurePerformance] public static Boolean IsUnique2(long datasetId, List<long> primaryKeys) { Hashtable hashtable = new Hashtable(); // load data DatasetManager datasetManager = new DatasetManager(); Dataset dataset = datasetManager.GetDataset(datasetId); DatasetVersion datasetVersion; List<long> dataTupleIds = new List<long>(); if (datasetManager.IsDatasetCheckedIn(datasetId)) { datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); #region load all datatuples first int size = 10000; int counter = 0; IEnumerable<long> dataTuplesIds; dataTuplesIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion); IEnumerable<long> currentIds; DataTuple dt; do { currentIds = dataTupleIds.Skip(counter * size).Take(size); //byte[] pKey; string pKey; foreach (long dtId in currentIds) { dt = datasetManager.DataTupleRepo.Query(d=>d.Id.Equals(dtId)).FirstOrDefault(); //pKey = getPrimaryKeysAsByteArray(dt, primaryKeys); pKey = pKey = getPrimaryKeysAsStringFromXml(dt, primaryKeys); if (pKey.Count() > 0) { try { //Debug.WriteLine(pKey +" : " +Utility.ComputeKey(pKey)); hashtable.Add(pKey, ""); //hashtable.Add(pKey, 0); } catch { return false; } } } counter++; } while (currentIds.Count() >= (size * counter)); #endregion } else { throw new Exception("Dataset is not checked in."); } return true; }