public GetDatasetVersionEffectiveTuples ( |
||
datasetVersion | The object representing the data set version requested. | |
return | List |
public List<Error> FinishUpload2(TaskManager taskManager) { List<Error> temp = new List<Error>(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID)) { long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]); DataStructureManager dsm = new DataStructureManager(); long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]); //datatuple list List<DataTuple> rows; DatasetManager dm = new DatasetManager(); Dataset ds = dm.GetDataset(id); DatasetVersion workingCopy = new DatasetVersion(); #region Progress Informations if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0); } if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE)) { TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0; } else { TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0); } #endregion #region structured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured)) { try { //Stopwatch fullTime = Stopwatch.StartNew(); //Stopwatch loadDT = Stopwatch.StartNew(); List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id)); //loadDT.Stop(); //Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString()); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd); dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables); #region excel reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm")) { int packageSize = 10000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; int counter = 0; ExcelReader reader = new ExcelReader(); //schleife dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!! if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault())) throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault())); workingCopy = dm.GetDatasetWorkingCopy(ds.Id); do { //Stopwatch packageTime = Stopwatch.StartNew(); counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; // open file Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //XXX Add packagesize to excel read function if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, rows, null, null); //Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); //Debug.WriteLine("----"); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { //Stopwatch split = Stopwatch.StartNew(); Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); //split.Stop(); //Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString()); //Stopwatch upload = Stopwatch.StartNew(); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); // upload.Stop(); // Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString()); // Debug.WriteLine("----"); } } } else { } } Stream.Close(); //packageTime.Stop(); //Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString()); } while (rows.Count() > 0); //fullTime.Stop(); //Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString()); } #endregion #region ascii reader if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") || TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt")) { // open file AsciiReader reader = new AsciiReader(); //Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); //DatasetManager dm = new DatasetManager(); //Dataset ds = dm.GetDataset(id); Stopwatch totalTime = Stopwatch.StartNew(); if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault())) { workingCopy = dm.GetDatasetWorkingCopy(ds.Id); int packageSize = 100000; TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize; //schleife int counter = 0; do { counter++; TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter; Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString()); rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize); Stream.Close(); if (reader.ErrorMessages.Count > 0) { //model.ErrorList = reader.errorMessages; } else { //model.Validated = true; Stopwatch dbTimer = Stopwatch.StartNew(); if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS)) { if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new")) { dm.EditDatasetVersion(workingCopy, rows, null, null); } if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit")) { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } } else { if (rows.Count() > 0) { Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>(); splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase); dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null); } } dbTimer.Stop(); Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString()); } } while (rows.Count() > 0); totalTime.Stop(); Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString()); } //Stream.Close(); } #endregion // start download generator // filepath //string path = ""; //if (workingCopy != null) //{ // path = GenerateDownloadFile(workingCopy); // dm.EditDatasetVersion(workingCopy, null, null, null); //} // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault()); LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated); } catch (Exception e) { temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message)); dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault()); } finally { } } #endregion #region unstructured data if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured)) { workingCopy = dm.GetDatasetLatestVersion(ds.Id); SaveFileInContentDiscriptor(workingCopy); dm.EditDatasetVersion(workingCopy, null, null, null); // ToDo: Get Comment from ui and users dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault()); } #endregion } else { temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected.")); } return temp; }
/// <summary> /// Shows the content of a specific dataset version /// </summary> /// <param name="id"></param> /// <returns></returns> public ActionResult Version(int id) { ViewBag.Title = PresentationModel.GetViewTitleForTenant("Version", this.Session.GetTenant()); DatasetManager dm = new DatasetManager(); DatasetVersion version = dm.DatasetVersionRepo.Get(p => p.Id == id).First(); var tuples = dm.GetDatasetVersionEffectiveTuples(version); ViewBag.VersionId = id; ViewBag.DatasetId = version.Dataset.Id; if (version.Dataset.DataStructure is StructuredDataStructure) { ViewBag.Variables = ((StructuredDataStructure) version.Dataset.DataStructure.Self).Variables.ToList(); } else { ViewBag.Variables = new List<Variable>(); } return View(tuples); }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="id"></param> /// <param name="metadataDoc"></param> /// <return></return> private void writeBexisIndex(long id, XmlDocument metadataDoc) { String docId = id.ToString();//metadataDoc.GetElementsByTagName("bgc:id")[0].InnerText; var dataset = new Document(); List<XmlNode> facetNodes = facetXmlNodeList; dataset.Add(new Field("doc_id", docId, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.NOT_ANALYZED)); foreach (XmlNode facet in facetNodes) { String multivalued = facet.Attributes.GetNamedItem("multivalued").Value; string[] metadataElementNames = facet.Attributes.GetNamedItem("metadata_name").Value.Split(','); String lucene_name = facet.Attributes.GetNamedItem("lucene_name").Value; foreach (string metadataElementName in metadataElementNames) { XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName); if (elemList != null) { for (int i = 0; i < elemList.Count; i++) { string eleme = elemList[i].InnerText; if (!elemList[i].InnerText.Trim().Equals("")) { dataset.Add(new Field("facet_" + lucene_name, elemList[i].InnerText, Lucene.Net.Documents.Field.Store.YES, Field.Index.NOT_ANALYZED)); dataset.Add(new Field("ng_all", elemList[i].InnerText, Lucene.Net.Documents.Field.Store.YES, Field.Index.ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText); writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText); } } } } } List<XmlNode> propertyNodes = propertyXmlNodeList; foreach (XmlNode property in propertyNodes) { String multivalued = property.Attributes.GetNamedItem("multivalued").Value; String lucene_name = property.Attributes.GetNamedItem("lucene_name").Value; string[] metadataElementNames = property.Attributes.GetNamedItem("metadata_name").Value.Split(','); foreach (string metadataElementName in metadataElementNames) { XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName); if (elemList != null) { String primitiveType = property.Attributes.GetNamedItem("primitive_type").Value; if (elemList[0] != null) { if (primitiveType.ToLower().Equals("string")) { dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText, Lucene.Net.Documents.Field.Store.YES, Field.Index.NOT_ANALYZED)); dataset.Add(new Field("ng_all", elemList[0].InnerText, Lucene.Net.Documents.Field.Store.YES, Field.Index.ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, elemList[0].InnerText); writeAutoCompleteIndex(docId, "ng_all", elemList[0].InnerText); } else if (primitiveType.ToLower().Equals("date")) { //DateTime MyDateTime = DateTime.Now; DateTime MyDateTime = new DateTime(); /*String dTFormatElementName = property.Attributes.GetNamedItem("date_format").Value; XmlNodeList dtFormatElements = metadataDoc.GetElementsByTagName(dTFormatElementName); String dateTimeFormat = dtFormatElements[0].InnerText;*/ if (DateTime.TryParse(elemList[0].InnerText, out MyDateTime)) { //MyDateTime = DateTime.ParseExact(elemList[0].InnerText, dateTimeFormat, // CultureInfo.InvariantCulture); long t = MyDateTime.Ticks; NumericField xyz = new NumericField("property_numeric_" + lucene_name).SetLongValue( MyDateTime.Ticks); String dateToString = MyDateTime.Date.ToString("d", CultureInfo.CreateSpecificCulture("en-US")); dataset.Add(xyz); dataset.Add(new Field("property_" + lucene_name, dateToString, Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, MyDateTime.Date.ToString()); writeAutoCompleteIndex(docId, "ng_all", MyDateTime.Date.ToString()); } } else if (primitiveType.ToLower().Equals("integer")) { dataset.Add( new NumericField("property_numeric" + lucene_name).SetIntValue( Convert.ToInt32(elemList[0].InnerText))); dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText, Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED)); // writeAutoCompleteIndex(lucene_name, elemList[0].InnerText); } else if (primitiveType.ToLower().Equals("double")) { dataset.Add( new NumericField("property_numeric" + lucene_name).SetDoubleValue( Convert.ToDouble(elemList[0].InnerText))); dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText, Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, elemList[0].InnerText); writeAutoCompleteIndex(docId, "ng_all", elemList[0].InnerText); } } } } } List<XmlNode> categoryNodes = categoryXmlNodeList; foreach (XmlNode category in categoryNodes) { if (category.Attributes.GetNamedItem("type").Value.Equals("primary_data_field")) { String primitiveType = category.Attributes.GetNamedItem("primitive_type").Value; String lucene_name = category.Attributes.GetNamedItem("lucene_name").Value; String analysing = category.Attributes.GetNamedItem("analysed").Value; float boosting = Convert.ToSingle(category.Attributes.GetNamedItem("boost").Value); var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED; if (analysing.ToLower().Equals("yes")) { toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED; } DatasetManager dm = new DatasetManager(); if (dm.IsDatasetCheckedIn(id)) { DatasetVersion dsv = dm.GetDatasetLatestVersion(id); DataStructureManager dsm = new DataStructureManager(); StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(dsv.Dataset.DataStructure.Id); // Javad: check if the dataset is "checked-in". If yes, then use the paging version of the GetDatasetVersionEffectiveTuples method // number of tuples for the for loop is also available via GetDatasetVersionEffectiveTupleCount // a proper fetch (page) size can be obtained by calling dm.PreferedBatchSize int fetchSize = dm.PreferedBatchSize; long tupleSize = dm.GetDatasetVersionEffectiveTupleCount(dsv); long noOfFetchs = tupleSize/fetchSize + 1; for (int round = 0; round < noOfFetchs; round++) { List<AbstractTuple> dsVersionTuples = dm.GetDatasetVersionEffectiveTuples(dsv, round, fetchSize); List<string> primaryDataStringToindex = generateStringFromTuples(dsVersionTuples, sds); if (primaryDataStringToindex != null) { foreach (string pDataValue in primaryDataStringToindex) // Loop through List with foreach { Field a = new Field("category_" + lucene_name, pDataValue, Lucene.Net.Documents.Field.Store.NO, toAnalyse); a.Boost = boosting; dataset.Add(a); dataset.Add(new Field("ng_" + lucene_name, pDataValue, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED)); dataset.Add(new Field("ng_all", pDataValue, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, pDataValue); writeAutoCompleteIndex(docId, "ng_all", pDataValue); } } GC.Collect(); } } } else { String multivalued = category.Attributes.GetNamedItem("multivalued").Value; String primitiveType = category.Attributes.GetNamedItem("primitive_type").Value; String lucene_name = category.Attributes.GetNamedItem("lucene_name").Value; String storing = category.Attributes.GetNamedItem("store").Value; String analysing = category.Attributes.GetNamedItem("analysed").Value; float boosting = Convert.ToSingle(category.Attributes.GetNamedItem("boost").Value); var toStore = Lucene.Net.Documents.Field.Store.NO; var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED; if (storing.ToLower().Equals("yes")) { toStore = Lucene.Net.Documents.Field.Store.YES; } if (analysing.ToLower().Equals("yes")) { toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED; } string[] metadataElementNames = category.Attributes.GetNamedItem("metadata_name").Value.Split(','); foreach (string metadataElementName in metadataElementNames) { XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName); if (elemList != null) { for (int i = 0; i < elemList.Count; i++) { Field a = new Field("category_" + lucene_name, elemList[i].InnerText, toStore, toAnalyse); a.Boost = boosting; dataset.Add(a); dataset.Add(new Field("ng_" + lucene_name, elemList[i].InnerText, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED)); dataset.Add(new Field("ng_all", elemList[i].InnerText, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText); writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText); } } } } } List<XmlNode> generalNodes = generalXmlNodeList; foreach (XmlNode general in generalNodes) { String multivalued = general.Attributes.GetNamedItem("multivalued").Value; String primitiveType = general.Attributes.GetNamedItem("primitive_type").Value; String lucene_name = general.Attributes.GetNamedItem("lucene_name").Value; String storing = general.Attributes.GetNamedItem("store").Value; String analysing = general.Attributes.GetNamedItem("analysed").Value; var toStore = Lucene.Net.Documents.Field.Store.NO; var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED; if (storing.ToLower().Equals("yes")) { toStore = Lucene.Net.Documents.Field.Store.YES; } if (analysing.ToLower().Equals("yes")) { toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED; } float boosting = Convert.ToSingle(general.Attributes.GetNamedItem("boost").Value); string[] metadataElementNames = general.Attributes.GetNamedItem("metadata_name").Value.Split(','); foreach (string metadataElementName in metadataElementNames) { XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName); for (int i = 0; i < elemList.Count; i++) { Field a = new Field(lucene_name, elemList[i].InnerText, toStore, toAnalyse); a.Boost = boosting; dataset.Add(a); dataset.Add(new Field("ng_all", elemList[i].InnerText, Lucene.Net.Documents.Field.Store.NO, Field.Index.ANALYZED)); writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText); writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText); } } } indexWriter.AddDocument(dataset); }
private void getEffectiveTuples(Int64 versionId) { DatasetManager dm = new DatasetManager(); DatasetVersion workingCopy = dm.GetDatasetVersion(versionId); var changed = dm.GetDatasetVersionEffectiveTuples(workingCopy); }
/// <summary> /// test unique of primary keys on a dataset /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="datasetId"></param> /// <param name="primaryKeys"></param> /// <returns></returns> ////[MeasurePerformance] public static Boolean IsUnique(long datasetId, List<long> primaryKeys) { Hashtable hashtable = new Hashtable(); // load data DatasetManager datasetManager = new DatasetManager(); Dataset dataset = datasetManager.GetDataset(datasetId); DatasetVersion datasetVersion; List<long> dataTupleIds = new List<long>(); if (datasetManager.IsDatasetCheckedIn(datasetId)) { datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId); #region load all datatuples first int size = 10000; int counter = 0; IEnumerable<AbstractTuple> dataTuples; do { dataTuples = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion,counter,size); //byte[] pKey; string pKey; foreach (DataTuple dt in dataTuples) { //pKey = getPrimaryKeysAsByteArray(dt, primaryKeys); pKey = getPrimaryKeysAsString(dt, primaryKeys); if (pKey.Count() > 0) { try { //Debug.WriteLine(pKey +" : " +Utility.ComputeKey(pKey)); hashtable.Add(pKey, ""); //hashtable.Add(pKey, 0); } catch { return false; } } } counter++; } while (dataTuples.Count() >= (size * counter)); #endregion } else { throw new Exception("Dataset is not checked in."); } return true; }