|
public GetDatasetVersionEffectiveTuples ( |
||
| datasetVersion | The object representing the data set version requested. | |
| return | List |
|
public List<Error> FinishUpload2(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
//datatuple list
List<DataTuple> rows;
DatasetManager dm = new DatasetManager();
Dataset ds = dm.GetDataset(id);
DatasetVersion workingCopy = new DatasetVersion();
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
//Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
}
} while (rows.Count() > 0);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
// start download generator
// filepath
//string path = "";
//if (workingCopy != null)
//{
// path = GenerateDownloadFile(workingCopy);
// dm.EditDatasetVersion(workingCopy, null, null, null);
//}
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated);
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
workingCopy = dm.GetDatasetLatestVersion(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
return temp;
}
/// <summary>
/// Shows the content of a specific dataset version
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ActionResult Version(int id)
{
ViewBag.Title = PresentationModel.GetViewTitleForTenant("Version", this.Session.GetTenant());
DatasetManager dm = new DatasetManager();
DatasetVersion version = dm.DatasetVersionRepo.Get(p => p.Id == id).First();
var tuples = dm.GetDatasetVersionEffectiveTuples(version);
ViewBag.VersionId = id;
ViewBag.DatasetId = version.Dataset.Id;
if (version.Dataset.DataStructure is StructuredDataStructure)
{
ViewBag.Variables = ((StructuredDataStructure) version.Dataset.DataStructure.Self).Variables.ToList();
}
else
{
ViewBag.Variables = new List<Variable>();
}
return View(tuples);
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="id"></param>
/// <param name="metadataDoc"></param>
/// <return></return>
private void writeBexisIndex(long id, XmlDocument metadataDoc)
{
String docId = id.ToString();//metadataDoc.GetElementsByTagName("bgc:id")[0].InnerText;
var dataset = new Document();
List<XmlNode> facetNodes = facetXmlNodeList;
dataset.Add(new Field("doc_id", docId, Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.NOT_ANALYZED));
foreach (XmlNode facet in facetNodes)
{
String multivalued = facet.Attributes.GetNamedItem("multivalued").Value;
string[] metadataElementNames = facet.Attributes.GetNamedItem("metadata_name").Value.Split(',');
String lucene_name = facet.Attributes.GetNamedItem("lucene_name").Value;
foreach (string metadataElementName in metadataElementNames)
{
XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName);
if (elemList != null)
{
for (int i = 0; i < elemList.Count; i++)
{
string eleme = elemList[i].InnerText;
if (!elemList[i].InnerText.Trim().Equals(""))
{
dataset.Add(new Field("facet_" + lucene_name, elemList[i].InnerText,
Lucene.Net.Documents.Field.Store.YES, Field.Index.NOT_ANALYZED));
dataset.Add(new Field("ng_all", elemList[i].InnerText,
Lucene.Net.Documents.Field.Store.YES, Field.Index.ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText);
writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText);
}
}
}
}
}
List<XmlNode> propertyNodes = propertyXmlNodeList;
foreach (XmlNode property in propertyNodes)
{
String multivalued = property.Attributes.GetNamedItem("multivalued").Value;
String lucene_name = property.Attributes.GetNamedItem("lucene_name").Value;
string[] metadataElementNames = property.Attributes.GetNamedItem("metadata_name").Value.Split(',');
foreach (string metadataElementName in metadataElementNames)
{
XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName);
if (elemList != null)
{
String primitiveType = property.Attributes.GetNamedItem("primitive_type").Value;
if (elemList[0] != null)
{
if (primitiveType.ToLower().Equals("string"))
{
dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText,
Lucene.Net.Documents.Field.Store.YES, Field.Index.NOT_ANALYZED));
dataset.Add(new Field("ng_all", elemList[0].InnerText,
Lucene.Net.Documents.Field.Store.YES, Field.Index.ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, elemList[0].InnerText);
writeAutoCompleteIndex(docId, "ng_all", elemList[0].InnerText);
}
else if (primitiveType.ToLower().Equals("date"))
{
//DateTime MyDateTime = DateTime.Now;
DateTime MyDateTime = new DateTime();
/*String dTFormatElementName = property.Attributes.GetNamedItem("date_format").Value;
XmlNodeList dtFormatElements = metadataDoc.GetElementsByTagName(dTFormatElementName);
String dateTimeFormat = dtFormatElements[0].InnerText;*/
if (DateTime.TryParse(elemList[0].InnerText, out MyDateTime))
{
//MyDateTime = DateTime.ParseExact(elemList[0].InnerText, dateTimeFormat,
// CultureInfo.InvariantCulture);
long t = MyDateTime.Ticks;
NumericField xyz =
new NumericField("property_numeric_" + lucene_name).SetLongValue(
MyDateTime.Ticks);
String dateToString = MyDateTime.Date.ToString("d",
CultureInfo.CreateSpecificCulture("en-US"));
dataset.Add(xyz);
dataset.Add(new Field("property_" + lucene_name, dateToString,
Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, MyDateTime.Date.ToString());
writeAutoCompleteIndex(docId, "ng_all", MyDateTime.Date.ToString());
}
}
else if (primitiveType.ToLower().Equals("integer"))
{
dataset.Add(
new NumericField("property_numeric" + lucene_name).SetIntValue(
Convert.ToInt32(elemList[0].InnerText)));
dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText,
Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED));
// writeAutoCompleteIndex(lucene_name, elemList[0].InnerText);
}
else if (primitiveType.ToLower().Equals("double"))
{
dataset.Add(
new NumericField("property_numeric" + lucene_name).SetDoubleValue(
Convert.ToDouble(elemList[0].InnerText)));
dataset.Add(new Field("property_" + lucene_name, elemList[0].InnerText,
Lucene.Net.Documents.Field.Store.NO, Field.Index.NOT_ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, elemList[0].InnerText);
writeAutoCompleteIndex(docId, "ng_all", elemList[0].InnerText);
}
}
}
}
}
List<XmlNode> categoryNodes = categoryXmlNodeList;
foreach (XmlNode category in categoryNodes)
{
if (category.Attributes.GetNamedItem("type").Value.Equals("primary_data_field"))
{
String primitiveType = category.Attributes.GetNamedItem("primitive_type").Value;
String lucene_name = category.Attributes.GetNamedItem("lucene_name").Value;
String analysing = category.Attributes.GetNamedItem("analysed").Value;
float boosting = Convert.ToSingle(category.Attributes.GetNamedItem("boost").Value);
var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED;
if (analysing.ToLower().Equals("yes"))
{
toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED;
}
DatasetManager dm = new DatasetManager();
if (dm.IsDatasetCheckedIn(id))
{
DatasetVersion dsv = dm.GetDatasetLatestVersion(id);
DataStructureManager dsm = new DataStructureManager();
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(dsv.Dataset.DataStructure.Id);
// Javad: check if the dataset is "checked-in". If yes, then use the paging version of the GetDatasetVersionEffectiveTuples method
// number of tuples for the for loop is also available via GetDatasetVersionEffectiveTupleCount
// a proper fetch (page) size can be obtained by calling dm.PreferedBatchSize
int fetchSize = dm.PreferedBatchSize;
long tupleSize = dm.GetDatasetVersionEffectiveTupleCount(dsv);
long noOfFetchs = tupleSize/fetchSize + 1;
for (int round = 0; round < noOfFetchs; round++)
{
List<AbstractTuple> dsVersionTuples = dm.GetDatasetVersionEffectiveTuples(dsv, round,
fetchSize);
List<string> primaryDataStringToindex = generateStringFromTuples(dsVersionTuples, sds);
if (primaryDataStringToindex != null)
{
foreach (string pDataValue in primaryDataStringToindex)
// Loop through List with foreach
{
Field a = new Field("category_" + lucene_name, pDataValue,
Lucene.Net.Documents.Field.Store.NO, toAnalyse);
a.Boost = boosting;
dataset.Add(a);
dataset.Add(new Field("ng_" + lucene_name, pDataValue,
Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
dataset.Add(new Field("ng_all", pDataValue, Lucene.Net.Documents.Field.Store.YES,
Lucene.Net.Documents.Field.Index.ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, pDataValue);
writeAutoCompleteIndex(docId, "ng_all", pDataValue);
}
}
GC.Collect();
}
}
}
else
{
String multivalued = category.Attributes.GetNamedItem("multivalued").Value;
String primitiveType = category.Attributes.GetNamedItem("primitive_type").Value;
String lucene_name = category.Attributes.GetNamedItem("lucene_name").Value;
String storing = category.Attributes.GetNamedItem("store").Value;
String analysing = category.Attributes.GetNamedItem("analysed").Value;
float boosting = Convert.ToSingle(category.Attributes.GetNamedItem("boost").Value);
var toStore = Lucene.Net.Documents.Field.Store.NO;
var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED;
if (storing.ToLower().Equals("yes"))
{
toStore = Lucene.Net.Documents.Field.Store.YES;
}
if (analysing.ToLower().Equals("yes"))
{
toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED;
}
string[] metadataElementNames = category.Attributes.GetNamedItem("metadata_name").Value.Split(',');
foreach (string metadataElementName in metadataElementNames)
{
XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName);
if (elemList != null)
{
for (int i = 0; i < elemList.Count; i++)
{
Field a = new Field("category_" + lucene_name, elemList[i].InnerText, toStore, toAnalyse);
a.Boost = boosting;
dataset.Add(a);
dataset.Add(new Field("ng_" + lucene_name, elemList[i].InnerText,
Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
dataset.Add(new Field("ng_all", elemList[i].InnerText,
Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText);
writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText);
}
}
}
}
}
List<XmlNode> generalNodes = generalXmlNodeList;
foreach (XmlNode general in generalNodes)
{
String multivalued = general.Attributes.GetNamedItem("multivalued").Value;
String primitiveType = general.Attributes.GetNamedItem("primitive_type").Value;
String lucene_name = general.Attributes.GetNamedItem("lucene_name").Value;
String storing = general.Attributes.GetNamedItem("store").Value;
String analysing = general.Attributes.GetNamedItem("analysed").Value;
var toStore = Lucene.Net.Documents.Field.Store.NO;
var toAnalyse = Lucene.Net.Documents.Field.Index.NOT_ANALYZED;
if (storing.ToLower().Equals("yes"))
{
toStore = Lucene.Net.Documents.Field.Store.YES;
}
if (analysing.ToLower().Equals("yes"))
{
toAnalyse = Lucene.Net.Documents.Field.Index.ANALYZED;
}
float boosting = Convert.ToSingle(general.Attributes.GetNamedItem("boost").Value);
string[] metadataElementNames = general.Attributes.GetNamedItem("metadata_name").Value.Split(',');
foreach (string metadataElementName in metadataElementNames)
{
XmlNodeList elemList = metadataDoc.SelectNodes(metadataElementName);
for (int i = 0; i < elemList.Count; i++)
{
Field a = new Field(lucene_name, elemList[i].InnerText, toStore, toAnalyse);
a.Boost = boosting;
dataset.Add(a);
dataset.Add(new Field("ng_all", elemList[i].InnerText, Lucene.Net.Documents.Field.Store.NO, Field.Index.ANALYZED));
writeAutoCompleteIndex(docId, lucene_name, elemList[i].InnerText);
writeAutoCompleteIndex(docId, "ng_all", elemList[i].InnerText);
}
}
}
indexWriter.AddDocument(dataset);
}
private void getEffectiveTuples(Int64 versionId)
{
DatasetManager dm = new DatasetManager();
DatasetVersion workingCopy = dm.GetDatasetVersion(versionId);
var changed = dm.GetDatasetVersionEffectiveTuples(workingCopy);
}
/// <summary>
/// test unique of primary keys on a dataset
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="datasetId"></param>
/// <param name="primaryKeys"></param>
/// <returns></returns>
////[MeasurePerformance]
public static Boolean IsUnique(long datasetId, List<long> primaryKeys)
{
Hashtable hashtable = new Hashtable();
// load data
DatasetManager datasetManager = new DatasetManager();
Dataset dataset = datasetManager.GetDataset(datasetId);
DatasetVersion datasetVersion;
List<long> dataTupleIds = new List<long>();
if (datasetManager.IsDatasetCheckedIn(datasetId))
{
datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
#region load all datatuples first
int size = 10000;
int counter = 0;
IEnumerable<AbstractTuple> dataTuples;
do
{
dataTuples = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion,counter,size);
//byte[] pKey;
string pKey;
foreach (DataTuple dt in dataTuples)
{
//pKey = getPrimaryKeysAsByteArray(dt, primaryKeys);
pKey = getPrimaryKeysAsString(dt, primaryKeys);
if (pKey.Count() > 0)
{
try
{
//Debug.WriteLine(pKey +" : " +Utility.ComputeKey(pKey));
hashtable.Add(pKey, "");
//hashtable.Add(pKey, 0);
}
catch
{
return false;
}
}
}
counter++;
}
while (dataTuples.Count() >= (size * counter));
#endregion
}
else
{
throw new Exception("Dataset is not checked in.");
}
return true;
}
