|
public GetDatasetVersionEffectiveTupleIds ( |
||
| datasetVersion | The object representing the data set version requested | |
| return | List |
|
private List<long> GetDataTuples(long datasetId)
{
DatasetManager datasetManager = new DatasetManager();
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
return datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
}
private void AddDatatuplesToFile(long datasetId, long dataStructureId, string path)
{
DatasetManager datasetManager = new DatasetManager();
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
List<long> tempDataTuplesIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
ExcelWriter excelWriter = new ExcelWriter();
excelWriter.AddDataTuplesToTemplate(datasetManager, tempDataTuplesIds, path, dataStructureId);
}
/// <summary>
/// CAUTION !!!!!!!!!!!!!!!!!
/// upload bezieht sich derzeit nur auf daten mit einem block (einer tabelle)
/// umsetzung von mehreren blöckes noch nicht geklärt.
/// (für einen wert gibt es eine weitere tabelle)
/// </summary>
/// <param name="dataSetID"></param>
/// <param name="DataBase"></param>
public void uploadData(string dataSetID, string DataBase)
{
DatasetManager datasetManager = new DatasetManager();
DataStructureManager dataStructureManager = new DataStructureManager();
XmlDataReader xmlDataReader = new XmlDataReader();
User user = new User();
string variableNames = "";
// query metadataAuthor and variable names from explorer.datasets
queryAuthorAndVariables(ref user, ref variableNames, dataSetID, DataBase);
List<string> varNames = variableNames.Split(',').ToList();
// get all dataStructures with equal variables count
List<StructuredDataStructure> dataStructures = dataStructureManager.StructuredDataStructureRepo.Get(s =>
varNames.Count().Equals(s.Variables.Count)).ToList();
// get all Ids of dataStructures with equal variables
List<long> dataStructureIds = new List<long>();
foreach (StructuredDataStructure dataStructure in dataStructures)
{
bool isSimilarStructure = true;
foreach (Variable variable in dataStructure.Variables)
{
if (!varNames.Contains(variable.Label))
{
isSimilarStructure &= false;
break;
}
}
if (isSimilarStructure)
dataStructureIds.Add(dataStructure.Id);
}
// get the wanted dataset by comparing the old B1 datasetId out of the datasets with similar dataStructure
Dataset dataset = null;
List<Dataset> datasets = datasetManager.DatasetRepo.Get(d => dataStructureIds.Contains(d.DataStructure.Id)).ToList();
foreach (Dataset ds in datasets)
{
string oldDatasetId = "";
try
{
XmlNode extraID = ds.Versions.FirstOrDefault().Metadata.SelectSingleNode("Metadata/general/general/id/id");
oldDatasetId = extraID.InnerText;
}
catch
{
}
if (oldDatasetId == dataSetID)
{
dataset = ds;
break;
}
}
if (dataset != null)
{
// get distinct and ascending ordered insertdates from DB
List<DB2TimeStamp> distInsertDates = queryDistInsertDates(dataSetID, DataBase);
bool checkObsIds = false;
foreach (DB2TimeStamp insertDate in distInsertDates)
{
List<DataTuple> createdDataTuples = new List<DataTuple>();
List<DataTuple> editedDataTuples = new List<DataTuple>();
List<DataTuple> deletedDataTuples = new List<DataTuple>();
DatasetVersion workingCopy = datasetManager.GetDatasetLatestVersion(dataset.Id); // get dataset
// get obsid, data, deleted and newest for each observation from DB
List<Observation> observations = queryObservation(dataSetID, DataBase, insertDate.ToString());
Dictionary<long, long> obsIdMapsToTupleId = new Dictionary<long, long>();
if (checkObsIds)
{
// get all EffectiveTupleIds
List<long> datasetTupleIds = datasetManager.GetDatasetVersionEffectiveTupleIds(workingCopy).ToList();
// id-mapping-list key=obsId, value=TupleId foreach EffectiveTupleId and obsId from Tuple.Extra
obsIdMapsToTupleId = idMapping(datasetTupleIds, ref datasetManager);
}
// observation counter
int observationIndex = 0;
bool isNewest = true;
foreach (Observation observation in observations) //////////////parallel
{
isNewest = (isNewest && observation.newest != 'Y') ? false : isNewest;
// create dataTuple with xmlDataReader
// split xml to string list and use DataReader.ReadRow
DataTuple dataRow = xmlDataReader.XmlRowReader(observation.data, dataset.DataStructure.Id, observationIndex);
// check if observation.obsid is in id-mapping-list
long TupleId;
if (checkObsIds && obsIdMapsToTupleId.TryGetValue(observation.obsid, out TupleId))
{
DataTuple dataTuple = datasetManager.DataTupleRepo.Get(TupleId);
if (observation.deleted != 'Y')
{
dataTuple.VariableValues = dataRow.VariableValues;
// edit tuple if observation exists as tuple in EffectiveTuple and observation is not deleted
editedDataTuples.Add(dataTuple);
}
else
{
// delete tuple if observation exists as tuple in EffectiveTuple and observ. is deleted
deletedDataTuples.Add(dataTuple);
}
}
else
{
// write the obsId and oldBExISdatasetId in Extra: <extra><obsid>[obsid]</obsid><oldBExISdatasetId>[dataSetID]</oldBExISdatasetId></extra>
dataRow.Extra = oldIdsIntoExtra(observation.obsid, dataSetID, dataRow.Extra);
// create tuple if observation exists not in EffectiveTuple
createdDataTuples.Add(dataRow);
}
observationIndex++; // observation counter
}
checkObsIds = (!checkObsIds && !isNewest) ? true : checkObsIds;
// checkOut
if (datasetManager.IsDatasetCheckedOutFor(dataset.Id, user.Name) || datasetManager.CheckOutDataset(dataset.Id, user.Name))
{
workingCopy = datasetManager.GetDatasetWorkingCopy(dataset.Id); // get dataset
datasetManager.EditDatasetVersion(workingCopy, createdDataTuples, editedDataTuples, deletedDataTuples); // edit dataset
datasetManager.CheckInDataset(dataset.Id, "Primary data row was submited.", user.Name); // checkIn
}
}
}
}
public List<Error> FinishUpload(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
DatasetManager dm = new DatasetManager();
DatasetVersion workingCopy = new DatasetVersion();
//datatuple list
List<DataTuple> rows = new List<DataTuple>();
Dataset ds = null;
bool inputWasAltered = false;
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
ds = dm.GetDataset(id);
// Javad: Please check if the dataset does exists!!
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
//workingCopy.ContentDescriptors = new List<ContentDescriptor>();
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
Stopwatch upload = Stopwatch.StartNew();
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
upload.Stop();
Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
upload.Stop();
Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
inputWasAltered = false;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
foreach (var err in reader.ErrorMessages)
{
temp.Add(new Error(ErrorType.Dataset, err.GetMessage()));
}
//return temp;
}
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>();
var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0 || inputWasAltered == true);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
#region contentdescriptors
//remove all contentdescriptors from the old version
//generatedTXT
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generatedCSV
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generated
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
#endregion
// ToDo: Get Comment from ui and users
MoveAndSaveOriginalFileInContentDiscriptor(workingCopy);
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
// checkout the dataset, apply the changes, and check it in.
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
if (temp.Count <= 0)
{
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
else
{
dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault());
}
return temp;
}
/// <summary>
/// test unique of primary keys on a dataset
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="datasetId"></param>
/// <param name="primaryKeys"></param>
/// <returns></returns>
////[MeasurePerformance]
public static Boolean IsUnique2(long datasetId, List<long> primaryKeys)
{
Hashtable hashtable = new Hashtable();
// load data
DatasetManager datasetManager = new DatasetManager();
Dataset dataset = datasetManager.GetDataset(datasetId);
DatasetVersion datasetVersion;
List<long> dataTupleIds = new List<long>();
if (datasetManager.IsDatasetCheckedIn(datasetId))
{
datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
#region load all datatuples first
int size = 10000;
int counter = 0;
IEnumerable<long> dataTuplesIds;
dataTuplesIds = datasetManager.GetDatasetVersionEffectiveTupleIds(datasetVersion);
IEnumerable<long> currentIds;
DataTuple dt;
do
{
currentIds = dataTupleIds.Skip(counter * size).Take(size);
//byte[] pKey;
string pKey;
foreach (long dtId in currentIds)
{
dt = datasetManager.DataTupleRepo.Query(d=>d.Id.Equals(dtId)).FirstOrDefault();
//pKey = getPrimaryKeysAsByteArray(dt, primaryKeys);
pKey = pKey = getPrimaryKeysAsStringFromXml(dt, primaryKeys);
if (pKey.Count() > 0)
{
try
{
//Debug.WriteLine(pKey +" : " +Utility.ComputeKey(pKey));
hashtable.Add(pKey, "");
//hashtable.Add(pKey, 0);
}
catch
{
return false;
}
}
}
counter++;
}
while (currentIds.Count() >= (size * counter));
#endregion
}
else
{
throw new Exception("Dataset is not checked in.");
}
return true;
}
