/// <summary>
/// Add Datatuples to a Excel Template file
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="dataTuples"> Datatuples to add</param>
/// <param name="filePath">Path of the excel template file</param>
/// <param name="dataStructureId">Id of datastructure</param>
/// <returns>List of Errors or null</returns>
public List <Error> AddDataTuples(DatasetManager datasetManager, List <long> dataTuplesIds, string filePath, long dataStructureId)
{
if (File.Exists(filePath))
{
using (DataStructureManager dataStructureManager = new DataStructureManager())
{
dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
// setup file
Init(filePath, dataStructureId);
// add header
StructuredDataStructure sds = GetDataStructure(dataStructureId);
AddHeader(sds);
// iterate over all input rows
DataTupleIterator tupleIterator = new DataTupleIterator(dataTuplesIds, datasetManager);
foreach (var tuple in tupleIterator)
{
// add row and increment current index
if (AddRow(tuple, rowIndex) && !tuple.Id.Equals(dataTuplesIds.Last()))
{
rowIndex += 1;
}
}
// close the excel file
Close();
}
}
return(ErrorMessages);
}
private void buildTheBody(DatasetManager datasetManager, List <long> tupleIds, DataTable dt, StructuredDataStructure sds)
{
DataTupleIterator tupleIterator = new DataTupleIterator(tupleIds, datasetManager);
foreach (var tuple in tupleIterator)
{
dt.Rows.Add(ConvertTupleIntoDataRow(dt, tuple, sds));
}
}
/// <summary>
/// Add Rows to a WorksheetPart
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="worksheetPart"></param>
/// <param name="startRow"></param>
/// <param name="endRow"></param>
/// <param name="dataTuplesIds"></param>
protected void AddRows(WorksheetPart worksheetPart, int startRow, int endRow, List<long> dataTuplesIds, DatasetManager datasetManager)
{
Worksheet worksheet = worksheetPart.Worksheet;
SheetData sheetData = worksheet.GetFirstChild<SheetData>();
int rowIndex = endRow;
DataTupleIterator tupleIterator = new DataTupleIterator(dataTuplesIds, datasetManager);
foreach (var tuple in tupleIterator)
{
// convert datatuple to row and add it to sheetdata
Row row = DatatupleToRow(tuple, rowIndex);
bool empty = true;
foreach (Cell c in row.Elements<Cell>().ToList())
{
if (!String.IsNullOrEmpty(c.InnerText))
{
empty = false;
break;
}
}
if (!empty)
{
sheetData.Append(row);
if (!tuple.Id.Equals(dataTuplesIds.Last()))
rowIndex++;
}
}
numOfDataRows = rowIndex;
}
/// <summary>
/// Add Datatuples and Datastructure to a Ascii file
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="dataTuples"> Datatuples to add</param>
/// <param name="filePath">Path of the excel template file</param>
/// <param name="dataStructureId">Id of datastructure</param>
/// <returns>List of Errors or null</returns>
public List<Error> AddDataTuples(DatasetManager datasetManager,List<long> dataTuplesIds, string filePath, long dataStructureId)
{
if (File.Exists(filePath))
{
StringBuilder data = new StringBuilder();
data.AppendLine(dataStructureToRow(dataStructureId));
DataTupleIterator tupleIterator = new DataTupleIterator(dataTuplesIds, datasetManager);
foreach (var tuple in tupleIterator)
{
string newline = datatupleToRow(tuple);
if (!String.IsNullOrEmpty(newline)) data.AppendLine(newline);
}
File.WriteAllText(filePath, data.ToString());
}
return ErrorMessages;
}
//temporary solution: norman :GetSplitDatatuples2
public Dictionary <string, List <DataTuple> > GetSplitDatatuples(List <DataTuple> incomingDatatuples, List <long> primaryKeys, DatasetVersion workingCopy, ref List <long> datatuplesFromDatabaseIds)
{
DatasetManager datasetManager = new DatasetManager();
Dictionary <string, List <DataTuple> > data = new Dictionary <string, List <DataTuple> >();
Dictionary <string, DataTuple> newDtList = new Dictionary <string, DataTuple>();
Dictionary <string, DataTuple> editDtList = new Dictionary <string, DataTuple>();
List <DataTuple> deleteDtList = new List <DataTuple>();
DataTupleIterator tupleIterator = new DataTupleIterator(datatuplesFromDatabaseIds, datasetManager, false);
// Keep the DB loop outer to reduce the number of DB queries
foreach (var existingTuple in tupleIterator)
{
if (existingTuple == null || existingTuple.Id < 0) // it is unlikely to happen, but just to reinforce it.
{
continue;
}
// iterating over the in-memory newDataTuples is faster
for (int counter = 0; counter < incomingDatatuples.Count(); counter++)
//foreach (var incomingTuple in newDatatuples)
{
DataTuple incomingTuple = incomingDatatuples[counter];
if (!IsEmpty(incomingTuple))
{
// we first assume that any incoming tuple is new, and then try to check if it is not.
// this reduces the iterations of the inner loop.
// because the search takes the DB tuple and lloks for it in the coming tuples, it is posssible for an incoming tuple to be added more than once.
// So they are added to a dictionary to avoid duplicates
string keysValueNewDataTuple = getPrimaryKeysAsString(incomingTuple, primaryKeys);
if (!newDtList.ContainsKey(keysValueNewDataTuple))
{
newDtList.Add(keysValueNewDataTuple, incomingTuple); // by default, assume that the incoming tuple is new (not in the DB)
}
string keysValueSourceDatatuple = getPrimaryKeysAsStringFromXml(existingTuple, primaryKeys);
if (keysValueNewDataTuple.Equals(keysValueSourceDatatuple)) // the incoming tuple exists in the DB
{
if (!Equal(incomingTuple, existingTuple)) // the incoming tuple is a changed version of an existing one
{
// the incoming tuple is found in the DB and brings some changes, therefore not NEW!
newDtList.Remove(keysValueNewDataTuple);
if (!editDtList.ContainsKey(keysValueNewDataTuple))
{
// apply the changes to the exisiting one and register is an edited tuple
editDtList.Add(keysValueNewDataTuple, Merge(incomingTuple, (DataTuple)existingTuple));
// remove the current incoming item to shorten the list for the next round
incomingDatatuples.RemoveAt(counter);
}
// the decision is made, hence break the inner loop
break;
}
else // the incoming tuple is found in the BD, but introduces no change., hence no action is needed.
{
// remove the incoming tuple from the list and from the new ones.
newDtList.Remove(keysValueNewDataTuple);
incomingDatatuples.RemoveAt(counter);
}
}
else // the incoming tuple does not match the PK, so it should be a new tuple, which is already added to the list.
{ // DO NOTHING
}
}
}
}
data.Add("new", newDtList.Values.ToList());
data.Add("edit", editDtList.Values.ToList());
return(data);
}
private DatasetVersion applyTupleChanges(DatasetVersion workingCopyVersion
, ref List<DataTupleVersion> tupleVersionsTobeAdded, ref List<DataTuple> tuplesTobeDeleted, ref List<DataTuple> tuplesTobeEdited
, ICollection<DataTuple> createdTuples, ICollection<DataTuple> editedTuples, ICollection<long> deletedTuples, ICollection<DataTuple> unchangedTuples = null)
{
// do nothing with unchanged for now
#region Process Newly Created Tuples
/// associate newly created tuples to the new version
/// try using bulk copy or stateless sessions for large amount of new tuples. it should also apply on deleted tuples.
/// Take care of automatic flushing and try to prevent or reduce it while the edit process is not finished.
if (createdTuples != null && createdTuples.Count() > 0)
{
// is not working cause of the item.Dematerialize();
//Parallel.ForEach(createdTuples, item =>
//{
// item.Dematerialize();
// // commented for the performance testing purpose. see the efects and uncomment if needed-> workingCopyVersion.PriliminaryTuples.Add(item);
// item.DatasetVersion = workingCopyVersion;
// item.TupleAction = TupleAction.Created;
// item.Timestamp = workingCopyVersion.Timestamp;
//});
foreach (var item in createdTuples)
{
//item.Dematerialize();
// commented for the performance testing purpose. see the efects and uncomment if needed-> workingCopyVersion.PriliminaryTuples.Add(item);
item.DatasetVersion = workingCopyVersion;
item.TupleAction = TupleAction.Created;
item.Timestamp = workingCopyVersion.Timestamp;
}
}
#endregion
if ((editedTuples != null && editedTuples.Count() > 0) || (deletedTuples != null && deletedTuples.Count() > 0))
{
// latest version is the latest checked in version. it is the previous version in comparison to the working copy version.
// the checks to see whether the dataset is checked out are considered to be done before
DatasetVersion latestCheckedInVersion = workingCopyVersion.Dataset.Versions.OrderByDescending(p => p.Timestamp).FirstOrDefault(p => p.Status == DatasetVersionStatus.CheckedIn);
if (latestCheckedInVersion == null) // there is no previous version, means its the first version. In this case there is no need to handle deleted and editedVersion items!
return (workingCopyVersion);
// the edit and delete candiates know the exact ID of the target tuple, so no need to load the whole tuples in advance. in worst case the number of single tuple match queries will increase
//List<DataTuple> latestVersionEffectiveTuples = getPrimaryTuples(workingCopyVersion); //latestVersionEffectiveTuples = DataTupleRepo.Get(p=>p.DatasetVersion == null).ToList();
#region Process Edited Tuples
/// manage editedVersion tuples:
/// 1: create a DataTupleVersion based on its previous version
/// 2: Remove the original from the original version
/// 3: add them to the version
/// 4: set timestamp for the editedVersion ones
if (editedTuples != null && editedTuples.Count() > 0)
{
// this part of the code loads the original tuples that are claimed edited. The edited collection is indeed one package of changes submitted in this round
// so its used to load only the relevant portion of the tuples. this avoid loading all the tuples asscoiated with the current version. also avoid N times querying the DB, each time to retreive one tuple
// Seems a moderate solution between the two extreme.
List<Int64> editedTupleIds = editedTuples.Select(t => t.Id).ToList(); // All the IDs of edited tuples of the current package
List<DataTuple> oraginalsOfEditedTuples = DataTupleRepo.Get(p => editedTupleIds.Contains(p.Id)).ToList(); // all the original tuples edited in the current package
//Parallel.ForEach(editedTuples, edited => // not able to use parallel for now, because the tuplesTobeEdited gets shared between threads for writting in it, which causes synch problems
foreach (var edited in editedTuples)
{
DataTuple orginalTuple = oraginalsOfEditedTuples.SingleOrDefault(p => p.Id == edited.Id); //DataTupleRepo.Get(edited.Id);// latestVersionEffectiveTuples.Where(p => p.Id == editedVersion.Id).Single();//maybe preliminary tuples are enough
if (orginalTuple == null || orginalTuple.Id <= 0) // maybe the tuple is in the edited list by a mistake!
continue;
//check if the history record for this data tuple has been created before. in cases of multiple edits in a single version for example
if (DataTupleVerionRepo.Query(p => p.OriginalTuple.Id == orginalTuple.Id && p.DatasetVersion.Id == orginalTuple.DatasetVersion.Id).Count() <= 0) // it is the first time the orginalTuple is getting editedVersion. so add a history record. the history record, keeps the tuple as was before the first edit!
{
DataTupleVersion tupleVersion = new DataTupleVersion()
{
TupleAction = TupleAction.Edited,
Extra = orginalTuple.Extra,
//Id = orginalTuple.Id,
OrderNo = orginalTuple.OrderNo,
Timestamp = orginalTuple.Timestamp,
XmlAmendments = orginalTuple.XmlAmendments,
XmlVariableValues = orginalTuple.XmlVariableValues,
OriginalTuple = orginalTuple,
DatasetVersion = orginalTuple.DatasetVersion, //latestCheckedInVersion,
ActingDatasetVersion = workingCopyVersion,
};
//DataTuple merged =
//orginalTuple.History.Add(tupleVersion);
}
//need a better way to preserve changes during the fetch of the original tuple. Maybe deep copy/ evict/ merge works
//XmlDocument xmlVariableValues = new XmlDocument();
//xmlVariableValues.LoadXml(editedVersion.XmlVariableValues.AsString());
// dematerialize just for the purpose of synching the xml fields with the object properties.
edited.Dematerialize();
orginalTuple.TupleAction = TupleAction.Edited;
orginalTuple.OrderNo = edited.OrderNo;
orginalTuple.XmlAmendments = null;
orginalTuple.XmlAmendments = edited.XmlAmendments;
orginalTuple.XmlVariableValues = null;
orginalTuple.XmlVariableValues = edited.XmlVariableValues;
//System.Diagnostics.Debug.Print(editedVersion.XmlVariableValues.AsString());
//editedVersion.VariableValues.ToList().ForEach(p => System.Diagnostics.Debug.Print(p.Value.ToString()));
//System.Diagnostics.Debug.Print(xmlVariableValues.AsString());
orginalTuple.DatasetVersion = workingCopyVersion;
orginalTuple.Timestamp = workingCopyVersion.Timestamp;
tuplesTobeEdited.Add(orginalTuple);
//workingCopyVersion.PriliminaryTuples.Add(detached);
//latestCheckedInVersion.PriliminaryTuples.Remove(orginalTuple);
//latestVersionEffectiveTuples.Remove(orginalTuple);
}
//); //parallel for each
}
#endregion
#region Process Deleted Tuples
/// manage deleted tuples:
/// 1: create a DataTupleVersion based on their previous version
/// 2: Remove them from the latest version
/// 3: DO NOT add them to the new version
/// 4: DO NOT set timestamp for the deleted ones
if (deletedTuples != null && deletedTuples.Count() > 0)
{
//Parallel.ForEach(deletedTuples, deleted => // the tuplesTobeDeleted gets shared between the threads!
// use the tuple iterator to reduce the # of DB fetchs
DataTupleIterator tupleIterator = new DataTupleIterator(deletedTuples.ToList(), this);
// load the ID all the tuple versions that are already linked to the tobe deleted tuples.
//This reduces the number of selects on the tuple versions, because most of the tuples have no version
List<long> tupleVersionIds = DataTupleVerionRepo.Query(p => deletedTuples.Contains(p.OriginalTuple.Id)).Select(p=>p.Id).ToList();
foreach (var deleted in tupleIterator)
{
DataTuple originalTuple = (DataTuple)deleted; // DataTupleRepo.Get(deleted.Id);// latestVersionEffectiveTuples.Where(p => p.Id == deleted.Id).Single();
// check if the tuple has a previous history record. for example may be it was first editedVersion and now is going to be deleted. in two different edits but in one version
DataTupleVersion tupleVersion;
// check if the tuple has a history record
if (tupleVersionIds.Contains(originalTuple.Id))
{
tupleVersion = DataTupleVerionRepo.Get(originalTuple.Id);
// there is a previous history record, with tuple action equal to Edit or even Delete!
tupleVersion.TupleAction = TupleAction.Deleted;
}
else // there is no previous record, so create one
{
tupleVersion = new DataTupleVersion()
{
TupleAction = TupleAction.Deleted,
Extra = originalTuple.Extra,
//Id = orginalTuple.Id,
OrderNo = originalTuple.OrderNo,
Timestamp = originalTuple.Timestamp,
XmlAmendments = originalTuple.XmlAmendments,
XmlVariableValues = originalTuple.XmlVariableValues,
//OriginalTuple = orginalTuple,
DatasetVersion = originalTuple.DatasetVersion, // latestCheckedInVersion,
ActingDatasetVersion = workingCopyVersion,
};
}
tupleVersion.OriginalTuple = null;
// /////////////////////////////////////////
// try avoid accessing to the PriliminaryTuples, they cause loading all the tuples!!
// ////////////////////////////////////////
// -> latestCheckedInVersion.PriliminaryTuples.Remove(originalTuple);
// check whether the deleted tuples are removed from the datatuples table!!!!!
//latestVersionEffectiveTuples.Remove(originalTuple);
// -> workingCopyVersion.PriliminaryTuples.Remove(originalTuple);
//try
//{
// //originalTuple.History.ToList().ForEach(p => p.OriginalTuple = null);
//}
//catch { }
//originalTuple.History.Clear();
originalTuple.DatasetVersion = null;
tuplesTobeDeleted.Add(originalTuple);
tupleVersionsTobeAdded.Add(tupleVersion);
}
//); // parralel for each loop
}
#endregion
}
return (workingCopyVersion);
}
public Dictionary <string, List <DataTuple> > GetSplitDatatuples(List <DataTuple> incomingDatatuples, List <long> primaryKeys, DatasetVersion workingCopy, ref List <long> datatuplesFromDatabaseIds)
{
using (DatasetManager datasetManager = new DatasetManager())
{
Dictionary <string, List <DataTuple> > data = new Dictionary <string, List <DataTuple> >();
Dictionary <string, DataTuple> newDtList = new Dictionary <string, DataTuple>();
Dictionary <string, DataTuple> editDtList = new Dictionary <string, DataTuple>();
List <DataTuple> deleteDtList = new List <DataTuple>();
Dictionary <string, DataTuple> newDTDic = new Dictionary <string, DataTuple>();
//iterating over incoming datatuples to gerenate the primary key and add it to a dictionary
for (int counter = 0; counter < incomingDatatuples.Count(); counter++)
{
DataTuple incomingTuple = incomingDatatuples[counter];
if (!IsEmpty(incomingTuple))
{
string keysValueNewDataTuple = getPrimaryKeysAsString(incomingTuple, primaryKeys);
newDTDic.Add(keysValueNewDataTuple, incomingTuple);
}
}
DataTupleIterator tupleIterator = new DataTupleIterator(datatuplesFromDatabaseIds, datasetManager, false);
// Keep the DB loop outer to reduce the number of DB queries
foreach (var existingTuple in tupleIterator)
{
if (existingTuple == null || existingTuple.Id < 0) // it is unlikely to happen, but just to reinforce it.
{
continue;
}
string keysValueSourceDatatuple = getPrimaryKeysAsString(existingTuple, primaryKeys);
// check if a datatuple exist in the incoming dictionary
try
{
DataTuple incomingTuple = newDTDic[keysValueSourceDatatuple];
// a incoming datatuple with this key exist, check if there are equal
if (!Equal(incomingTuple, existingTuple)) // the incoming tuple is a changed version of an existing one
{
if (!editDtList.ContainsKey(keysValueSourceDatatuple))
{
// apply the changes to the exisiting one and register is an edited tuple
editDtList.Add(keysValueSourceDatatuple, Merge(incomingTuple, (DataTuple)existingTuple));
// remove the current incoming item to shorten the list for the next round
}
}
// the incoming tuple is found in the DB and brings some changes, therefore not NEW!
// so remove it from the dictionary
newDTDic.Remove(keysValueSourceDatatuple);
}
catch//if
{
//there is no datatuple incoming theat matches the db datatuple
}
}
// the rest of the incoming datatuples are in the new datatuples, all existing datatuples will be removed from that list
data.Add("new", newDTDic.Values.ToList());
data.Add("edit", editDtList.Values.ToList());
return(data);
}
}
//temporary solution: norman :GetSplitDatatuples2
public static Dictionary<string, List<DataTuple>> GetSplitDatatuples(List<DataTuple> incomingDatatuples, List<long> primaryKeys, DatasetVersion workingCopy, ref List<long> datatuplesFromDatabaseIds)
{
Dictionary<string, List<DataTuple>> data = new Dictionary<string, List<DataTuple>>();
Dictionary<string, DataTuple> newDtList = new Dictionary<string, DataTuple>();
Dictionary<string, DataTuple> editDtList = new Dictionary<string, DataTuple>();
List<DataTuple> deleteDtList = new List<DataTuple>();
DatasetManager datasetManager = new DatasetManager();
DataTupleIterator tupleIterator = new DataTupleIterator(datatuplesFromDatabaseIds, datasetManager, false);
// Keep the DB loop outer to reduce the number of DB queries
foreach (var existingTuple in tupleIterator)
{
if (existingTuple == null || existingTuple.Id < 0) // it is unlikely to happen, but just to reinforce it.
continue;
// iterating over the in-memory newDataTuples is faster
for (int counter = 0; counter < incomingDatatuples.Count(); counter++)
//foreach (var incomingTuple in newDatatuples)
{
DataTuple incomingTuple = incomingDatatuples[counter];
if (!IsEmpty(incomingTuple))
{
// we first assume that any incoming tuple is new, and then try to check if it is not.
// this reduces the iterations of the inner loop.
// because the search takes the DB tuple and lloks for it in the coming tuples, it is posssible for an incoming tuple to be added more than once.
// So they are added to a dictionary to avoid duplicates
string keysValueNewDataTuple = getPrimaryKeysAsString(incomingTuple, primaryKeys);
if (!newDtList.ContainsKey(keysValueNewDataTuple))
{
newDtList.Add(keysValueNewDataTuple, incomingTuple); // by default, assume that the incoming tuple is new (not in the DB)
}
string keysValueSourceDatatuple = getPrimaryKeysAsStringFromXml(existingTuple, primaryKeys);
if (keysValueNewDataTuple.Equals(keysValueSourceDatatuple)) // the incoming tuple exists in the DB
{
if (!Equal(incomingTuple, existingTuple)) // the incoming tuple is a changed version of an existing one
{
// the incoming tuple is found in the DB and brings some changes, therefore not NEW!
newDtList.Remove(keysValueNewDataTuple);
if (!editDtList.ContainsKey(keysValueNewDataTuple))
{
// apply the changes to the exisiting one and register is an edited tuple
editDtList.Add(keysValueNewDataTuple, Merge(incomingTuple, (DataTuple)existingTuple));
// remove the current incoming item to shorten the list for the next round
incomingDatatuples.RemoveAt(counter);
}
// the decision is made, hence break the inner loop
break;
}
else // the incoming tuple is found in the BD, but introduces no change., hence no action is needed.
{
// remove the incoming tuple from the list and from the new ones.
newDtList.Remove(keysValueNewDataTuple);
incomingDatatuples.RemoveAt(counter);
}
}
else // the incoming tuple does not match the PK, so it should be a new tuple, which is already added to the list.
{ // DO NOTHING
}
}
}
}
data.Add("new", newDtList.Values.ToList());
data.Add("edit", editDtList.Values.ToList());
return data;
}
