public static DatasetGridRowModel Convert(DatasetVersion datasetVersion, bool isPublic)
{
return new DatasetGridRowModel()
{
Id = datasetVersion.Dataset.Id,
Title = XmlDatasetHelper.GetInformation(datasetVersion, NameAttributeValues.title),
IsPublic = isPublic
};
}
private static string storeGeneratedFilePathToContentDiscriptor(long datasetId, DatasetVersion datasetVersion, string title, string ext)
{
string name = "";
string mimeType = "";
if (ext.Contains("csv"))
{
name = "datastructure";
mimeType = "text/comma-separated-values";
}
// create the generated FileStream and determine its location
string dynamicPath = OutputDatasetManager.GetDynamicDatasetStorePath(datasetId, datasetVersion.Id, title, ext);
//Register the generated data FileStream as a resource of the current dataset version
//ContentDescriptor generatedDescriptor = new ContentDescriptor()
//{
// OrderNo = 1,
// Name = name,
// MimeType = mimeType,
// URI = dynamicPath,
// DatasetVersion = datasetVersion,
//};
DatasetManager dm = new DatasetManager();
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(name)) > 0)
{ // remove the one contentdesciptor
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
if (cd.Name == name)
{
cd.URI = dynamicPath;
dm.UpdateContentDescriptor(cd);
}
}
}
else
{
// add current contentdesciptor to list
//datasetVersion.ContentDescriptors.Add(generatedDescriptor);
dm.CreateContentDescriptor(name, mimeType, dynamicPath, 1, datasetVersion);
}
//dm.EditDatasetVersion(datasetVersion, null, null, null);
return dynamicPath;
}
/// <summary>
/// Each dataset may have more than one versions, each having their own data tuples. The data tuples of the latest version are kept in a separate collection,
/// but the previous versions are scattered among the data tuple and historical tuple collections. The later is the place that acts as the place to keep record of
/// all the previous actions done on the dataset and its their results.
/// This method may be called to get the tuples of the version 2 while the current version is i.e. 10.
/// Based on the status of the requested version, the method may use the tuple collection alone or in combination with the history records to rebuild the version as it was at its check-in time.
/// The versions are stored in the tuple collection in a differential way, so that the version 3 computes the differences to the version 2 and applies the difference only.
/// So retrieving algorithm in this method rebuilds the requested version from its own and previous versions' tuples.
/// If the latest version is requested and the dataset is checked in, the algorithm retrieves all tuples in the tuple collection associated with the current and all previous versions.
/// If the latest version is requested and the dataset is in checked-out state, the method retrieves the working copy tuples.
/// </summary>
/// <param name="datasetVersion">The object representing the data set version requested.</param>
/// <returns>A list of data tuples representing the associated data of the version requested.</returns>
/// <remarks>The returned list may contain normal and historic tuples, if the requested version is not the latest. All the tuples are <b>materialized</b>.</remarks>
public List<AbstractTuple> GetDatasetVersionEffectiveTuples(DatasetVersion datasetVersion)
{
return getDatasetVersionEffectiveTuples(datasetVersion);
}
private string MoveAndSaveOriginalFileInContentDiscriptor(DatasetVersion datasetVersion)
{
TaskManager TaskManager = (TaskManager)Session["TaskManager"];
//dataset id and data structure id are available via datasetVersion properties,why you are passing them via the BUS? Javad
long datasetId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASET_ID]);
long dataStructureId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
DatasetManager datasetManager = new DatasetManager();
string title = TaskManager.Bus[TaskManager.DATASET_TITLE].ToString();
string ext = ".xlsm";// TaskManager.Bus[TaskManager.EXTENTION].ToString();
ExcelWriter excelWriter = new ExcelWriter();
// Move Original File to its permanent location
String tempPath = TaskManager.Bus[TaskManager.FILEPATH].ToString();
string originalFileName = TaskManager.Bus[TaskManager.FILENAME].ToString();
string storePath = excelWriter.GetFullStorePathOriginalFile(datasetId, datasetVersion.Id, originalFileName);
string dynamicStorePath = excelWriter.GetDynamicStorePathOriginalFile(datasetId, datasetVersion.VersionNo, originalFileName);
//Why using the excel writer, isn't any function available in System.IO.File/ Directory, etc. Javad
FileHelper.MoveFile(tempPath, storePath);
//Register the original data as a resource of the current dataset version
ContentDescriptor originalDescriptor = new ContentDescriptor()
{
OrderNo = 1,
Name = "original",
MimeType = "application/xlsm",
URI = dynamicStorePath,
DatasetVersion = datasetVersion,
};
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(originalDescriptor.Name)) > 0)
{ // remove the one contentdesciptor
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
if (cd.Name == originalDescriptor.Name)
{
cd.URI = originalDescriptor.URI;
}
}
}
else
{
// add current contentdesciptor to list
datasetVersion.ContentDescriptors.Add(originalDescriptor);
}
return storePath;
}
public List<Error> FinishUpload(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
DatasetManager dm = new DatasetManager();
DatasetVersion workingCopy = new DatasetVersion();
//datatuple list
List<DataTuple> rows = new List<DataTuple>();
Dataset ds = null;
bool inputWasAltered = false;
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
ds = dm.GetDataset(id);
// Javad: Please check if the dataset does exists!!
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<long> datatupleFromDatabaseIds = dm.GetDatasetVersionEffectiveTupleIds(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
//workingCopy.ContentDescriptors = new List<ContentDescriptor>();
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
Stopwatch upload = Stopwatch.StartNew();
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
upload.Stop();
Debug.WriteLine("ReadFile: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
upload.Stop();
Debug.WriteLine("EditDatasetVersion: " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
inputWasAltered = false;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
foreach (var err in reader.ErrorMessages)
{
temp.Add(new Error(ErrorType.Dataset, err.GetMessage()));
}
//return temp;
}
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<AbstractTuple>>();
var splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabaseIds);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
inputWasAltered = true;
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0 || inputWasAltered == true);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
#region contentdescriptors
//remove all contentdescriptors from the old version
//generatedTXT
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedTXT")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedTXT"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generatedCSV
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generatedCSV")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generatedCSV"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
//generated
if (workingCopy.ContentDescriptors.Any(c => c.Name.Equals("generated")))
{
ContentDescriptor tmp =
workingCopy.ContentDescriptors.Where(c => c.Name.Equals("generated"))
.FirstOrDefault();
dm.DeleteContentDescriptor(tmp);
}
#endregion
// ToDo: Get Comment from ui and users
MoveAndSaveOriginalFileInContentDiscriptor(workingCopy);
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
// checkout the dataset, apply the changes, and check it in.
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
if (temp.Count <= 0)
{
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
else
{
dm.UndoCheckoutDataset(ds.Id, GetUsernameOrDefault());
}
return temp;
}
private string getMappingFileName(DatasetVersion datasetVersion, TransmissionType convertType, string name)
{
return XmlMetadataImportHelper.GetMappingFileName(datasetVersion.Dataset.MetadataStructure.Id, convertType, name);
}
/// <summary>
/// An extended property is a custom property that is assigned to a dataset version in addition to the predefined properties. Then each dataset version owner/ accessor
/// can provide a value for the attached properties.
/// </summary>
/// <param name="extendedPropertyId">The identifier of the extended property.</param>
/// <param name="value">The value to be assigned to the extended property of the dataset version.</param>
/// <param name="note"><see cref="DataValue"/></param>
/// <param name="samplingTime"><see cref="DataValue"/></param>
/// <param name="resultTime"><see cref="DataValue"/></param>
/// <param name="obtainingMethod"><see cref="DataValue"/></param>
/// <param name="datasetVersion">The dataset version receiving the property value</param>
/// <returns>The extended property value linked to its <see cref="ExtendedProperty"/> and the <see cref="DatasetVersion"/></returns>
public ExtendedPropertyValue CreateExtendedPropertyValue(Int64 extendedPropertyId, string value, string note, DateTime samplingTime, DateTime resultTime, ObtainingMethod obtainingMethod,
DatasetVersion datasetVersion)
{
Contract.Requires(!string.IsNullOrWhiteSpace(value));
Contract.Requires(extendedPropertyId > 0);
Contract.Requires(datasetVersion != null);
Contract.Ensures(Contract.Result<ExtendedPropertyValue>() != null);
ExtendedPropertyValue e = new ExtendedPropertyValue()
{
Value = value,
Note = note,
SamplingTime = samplingTime,
ResultTime = resultTime,
ObtainingMethod = obtainingMethod,
ExtendedPropertyId = extendedPropertyId,
DatasetVersion = datasetVersion, // subject to delete
};
e.DatasetVersion.ExtendedPropertyValues.Add(e);
//using (IUnitOfWork uow = this.GetUnitOfWork())
//{
// IRepository<ExtendedPropertyValue> repo = uow.GetRepository<ExtendedPropertyValue>();
// repo.Put(e);
// uow.Commit();
//}
return (e);
}
/// <summary>
/// Resource descriptors are the way to link resources to data set versions (and some other entity types, too).
/// The resources can be persisted in the local or a remote file system or any other location reachable via a URL.
/// The resource itself can be any type of file, service returning a resource, a normal webpage, and so on.
/// The method creates a resource descriptor, links it to the provided data set version and persists the descriptor.
/// </summary>
/// <param name="name">A friendly name for the resource, mainly used in the UI</param>
/// <param name="mimeType">The type of the resource. Used in the methods that transfer and/or process the resource</param>
/// <param name="uri">The URI of the resource, may contain protocol, access method, authorization information , etc.</param>
/// <param name="orderNo">The order of the resource in the list of all resources associated to the same dataset version.</param>
/// <param name="datasetVersion"></param>
/// <returns>A persisted <seealso cref="ContentDescriptor"/> object linked to the <paramref name="datasetVersion"/></returns>
/// <remarks>The method does not have access to the resource itself, and does not persist it.</remarks>
public ContentDescriptor CreateContentDescriptor(string name, string mimeType, string uri, Int32 orderNo, DatasetVersion datasetVersion)
{
Contract.Requires(!string.IsNullOrWhiteSpace(name));
Contract.Requires(!string.IsNullOrWhiteSpace(mimeType));
Contract.Requires(!string.IsNullOrWhiteSpace(uri));
Contract.Requires(datasetVersion != null);
// check whether is it needed that the dataset is checked out to add descriptor
Contract.Ensures(Contract.Result<ContentDescriptor>() != null);
ContentDescriptor e = new ContentDescriptor()
{
Name = name,
MimeType = mimeType,
OrderNo = orderNo,
URI = uri,
DatasetVersion = datasetVersion,
};
e.DatasetVersion.ContentDescriptors.Add(e);
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository<ContentDescriptor> repo = uow.GetRepository<ContentDescriptor>();
repo.Put(e);
uow.Commit();
}
return (e);
}
private List<AbstractTuple> getHistoricTuples(DatasetVersion datasetVersion)
{
//get previous versions including the version specified, because the data tuples belong to all versions greater or equal to their original versions.
List<Int64> versionIds = getPreviousVersionIds(datasetVersion); //get all tuples from the main tuples table belonging to one of the previous versions + the current version
List<DataTuple> tuples = DataTupleRepo.Get(p => versionIds.Contains(p.DatasetVersion.Id)).ToList();
// get those history tuples that represent editedVersion versions of data tuples changed from at least one of the effective versions and not committed to them (them: the effective versions).
// any single data tuple can be editedVersion by a specific version once at most.
// it is possible for a tuple to have beed changed many times between any given two versions v(x) and v(y), so it is required to group the tuples based on their original ID and then select the record corresponding to the max version
var editedTupleVersionsGrouped = DataTupleVerionRepo.Query(p => (p.TupleAction == TupleAction.Edited)
&& (versionIds.Contains(p.DatasetVersion.Id))
&& !(versionIds.Contains(p.ActingDatasetVersion.Id)))
.GroupBy(p => p.OriginalTuple.Id)
.Select(p => new { OriginalTupleId = p.Key, MaxVersionOfTheTuple = p.Max(l => l.DatasetVersion.Id) })
.ToList();
IList<DataTupleVersion> editedTuples = new List<DataTupleVersion>();
// having a list of original tuple id and related max version, now its time to build a proper query to fetch the actual data tuple versions from the database, the following block builds a dynamic predicate
// to be passed to the where clause of the data retrieval method at: DataTupleVerionRepo.Query(...)
if (editedTupleVersionsGrouped.Count >= 1)
{
var param1 = Expression.Parameter(typeof(DataTupleVersion), "p");
var exp1 =
Expression.AndAlso(
Expression.Equal(
Expression.Property(Expression.Property(param1, "OriginalTuple"), "Id"),
Expression.Constant(editedTupleVersionsGrouped.First().OriginalTupleId)
),
Expression.Equal(
Expression.Property(Expression.Property(param1, "DatasetVersion"), "Id"),
Expression.Constant(editedTupleVersionsGrouped.First().MaxVersionOfTheTuple)
)
);
if (editedTupleVersionsGrouped.Count > 1)
{
foreach (var item in editedTupleVersionsGrouped.Skip(1))
{
//var param = Expression.Parameter(typeof(DataTupleVersion), "p");
var exp =
Expression.AndAlso(
Expression.Equal(
Expression.Property(Expression.Property(param1, "OriginalTuple"), "Id"),
Expression.Constant(item.OriginalTupleId)
),
Expression.Equal(
Expression.Property(Expression.Property(param1, "DatasetVersion"), "Id"),
Expression.Constant(item.MaxVersionOfTheTuple)
)
);
exp1 = Expression.OrElse(exp1, exp); ;
}
}
var typedExpression = Expression.Lambda<Func<DataTupleVersion, bool>>(exp1, new ParameterExpression[] { param1 });
editedTuples = DataTupleVerionRepo.Query(typedExpression).ToList();
}
var deletedTuples = DataTupleVerionRepo.Get(p => (p.TupleAction == TupleAction.Deleted)
&& (versionIds.Contains(p.DatasetVersion.Id))
&& !(versionIds.Contains(p.ActingDatasetVersion.Id)))
.Cast<AbstractTuple>()
.ToList();
List<AbstractTuple> result = tuples
.Union(editedTuples.Cast<AbstractTuple>())
.Union(deletedTuples)
// there is no guarantee that the overall list is ordered as its original order! because 1: OrderNo is not set yet. 2: OrderNo is not managed during the changes and so on,
// 3: The timestamp of the current tuples is indeed the timestamp of the change made by their latest acting version, but history record are carrying the original timestamp. but as there should be no overlap between the two table records
// and history records have smaller timestamps, no side effect is expected. 4: I don't know why but ...
.OrderBy(p => p.OrderNo).OrderBy(p => p.Timestamp)
.ToList();
return (result);
}
private List<AbstractTuple> getDatasetVersionEffectiveTuples(DatasetVersion datasetVersion, int pageNumber, int pageSize)
{
List<AbstractTuple> tuples = new List<AbstractTuple>();
Dataset dataset = datasetVersion.Dataset;
if (dataset.Status == DatasetStatus.Deleted)
throw new Exception(string.Format("Provided dataset version {0} belongs to deleted dataset {1}.", datasetVersion.Id, dataset.Id));
Int64 latestVersionId = dataset.Versions.OrderByDescending(t => t.Timestamp).Where(p => p.Timestamp <= dataset.LastCheckIOTimestamp).First().Id; // no need to replace it with the STATUS version
if (datasetVersion.Id > latestVersionId)
throw new Exception(string.Format("Invalid version id. The dataset version id {0} is greater than the latest version number!", datasetVersion.Id));
if (latestVersionId == datasetVersion.Id && dataset.Status == DatasetStatus.CheckedOut) // its a request for the working copy
{
tuples = getWorkingCopyTuples(datasetVersion, pageNumber, pageSize).Cast<AbstractTuple>().ToList();
}
else if (latestVersionId == datasetVersion.Id && dataset.Status == DatasetStatus.CheckedIn) // its a request for the latest checked-in version that should be served from the Tuples table
{
tuples = getPrimaryTuples(datasetVersion, pageNumber, pageSize).Cast<AbstractTuple>().ToList();
}
else
{
tuples = getHistoricTuples(datasetVersion, pageNumber, pageSize).Cast<AbstractTuple>().ToList(); // its a request for version earlier than the current version, whether the latest version is check-out or in.
}
//Parallel.ForEach(tuples, p => p.Materialize()); // causes object serialization issues
tuples .ForEach(p => p.Materialize());
return (tuples);
}
private List<Int64> getDatasetVersionEffectiveTupleIds(DatasetVersion datasetVersion)
{
List<Int64> tuples = new List<Int64>();
Dataset dataset = datasetVersion.Dataset;
if (dataset.Status == DatasetStatus.Deleted)
throw new Exception(string.Format("Provided dataset version {0} belongs to deleted dataset {1}.", datasetVersion.Id, dataset.Id));
Int64 latestVersionId = dataset.Versions.OrderByDescending(t => t.Timestamp).Where(p => p.Timestamp <= dataset.LastCheckIOTimestamp).First().Id; // no need to replace it with the STATUS version
if (datasetVersion.Id > latestVersionId)
throw new Exception(string.Format("Invalid version id. The dataset version id {0} is greater than the latest version number!", datasetVersion.Id));
if (latestVersionId == datasetVersion.Id && dataset.Status == DatasetStatus.CheckedOut) // its a request for the working copy
{
tuples = getWorkingCopyTupleIds(datasetVersion);
}
else if (latestVersionId == datasetVersion.Id && dataset.Status == DatasetStatus.CheckedIn) // its a request for the latest checked in version that should be served from the Tuples table
{
tuples = getPrimaryTupleIds(datasetVersion);
}
else
{
throw new NotSupportedException(string.Format("Invalid version id. The dataset version id {0} is not referring to the latest or working versions. This function is able to access historical versions, use GetDatasetVersionEffectiveTuples function instead!", datasetVersion.Id));
}
return (tuples);
}
//[MeasurePerformance]
private DatasetVersion editDatasetVersion(DatasetVersion workingCopyDatasetVersion, List<DataTuple> createdTuples, ICollection<DataTuple> editedTuples, ICollection<long> deletedTuples, ICollection<DataTuple> unchangedTuples)
{
Contract.Requires(workingCopyDatasetVersion.Dataset != null && workingCopyDatasetVersion.Dataset.Id >= 0);
Contract.Requires(workingCopyDatasetVersion.Dataset.Status == DatasetStatus.CheckedOut);
Contract.Ensures(Contract.Result<DatasetVersion>() != null && Contract.Result<DatasetVersion>().Id >= 0);
// be sure you are working on the latest version (working copy). applyTupleChanges takes the working copy from the DB
List<DataTupleVersion> tobeAdded = new List<DataTupleVersion>();
List<DataTuple> tobeDeleted = new List<DataTuple>();
List<DataTuple> tobeEdited = new List<DataTuple>();
DatasetVersion editedVersion = applyTupleChanges(workingCopyDatasetVersion, ref tobeAdded, ref tobeDeleted, ref tobeEdited, createdTuples, editedTuples, deletedTuples, unchangedTuples);
#region main code
//using (IUnitOfWork uow = this.GetUnitOfWork())
//{
// IRepository<DatasetVersion> repo = uow.GetRepository<DatasetVersion>();
// IRepository<DataTupleVersion> tupleVersionRepo = uow.GetRepository<DataTupleVersion>();
// IRepository<DataTuple> tupleRepo = uow.GetRepository<DataTuple>();
// // depends on how applyTupleChanges adds the tuples to its PriliminaryTuples!
// if (createdTuples != null)
// {
// foreach (DataTuple tuple in createdTuples)
// {
// tupleRepo.Put(tuple);
// }
// }
// if (tobeAdded != null)
// {
// foreach (DataTupleVersion dtv in tobeAdded)
// {
// tupleVersionRepo.Put(dtv);
// }
// }
// //foreach (var editedTuple in tobeEdited)
// //{
// // editedTuple.VariableValues.ToList().ForEach(p => System.Diagnostics.Debug.Print(p.Value.ToString()));
// // System.Diagnostics.Debug.Print(editedTuple.XmlVariableValues.AsString());
// //}
// if (tobeDeleted != null)
// {
// foreach (DataTuple tuple in tobeDeleted)
// {
// tupleRepo.Delete(tuple);
// }
// }
// // check whether the changes to the latest version, which is changed in the applyTupleChanges , are committed too!
// repo.Put(editedVersion);
// uow.Commit();
//}
#endregion
#region <<------ experimental code ------>>
// Check the same scenario using stateless session/ BulkUnitOfWork
using (IUnitOfWork uow = this.GetBulkUnitOfWork())
{
IRepository<DataTupleVersion> tupleVersionRepo = uow.GetRepository<DataTupleVersion>();
IRepository<DataTuple> tupleRepo = uow.GetRepository<DataTuple>();
// depends on how applyTupleChanges adds the tuples to its PriliminaryTuples!
if (createdTuples != null && createdTuples.Count >0)
{
int batchSize = uow.PersistenceManager.PreferredPushSize;
List<DataTuple> processedTuples = null;
long iterations = createdTuples.Count / batchSize;
if (iterations * batchSize < createdTuples.Count)
iterations++;
for (int round = 0; round < iterations; round++)
{
processedTuples = createdTuples.Skip(round * batchSize).Take(batchSize).ToList();
processedTuples.ForEach(tuple => tuple.Dematerialize());
tupleRepo.Put(processedTuples);
uow.ClearCache(true); //flushes one batch of the tuples to the DB
processedTuples.Clear();
GC.Collect();
}
}
if (tobeAdded != null && tobeAdded.Count > 0)
{
int batchSize = uow.PersistenceManager.PreferredPushSize;
List<DataTupleVersion> processedTuples = null;
long iterations = tobeAdded.Count / batchSize;
if (iterations * batchSize < tobeAdded.Count)
iterations++;
for(int round = 0; round < iterations; round++)
{
processedTuples = tobeAdded.Skip(round * batchSize).Take(batchSize).ToList();
tupleVersionRepo.Put(processedTuples);
uow.ClearCache(true); //flushes one batch of tuples
processedTuples.Clear();
GC.Collect();
}
}
//foreach (var editedTuple in tobeEdited)
//{
// editedTuple.VariableValues.ToList().ForEach(p => System.Diagnostics.Debug.Print(p.Value.ToString()));
// System.Diagnostics.Debug.Print(editedTuple.XmlVariableValues.AsString());
//}
if (tobeDeleted != null && tobeDeleted.Count > 0)
{
int batchSize = uow.PersistenceManager.PreferredPushSize;
List<DataTupleVersion> processedTuples = null;
long iterations = tobeAdded.Count / batchSize;
if (iterations * batchSize < tobeAdded.Count)
iterations++;
for (int round = 0; round < iterations; round++)
{
processedTuples = tobeAdded.Skip(round * batchSize).Take(batchSize).ToList();
tupleVersionRepo.Delete(processedTuples);
uow.ClearCache(true); //flushes one batch of tuples
processedTuples.Clear();
GC.Collect();
}
}
// check whether the changes to the latest version, which is changed in the applyTupleChanges , are committed too!
uow.Commit();
}
#endregion
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository<DatasetVersion> repo = uow.GetRepository<DatasetVersion>();
repo.Put(editedVersion); // must be updated in a tracked session
uow.Commit();
}
return (editedVersion);
}
/// <summary>
/// checks out the dataset and creates a new version on it. The new version acts like a working copy while it is not committed, hence editable.
/// </summary>
/// <param name="datasetId"></param>
/// <param name="username"></param>
private bool checkOutDataset(Int64 datasetId, string username, DateTime timestamp)
{
bool checkedOut = false;
//XmlDocument doc = new XmlDocument();
//doc.LoadXml(@"<Metadata>Empty</Metadata>");
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository<Dataset> repo = uow.GetRepository<Dataset>();
var q = repo.Query(p => p.Id == datasetId && p.Status == DatasetStatus.CheckedIn && (p.CheckOutUser.Equals(string.Empty) || p.CheckOutUser == null));
Dataset ds = q.FirstOrDefault();
if (ds != null)
{
DatasetVersion dsNewVersion = new DatasetVersion()
{
Timestamp = timestamp,
//Metadata = doc,
//ExtendedPropertyValues = new List<ExtendedPropertyValue>(),
//ContentDescriptors = new List<ContentDescriptor>(),
Status = DatasetVersionStatus.CheckedOut,
Dataset = ds,
};
// if there is a previous version, copy its metadata, content descriptors and extended property values to the newly created version
if (ds.Versions.Count() > 0)
{
var previousCheckedInVersion = ds.Versions.Where(p => p.Status == DatasetVersionStatus.CheckedIn).First();
if (previousCheckedInVersion != null && previousCheckedInVersion.Timestamp >= timestamp) // it is an error
{
throw new Exception(string.Format("The provided timestamp {0} is earlier than the timestamp of the latest checked in version!", timestamp));
}
if (previousCheckedInVersion != null)
{
dsNewVersion.Metadata = previousCheckedInVersion.Metadata;
dsNewVersion.ExtendedPropertyValues = previousCheckedInVersion.ExtendedPropertyValues;
foreach (var item in previousCheckedInVersion.ContentDescriptors)
{
ContentDescriptor cd = new ContentDescriptor()
{
MimeType = item.MimeType,
Name = item.Name,
OrderNo = item.OrderNo,
URI = item.URI,
DatasetVersion = dsNewVersion,
};
dsNewVersion.ContentDescriptors.Add(cd);
}
}
}
ds.Status = DatasetStatus.CheckedOut;
ds.LastCheckIOTimestamp = timestamp;
ds.CheckOutUser = getUserIdentifier(username);
ds.Versions.Add(dsNewVersion);
repo.Put(ds);
uow.Commit();
checkedOut = true;
}
}
return (checkedOut);
}
private DatasetVersion applyTupleChanges(DatasetVersion workingCopyVersion
, ref List<DataTupleVersion> tupleVersionsTobeAdded, ref List<DataTuple> tuplesTobeDeleted, ref List<DataTuple> tuplesTobeEdited
, ICollection<DataTuple> createdTuples, ICollection<DataTuple> editedTuples, ICollection<long> deletedTuples, ICollection<DataTuple> unchangedTuples = null)
{
// do nothing with unchanged for now
#region Process Newly Created Tuples
/// associate newly created tuples to the new version
/// try using bulk copy or stateless sessions for large amount of new tuples. it should also apply on deleted tuples.
/// Take care of automatic flushing and try to prevent or reduce it while the edit process is not finished.
if (createdTuples != null && createdTuples.Count() > 0)
{
// is not working cause of the item.Dematerialize();
//Parallel.ForEach(createdTuples, item =>
//{
// item.Dematerialize();
// // commented for the performance testing purpose. see the efects and uncomment if needed-> workingCopyVersion.PriliminaryTuples.Add(item);
// item.DatasetVersion = workingCopyVersion;
// item.TupleAction = TupleAction.Created;
// item.Timestamp = workingCopyVersion.Timestamp;
//});
foreach (var item in createdTuples)
{
//item.Dematerialize();
// commented for the performance testing purpose. see the efects and uncomment if needed-> workingCopyVersion.PriliminaryTuples.Add(item);
item.DatasetVersion = workingCopyVersion;
item.TupleAction = TupleAction.Created;
item.Timestamp = workingCopyVersion.Timestamp;
}
}
#endregion
if ((editedTuples != null && editedTuples.Count() > 0) || (deletedTuples != null && deletedTuples.Count() > 0))
{
// latest version is the latest checked in version. it is the previous version in comparison to the working copy version.
// the checks to see whether the dataset is checked out are considered to be done before
DatasetVersion latestCheckedInVersion = workingCopyVersion.Dataset.Versions.OrderByDescending(p => p.Timestamp).FirstOrDefault(p => p.Status == DatasetVersionStatus.CheckedIn);
if (latestCheckedInVersion == null) // there is no previous version, means its the first version. In this case there is no need to handle deleted and editedVersion items!
return (workingCopyVersion);
// the edit and delete candiates know the exact ID of the target tuple, so no need to load the whole tuples in advance. in worst case the number of single tuple match queries will increase
//List<DataTuple> latestVersionEffectiveTuples = getPrimaryTuples(workingCopyVersion); //latestVersionEffectiveTuples = DataTupleRepo.Get(p=>p.DatasetVersion == null).ToList();
#region Process Edited Tuples
/// manage editedVersion tuples:
/// 1: create a DataTupleVersion based on its previous version
/// 2: Remove the original from the original version
/// 3: add them to the version
/// 4: set timestamp for the editedVersion ones
if (editedTuples != null && editedTuples.Count() > 0)
{
// this part of the code loads the original tuples that are claimed edited. The edited collection is indeed one package of changes submitted in this round
// so its used to load only the relevant portion of the tuples. this avoid loading all the tuples asscoiated with the current version. also avoid N times querying the DB, each time to retreive one tuple
// Seems a moderate solution between the two extreme.
List<Int64> editedTupleIds = editedTuples.Select(t => t.Id).ToList(); // All the IDs of edited tuples of the current package
List<DataTuple> oraginalsOfEditedTuples = DataTupleRepo.Get(p => editedTupleIds.Contains(p.Id)).ToList(); // all the original tuples edited in the current package
//Parallel.ForEach(editedTuples, edited => // not able to use parallel for now, because the tuplesTobeEdited gets shared between threads for writting in it, which causes synch problems
foreach (var edited in editedTuples)
{
DataTuple orginalTuple = oraginalsOfEditedTuples.SingleOrDefault(p => p.Id == edited.Id); //DataTupleRepo.Get(edited.Id);// latestVersionEffectiveTuples.Where(p => p.Id == editedVersion.Id).Single();//maybe preliminary tuples are enough
if (orginalTuple == null || orginalTuple.Id <= 0) // maybe the tuple is in the edited list by a mistake!
continue;
//check if the history record for this data tuple has been created before. in cases of multiple edits in a single version for example
if (DataTupleVerionRepo.Query(p => p.OriginalTuple.Id == orginalTuple.Id && p.DatasetVersion.Id == orginalTuple.DatasetVersion.Id).Count() <= 0) // it is the first time the orginalTuple is getting editedVersion. so add a history record. the history record, keeps the tuple as was before the first edit!
{
DataTupleVersion tupleVersion = new DataTupleVersion()
{
TupleAction = TupleAction.Edited,
Extra = orginalTuple.Extra,
//Id = orginalTuple.Id,
OrderNo = orginalTuple.OrderNo,
Timestamp = orginalTuple.Timestamp,
XmlAmendments = orginalTuple.XmlAmendments,
XmlVariableValues = orginalTuple.XmlVariableValues,
OriginalTuple = orginalTuple,
DatasetVersion = orginalTuple.DatasetVersion, //latestCheckedInVersion,
ActingDatasetVersion = workingCopyVersion,
};
//DataTuple merged =
//orginalTuple.History.Add(tupleVersion);
}
//need a better way to preserve changes during the fetch of the original tuple. Maybe deep copy/ evict/ merge works
//XmlDocument xmlVariableValues = new XmlDocument();
//xmlVariableValues.LoadXml(editedVersion.XmlVariableValues.AsString());
// dematerialize just for the purpose of synching the xml fields with the object properties.
edited.Dematerialize();
orginalTuple.TupleAction = TupleAction.Edited;
orginalTuple.OrderNo = edited.OrderNo;
orginalTuple.XmlAmendments = null;
orginalTuple.XmlAmendments = edited.XmlAmendments;
orginalTuple.XmlVariableValues = null;
orginalTuple.XmlVariableValues = edited.XmlVariableValues;
//System.Diagnostics.Debug.Print(editedVersion.XmlVariableValues.AsString());
//editedVersion.VariableValues.ToList().ForEach(p => System.Diagnostics.Debug.Print(p.Value.ToString()));
//System.Diagnostics.Debug.Print(xmlVariableValues.AsString());
orginalTuple.DatasetVersion = workingCopyVersion;
orginalTuple.Timestamp = workingCopyVersion.Timestamp;
tuplesTobeEdited.Add(orginalTuple);
//workingCopyVersion.PriliminaryTuples.Add(detached);
//latestCheckedInVersion.PriliminaryTuples.Remove(orginalTuple);
//latestVersionEffectiveTuples.Remove(orginalTuple);
}
//); //parallel for each
}
#endregion
#region Process Deleted Tuples
/// manage deleted tuples:
/// 1: create a DataTupleVersion based on their previous version
/// 2: Remove them from the latest version
/// 3: DO NOT add them to the new version
/// 4: DO NOT set timestamp for the deleted ones
if (deletedTuples != null && deletedTuples.Count() > 0)
{
//Parallel.ForEach(deletedTuples, deleted => // the tuplesTobeDeleted gets shared between the threads!
// use the tuple iterator to reduce the # of DB fetchs
DataTupleIterator tupleIterator = new DataTupleIterator(deletedTuples.ToList(), this);
// load the ID all the tuple versions that are already linked to the tobe deleted tuples.
//This reduces the number of selects on the tuple versions, because most of the tuples have no version
List<long> tupleVersionIds = DataTupleVerionRepo.Query(p => deletedTuples.Contains(p.OriginalTuple.Id)).Select(p=>p.Id).ToList();
foreach (var deleted in tupleIterator)
{
DataTuple originalTuple = (DataTuple)deleted; // DataTupleRepo.Get(deleted.Id);// latestVersionEffectiveTuples.Where(p => p.Id == deleted.Id).Single();
// check if the tuple has a previous history record. for example may be it was first editedVersion and now is going to be deleted. in two different edits but in one version
DataTupleVersion tupleVersion;
// check if the tuple has a history record
if (tupleVersionIds.Contains(originalTuple.Id))
{
tupleVersion = DataTupleVerionRepo.Get(originalTuple.Id);
// there is a previous history record, with tuple action equal to Edit or even Delete!
tupleVersion.TupleAction = TupleAction.Deleted;
}
else // there is no previous record, so create one
{
tupleVersion = new DataTupleVersion()
{
TupleAction = TupleAction.Deleted,
Extra = originalTuple.Extra,
//Id = orginalTuple.Id,
OrderNo = originalTuple.OrderNo,
Timestamp = originalTuple.Timestamp,
XmlAmendments = originalTuple.XmlAmendments,
XmlVariableValues = originalTuple.XmlVariableValues,
//OriginalTuple = orginalTuple,
DatasetVersion = originalTuple.DatasetVersion, // latestCheckedInVersion,
ActingDatasetVersion = workingCopyVersion,
};
}
tupleVersion.OriginalTuple = null;
// /////////////////////////////////////////
// try avoid accessing to the PriliminaryTuples, they cause loading all the tuples!!
// ////////////////////////////////////////
// -> latestCheckedInVersion.PriliminaryTuples.Remove(originalTuple);
// check whether the deleted tuples are removed from the datatuples table!!!!!
//latestVersionEffectiveTuples.Remove(originalTuple);
// -> workingCopyVersion.PriliminaryTuples.Remove(originalTuple);
//try
//{
// //originalTuple.History.ToList().ForEach(p => p.OriginalTuple = null);
//}
//catch { }
//originalTuple.History.Clear();
originalTuple.DatasetVersion = null;
tuplesTobeDeleted.Add(originalTuple);
tupleVersionsTobeAdded.Add(tupleVersion);
}
//); // parralel for each loop
}
#endregion
}
return (workingCopyVersion);
}
/// <summary>
/// Returns one page of the data tuples of the dataset version requested. See <see cref="GetDatasetVersionEffectiveTuples"/> for more details about the effective tuples of a dataset.
/// </summary>
/// <param name="datasetVersion"></param>
/// <param name="pageNumber"></param>
/// <param name="pageSize"></param>
/// <returns>A list containing one page of maximum length of <paramref name="pageSize"/> from the effective tuples of the <paramref name="datasetVersion"/>.</returns>
/// <remarks>The actual returned tuples depend on the status of the dataset and whether the requested version is the latest.</remarks>
//[MeasurePerformance]
public List<AbstractTuple> GetDatasetVersionEffectiveTuples(DatasetVersion datasetVersion, int pageNumber, int pageSize)
{
return getDatasetVersionEffectiveTuples(datasetVersion, pageNumber, pageSize);
}
private List<DataTuple> getWorkingCopyTuples(DatasetVersion datasetVersion, int pageNumber, int pageSize)
{
// effective tuples of the working copy are similar to latest checked in version. They are in DataTuples table but they belong to the latest and previous versions
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
List<DataTuple> tuples = (versionIds == null || versionIds.Count() <= 0) ? new List<DataTuple>() :
DataTupleRepo.Query(p => versionIds.Contains(((DataTuple)p).DatasetVersion.Id))
.Skip(pageNumber*pageSize).Take(pageSize)
.ToList();
return (tuples);
}
private DatasetVersion undoTupleChanges(DatasetVersion workingCopyVersion)
{
// delete newly created tuples
// undo edit
//undo delete
return (workingCopyVersion);
}
private List<DataTuple> getHistoricTuples(DatasetVersion datasetVersion, int pageNumber, int pageSize)
{
//get previous versions including the version specified
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
//get all tuples from the main tuples table belonging to one of the previous versions + the current version
List<DataTuple> tuples = DataTupleRepo.Get(p => versionIds.Contains(p.DatasetVersion.Id)).ToList();
List<DataTuple> editedTuples = DataTupleVerionRepo.Query(p => (p.TupleAction == TupleAction.Edited)
&& (p.DatasetVersion.Id == datasetVersion.Id)
&& !(versionIds.Contains(p.ActingDatasetVersion.Id)))
.Skip(pageNumber * pageSize).Take(pageSize)
.Cast<DataTuple>().ToList();
List<DataTuple> deletedTuples = DataTupleVerionRepo.Query(p => (p.TupleAction == TupleAction.Deleted)
&& (versionIds.Contains(p.DatasetVersion.Id))
&& !(versionIds.Contains(p.ActingDatasetVersion.Id)))
.Skip(pageNumber * pageSize).Take(pageSize)
.Cast<DataTuple>().ToList();
// the resulting union-ned list is made by a page from editedVersion and a page from the deleted ones, so it is maximum 2 pages, but should be reduced to a page.
// for this reason the union is sorted by timestamp and then the first page is taken.
List<DataTuple> unioned = tuples.Union(editedTuples).Union(deletedTuples)
.OrderBy(p => p.Timestamp)
.Take(pageSize)
.ToList();
return (unioned);
}
/// <summary>
/// Using the provided values creates a data tuple, attaches it to the version and persists it in the database.
/// This method does not affect the status of the dataset version.
/// </summary>
/// <param name="orderNo">The order of the data tuple in the list of tuples</param>
/// <param name="variableValues">The values to be considered as the data tuple. They must be attached to their corresponding variables according to the dataset's data structure.</param>
/// <param name="amendments">Each data tuple can have amendments and if provided, the method attaches them to the data tuple.</param>
/// <param name="datasetVersion">The version of the dataset the data tuple is attached to. The version must be checked-out.</param>
/// <returns>the created data tuple</returns>
/// <exception cref="Exception">throws and exception if the dataset version is not checked-out.</exception>
public DataTuple CreateDataTuple(int orderNo, ICollection<VariableValue> variableValues, ICollection<Amendment> amendments, DatasetVersion datasetVersion)
{
//Contract.Requires(!string.IsNullOrWhiteSpace(name));
Contract.Requires(datasetVersion != null);
Contract.Ensures(Contract.Result<DataTuple>() != null && Contract.Result<DataTuple>().Id >= 0);
if (datasetVersion.Status != DatasetVersionStatus.CheckedOut)
{
throw new Exception(string.Format("The dataset version {0} must be checked-out!", datasetVersion.Id));
}
DataTuple e = new DataTuple()
{
OrderNo = orderNo,
DatasetVersion = datasetVersion,
VariableValues = new List<VariableValue>(variableValues),
Amendments = new List<Amendment>(amendments),
};
e.DatasetVersion.PriliminaryTuples.Add(e);
e.Amendments.ToList().ForEach(ex => ex.Tuple = e);
//e.VariableValues.ToList().ForEach(ex => ex.Tuple = e);
// check to see if all variable values and their parameter values are defined in the data structure
using (IUnitOfWork uow = this.GetUnitOfWork())
{
IRepository<DataTuple> repo = uow.GetRepository<DataTuple>();
repo.Put(e);
uow.Commit();
}
return (e);
}
private List<Int64> getPreviousVersionIds(DatasetVersion datasetVersion)
{
List<Int64> versionIds = datasetVersion.Dataset.Versions
.Where(p => p.Timestamp <= datasetVersion.Timestamp)
.Select(p => p.Id)
.ToList();
return versionIds;
}
/// <summary>
/// Applies the submitted changes to the working copy and persists the changes but does <b>NOT</b> check-in the dataset.
/// The changes are coming in the form of the tuples to be added, deleted, or editedVersion.
/// The general procedure of making changes is CheckOut, Edit (one or more times), CheckIn or Rollback.
/// there is no need to pass metadata, extendedPropertyValues, contentDescriptors .. as they can be assigned to the working copy version before sending it to the method.
/// Just if they are null, they will not affect the version.
/// The general procedure is CheckOut, Edit*, CheckIn or Rollback
/// While the dataset is checked out, all the changes go to the latest+1 version which acts like a working copy
/// </summary>
/// <param name="workingCopyDatasetVersion">The working copy version that accepts the changes.</param>
/// <param name="createdTuples">The list of the new tuples to be added to the working copy.</param>
/// <param name="editedTuples">The list of the tuples whose values have been changed and the changes should be considered in the working copy.</param>
/// <param name="deletedTuples">The list of existing tuples to be deleted from the working copy.</param>
/// <param name="unchangedTuples">to be removed</param>
/// <returns>The working copy having the changes applied on it.</returns>
public DatasetVersion EditDatasetVersion(DatasetVersion workingCopyDatasetVersion,
List<DataTuple> createdTuples, ICollection<DataTuple> editedTuples, ICollection<long> deletedTuples, ICollection<DataTuple> unchangedTuples = null
//,ICollection<ExtendedPropertyValue> extendedPropertyValues, ICollection<ContentDescriptor> contentDescriptors
)
{
workingCopyDatasetVersion.Dematerialize(false);
//preserve metadata and XmlExtendedPropertyValues for later use
var workingCopyDatasetVersionId = workingCopyDatasetVersion.Id;
var metadata = workingCopyDatasetVersion.Metadata;
var xmlExtendedPropertyValues = workingCopyDatasetVersion.XmlExtendedPropertyValues;
var contentDescriptors = workingCopyDatasetVersion.ContentDescriptors;
// do not move them to editDatasetVersion function
this.DatasetRepo.Evict();
this.DatasetVersionRepo.Evict();
this.DataTupleRepo.Evict();
this.DataTupleVerionRepo.Evict();
this.DatasetRepo.UnitOfWork.ClearCache();
// maybe its better to use Merge function ...
workingCopyDatasetVersion = this.DatasetVersionRepo.Get(workingCopyDatasetVersionId);
if (metadata != null)
workingCopyDatasetVersion.Metadata = metadata;
if (xmlExtendedPropertyValues != null)
workingCopyDatasetVersion.XmlExtendedPropertyValues = xmlExtendedPropertyValues;
if (contentDescriptors != null)
workingCopyDatasetVersion.ContentDescriptors = contentDescriptors;
return editDatasetVersion(workingCopyDatasetVersion, createdTuples, editedTuples, deletedTuples, unchangedTuples);
}
private List<Int64> getPreviousVersionIdsOrdered(DatasetVersion datasetVersion)
{
List<Int64> versionIds = datasetVersion.Dataset.Versions
.Where(p => p.Timestamp <= datasetVersion.Timestamp)
.OrderByDescending(t => t.Timestamp)
.Select(p => p.Id)
.ToList();
return versionIds;
}
private List<Int64> getPrimaryTupleIds(DatasetVersion datasetVersion)
{
// effective tuples of the latest checked in version are in DataTuples table but they belong to the latest and previous versions
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
List<Int64> tuples = (versionIds == null || versionIds.Count() <= 0) ?
new List<Int64>()
: DataTupleRepo.Query(p => versionIds.Contains(p.DatasetVersion.Id)).Select(p=> p.Id)
.ToList();
return (tuples);
}
private List<DataTuple> getPrimaryTuples(DatasetVersion datasetVersion)
{
// effective tuples of the latest checked in version are in DataTuples table but they belong to the latest and previous versions
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
List<DataTuple> tuples;
// experimental code, the stateles session fails to allow fetching the object graphs later. tuple.value.variable...
//using (IUnitOfWork uow = this.GetBulkUnitOfWork())
//{
// IReadOnlyRepository<DataTuple> tuplesRepoTemp = uow.GetReadOnlyRepository<DataTuple>();
// tuples = (versionIds == null || versionIds.Count() <= 0) ?
// new List<DataTuple>() :
// tuplesRepoTemp.Get(p => versionIds.Contains(p.DatasetVersion.Id)).ToList();
//}
tuples = (versionIds == null || versionIds.Count() <= 0) ?
new List<DataTuple>() :
DataTupleRepo.Get(p => versionIds.Contains(p.DatasetVersion.Id)).ToList();
////Dictionary<string, object> parameters = new Dictionary<string, object>() { { "datasetVersionId", datasetVersion.Id } };
////List<DataTuple> tuples = DataTupleRepo.Get("getLatestCheckedInTuples", parameters).ToList();
return (tuples);
}
public List<Error> FinishUpload2(TaskManager taskManager)
{
List<Error> temp = new List<Error>();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
DataStructureManager dsm = new DataStructureManager();
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
//datatuple list
List<DataTuple> rows;
DatasetManager dm = new DatasetManager();
Dataset ds = dm.GetDataset(id);
DatasetVersion workingCopy = new DatasetVersion();
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List<AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
//Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary<string, List<DataTuple>> splittedDatatuples = new Dictionary<string, List<DataTuple>>();
splittedDatatuples = UploadWizardHelper.GetSplitDatatuples2(rows, (List<long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
}
} while (rows.Count() > 0);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
// start download generator
// filepath
//string path = "";
//if (workingCopy != null)
//{
// path = GenerateDownloadFile(workingCopy);
// dm.EditDatasetVersion(workingCopy, null, null, null);
//}
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated);
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault());
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
workingCopy = dm.GetDatasetLatestVersion(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault());
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
return temp;
}
private List<DataTuple> getPrimaryTuples(DatasetVersion datasetVersion, int pageNumber, int pageSize)
{
// effective tuples of the latest checked in version are in DataTuples table but they belong to the latest and previous versions
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
List<DataTuple> tuples;
try
{
using (IUnitOfWork uow = this.GetBulkUnitOfWork())
{
IReadOnlyRepository<DataTuple> tuplesRepoTemp = uow.GetReadOnlyRepository<DataTuple>();
tuples = (versionIds == null || versionIds.Count() <= 0) ?
new List<DataTuple>() :
tuplesRepoTemp.Query(p => versionIds.Contains(p.DatasetVersion.Id))
.Skip(pageNumber * pageSize)
.Take(pageSize)
.ToList();
}
//tuples = (versionIds == null || versionIds.Count() <= 0) ? new List<DataTuple>() :
// DataTupleRepo.Query(p => versionIds.Contains(p.DatasetVersion.Id))
// .Skip(pageNumber * pageSize).Take(pageSize)
// .ToList();
//Dictionary<string, object> parameters = new Dictionary<string, object>() { { "datasetVersionId", datasetVersion.Id } };
//List<DataTuple> tuples = DataTupleRepo.Get("getLatestCheckedInTuples", parameters).ToList();
return (tuples);
} catch (Exception ex)
{
return null;
}
}
private string GenerateDownloadFile(DatasetVersion datasetVersion)
{
TaskManager TaskManager = (TaskManager)Session["TaskManager"];
//dataset id and data structure id are available via datasetVersion properties,why you are passing them via the BUS? Javad
long datasetId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASET_ID]);
long dataStructureId = Convert.ToInt64(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
DatasetManager datasetManager = new DatasetManager();
string title = TaskManager.Bus[TaskManager.DATASET_TITLE].ToString();
string ext = ".xlsm";// TaskManager.Bus[TaskManager.EXTENTION].ToString();
ExcelWriter excelWriter = new ExcelWriter();
// create the generated file and determine its location
string path = excelWriter.CreateFile(datasetId, datasetVersion.VersionNo, dataStructureId, title, ext);
string dynamicPath = excelWriter.GetDynamicStorePath(datasetId, datasetVersion.VersionNo, title, ext);
//Register the generated data file as a resource of the current dataset version
ContentDescriptor generatedDescriptor = new ContentDescriptor()
{
OrderNo = 1,
Name = "generated",
MimeType = "application/xlsm",
URI = dynamicPath,
DatasetVersion = datasetVersion,
};
if (datasetVersion.ContentDescriptors.Count(p => p.Name.Equals(generatedDescriptor.Name)) > 0)
{ // remove the one contentdesciptor
foreach (ContentDescriptor cd in datasetVersion.ContentDescriptors)
{
if (cd.Name == generatedDescriptor.Name)
{
cd.URI = generatedDescriptor.URI;
}
}
}
else
{
// add current contentdesciptor to list
datasetVersion.ContentDescriptors.Add(generatedDescriptor);
}
// note: the descriptors are not persisted yet, they will be persisted if the caller of this method persists the datasetVersion object.
return path;
}
private Int32 getWorkingCopyTupleCount(DatasetVersion datasetVersion)
{
List<Int64> versionIds = getPreviousVersionIds(datasetVersion);
Int32 tupleCount = (versionIds == null || versionIds.Count() <= 0) ? 0 : DataTupleRepo.Query(p => versionIds.Contains(((DataTuple)p).DatasetVersion.Id)).Select(p => p.Id).Count();
return (tupleCount);
}
private string SaveFileInContentDiscriptor(DatasetVersion datasetVersion)
{
try
{
//XXX put function like GetStorePathOriginalFile or GetDynamicStorePathOriginalFile
// the function is available in the abstract class datawriter
ExcelWriter excelWriter = new ExcelWriter();
// Move Original File to its permanent location
String tempPath = TaskManager.Bus[TaskManager.FILEPATH].ToString();
string originalFileName = TaskManager.Bus[TaskManager.FILENAME].ToString();
string storePath = excelWriter.GetFullStorePathOriginalFile(datasetVersion.Dataset.Id, datasetVersion.VersionNo, originalFileName);
string dynamicStorePath = excelWriter.GetDynamicStorePathOriginalFile(datasetVersion.Dataset.Id, datasetVersion.VersionNo, originalFileName);
string extention = TaskManager.Bus[TaskManager.EXTENTION].ToString();
Debug.WriteLine("extention : " + extention);
//Why using the excel writer, isn't any function available in System.IO.File/ Directory, etc. Javad
FileHelper.MoveFile(tempPath, storePath);
string mimeType = MimeMapping.GetMimeMapping(originalFileName);
//Register the original data as a resource of the current dataset version
ContentDescriptor originalDescriptor = new ContentDescriptor()
{
OrderNo = 1,
Name = "unstructuredData",
MimeType = mimeType,
URI = dynamicStorePath,
DatasetVersion = datasetVersion,
};
// add current contentdesciptor to list
datasetVersion.ContentDescriptors.Add(originalDescriptor);
return storePath;
}
catch(Exception e)
{
return "";
}
}
/// <summary>
/// Returns a list of <b>identifiers</b> of the effective tuples of the dataset version requested. See <see cref="GetDatasetVersionEffectiveTuples"/> for more details about the effective tuples of a dataset.
/// </summary>
/// <param name="datasetVersion">The object representing the data set version requested</param>
/// <returns>The list of identifiers of the specified version</returns>
public List<Int64> GetDatasetVersionEffectiveTupleIds(DatasetVersion datasetVersion)
{
return getDatasetVersionEffectiveTupleIds(datasetVersion);
}