public void GetDatasetVersionEffectiveDataTuples_CalledOlderVersionWithPaging_ReturnListOfAbstractTuples()
{
//Arrange
DatasetManager datasetManager = null;
int pageSize = 4;
int pageNumber = 2;
try
{
datasetManager = new DatasetManager();
var latestDatasetVersionId = datasetManager.GetDatasetLatestVersionId(datasetId);
//get latest datatupleid before create a new dataset and data
using (var uow = this.GetUnitOfWork())
{
var latestDataTuple = uow.GetReadOnlyRepository <DataTuple>().Get().LastOrDefault();
var firstDataTuple = uow.GetReadOnlyRepository <DataTuple>().Get().Where(dt => dt.DatasetVersion.Id.Equals(latestDatasetVersionId)).FirstOrDefault();
if (latestDataTuple != null)
{
latestDataTupleId = latestDataTuple.Id;
}
if (firstDataTuple != null)
{
firstDataTupleId = firstDataTuple.Id;
}
}
var dataset = datasetManager.GetDataset(datasetId);
dataset = dsHelper.UpdateOneTupleForDataset(dataset, (StructuredDataStructure)dataset.DataStructure, firstDataTupleId, 1, datasetManager);
datasetManager.CheckInDataset(dataset.Id, "for testing datatuples with versions", username, ViewCreationBehavior.None);
dataset = dsHelper.UpdateOneTupleForDataset(dataset, (StructuredDataStructure)dataset.DataStructure, latestDataTupleId, 2, datasetManager);
datasetManager.CheckInDataset(dataset.Id, "for testing datatuples with versions", username, ViewCreationBehavior.None);
//Act
List <DatasetVersion> datasetversions = datasetManager.GetDatasetVersions(datasetId).OrderBy(d => d.Timestamp).ToList();
var resultAll = datasetManager.GetDatasetVersionEffectiveTuples(datasetversions.ElementAt(datasetversions.Count - 2));
List <long> comapreIds = resultAll.OrderBy(dt => dt.OrderNo).Skip(pageNumber * pageSize).Take(pageSize).Select(dt => dt.Id).ToList();
var result = datasetManager.GetDatasetVersionEffectiveTuples(datasetversions.ElementAt(datasetversions.Count - 2), pageNumber, pageSize); // get datatuples from the one before the latest
var resultIds = result.Select(dt => dt.Id).ToList();
//Assert
Assert.That(comapreIds, Is.EquivalentTo(resultIds));
}
finally
{
datasetManager.Dispose();
}
}
public void TransformDatatupleToJson()
{
//Arrange
DatasetManager datasetManager = new DatasetManager();
try
{
//Act
DatasetVersion datasetversion = datasetManager.GetDatasetLatestVersion(datasetId);
var result = datasetManager.GetDatasetVersionEffectiveTuples(datasetversion);
foreach (var dt in result)
{
dt.Materialize();
dt.Dematerialize();
}
var newDts = result.Cast <DataTuple>();
foreach (var dt in newDts)
{
//dt.Materialize2();
//dt.Dematerialize2(); // convert variablevalues 1 to json
}
//Assert
Assert.That(result.Count(), Is.EqualTo(numberOfTuples));
foreach (var dt in newDts)
{
for (int i = 0; i < dt.VariableValues.Count; i++)
{
var vv1 = dt.VariableValues.ElementAt(i);
//var vv2 = dt.VariableValues2.ElementAt(i);
//Assert.That(vv1.DataAttribute.Id , Is.EqualTo(vv2.DataAttribute.Id));
//Assert.That(vv1.Note, Is.EqualTo(vv2.Note));
//Assert.That(vv1.ObtainingMethod, Is.EqualTo(vv2.ObtainingMethod));
//Assert.That(vv1.ParameterValues, Is.EqualTo(vv2.ParameterValues));
//Assert.That(vv1.ResultTime, Is.EqualTo(vv2.ResultTime));
//Assert.That(vv1.SamplingTime, Is.EqualTo(vv2.SamplingTime));
//Assert.That(vv1.Tuple.Id, Is.EqualTo(vv2.Tuple.Id));
//Assert.That(vv1.Value, Is.EqualTo(vv2.Value));
//Assert.That(vv1.Variable.Id, Is.EqualTo(vv2.Variable.Id));
//Assert.That(vv1.VariableId, Is.EqualTo(vv2.VariableId));
}
}
}
catch (Exception ex)
{
throw (ex);
}
finally
{
datasetManager.Dispose();
}
}
private DataTable getHistoryData(long versionId)
{
DatasetManager dm = new DatasetManager();
try
{
DatasetVersion dsv = dm.GetDatasetVersion(versionId);
DataTable table = null;
long rowCount = dm.GetDatasetVersionEffectiveTuples(dsv).Count;
table = dm.GetDatasetVersionTuples(versionId, 0, (int)rowCount);
return(table);
}
finally
{
dm.Dispose();
}
}
public void GetDatasetVersionEffectiveDataTuples_AllDataTuplesFromLatestVersion_ReturnListOfAbstractTuples()
{
//Arrange
DatasetManager datasetManager = new DatasetManager();
try
{
//Act
DatasetVersion datasetversion = datasetManager.GetDatasetLatestVersion(datasetId);
var result = datasetManager.GetDatasetVersionEffectiveTuples(datasetversion);
//Assert
Assert.That(result.Count(), Is.EqualTo(numberOfTuples));
}
finally
{
datasetManager.Dispose();
}
}
/// <summary>
/// Shows the content of a specific dataset version
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ActionResult Version(int id)
{
ViewBag.Title = PresentationModel.GetViewTitleForTenant("Version", Session.GetTenant());
DatasetManager dm = new DatasetManager();
DatasetVersion version = dm.DatasetVersionRepo.Get(p => p.Id == id).First();
var tuples = dm.GetDatasetVersionEffectiveTuples(version);
ViewBag.VersionId = id;
ViewBag.DatasetId = version.Dataset.Id;
if (version.Dataset.DataStructure is StructuredDataStructure)
{
ViewBag.Variables = ((StructuredDataStructure)version.Dataset.DataStructure.Self).Variables.ToList();
}
else
{
ViewBag.Variables = new List <Variable>();
}
return(View(tuples));
}
public void GetDatasetVersionEffectiveDataTuples_WhenCalledOlderVersionValid_ReturnListOfAbstractTuples()
{
//Arrange
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
//get latest datatupleid before create a new dataset and data
using (var uow = this.GetUnitOfWork())
{
var latestDataTuple = uow.GetReadOnlyRepository <DataTuple>().Get().LastOrDefault();
if (latestDataTuple != null)
{
latestDataTupleId = latestDataTuple.Id;
}
}
var dataset = datasetManager.GetDataset(datasetId);
dataset = dsHelper.UpdateOneTupleForDataset(dataset, (StructuredDataStructure)dataset.DataStructure, latestDataTupleId, 1, datasetManager);
datasetManager.CheckInDataset(dataset.Id, "for testing datatuples with versions", username, ViewCreationBehavior.None);
//Act
List <DatasetVersion> datasetversions = datasetManager.GetDatasetVersions(datasetId).OrderBy(d => d.Timestamp).ToList();
var result = datasetManager.GetDatasetVersionEffectiveTuples(datasetversions.ElementAt(datasetversions.Count - 2)); // get datatuples from the one before the latest
//Assert
Assert.That(result.Count(), Is.EqualTo(numberOfTuples));
}
finally
{
datasetManager.Dispose();
}
}
public DataStructureResultStruct(long dataStructureId)
{
using (DataStructureManager dataStructureManager = new DataStructureManager())
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
if (structuredDataStructure != null)
{
this.Id = structuredDataStructure.Id;
this.Title = structuredDataStructure.Name;
this.Description = structuredDataStructure.Description;
this.inUse = false;
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
foreach (Dataset d in structuredDataStructure.Datasets)
{
if (datasetManager.RowAny(d.Id))
{
this.inUse = true;
break;
}
else
{
foreach (DatasetVersion dv in d.Versions)
{
if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any())
{
this.inUse = true;
break;
}
}
}
}
}
finally
{
datasetManager.Dispose();
}
this.Structured = true;
this.Preview = false;
}
else
{
UnStructuredDataStructure unStructuredDataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId);
if (unStructuredDataStructure != null)
{
this.Id = unStructuredDataStructure.Id;
this.Title = unStructuredDataStructure.Name;
this.Description = unStructuredDataStructure.Description;
if (unStructuredDataStructure.Datasets != null && unStructuredDataStructure.Datasets.Any())
{
this.inUse = true;
}
else
{
this.inUse = false;
}
this.Structured = false;
this.Preview = false;
}
else
{
new DataStructureResultStruct();
}
}
}
}
public DataStructurePreviewModel fill(long dataStructureId)
{
if (dataStructureId > 0)
{
DataStructureManager dataStructureManager = null;
try
{
dataStructureManager = new DataStructureManager();
if (dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId) != null)
{
StructuredDataStructure dataStructure = dataStructureManager.StructuredDataStructureRepo.Get(dataStructureId);
VariablePreviewStruct variablePreview;
this.Id = dataStructure.Id;
this.Name = dataStructure.Name;
this.Description = dataStructure.Description;
if (dataStructure.Datasets.Any())
{
DatasetManager datasetManager = null;
try
{
datasetManager = new DatasetManager();
foreach (Dataset d in dataStructure.Datasets)
{
if (datasetManager.RowAny(d.Id))
{
this.inUse = true;
break;
}
else
{
foreach (DatasetVersion dv in d.Versions)
{
if (datasetManager.GetDatasetVersionEffectiveTuples(dv).Any())
{
this.inUse = true;
break;
}
}
}
}
}
finally
{
datasetManager.Dispose();
}
}
foreach (Variable v in DataStructureIO.getOrderedVariables(dataStructure))
{
variablePreview = new VariablePreviewStruct().fill(v);
this.VariablePreviews.Add(variablePreview);
}
}
else if (dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId) != null)
{
UnStructuredDataStructure dataStructure = dataStructureManager.UnStructuredDataStructureRepo.Get(dataStructureId);
this.Id = dataStructure.Id;
this.Name = dataStructure.Name;
this.Description = dataStructure.Description;
this.VariablePreviews = null;
if (dataStructure.Datasets.Any())
{
this.inUse = true;
}
}
return(this);
}
finally
{
dataStructureManager.Dispose();
}
}
else
{
return(new DataStructurePreviewModel());
}
}
private List <AbstractTuple> GetFilteredDataTuples(DatasetVersion datasetVersion)
{
DatasetManager datasetManager = new DatasetManager();
try
{
List <AbstractTuple> datatuples = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion);
if (Session["Filter"] != null)
{
GridCommand command = (GridCommand)Session["Filter"];
List <AbstractTuple> dataTupleList = datatuples;
if (command.FilterDescriptors.Count > 0)
{
foreach (IFilterDescriptor filter in command.FilterDescriptors)
{
var test = filter;
// one filter is set
if (filter.GetType() == typeof(FilterDescriptor))
{
FilterDescriptor filterDescriptor = (FilterDescriptor)filter;
// get id as long from filtername
Regex r = new Regex("(\\d+)");
long id = Convert.ToInt64(r.Match(filterDescriptor.Member).Value);
var list = from datatuple in dataTupleList
let val = datatuple.VariableValues.Where(p => p.Variable.Id.Equals(id)).FirstOrDefault()
where GridHelper.ValueComparion(val, filterDescriptor.Operator, filterDescriptor.Value)
select datatuple;
dataTupleList = list.ToList();
}
else
// more than one filter is set
if (filter.GetType() == typeof(CompositeFilterDescriptor))
{
CompositeFilterDescriptor filterDescriptor = (CompositeFilterDescriptor)filter;
List <AbstractTuple> temp = new List <AbstractTuple>();
foreach (IFilterDescriptor f in filterDescriptor.FilterDescriptors)
{
if ((FilterDescriptor)f != null)
{
FilterDescriptor fd = (FilterDescriptor)f;
// get id as long from filtername
Regex r = new Regex("(\\d+)");
long id = Convert.ToInt64(r.Match(fd.Member).Value);
var list = from datatuple in dataTupleList
let val = datatuple.VariableValues.Where(p => p.Variable.Id.Equals(id)).FirstOrDefault()
where GridHelper.ValueComparion(val, fd.Operator, fd.Value)
select datatuple;
//temp = list.Intersect<AbstractTuple>(temp as IEnumerable<AbstractTuple>).ToList();
dataTupleList = list.ToList();
}
}
//dataTupleList = temp;
}
}
}
if (command.SortDescriptors.Count > 0)
{
foreach (SortDescriptor sort in command.SortDescriptors)
{
string direction = sort.SortDirection.ToString();
// get id as long from filtername
Regex r = new Regex("(\\d+)");
long id = Convert.ToInt64(r.Match(sort.Member).Value);
if (direction.Equals("Ascending"))
{
var list = from datatuple in dataTupleList
let val = datatuple.VariableValues.Where(p => p.Variable.Id.Equals(id)).FirstOrDefault()
orderby GridHelper.CastVariableValue(val.Value, val.DataAttribute.DataType.SystemType) ascending
select datatuple;
dataTupleList = list.ToList();
}
else
if (direction.Equals("Descending"))
{
var list = from datatuple in dataTupleList
let val = datatuple.VariableValues.Where(p => p.Variable.Id.Equals(id)).FirstOrDefault()
orderby GridHelper.CastVariableValue(val.Value, val.DataAttribute.DataType.SystemType) descending
select datatuple;
dataTupleList = list.ToList();
}
}
}
return(dataTupleList);
}
return(null);
}
finally
{
datasetManager.Dispose();
}
}
public List <Error> FinishUpload2(TaskManager taskManager)
{
DataStructureManager dsm = new DataStructureManager();
try
{
List <Error> temp = new List <Error>();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_ID) && TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_ID))
{
long id = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASET_ID]);
long iddsd = Convert.ToInt32(TaskManager.Bus[TaskManager.DATASTRUCTURE_ID]);
//datatuple list
List <DataTuple> rows;
DatasetManager dm = new DatasetManager();
Dataset ds = dm.GetDataset(id);
DatasetVersion workingCopy = new DatasetVersion();
#region Progress Informations
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGESIZE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGESIZE, 0);
}
if (TaskManager.Bus.ContainsKey(TaskManager.CURRENTPACKAGE))
{
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = 0;
}
else
{
TaskManager.Bus.Add(TaskManager.CURRENTPACKAGE, 0);
}
#endregion
#region structured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Structured))
{
try
{
//Stopwatch fullTime = Stopwatch.StartNew();
//Stopwatch loadDT = Stopwatch.StartNew();
List <AbstractTuple> datatupleFromDatabase = dm.GetDatasetVersionEffectiveTuples(dm.GetDatasetLatestVersion(ds.Id));
//loadDT.Stop();
//Debug.WriteLine("Load DT From Db Time " + loadDT.Elapsed.TotalSeconds.ToString());
StructuredDataStructure sds = dsm.StructuredDataStructureRepo.Get(iddsd);
dsm.StructuredDataStructureRepo.LoadIfNot(sds.Variables);
#region excel reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".xlsm"))
{
int packageSize = 10000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
int counter = 0;
ExcelReader reader = new ExcelReader();
//schleife
dm.CheckOutDatasetIfNot(ds.Id, GetUsernameOrDefault()); // there are cases, the dataset does not get checked out!!
if (!dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()))
{
throw new Exception(string.Format("Not able to checkout dataset '{0}' for user '{1}'!", ds.Id, GetUsernameOrDefault()));
}
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
do
{
//Stopwatch packageTime = Stopwatch.StartNew();
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
// open file
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), sds, (int)id, packageSize);
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//XXX Add packagesize to excel read function
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, rows, null, null);
//Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
//Debug.WriteLine("----");
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
//Stopwatch split = Stopwatch.StartNew();
Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >();
splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
//split.Stop();
//Debug.WriteLine("Split : " + counter + " Time " + split.Elapsed.TotalSeconds.ToString());
//Stopwatch upload = Stopwatch.StartNew();
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
// upload.Stop();
// Debug.WriteLine("Upload : " + counter + " Time " + upload.Elapsed.TotalSeconds.ToString());
// Debug.WriteLine("----");
}
}
}
else
{
}
}
Stream.Close();
//packageTime.Stop();
//Debug.WriteLine("Package : " + counter + " packageTime Time " + packageTime.Elapsed.TotalSeconds.ToString());
} while (rows.Count() > 0);
//fullTime.Stop();
//Debug.WriteLine("FullTime " + fullTime.Elapsed.TotalSeconds.ToString());
}
#endregion
#region ascii reader
if (TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".csv") ||
TaskManager.Bus[TaskManager.EXTENTION].ToString().Equals(".txt"))
{
// open file
AsciiReader reader = new AsciiReader();
//Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
//DatasetManager dm = new DatasetManager();
//Dataset ds = dm.GetDataset(id);
Stopwatch totalTime = Stopwatch.StartNew();
if (dm.IsDatasetCheckedOutFor(ds.Id, GetUsernameOrDefault()) || dm.CheckOutDataset(ds.Id, GetUsernameOrDefault()))
{
workingCopy = dm.GetDatasetWorkingCopy(ds.Id);
int packageSize = 100000;
TaskManager.Bus[TaskManager.CURRENTPACKAGESIZE] = packageSize;
//schleife
int counter = 0;
do
{
counter++;
TaskManager.Bus[TaskManager.CURRENTPACKAGE] = counter;
Stream = reader.Open(TaskManager.Bus[TaskManager.FILEPATH].ToString());
rows = reader.ReadFile(Stream, TaskManager.Bus[TaskManager.FILENAME].ToString(), (AsciiFileReaderInfo)TaskManager.Bus[TaskManager.FILE_READER_INFO], sds, id, packageSize);
Stream.Close();
if (reader.ErrorMessages.Count > 0)
{
//model.ErrorList = reader.errorMessages;
}
else
{
//model.Validated = true;
Stopwatch dbTimer = Stopwatch.StartNew();
if (TaskManager.Bus.ContainsKey(TaskManager.DATASET_STATUS))
{
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("new"))
{
dm.EditDatasetVersion(workingCopy, rows, null, null);
}
if (TaskManager.Bus[TaskManager.DATASET_STATUS].ToString().Equals("edit"))
{
if (rows.Count() > 0)
{
Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >();
splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
}
else
{
if (rows.Count() > 0)
{
Dictionary <string, List <DataTuple> > splittedDatatuples = new Dictionary <string, List <DataTuple> >();
splittedDatatuples = uploadWizardHelper.GetSplitDatatuples2(rows, (List <long>)TaskManager.Bus[TaskManager.PRIMARY_KEYS], workingCopy, ref datatupleFromDatabase);
dm.EditDatasetVersion(workingCopy, splittedDatatuples["new"], splittedDatatuples["edit"], null);
}
}
dbTimer.Stop();
Debug.WriteLine(" db time" + dbTimer.Elapsed.TotalSeconds.ToString());
}
} while (rows.Count() > 0);
totalTime.Stop();
Debug.WriteLine(" Total Time " + totalTime.Elapsed.TotalSeconds.ToString());
}
//Stream.Close();
}
#endregion
// start download generator
// filepath
//string path = "";
//if (workingCopy != null)
//{
// path = GenerateDownloadFile(workingCopy);
// dm.EditDatasetVersion(workingCopy, null, null, null);
//}
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload data from upload wizard", GetUsernameOrDefault());
LoggerFactory.LogData(id.ToString(), typeof(Dataset).Name, Vaiona.Entities.Logging.CrudState.Updated);
}
catch (Exception e)
{
temp.Add(new Error(ErrorType.Other, "Can not upload. : " + e.Message));
dm.CheckInDataset(ds.Id, "checked in but no update on data tuples", GetUsernameOrDefault(), ViewCreationBehavior.None);
}
finally
{
}
}
#endregion
#region unstructured data
if (TaskManager.Bus.ContainsKey(TaskManager.DATASTRUCTURE_TYPE) && TaskManager.Bus[TaskManager.DATASTRUCTURE_TYPE].Equals(DataStructureType.Unstructured))
{
workingCopy = dm.GetDatasetLatestVersion(ds.Id);
SaveFileInContentDiscriptor(workingCopy);
dm.EditDatasetVersion(workingCopy, null, null, null);
// ToDo: Get Comment from ui and users
dm.CheckInDataset(ds.Id, "upload unstructured data", GetUsernameOrDefault(), ViewCreationBehavior.None);
}
#endregion
}
else
{
temp.Add(new Error(ErrorType.Dataset, "Dataset is not selected."));
}
return(temp);
}
finally
{
dsm.Dispose();
}
}
/// <summary>
/// test unique of primary keys on a dataset
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="datasetId"></param>
/// <param name="primaryKeys"></param>
/// <returns></returns>
////[MeasurePerformance]
public Boolean IsUnique(long datasetId, List <long> primaryKeys)
{
DatasetManager datasetManager = new DatasetManager();
try
{
Hashtable hashtable = new Hashtable();
// load data
Dataset dataset = datasetManager.GetDataset(datasetId);
DatasetVersion datasetVersion;
List <long> dataTupleIds = new List <long>();
if (datasetManager.IsDatasetCheckedIn(datasetId))
{
datasetVersion = datasetManager.GetDatasetLatestVersion(datasetId);
#region load all datatuples first
int size = 10000;
int counter = 0;
IEnumerable <AbstractTuple> dataTuples;
do
{
dataTuples = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion, counter, size);
//byte[] pKey;
string pKey;
foreach (DataTuple dt in dataTuples)
{
//pKey = getPrimaryKeysAsByteArray(dt, primaryKeys);
pKey = getPrimaryKeysAsString(dt, primaryKeys);
if (pKey.Count() > 0)
{
try
{
//Debug.WriteLine(pKey +" : " +Utility.ComputeKey(pKey));
hashtable.Add(pKey, "");
//hashtable.Add(pKey, 0);
}
catch
{
return(false);
}
}
}
counter++;
}while (dataTuples.Count() >= (size * counter));
#endregion
}
else
{
throw new Exception("Dataset is not checked in.");
}
return(true);
}
finally
{
datasetManager.Dispose();
}
}
private HttpResponseMessage getData(long id, int version, string token, string projection = null, string selection = null)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
// If the requested version is -1 or the last version of the dataset, then the data will be loaded in a
// different way than when loading the data from an older version
bool isLatestVersion = false;
if (version == -1 || dataset.Versions.Count == version)
{
isLatestVersion = true;
}
if (isLatestVersion)
{
#region get data from the latest version of a dataset
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
//FilterExpression filter = null;
//OrderByExpression orderBy = null;
//ProjectionExpression projectionExpression = GetProjectionExpression(projection);
// apply selection and projection
long count = datasetManager.RowCount(id);
DataTable dt = datasetManager.GetLatestDatasetVersionTuples(id, null, null, null, 0, (int)count);
dt.Strip();
if (!string.IsNullOrEmpty(selection))
{
dt = OutputDataManager.SelectionOnDataTable(dt, selection, true);
}
if (!string.IsNullOrEmpty(projection))
{
// make the header names upper case to make them case insensitive
dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(','));
}
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse();
response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName));
response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/csv");
//set headers on the "response"
return(response);
#endregion get data from the latest version of a dataset
//return model;
}
else
{
return(Request.CreateResponse());
}
}
else
{
#region load data of a older version of a dataset
int index = version - 1;
if (version >= dataset.Versions.Count)
{
return(Request.CreateResponse(HttpStatusCode.PreconditionFailed, String.Format("This version ({0}) is not available for the dataset", version)));
}
DatasetVersion datasetVersion = dataset.Versions.OrderBy(d => d.Timestamp).ElementAt(version - 1);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
//FilterExpression filter = null;
//OrderByExpression orderBy = null;
// apply selection and projection
int count = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion).Count;
DataTable dt = datasetManager.GetDatasetVersionTuples(datasetVersion.Id, 0, count);
dt.Strip();
if (!string.IsNullOrEmpty(selection))
{
dt = OutputDataManager.SelectionOnDataTable(dt, selection);
}
if (!string.IsNullOrEmpty(projection))
{
// make the header names upper case to make them case insensitive
dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(','));
}
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse();
response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName));
response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/csv");
//set headers on the "response"
return(response);
}
else // return files of the unstructure dataset
{
return(Request.CreateResponse());
}
#endregion load data of a older version of a dataset
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
}
}
