/// <summary>
/// Initializes a new instance of the <see cref="Dataset"/> class.
/// </summary>
/// <param name="datasetSkeleton">The dataset skeleton corresponding to the features</param>
/// <param name="flocks">An array of feature flocks</param>
public Dataset(DatasetSkeleton datasetSkeleton, FeatureFlockBase[] flocks)
{
Contracts.AssertValue(datasetSkeleton);
Contracts.AssertValue(flocks);
Contracts.Assert(flocks.All(f => f.Examples == datasetSkeleton.NumDocs));
_datasetSkeleton = datasetSkeleton;
_maxDocsPerQuery = -1;
_flocks = flocks;
_flockToFirstFeature = new int[_flocks.Length];
if (_flocks.Length > 0)
{
for (int i = 1; i < _flocks.Length; ++i)
{
Contracts.AssertValue(_flocks[i - 1]);
_flockToFirstFeature[i] = _flockToFirstFeature[i - 1] + _flocks[i - 1].Count;
}
var lastFlock = _flocks[_flocks.Length - 1];
Contracts.AssertValue(lastFlock);
int numFeatures = _flockToFirstFeature[_flockToFirstFeature.Length - 1] + lastFlock.Count;
Contracts.Assert(numFeatures == _flocks.Sum(f => f.Count));
_featureToFlock = new int[numFeatures];
for (int flock = 0; flock < _flockToFirstFeature.Length; ++flock)
{
int min = _flockToFirstFeature[flock];
int lim = min + _flocks[flock].Count;
for (int feat = min; feat < lim; ++feat)
{
_featureToFlock[feat] = flock;
}
}
}
else
{
_featureToFlock = new int[0];
}
}
/// <summary>
/// Takes an array of DatasetSkeleton objects and concatenates them into one big DatasetSkeleton
/// </summary>
/// <param name="parts">An array of DatasetSkeletons</param>
/// <returns>A concatenated DatasetSkeleton</returns>
public static DatasetSkeleton Concat(DatasetSkeleton[] parts)
{
int concatNumDocs = parts.Sum(x => x.NumDocs);
int concatNumQueries = parts.Sum(x => x.NumQueries);
// allocate
short[] concatRatings = new short[concatNumDocs];
ulong[] concatDocIds = new ulong[concatNumDocs];
ulong[] concatQueryIds = new ulong[concatNumQueries];
int[] concatBoundaries = new int[concatNumQueries + 1];
// copy components into new arrays
int docBegin = 0;
int queryBegin = 0;
for (int p = 0; p < parts.Length; ++p)
{
int numDocs = parts[p].NumDocs;
int numQueries = parts[p].NumQueries;
Array.Copy(parts[p].Ratings, 0, concatRatings, docBegin, numDocs);
Array.Copy(parts[p].DocIds, 0, concatDocIds, docBegin, numDocs);
Array.Copy(parts[p].QueryIds, 0, concatQueryIds, queryBegin, numQueries);
for (int q = 0; q < numQueries; ++q)
{
concatBoundaries[queryBegin + q] = parts[p].Boundaries[q] + docBegin;
}
docBegin += numDocs;
queryBegin += numQueries;
}
concatBoundaries[queryBegin] = docBegin;
DatasetSkeleton skel = new DatasetSkeleton(concatRatings, concatBoundaries, concatQueryIds, concatDocIds);
SetConcatenatedAuxiliaryData(parts, skel);
return(skel);
}
/// <summary>
/// Given the auxiliary data in a bunch of parts, set the concatenated dataset appropriately.
/// </summary>
/// <param name="parts">The individual parts of the dataset</param>
/// <param name="concat">The concatenated version of this dataset</param>
private static void SetConcatenatedAuxiliaryData(DatasetSkeleton[] parts, DatasetSkeleton concat)
{
// Get the union of all the auxiliary data names.
Dictionary <string, bool> auxNames = new Dictionary <string, bool>();
foreach (DatasetSkeleton part in parts)
{
foreach (string name in part.AuxiliaryData.Keys)
{
auxNames[name] = true;
}
}
DatasetSkeletonQueryDocData[] partsDatas = new DatasetSkeletonQueryDocData[parts.Length];
int[] docLengths = parts.Select(x => x.NumDocs).ToArray();
int[] queryLengths = parts.Select(x => x.NumQueries).ToArray();
foreach (string name in auxNames.Keys)
{
for (int p = 0; p < parts.Length; ++p)
{
partsDatas[p] = parts[p].AuxiliaryData.ContainsKey(name) ? parts[p].AuxiliaryData[name] : default(DatasetSkeletonQueryDocData);
}
bool isQuery = partsDatas.First(pd => pd.Data != null).IsQueryLevel;
if (partsDatas.Any(pd => pd.Data != null && pd.IsQueryLevel != isQuery))
{
throw Contracts.Except("On auxiliary data {0}, disagreement on whether this is query/doc", name);
}
Array concatArray = ConcatArrays(partsDatas.Select(pd => pd.Data).ToArray(), isQuery ? queryLengths : docLengths, name);
concat.SetData(name, concatArray, isQuery);
}
}
public Dataset GetSubDataset(int[] docIndices, bool destroyThisDataset, FileObjectStore <IntArrayFormatter> newBinsCache)
{
#endif
int[] queryIndices = docIndices.Select(d => DocToQuery[d]).ToArray();
ulong[] uniqueQueryIds = queryIndices.Distinct().Select(q => QueryIds[q]).ToArray();
// calculate boundaries
int[] boundaries = new int[uniqueQueryIds.Length + 1];
boundaries[0] = 0;
int queryIndex = 1;
for (int q = 1; q < queryIndices.Length; ++q)
{
if (queryIndices[q] != queryIndices[q - 1])
{
boundaries[queryIndex++] = q;
}
}
boundaries[uniqueQueryIds.Length] = queryIndices.Length;
// construct skeleton
DatasetSkeleton datasetSkeleton = new DatasetSkeleton(docIndices.Select(d => Ratings[d]).ToArray(),
boundaries,
uniqueQueryIds,
docIndices.Select(d => DocIds[d]).ToArray());
// create features
FeatureFlockBase[] features = new FeatureFlockBase[NumFlocks];
int[][] assignment = new int[][] { docIndices };
Parallel.For(0, NumFlocks, new ParallelOptions {
MaxDegreeOfParallelism = BlockingThreadPool.NumThreads
},
(int flockIndex) =>
{
#if !NO_STORE
GetSubDataset_ThreadWorker(features, flockIndex, assignment, destroyThisDataset, newBinsCache);
#else
GetSubDatasetThreadWorker(features, flockIndex, assignment, destroyThisDataset);
#endif
});
uint[] filteredDupeIds = null;
// Filter the dupe ids, if any
if (DupeIds != null)
{
uint[] dupeIds = DupeIds;
filteredDupeIds = docIndices.Select(i => dupeIds[i]).ToArray();
}
// auxiliary data
Dictionary <string, DatasetSkeletonQueryDocData> auxData = _datasetSkeleton.AuxiliaryData;
Dictionary <string, DatasetSkeletonQueryDocData> newAuxData = new Dictionary <string, DatasetSkeletonQueryDocData>();
foreach (KeyValuePair <string, DatasetSkeletonQueryDocData> pair in auxData)
{
newAuxData[pair.Key] = pair.Value.GetSubset(pair.Value.IsQueryLevel ? queryIndices.Distinct().ToArray() : docIndices);
}
datasetSkeleton.AuxiliaryData = newAuxData;
// create new Dataset
Dataset dataset = new Dataset(datasetSkeleton, features);
dataset.DupeIds = filteredDupeIds;
return(dataset);
}