private static SparseIndexing GenerateIndexes(SortStruct[] data)
{
SparseIndexing Indexes = new SparseIndexing();
List <SparseSet> elements = new List <SparseSet>();
var length = data.Length;
SparseSet current = default(SparseSet);
if (length == 0)
{
return(Indexes);
}
current.Start = data[0].SparseSpace;
int expected = data[0].SparseSpace + 1;
for (int i = 1; i < length; i++)
{
if (data[i].SparseSpace != expected)
{
current.Stop = data[i - 1].SparseSpace;
current.BaseLocation = 0;
elements.Add(current);
current.Start = data[i].SparseSpace;
}
expected = data[i].SparseSpace + 1;
}
current.Stop = data[length - 1].SparseSpace;
elements.Add(current);
Indexes.Indexes = elements.ToArray();
return(Indexes);
}
private static SparseIndexing ConvertToIndexes(List <SparseSet> processedIndexes, out T[][] Data, T[] data, SortStruct[] index)
{
SparseIndexing start = new SparseIndexing();
var iLength = processedIndexes.Count;
Data = new T[iLength][];
start.Indexes = new SparseSet[iLength];
int dataProcessed = 0;
for (int i = 0; i < iLength; i++)
{
start.Indexes[i] = processedIndexes[i];
var jLength = 0;
var jSections = processedIndexes[i].SubIndex.Indexes.Length;
for (int jSection = 0; jSection < jSections; jSection++)
{
var indexStop = processedIndexes[i].SubIndex.Indexes[jSection].Stop;
var indexStart = processedIndexes[i].SubIndex.Indexes[jSection].Start;
start.Indexes[i].SubIndex.Indexes[jSection].Start = indexStart;
start.Indexes[i].SubIndex.Indexes[jSection].Stop = indexStop;
jLength += indexStop - indexStart + 1;
}
Data[i] = new T[jLength];
for (int j = 0; j < jLength; j++)
{
Data[i][j] = data[index[dataProcessed++].DataSpace];
}
}
return(start);
}
public SparseArray(SparseIndexing indexing, T[] rawData = null)
{
this.Indexing = indexing;
if (rawData != null)
{
this.Data = rawData;
}
GenerateStructure();
}
public SparseTriIndex(SparseIndexing indexes, T[][][] data = null)
{
Indexes = indexes;
if (data != null)
{
Data = data;
}
GenerateStructure();
}
public SparseTwinIndex(SparseIndexing indexes, T[][] data = null)
{
this.Indexes = indexes;
if (data != null)
{
this.Data = data;
}
if (indexes.Indexes != null)
{
GenerateStructure();
}
}
public static SparseTwinIndex <T> CreateSimilarArray <TFirst, TSecond>(SparseArray <TFirst> first, SparseArray <TSecond> second)
{
var indexes = new SparseIndexing();
indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone();
var length = indexes.Indexes.Length;
for (var i = 0; i < length; i++)
{
indexes.Indexes[i].SubIndex = new SparseIndexing()
{
Indexes = second.Indexing.Indexes.Clone() as SparseSet[]
};
}
return(new SparseTwinIndex <T>(indexes));
}
public static SparseTwinIndex <T> CreateSimilarArray <J, K>(SparseArray <J> first, SparseArray <K> second)
{
SparseIndexing indexes = new SparseIndexing();
indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[];
var length = indexes.Indexes.Length;
for (int i = 0; i < length; i++)
{
indexes.Indexes[i].SubIndex = new SparseIndexing()
{
Indexes = second.Indexing.Indexes.Clone() as SparseSet[]
};
}
return(new SparseTwinIndex <T>(indexes));
}
public SparseTwinIndex(SparseIndexing indexes, T[][] data = null)
{
Indexes = indexes;
if (data != null)
{
Data = data;
}
if (indexes.Indexes != null)
{
GenerateStructure();
}
//Generate _Count
var count = 0;
for (var i = 0; i < Data.Length; i++)
{
count += Data[i].Length;
}
Count = count;
}
public static SparseTriIndex <T> CreateSimilarArray <TFirst, TSecond, TThird>(SparseArray <TFirst> first, SparseArray <TSecond> second, SparseArray <TThird> third)
{
var indexes = new SparseIndexing();
indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone();
var length = indexes.Indexes.Length;
for (var i = 0; i < length; i++)
{
SparseSet[] subIndexes;
indexes.Indexes[i].SubIndex = new SparseIndexing()
{
Indexes = (subIndexes = (SparseSet[])second.Indexing.Indexes.Clone())
};
var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length;
for (var j = 0; j < subindexLength; j++)
{
subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[];
}
}
return(new SparseTriIndex <T>(indexes));
}
private static SparseIndexing ConvertToIndexes(List <Pair <int, List <SparseSet> > > processedIndexes, out T[][][] outputData, T[] data, SortStruct[] index)
{
var start = new SparseIndexing();
var iLength = processedIndexes.Count;
outputData = new T[iLength][][];
start.Indexes = new SparseSet[iLength];
var dataProcessed = 0;
for (var i = 0; i < iLength; i++)
{
var jLength = processedIndexes[i].Second.Count;
outputData[i] = new T[jLength][];
start.Indexes[i].Start = start.Indexes[i].Stop = processedIndexes[i].First;
start.Indexes[i].SubIndex.Indexes = new SparseSet[jLength];
for (var j = 0; j < jLength; j++)
{
var totalK = 0;
var currentJ = processedIndexes[i].Second[j];
start.Indexes[i].SubIndex.Indexes[j] = currentJ;
var kSections = currentJ.SubIndex.Indexes.Length;
for (var kSection = 0; kSection < kSections; kSection++)
{
totalK += (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Stop = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Stop)
- (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Start = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Start) + 1;
}
outputData[i][j] = new T[totalK];
for (var k = 0; k < totalK; k++)
{
outputData[i][j][k] = data[index[dataProcessed++].DataSpace];
}
}
}
if (dataProcessed != data.Length)
{
throw new InvalidOperationException("The data processed should always be of the data's length!");
}
return(start);
}
public static SparseTriIndex <T> CreateSimilarArray <J, K, L>(SparseArray <J> first, SparseArray <K> second, SparseArray <L> third)
{
SparseIndexing indexes = new SparseIndexing();
indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[];
var length = indexes.Indexes.Length;
for (int i = 0; i < length; i++)
{
SparseSet[] subIndexes;
indexes.Indexes[i].SubIndex = new SparseIndexing()
{
Indexes = (subIndexes = second.Indexing.Indexes.Clone() as SparseSet[])
};
var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length;
for (int j = 0; j < subindexLength; j++)
{
subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[];
}
}
return(new SparseTriIndex <T>(indexes));
}
public SparseTwinIndex <float[]> StoreAll()
{
if (Indexes == null)
{
return(null);
}
int types = Types * Times;
SparseIndexing index = new SparseIndexing();
int firstLength = Indexes.Length;
int iTotal = 0;
index.Indexes = new SparseSet[Indexes.Length];
float[][][] data = null;
byte[] tempReader = new byte[(int)(Reader.BaseStream.Length - Reader.BaseStream.Position)];
BlockingCollection <LoadRequest> requests = new BlockingCollection <LoadRequest>(50);
// build all of the data / mallocs in a different thread
Task.Factory.StartNew(() =>
{
for (int i = 0; i < firstLength; i++)
{
index.Indexes[i].Start = Indexes[i].Start;
index.Indexes[i].Stop = Indexes[i].End;
iTotal += Indexes[i].End - Indexes[i].Start + 1;
}
int iSoFar = 0;
data = new float[iTotal][][];
for (int i = 0; i < firstLength; i++)
{
var sub = Indexes[i].SubIndexes;
var subLength = sub.Length;
index.Indexes[i].SubIndex = new SparseIndexing();
index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength];
int jTotal = 0;
var ithIndex = index.Indexes[i].SubIndex.Indexes;
for (int j = 0; j < subLength; j++)
{
ithIndex[j].Start = sub[j].Start;
ithIndex[j].Stop = sub[j].End;
jTotal += sub[j].End - sub[j].Start + 1;
}
int iSectionTotal = Indexes[i].End - Indexes[i].Start + 1;
for (int k = 0; k < iSectionTotal; k++)
{
data[iSoFar + k] = new float[jTotal][];
}
requests.Add(new LoadRequest()
{
OriginSectionTotal = iSectionTotal, DestinationSectionTotal = jTotal, DataIndex = iSoFar
});
iSoFar += iSectionTotal;
}
requests.CompleteAdding();
});
// then we can read everything into memory
int readInSoFar = 0;
Reader.BaseStream.Position = Indexes[0].SubIndexes[0].Location;
// store everything into memory
while (readInSoFar < tempReader.Length)
{
readInSoFar += Reader.Read(tempReader, readInSoFar, tempReader.Length - readInSoFar);
}
int currentIndex = 0;
// process the data from the stream
foreach (var request in requests.GetConsumingEnumerable())
{
for (int k = 0; k < request.OriginSectionTotal; k++)
{
for (int l = 0; l < request.DestinationSectionTotal; l++)
{
var row = data[request.DataIndex + k][l] = new float[types];
Buffer.BlockCopy(tempReader, currentIndex * sizeof(float), row, 0, types * sizeof(float));
currentIndex += types;
}
}
}
return(new SparseTwinIndex <float[]>(index, data));
}
public SparseTwinIndex<float[]> StoreAll()
{
if ( this.Indexes == null )
{
return null;
}
int types = this.Types * this.Times;
SparseIndexing index = new SparseIndexing();
int firstLength = this.Indexes.Length;
int iTotal = 0;
index.Indexes = new SparseSet[this.Indexes.Length];
float[][][] data = null;
byte[] tempReader = new byte[(int)( this.Reader.BaseStream.Length - this.Reader.BaseStream.Position )];
BlockingCollection<LoadRequest> requests = new BlockingCollection<LoadRequest>( 50 );
// build all of the data / mallocs in a different thread
Task.Factory.StartNew( () =>
{
for ( int i = 0; i < firstLength; i++ )
{
index.Indexes[i].Start = this.Indexes[i].Start;
index.Indexes[i].Stop = this.Indexes[i].End;
iTotal += this.Indexes[i].End - this.Indexes[i].Start + 1;
}
int iSoFar = 0;
data = new float[iTotal][][];
for ( int i = 0; i < firstLength; i++ )
{
var sub = this.Indexes[i].SubIndex;
var subLength = sub.Length;
index.Indexes[i].SubIndex = new SparseIndexing();
index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength];
int jTotal = 0;
var ithIndex = index.Indexes[i].SubIndex.Indexes;
for ( int j = 0; j < subLength; j++ )
{
ithIndex[j].Start = sub[j].Start;
ithIndex[j].Stop = sub[j].End;
jTotal += sub[j].End - sub[j].Start + 1;
}
int iSectionTotal = this.Indexes[i].End - this.Indexes[i].Start + 1;
for ( int k = 0; k < iSectionTotal; k++ )
{
data[iSoFar + k] = new float[jTotal][];
}
requests.Add( new LoadRequest() { ISectionTotal = iSectionTotal, JSectionTotal = jTotal, DataIndex = iSoFar } );
iSoFar += iSectionTotal;
}
requests.CompleteAdding();
} );
// then we can read everything into memory
int readInSoFar = 0;
this.Reader.BaseStream.Position = this.Indexes[0].SubIndex[0].Location;
// store everything into memory
while ( readInSoFar < tempReader.Length )
{
readInSoFar += this.Reader.Read( tempReader, readInSoFar, tempReader.Length - readInSoFar );
}
int currentIndex = 0;
// process the data from the stream
foreach ( var request in requests.GetConsumingEnumerable() )
{
for ( int k = 0; k < request.ISectionTotal; k++ )
{
for ( int l = 0; l < request.JSectionTotal; l++ )
{
var row = data[request.DataIndex + k][l] = new float[types];
Buffer.BlockCopy( tempReader, currentIndex * sizeof( float ), row, 0, types * sizeof( float ) );
currentIndex += types;
}
}
}
return new SparseTwinIndex<float[]>( index, data );
}