private static SparseIndexing GenerateIndexes(SortStruct[] data) { SparseIndexing Indexes = new SparseIndexing(); List <SparseSet> elements = new List <SparseSet>(); var length = data.Length; SparseSet current = default(SparseSet); if (length == 0) { return(Indexes); } current.Start = data[0].SparseSpace; int expected = data[0].SparseSpace + 1; for (int i = 1; i < length; i++) { if (data[i].SparseSpace != expected) { current.Stop = data[i - 1].SparseSpace; current.BaseLocation = 0; elements.Add(current); current.Start = data[i].SparseSpace; } expected = data[i].SparseSpace + 1; } current.Stop = data[length - 1].SparseSpace; elements.Add(current); Indexes.Indexes = elements.ToArray(); return(Indexes); }
private static SparseIndexing ConvertToIndexes(List <SparseSet> processedIndexes, out T[][] Data, T[] data, SortStruct[] index) { SparseIndexing start = new SparseIndexing(); var iLength = processedIndexes.Count; Data = new T[iLength][]; start.Indexes = new SparseSet[iLength]; int dataProcessed = 0; for (int i = 0; i < iLength; i++) { start.Indexes[i] = processedIndexes[i]; var jLength = 0; var jSections = processedIndexes[i].SubIndex.Indexes.Length; for (int jSection = 0; jSection < jSections; jSection++) { var indexStop = processedIndexes[i].SubIndex.Indexes[jSection].Stop; var indexStart = processedIndexes[i].SubIndex.Indexes[jSection].Start; start.Indexes[i].SubIndex.Indexes[jSection].Start = indexStart; start.Indexes[i].SubIndex.Indexes[jSection].Stop = indexStop; jLength += indexStop - indexStart + 1; } Data[i] = new T[jLength]; for (int j = 0; j < jLength; j++) { Data[i][j] = data[index[dataProcessed++].DataSpace]; } } return(start); }
public SparseArray(SparseIndexing indexing, T[] rawData = null) { this.Indexing = indexing; if (rawData != null) { this.Data = rawData; } GenerateStructure(); }
public SparseTriIndex(SparseIndexing indexes, T[][][] data = null) { Indexes = indexes; if (data != null) { Data = data; } GenerateStructure(); }
public SparseTwinIndex(SparseIndexing indexes, T[][] data = null) { this.Indexes = indexes; if (data != null) { this.Data = data; } if (indexes.Indexes != null) { GenerateStructure(); } }
public static SparseTwinIndex <T> CreateSimilarArray <TFirst, TSecond>(SparseArray <TFirst> first, SparseArray <TSecond> second) { var indexes = new SparseIndexing(); indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone(); var length = indexes.Indexes.Length; for (var i = 0; i < length; i++) { indexes.Indexes[i].SubIndex = new SparseIndexing() { Indexes = second.Indexing.Indexes.Clone() as SparseSet[] }; } return(new SparseTwinIndex <T>(indexes)); }
public static SparseTwinIndex <T> CreateSimilarArray <J, K>(SparseArray <J> first, SparseArray <K> second) { SparseIndexing indexes = new SparseIndexing(); indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[]; var length = indexes.Indexes.Length; for (int i = 0; i < length; i++) { indexes.Indexes[i].SubIndex = new SparseIndexing() { Indexes = second.Indexing.Indexes.Clone() as SparseSet[] }; } return(new SparseTwinIndex <T>(indexes)); }
public SparseTwinIndex(SparseIndexing indexes, T[][] data = null) { Indexes = indexes; if (data != null) { Data = data; } if (indexes.Indexes != null) { GenerateStructure(); } //Generate _Count var count = 0; for (var i = 0; i < Data.Length; i++) { count += Data[i].Length; } Count = count; }
public static SparseTriIndex <T> CreateSimilarArray <TFirst, TSecond, TThird>(SparseArray <TFirst> first, SparseArray <TSecond> second, SparseArray <TThird> third) { var indexes = new SparseIndexing(); indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone(); var length = indexes.Indexes.Length; for (var i = 0; i < length; i++) { SparseSet[] subIndexes; indexes.Indexes[i].SubIndex = new SparseIndexing() { Indexes = (subIndexes = (SparseSet[])second.Indexing.Indexes.Clone()) }; var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length; for (var j = 0; j < subindexLength; j++) { subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[]; } } return(new SparseTriIndex <T>(indexes)); }
private static SparseIndexing ConvertToIndexes(List <Pair <int, List <SparseSet> > > processedIndexes, out T[][][] outputData, T[] data, SortStruct[] index) { var start = new SparseIndexing(); var iLength = processedIndexes.Count; outputData = new T[iLength][][]; start.Indexes = new SparseSet[iLength]; var dataProcessed = 0; for (var i = 0; i < iLength; i++) { var jLength = processedIndexes[i].Second.Count; outputData[i] = new T[jLength][]; start.Indexes[i].Start = start.Indexes[i].Stop = processedIndexes[i].First; start.Indexes[i].SubIndex.Indexes = new SparseSet[jLength]; for (var j = 0; j < jLength; j++) { var totalK = 0; var currentJ = processedIndexes[i].Second[j]; start.Indexes[i].SubIndex.Indexes[j] = currentJ; var kSections = currentJ.SubIndex.Indexes.Length; for (var kSection = 0; kSection < kSections; kSection++) { totalK += (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Stop = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Stop) - (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Start = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Start) + 1; } outputData[i][j] = new T[totalK]; for (var k = 0; k < totalK; k++) { outputData[i][j][k] = data[index[dataProcessed++].DataSpace]; } } } if (dataProcessed != data.Length) { throw new InvalidOperationException("The data processed should always be of the data's length!"); } return(start); }
public static SparseTriIndex <T> CreateSimilarArray <J, K, L>(SparseArray <J> first, SparseArray <K> second, SparseArray <L> third) { SparseIndexing indexes = new SparseIndexing(); indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[]; var length = indexes.Indexes.Length; for (int i = 0; i < length; i++) { SparseSet[] subIndexes; indexes.Indexes[i].SubIndex = new SparseIndexing() { Indexes = (subIndexes = second.Indexing.Indexes.Clone() as SparseSet[]) }; var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length; for (int j = 0; j < subindexLength; j++) { subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[]; } } return(new SparseTriIndex <T>(indexes)); }
public SparseTwinIndex <float[]> StoreAll() { if (Indexes == null) { return(null); } int types = Types * Times; SparseIndexing index = new SparseIndexing(); int firstLength = Indexes.Length; int iTotal = 0; index.Indexes = new SparseSet[Indexes.Length]; float[][][] data = null; byte[] tempReader = new byte[(int)(Reader.BaseStream.Length - Reader.BaseStream.Position)]; BlockingCollection <LoadRequest> requests = new BlockingCollection <LoadRequest>(50); // build all of the data / mallocs in a different thread Task.Factory.StartNew(() => { for (int i = 0; i < firstLength; i++) { index.Indexes[i].Start = Indexes[i].Start; index.Indexes[i].Stop = Indexes[i].End; iTotal += Indexes[i].End - Indexes[i].Start + 1; } int iSoFar = 0; data = new float[iTotal][][]; for (int i = 0; i < firstLength; i++) { var sub = Indexes[i].SubIndexes; var subLength = sub.Length; index.Indexes[i].SubIndex = new SparseIndexing(); index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength]; int jTotal = 0; var ithIndex = index.Indexes[i].SubIndex.Indexes; for (int j = 0; j < subLength; j++) { ithIndex[j].Start = sub[j].Start; ithIndex[j].Stop = sub[j].End; jTotal += sub[j].End - sub[j].Start + 1; } int iSectionTotal = Indexes[i].End - Indexes[i].Start + 1; for (int k = 0; k < iSectionTotal; k++) { data[iSoFar + k] = new float[jTotal][]; } requests.Add(new LoadRequest() { OriginSectionTotal = iSectionTotal, DestinationSectionTotal = jTotal, DataIndex = iSoFar }); iSoFar += iSectionTotal; } requests.CompleteAdding(); }); // then we can read everything into memory int readInSoFar = 0; Reader.BaseStream.Position = Indexes[0].SubIndexes[0].Location; // store everything into memory while (readInSoFar < tempReader.Length) { readInSoFar += Reader.Read(tempReader, readInSoFar, tempReader.Length - readInSoFar); } int currentIndex = 0; // process the data from the stream foreach (var request in requests.GetConsumingEnumerable()) { for (int k = 0; k < request.OriginSectionTotal; k++) { for (int l = 0; l < request.DestinationSectionTotal; l++) { var row = data[request.DataIndex + k][l] = new float[types]; Buffer.BlockCopy(tempReader, currentIndex * sizeof(float), row, 0, types * sizeof(float)); currentIndex += types; } } } return(new SparseTwinIndex <float[]>(index, data)); }
public SparseTwinIndex<float[]> StoreAll() { if ( this.Indexes == null ) { return null; } int types = this.Types * this.Times; SparseIndexing index = new SparseIndexing(); int firstLength = this.Indexes.Length; int iTotal = 0; index.Indexes = new SparseSet[this.Indexes.Length]; float[][][] data = null; byte[] tempReader = new byte[(int)( this.Reader.BaseStream.Length - this.Reader.BaseStream.Position )]; BlockingCollection<LoadRequest> requests = new BlockingCollection<LoadRequest>( 50 ); // build all of the data / mallocs in a different thread Task.Factory.StartNew( () => { for ( int i = 0; i < firstLength; i++ ) { index.Indexes[i].Start = this.Indexes[i].Start; index.Indexes[i].Stop = this.Indexes[i].End; iTotal += this.Indexes[i].End - this.Indexes[i].Start + 1; } int iSoFar = 0; data = new float[iTotal][][]; for ( int i = 0; i < firstLength; i++ ) { var sub = this.Indexes[i].SubIndex; var subLength = sub.Length; index.Indexes[i].SubIndex = new SparseIndexing(); index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength]; int jTotal = 0; var ithIndex = index.Indexes[i].SubIndex.Indexes; for ( int j = 0; j < subLength; j++ ) { ithIndex[j].Start = sub[j].Start; ithIndex[j].Stop = sub[j].End; jTotal += sub[j].End - sub[j].Start + 1; } int iSectionTotal = this.Indexes[i].End - this.Indexes[i].Start + 1; for ( int k = 0; k < iSectionTotal; k++ ) { data[iSoFar + k] = new float[jTotal][]; } requests.Add( new LoadRequest() { ISectionTotal = iSectionTotal, JSectionTotal = jTotal, DataIndex = iSoFar } ); iSoFar += iSectionTotal; } requests.CompleteAdding(); } ); // then we can read everything into memory int readInSoFar = 0; this.Reader.BaseStream.Position = this.Indexes[0].SubIndex[0].Location; // store everything into memory while ( readInSoFar < tempReader.Length ) { readInSoFar += this.Reader.Read( tempReader, readInSoFar, tempReader.Length - readInSoFar ); } int currentIndex = 0; // process the data from the stream foreach ( var request in requests.GetConsumingEnumerable() ) { for ( int k = 0; k < request.ISectionTotal; k++ ) { for ( int l = 0; l < request.JSectionTotal; l++ ) { var row = data[request.DataIndex + k][l] = new float[types]; Buffer.BlockCopy( tempReader, currentIndex * sizeof( float ), row, 0, types * sizeof( float ) ); currentIndex += types; } } } return new SparseTwinIndex<float[]>( index, data ); }