Esempio n. 1
0
        private static SparseIndexing GenerateIndexes(SortStruct[] data)
        {
            SparseIndexing   Indexes  = new SparseIndexing();
            List <SparseSet> elements = new List <SparseSet>();
            var       length          = data.Length;
            SparseSet current         = default(SparseSet);

            if (length == 0)
            {
                return(Indexes);
            }
            current.Start = data[0].SparseSpace;
            int expected = data[0].SparseSpace + 1;

            for (int i = 1; i < length; i++)
            {
                if (data[i].SparseSpace != expected)
                {
                    current.Stop         = data[i - 1].SparseSpace;
                    current.BaseLocation = 0;
                    elements.Add(current);
                    current.Start = data[i].SparseSpace;
                }
                expected = data[i].SparseSpace + 1;
            }
            current.Stop = data[length - 1].SparseSpace;
            elements.Add(current);
            Indexes.Indexes = elements.ToArray();
            return(Indexes);
        }
Esempio n. 2
0
        private static SparseIndexing ConvertToIndexes(List <SparseSet> processedIndexes, out T[][] Data, T[] data, SortStruct[] index)
        {
            SparseIndexing start   = new SparseIndexing();
            var            iLength = processedIndexes.Count;

            Data          = new T[iLength][];
            start.Indexes = new SparseSet[iLength];
            int dataProcessed = 0;

            for (int i = 0; i < iLength; i++)
            {
                start.Indexes[i] = processedIndexes[i];
                var jLength   = 0;
                var jSections = processedIndexes[i].SubIndex.Indexes.Length;
                for (int jSection = 0; jSection < jSections; jSection++)
                {
                    var indexStop  = processedIndexes[i].SubIndex.Indexes[jSection].Stop;
                    var indexStart = processedIndexes[i].SubIndex.Indexes[jSection].Start;
                    start.Indexes[i].SubIndex.Indexes[jSection].Start = indexStart;
                    start.Indexes[i].SubIndex.Indexes[jSection].Stop  = indexStop;
                    jLength += indexStop - indexStart + 1;
                }
                Data[i] = new T[jLength];
                for (int j = 0; j < jLength; j++)
                {
                    Data[i][j] = data[index[dataProcessed++].DataSpace];
                }
            }
            return(start);
        }
Esempio n. 3
0
 public SparseArray(SparseIndexing indexing, T[] rawData = null)
 {
     this.Indexing = indexing;
     if (rawData != null)
     {
         this.Data = rawData;
     }
     GenerateStructure();
 }
Esempio n. 4
0
 public SparseTriIndex(SparseIndexing indexes, T[][][] data = null)
 {
     Indexes = indexes;
     if (data != null)
     {
         Data = data;
     }
     GenerateStructure();
 }
Esempio n. 5
0
 public SparseTwinIndex(SparseIndexing indexes, T[][] data = null)
 {
     this.Indexes = indexes;
     if (data != null)
     {
         this.Data = data;
     }
     if (indexes.Indexes != null)
     {
         GenerateStructure();
     }
 }
Esempio n. 6
0
        public static SparseTwinIndex <T> CreateSimilarArray <TFirst, TSecond>(SparseArray <TFirst> first, SparseArray <TSecond> second)
        {
            var indexes = new SparseIndexing();

            indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone();
            var length = indexes.Indexes.Length;

            for (var i = 0; i < length; i++)
            {
                indexes.Indexes[i].SubIndex = new SparseIndexing()
                {
                    Indexes = second.Indexing.Indexes.Clone() as SparseSet[]
                };
            }
            return(new SparseTwinIndex <T>(indexes));
        }
Esempio n. 7
0
        public static SparseTwinIndex <T> CreateSimilarArray <J, K>(SparseArray <J> first, SparseArray <K> second)
        {
            SparseIndexing indexes = new SparseIndexing();

            indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[];
            var length = indexes.Indexes.Length;

            for (int i = 0; i < length; i++)
            {
                indexes.Indexes[i].SubIndex = new SparseIndexing()
                {
                    Indexes = second.Indexing.Indexes.Clone() as SparseSet[]
                };
            }
            return(new SparseTwinIndex <T>(indexes));
        }
Esempio n. 8
0
        public SparseTwinIndex(SparseIndexing indexes, T[][] data = null)
        {
            Indexes = indexes;
            if (data != null)
            {
                Data = data;
            }
            if (indexes.Indexes != null)
            {
                GenerateStructure();
            }
            //Generate _Count
            var count = 0;

            for (var i = 0; i < Data.Length; i++)
            {
                count += Data[i].Length;
            }
            Count = count;
        }
Esempio n. 9
0
        public static SparseTriIndex <T> CreateSimilarArray <TFirst, TSecond, TThird>(SparseArray <TFirst> first, SparseArray <TSecond> second, SparseArray <TThird> third)
        {
            var indexes = new SparseIndexing();

            indexes.Indexes = (SparseSet[])first.Indexing.Indexes.Clone();
            var length = indexes.Indexes.Length;

            for (var i = 0; i < length; i++)
            {
                SparseSet[] subIndexes;
                indexes.Indexes[i].SubIndex = new SparseIndexing()
                {
                    Indexes = (subIndexes = (SparseSet[])second.Indexing.Indexes.Clone())
                };
                var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length;
                for (var j = 0; j < subindexLength; j++)
                {
                    subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[];
                }
            }
            return(new SparseTriIndex <T>(indexes));
        }
Esempio n. 10
0
        private static SparseIndexing ConvertToIndexes(List <Pair <int, List <SparseSet> > > processedIndexes, out T[][][] outputData, T[] data, SortStruct[] index)
        {
            var start   = new SparseIndexing();
            var iLength = processedIndexes.Count;

            outputData    = new T[iLength][][];
            start.Indexes = new SparseSet[iLength];
            var dataProcessed = 0;

            for (var i = 0; i < iLength; i++)
            {
                var jLength = processedIndexes[i].Second.Count;
                outputData[i]                     = new T[jLength][];
                start.Indexes[i].Start            = start.Indexes[i].Stop = processedIndexes[i].First;
                start.Indexes[i].SubIndex.Indexes = new SparseSet[jLength];
                for (var j = 0; j < jLength; j++)
                {
                    var totalK   = 0;
                    var currentJ = processedIndexes[i].Second[j];
                    start.Indexes[i].SubIndex.Indexes[j] = currentJ;
                    var kSections = currentJ.SubIndex.Indexes.Length;
                    for (var kSection = 0; kSection < kSections; kSection++)
                    {
                        totalK += (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Stop = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Stop)
                                  - (start.Indexes[i].SubIndex.Indexes[j].SubIndex.Indexes[kSection].Start = processedIndexes[i].Second[j].SubIndex.Indexes[kSection].Start) + 1;
                    }
                    outputData[i][j] = new T[totalK];
                    for (var k = 0; k < totalK; k++)
                    {
                        outputData[i][j][k] = data[index[dataProcessed++].DataSpace];
                    }
                }
            }
            if (dataProcessed != data.Length)
            {
                throw new InvalidOperationException("The data processed should always be of the data's length!");
            }
            return(start);
        }
Esempio n. 11
0
        public static SparseTriIndex <T> CreateSimilarArray <J, K, L>(SparseArray <J> first, SparseArray <K> second, SparseArray <L> third)
        {
            SparseIndexing indexes = new SparseIndexing();

            indexes.Indexes = first.Indexing.Indexes.Clone() as SparseSet[];
            var length = indexes.Indexes.Length;

            for (int i = 0; i < length; i++)
            {
                SparseSet[] subIndexes;
                indexes.Indexes[i].SubIndex = new SparseIndexing()
                {
                    Indexes = (subIndexes = second.Indexing.Indexes.Clone() as SparseSet[])
                };
                var subindexLength = indexes.Indexes[i].SubIndex.Indexes.Length;
                for (int j = 0; j < subindexLength; j++)
                {
                    subIndexes[j].SubIndex.Indexes = third.Indexing.Indexes.Clone() as SparseSet[];
                }
            }
            return(new SparseTriIndex <T>(indexes));
        }
Esempio n. 12
0
        public SparseTwinIndex <float[]> StoreAll()
        {
            if (Indexes == null)
            {
                return(null);
            }
            int            types       = Types * Times;
            SparseIndexing index       = new SparseIndexing();
            int            firstLength = Indexes.Length;
            int            iTotal      = 0;

            index.Indexes = new SparseSet[Indexes.Length];
            float[][][] data       = null;
            byte[]      tempReader = new byte[(int)(Reader.BaseStream.Length - Reader.BaseStream.Position)];
            BlockingCollection <LoadRequest> requests = new BlockingCollection <LoadRequest>(50);

            // build all of the data / mallocs in a different thread
            Task.Factory.StartNew(() =>
            {
                for (int i = 0; i < firstLength; i++)
                {
                    index.Indexes[i].Start = Indexes[i].Start;
                    index.Indexes[i].Stop  = Indexes[i].End;
                    iTotal += Indexes[i].End - Indexes[i].Start + 1;
                }
                int iSoFar = 0;
                data       = new float[iTotal][][];
                for (int i = 0; i < firstLength; i++)
                {
                    var sub                           = Indexes[i].SubIndexes;
                    var subLength                     = sub.Length;
                    index.Indexes[i].SubIndex         = new SparseIndexing();
                    index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength];
                    int jTotal                        = 0;
                    var ithIndex                      = index.Indexes[i].SubIndex.Indexes;
                    for (int j = 0; j < subLength; j++)
                    {
                        ithIndex[j].Start = sub[j].Start;
                        ithIndex[j].Stop  = sub[j].End;
                        jTotal           += sub[j].End - sub[j].Start + 1;
                    }
                    int iSectionTotal = Indexes[i].End - Indexes[i].Start + 1;
                    for (int k = 0; k < iSectionTotal; k++)
                    {
                        data[iSoFar + k] = new float[jTotal][];
                    }
                    requests.Add(new LoadRequest()
                    {
                        OriginSectionTotal = iSectionTotal, DestinationSectionTotal = jTotal, DataIndex = iSoFar
                    });
                    iSoFar += iSectionTotal;
                }
                requests.CompleteAdding();
            });
            // then we can read everything into memory
            int readInSoFar = 0;

            Reader.BaseStream.Position = Indexes[0].SubIndexes[0].Location;
            // store everything into memory
            while (readInSoFar < tempReader.Length)
            {
                readInSoFar += Reader.Read(tempReader, readInSoFar, tempReader.Length - readInSoFar);
            }
            int currentIndex = 0;

            // process the data from the stream
            foreach (var request in requests.GetConsumingEnumerable())
            {
                for (int k = 0; k < request.OriginSectionTotal; k++)
                {
                    for (int l = 0; l < request.DestinationSectionTotal; l++)
                    {
                        var row = data[request.DataIndex + k][l] = new float[types];
                        Buffer.BlockCopy(tempReader, currentIndex * sizeof(float), row, 0, types * sizeof(float));
                        currentIndex += types;
                    }
                }
            }

            return(new SparseTwinIndex <float[]>(index, data));
        }
Esempio n. 13
0
        public SparseTwinIndex<float[]> StoreAll()
        {
            if ( this.Indexes == null )
            {
                return null;
            }
            int types = this.Types * this.Times;
            SparseIndexing index = new SparseIndexing();
            int firstLength = this.Indexes.Length;
            int iTotal = 0;
            index.Indexes = new SparseSet[this.Indexes.Length];
            float[][][] data = null;
            byte[] tempReader = new byte[(int)( this.Reader.BaseStream.Length - this.Reader.BaseStream.Position )];
            BlockingCollection<LoadRequest> requests = new BlockingCollection<LoadRequest>( 50 );
            // build all of the data / mallocs in a different thread
            Task.Factory.StartNew( () =>
            {
                for ( int i = 0; i < firstLength; i++ )
                {
                    index.Indexes[i].Start = this.Indexes[i].Start;
                    index.Indexes[i].Stop = this.Indexes[i].End;
                    iTotal += this.Indexes[i].End - this.Indexes[i].Start + 1;
                }
                int iSoFar = 0;
                data = new float[iTotal][][];
                for ( int i = 0; i < firstLength; i++ )
                {
                    var sub = this.Indexes[i].SubIndex;
                    var subLength = sub.Length;
                    index.Indexes[i].SubIndex = new SparseIndexing();
                    index.Indexes[i].SubIndex.Indexes = new SparseSet[subLength];
                    int jTotal = 0;
                    var ithIndex = index.Indexes[i].SubIndex.Indexes;
                    for ( int j = 0; j < subLength; j++ )
                    {
                        ithIndex[j].Start = sub[j].Start;
                        ithIndex[j].Stop = sub[j].End;
                        jTotal += sub[j].End - sub[j].Start + 1;
                    }
                    int iSectionTotal = this.Indexes[i].End - this.Indexes[i].Start + 1;
                    for ( int k = 0; k < iSectionTotal; k++ )
                    {
                        data[iSoFar + k] = new float[jTotal][];
                    }
                    requests.Add( new LoadRequest() { ISectionTotal = iSectionTotal, JSectionTotal = jTotal, DataIndex = iSoFar } );
                    iSoFar += iSectionTotal;
                }
                requests.CompleteAdding();
            } );
            // then we can read everything into memory
            int readInSoFar = 0;
            this.Reader.BaseStream.Position = this.Indexes[0].SubIndex[0].Location;
            // store everything into memory
            while ( readInSoFar < tempReader.Length )
            {
                readInSoFar += this.Reader.Read( tempReader, readInSoFar, tempReader.Length - readInSoFar );
            }
            int currentIndex = 0;
            // process the data from the stream
            foreach ( var request in requests.GetConsumingEnumerable() )
            {
                for ( int k = 0; k < request.ISectionTotal; k++ )
                {
                    for ( int l = 0; l < request.JSectionTotal; l++ )
                    {
                        var row = data[request.DataIndex + k][l] = new float[types];
                        Buffer.BlockCopy( tempReader, currentIndex * sizeof( float ), row, 0, types * sizeof( float ) );
                        currentIndex += types;
                    }
                }
            }

            return new SparseTwinIndex<float[]>( index, data );
        }