Beispiel #1
0
        /// <summary>
        /// Creates a new <see cref="DataSet"/> from the binary reader provided.
        /// </summary>
        /// <param name="dataSet">The data set to be loaded with data from the reader</param>
        /// <param name="reader">
        /// Reader connected to the source data structure and positioned to start reading
        /// </param>
        /// <param name="init">
        /// True to indicate that the data set should be fulling initialised
        /// </param>
        /// <para>
        /// A <see cref="DataSet"/> is constructed using the reader to retrieve
        /// the header information. This is then passed to the Read methods to create the
        /// lists before reading the data into memory. Finally it initialise is required
        /// references between entities are worked out and stored.
        /// </para>
        internal static void Load(DataSet dataSet, Reader reader, bool init)
        {
            CommonFactory.LoadHeader(dataSet, reader);

            var strings = new MemoryVariableList <AsciiString>(dataSet, reader, new AsciiStringFactory());
            MemoryFixedList <Component> components = null;

            switch (dataSet.VersionEnum)
            {
            case BinaryConstants.FormatVersions.PatternV31:
                components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV31());
                break;

            case BinaryConstants.FormatVersions.PatternV32:
                components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV32());
                break;
            }
            var maps       = new MemoryFixedList <Map>(dataSet, reader, new MapFactory());
            var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
            var values     = new MemoryFixedList <Value>(dataSet, reader, new ValueFactory());
            var profiles   = new MemoryVariableList <Entities.Profile>(dataSet, reader, new ProfileMemoryFactory());
            MemoryFixedList <Signature> signatures                 = null;
            MemoryFixedList <Integer>   signatureNodeOffsets       = null;
            MemoryFixedList <Integer>   nodeRankedSignatureIndexes = null;

            switch (dataSet.VersionEnum)
            {
            case BinaryConstants.FormatVersions.PatternV31:
                signatures = new MemoryFixedList <Signature>(dataSet, reader, new SignatureFactoryV31(dataSet));
                break;

            case BinaryConstants.FormatVersions.PatternV32:
                signatures                 = new MemoryFixedList <Signature>(dataSet, reader, new SignatureFactoryV32(dataSet));
                signatureNodeOffsets       = new MemoryFixedList <Integer>(dataSet, reader, new IntegerFactory());
                nodeRankedSignatureIndexes = new MemoryFixedList <Integer>(dataSet, reader, new IntegerFactory());
                break;
            }
            var rankedSignatureIndexes = new MemoryFixedList <Integer>(
                dataSet, reader, new IntegerFactory());
            MemoryVariableList <Entities.Node> nodes = null;

            switch (dataSet.VersionEnum)
            {
            case BinaryConstants.FormatVersions.PatternV31:
                nodes = new MemoryVariableList <Entities.Node>(dataSet, reader, new NodeMemoryFactoryV31());
                break;

            case BinaryConstants.FormatVersions.PatternV32:
                nodes = new MemoryVariableList <Entities.Node>(dataSet, reader, new NodeMemoryFactoryV32());
                break;
            }
            var rootNodes      = new MemoryFixedList <Entities.Node>(dataSet, reader, new RootNodeFactory());
            var profileOffsets = new MemoryFixedList <ProfileOffset>(dataSet, reader, new ProfileOffsetFactory());

            dataSet.Strings                 = strings;
            dataSet._components             = components;
            dataSet._maps                   = maps;
            dataSet._properties             = properties;
            dataSet._values                 = values;
            dataSet.Profiles                = profiles;
            dataSet._signatures             = signatures;
            dataSet._rankedSignatureIndexes = rankedSignatureIndexes;
            switch (dataSet.VersionEnum)
            {
            case BinaryConstants.FormatVersions.PatternV32:
                dataSet._signatureNodeOffsets       = signatureNodeOffsets;
                dataSet._nodeRankedSignatureIndexes = nodeRankedSignatureIndexes;
                break;
            }
            dataSet.Nodes           = nodes;
            dataSet.RootNodes       = rootNodes;
            dataSet._profileOffsets = profileOffsets;

            strings.Read(reader);
            components.Read(reader);
            maps.Read(reader);
            properties.Read(reader);
            values.Read(reader);
            profiles.Read(reader);
            signatures.Read(reader);
            switch (dataSet.VersionEnum)
            {
            case BinaryConstants.FormatVersions.PatternV32:
                signatureNodeOffsets.Read(reader);
                nodeRankedSignatureIndexes.Read(reader);
                break;
            }
            rankedSignatureIndexes.Read(reader);
            nodes.Read(reader);
            rootNodes.Read(reader);
            profileOffsets.Read(reader);

            if (init)
            {
                // Set references between objects.
                dataSet.Init();

                // The following lists will not be needed anymore
                // so they can be freed.
                dataSet._signatureNodeOffsets       = null;
                dataSet._nodeRankedSignatureIndexes = null;

                // Force garbage collection as a lot of memory has been freed.
                GC.Collect();
            }
        }
Beispiel #2
0
        /// <summary>
        /// Load the necessary values from the data file in to the DataSet.
        /// Initially, this will only load data such as file headers
        /// and the smaller lists.
        /// </summary>
        /// <param name="dataSet">The dataset object to load in to</param>
        /// <param name="cacheConfiguration">the cache configuration to use when creating the caches</param>
        private static void LoadForStreaming(IndirectDataSet dataSet, Dictionary <CacheType, ICacheOptions> cacheConfiguration)
        {
            var reader = dataSet.Pool.GetReader();

            try
            {
                CacheMap cacheMap = BuildCaches(cacheConfiguration);
                dataSet.CacheMap = cacheMap;

                reader.BaseStream.Position = 0;
                CommonFactory.LoadHeader(dataSet, reader);

                // ---- Create AsciiString list
                var stringLoader = EntityLoaderFactory.GetLoaderFor(
                    new Header(reader),
                    cacheMap.StringCache,
                    dataSet,
                    new StreamAsciiStringFactory());
                dataSet.Strings = new StreamList <AsciiString, IStreamDataSet>(stringLoader);

                MemoryFixedList <Component, DataSet> components = null;
                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    components = new MemoryFixedList <Component, DataSet>(
                        dataSet,
                        reader,
                        new ComponentFactoryV31());
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    components = new MemoryFixedList <Component, DataSet>(
                        dataSet,
                        reader,
                        new ComponentFactoryV32());
                    break;
                }
                dataSet._components = components;
                var maps = new MemoryFixedList <Map, DataSet>(
                    dataSet,
                    reader,
                    new MapFactory());
                dataSet._maps = maps;
                var properties = new PropertiesList(
                    dataSet,
                    reader,
                    new PropertyFactory());
                dataSet._properties = properties;

                // ---- Create Value list
                var valueLoader = EntityLoaderFactory.GetLoaderFor(new Header(reader),
                                                                   cacheMap.ValueCache,
                                                                   dataSet,
                                                                   new ValueFactory <IStreamDataSet>());
                dataSet._values = new StreamList <Value, IStreamDataSet>(valueLoader);

                // ---- Create Profile list
                var profileLoader = EntityLoaderFactory.GetLoaderFor(new Header(reader),
                                                                     cacheMap.ProfileCache,
                                                                     dataSet,
                                                                     new ProfileStreamFactory(dataSet.Pool));
                dataSet.Profiles = new StreamList <Entities.Profile, IStreamDataSet>(profileLoader);

                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    // ---- Create Signature list for V31
                    var signatureLoaderV31 = EntityLoaderFactory.GetLoaderFor(
                        new Header(reader),
                        cacheMap.SignatureCache,
                        dataSet,
                        new SignatureFactoryV31 <IndirectDataSet>(dataSet));
                    dataSet._signatures = new StreamList <Signature, IndirectDataSet>(
                        signatureLoaderV31);
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    // ---- Create Signature list for V32
                    var signatureLoaderV32 = EntityLoaderFactory.GetLoaderFor(
                        new Header(reader),
                        cacheMap.SignatureCache,
                        dataSet,
                        new SignatureFactoryV32 <IndirectDataSet>(dataSet));
                    dataSet._signatures = new StreamList <Signature, IndirectDataSet>(
                        signatureLoaderV32);
                    dataSet._signatureNodeOffsets       = new IntegerList(dataSet, reader);
                    dataSet._nodeRankedSignatureIndexes = new IntegerList(dataSet, reader);
                    break;
                }
                dataSet._rankedSignatureIndexes = new IntegerList(dataSet, reader);
                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    // ---- Create Nodes list for V31
                    var nodeLoaderV31 = EntityLoaderFactory.GetLoaderFor(
                        new Header(reader),
                        cacheMap.NodeCache,
                        dataSet,
                        new NodeStreamFactoryV31(dataSet.Pool));
                    dataSet.Nodes = new StreamList <Entities.Node, IStreamDataSet>(
                        nodeLoaderV31);
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    // ---- Create Nodes list for V31
                    var nodeLoaderV32 = EntityLoaderFactory.GetLoaderFor(
                        new Header(reader),
                        cacheMap.NodeCache,
                        dataSet,
                        new NodeStreamFactoryV32(dataSet.Pool));
                    dataSet.Nodes = new StreamList <Entities.Node, IStreamDataSet>(
                        nodeLoaderV32);
                    break;
                }
                var rootNodes = new MemoryFixedList <Entities.Node, DataSet>(
                    dataSet,
                    reader,
                    new RootNodeFactory());
                dataSet.RootNodes = rootNodes;
                var profileOffsets = new MemoryFixedList <ProfileOffset, DataSet>(
                    dataSet,
                    reader,
                    new ProfileOffsetFactory());
                dataSet._profileOffsets = profileOffsets;

                // Read into memory all the small lists which are frequently accessed.
                reader.BaseStream.Position = components.Header.StartPosition;
                components.Read(reader);
                reader.BaseStream.Position = maps.Header.StartPosition;
                maps.Read(reader);
                reader.BaseStream.Position = properties.Header.StartPosition;
                properties.Read(reader);
                reader.BaseStream.Position = rootNodes.Header.StartPosition;
                rootNodes.Read(reader);
                reader.BaseStream.Position = profileOffsets.Header.StartPosition;
                profileOffsets.Read(reader);
            }
            finally
            {
                dataSet.Pool.Release(reader);
            }
        }
        /// <summary>
        /// Initialises the dataset <see cref="DataSet"/> using the source of the data set.
        /// </summary>
        /// <para>
        /// A <see cref="DataSet"/> is initialised using the reader to retrieve 
        /// entity information. The data is only loaded when required by the detection
        /// process.
        /// </para>
        /// <param name="dataSet">A data set to be initialised ready for detection</param>
        private static void Load(DataSet dataSet)
        {
            var reader = dataSet.Pool.GetReader();
            try
            {
                reader.BaseStream.Position = 0;
                CommonFactory.LoadHeader(dataSet, reader);
                dataSet.Strings = new VariableList<AsciiString<Entities.DataSet>, DataSet>(
                    dataSet, reader, new StreamAsciiStringFactory(), Constants.StringsCacheSize);
                MemoryFixedList<Component, Entities.DataSet> components = null;
                switch (dataSet.VersionEnum)
                {
                    case BinaryConstants.FormatVersions.PatternV31:
                        components = new MemoryFixedList<Component, Entities.DataSet>(dataSet, reader, new ComponentFactoryV31());
                        break;
                    case BinaryConstants.FormatVersions.PatternV32:
                        components = new MemoryFixedList<Component, Entities.DataSet>(dataSet, reader, new ComponentFactoryV32());
                        break;
                }
                dataSet._components = components;
                var maps = new MemoryFixedList<Map, Entities.DataSet>(dataSet, reader, new MapFactory());
                dataSet._maps = maps;
                var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
                dataSet._properties = properties;
                dataSet._values = new FixedCacheList<Value, DataSet>(dataSet, reader, new ValueFactory<DataSet>(), Constants.ValuesCacheSize);
                dataSet.Profiles = new VariableList<Entities.Profile, DataSet>(dataSet, reader, new ProfileStreamFactory(dataSet.Pool), Constants.ProfilesCacheSize);
                switch (dataSet.VersionEnum)
                {
                    case BinaryConstants.FormatVersions.PatternV31:
                        dataSet._signatures = new FixedCacheList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV31<DataSet>(dataSet), Constants.SignaturesCacheSize);
                        break;
                    case BinaryConstants.FormatVersions.PatternV32:
                        dataSet._signatures = new FixedCacheList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV32<DataSet>(dataSet), Constants.SignaturesCacheSize);
                        dataSet._signatureNodeOffsets = new IntegerList(dataSet, reader);
                        dataSet._nodeRankedSignatureIndexes = new IntegerList(dataSet, reader);
                        break;
                }
                dataSet._rankedSignatureIndexes = new IntegerList(dataSet, reader);
                switch (dataSet.VersionEnum)
                {
                    case BinaryConstants.FormatVersions.PatternV31:
                        dataSet.Nodes = new VariableList<Entities.Node, DataSet>(dataSet, reader, new NodeStreamFactoryV31(dataSet.Pool), Constants.NodesCacheSize);
                        break;
                    case BinaryConstants.FormatVersions.PatternV32:
                        dataSet.Nodes = new VariableList<Entities.Node, DataSet>(dataSet, reader, new NodeStreamFactoryV32(dataSet.Pool), Constants.NodesCacheSize);
                        break;
                }
                var rootNodes = new MemoryFixedList<Entities.Node, Entities.DataSet>(dataSet, reader, new RootNodeFactory());
                dataSet.RootNodes = rootNodes;
                var profileOffsets = new MemoryFixedList<ProfileOffset, Entities.DataSet>(dataSet, reader, new ProfileOffsetFactory());
                dataSet._profileOffsets = profileOffsets;

                // Read into memory all the small lists which are frequently accessed.
                reader.BaseStream.Position = components.Header.StartPosition;
                components.Read(reader);
                reader.BaseStream.Position = maps.Header.StartPosition;
                maps.Read(reader);
                reader.BaseStream.Position = properties.Header.StartPosition;
                properties.Read(reader);
                reader.BaseStream.Position = rootNodes.Header.StartPosition;
                rootNodes.Read(reader);
                reader.BaseStream.Position = profileOffsets.Header.StartPosition;
                profileOffsets.Read(reader);
            }
            finally
            {
                dataSet.Pool.Release(reader);
            }
        }
        /// <summary>
        /// Creates a new <see cref="DataSet"/> from the binary reader provided.
        /// </summary>
        /// <param name="dataSet">The data set to be loaded with data from the reader</param>
        /// <param name="reader">
        /// Reader connected to the source data structure and positioned to start reading
        /// </param>
        /// <param name="init">
        /// True to indicate that the data set should be fulling initialised
        /// </param>
        /// <para>
        /// A <see cref="DataSet"/> is constructed using the reader to retrieve 
        /// the header information. This is then passed to the Read methods to create the
        /// lists before reading the data into memory. Finally it initialise is required
        /// references between entities are worked out and stored.
        /// </para>
        internal static void Load(DataSet dataSet, Reader reader, bool init)
        {
            CommonFactory.LoadHeader(dataSet, reader);

            var strings = new MemoryVariableList<AsciiString<DataSet>, DataSet>(dataSet, reader, new MemoryAsciiStringFactory());
            MemoryFixedList<Component, DataSet> components = null;
            switch(dataSet.VersionEnum)
            {
                case BinaryConstants.FormatVersions.PatternV31:
                    components = new MemoryFixedList<Component, DataSet>(dataSet, reader, new ComponentFactoryV31());
                    break;
                case BinaryConstants.FormatVersions.PatternV32:
                    components = new MemoryFixedList<Component, DataSet>(dataSet, reader, new ComponentFactoryV32());
                    break;
            }
            var maps = new MemoryFixedList<Map, DataSet>(dataSet, reader, new MapFactory());
            var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
            var values = new MemoryFixedList<Value, DataSet>(dataSet, reader, new ValueFactory<DataSet>());
            var profiles = new MemoryVariableList<Entities.Profile, DataSet>(dataSet, reader, new ProfileMemoryFactory());
            MemoryFixedList<Signature, DataSet> signatures = null;
            MemoryIntegerList signatureNodeOffsets = null;
            MemoryIntegerList nodeRankedSignatureIndexes = null;
            switch(dataSet.VersionEnum)
            {
                case BinaryConstants.FormatVersions.PatternV31:
                    signatures = new MemoryFixedList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV31<DataSet>(dataSet));
                    break;
                case BinaryConstants.FormatVersions.PatternV32:
                    signatures = new MemoryFixedList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV32<DataSet>(dataSet));
                    signatureNodeOffsets = new MemoryIntegerList(reader);
                    nodeRankedSignatureIndexes = new MemoryIntegerList(reader);
                    break;
            }
            var rankedSignatureIndexes = new MemoryIntegerList(reader);
            MemoryVariableList<Entities.Node, DataSet> nodes = null;
            switch (dataSet.VersionEnum)
            {
                case BinaryConstants.FormatVersions.PatternV31:
                    nodes = new MemoryVariableList<Entities.Node, DataSet>(dataSet, reader, new NodeMemoryFactoryV31());
                    break;
                case BinaryConstants.FormatVersions.PatternV32:
                    nodes = new MemoryVariableList<Entities.Node, DataSet>(dataSet, reader, new NodeMemoryFactoryV32());
                    break;
            }
            var rootNodes = new MemoryFixedList<Entities.Node, DataSet>(dataSet, reader, new RootNodeFactory());
            var profileOffsets = new MemoryFixedList<ProfileOffset, DataSet>(dataSet, reader, new ProfileOffsetFactory());

            dataSet.Strings = strings;
            dataSet._components = components;
            dataSet._maps = maps;
            dataSet._properties = properties;
            dataSet._values = values;
            dataSet.Profiles = profiles;
            dataSet._signatures = signatures;
            dataSet._rankedSignatureIndexes = rankedSignatureIndexes;
            switch (dataSet.VersionEnum)
            {
                case BinaryConstants.FormatVersions.PatternV32:
                    dataSet._signatureNodeOffsets = signatureNodeOffsets;
                    dataSet._nodeRankedSignatureIndexes = nodeRankedSignatureIndexes;
                    break;
            }
            dataSet.Nodes = nodes;
            dataSet.RootNodes = rootNodes;
            dataSet._profileOffsets = profileOffsets;

            strings.Read(reader);
            components.Read(reader);
            maps.Read(reader);
            properties.Read(reader);
            values.Read(reader);
            profiles.Read(reader);
            signatures.Read(reader);
            switch (dataSet.VersionEnum)
            {
                case BinaryConstants.FormatVersions.PatternV32:
                    signatureNodeOffsets.Read(reader);
                    nodeRankedSignatureIndexes.Read(reader);
                    break;
            }
            rankedSignatureIndexes.Read(reader);
            nodes.Read(reader);
            rootNodes.Read(reader);
            profileOffsets.Read(reader);

            if (init)
            {
                // Set references between objects.
                dataSet.Init();

                // The following lists will not be needed anymore
                // so they can be freed.
                dataSet._signatureNodeOffsets = null;
                dataSet._nodeRankedSignatureIndexes = null;

                // Force garbage collection as a lot of memory has been freed.
                GC.Collect();
            }
        }
Beispiel #5
0
        /// <summary>
        /// Initialises the dataset <see cref="DataSet"/> using the source of the data set.
        /// </summary>
        /// <para>
        /// A <see cref="DataSet"/> is initialised using the reader to retrieve
        /// entity information. The data is only loaded when required by the detection
        /// process.
        /// </para>
        /// <param name="dataSet">A data set to be initialised ready for detection</param>
        private static void Load(DataSet dataSet)
        {
            var reader = dataSet.Pool.GetReader();

            try
            {
                reader.BaseStream.Position = 0;
                CommonFactory.LoadHeader(dataSet, reader);
                dataSet.Strings = new VariableList <AsciiString>(dataSet, reader, new AsciiStringFactory(), Constants.StringsCacheSize);
                MemoryFixedList <Component> components = null;
                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV31());
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV32());
                    break;
                }
                dataSet._components = components;
                var maps = new MemoryFixedList <Map>(dataSet, reader, new MapFactory());
                dataSet._maps = maps;
                var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
                dataSet._properties = properties;
                dataSet._values     = new FixedCacheList <Value>(dataSet, reader, new ValueFactory(), Constants.ValuesCacheSize);
                dataSet.Profiles    = new VariableList <Entities.Profile>(dataSet, reader, new ProfileStreamFactory(dataSet.Pool), Constants.ProfilesCacheSize);
                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    dataSet._signatures = new FixedCacheList <Signature>(dataSet, reader, new SignatureFactoryV31(dataSet), Constants.SignaturesCacheSize);
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    dataSet._signatures                 = new FixedCacheList <Signature>(dataSet, reader, new SignatureFactoryV32(dataSet), Constants.SignaturesCacheSize);
                    dataSet._signatureNodeOffsets       = new FixedList <Integer>(dataSet, reader, new IntegerFactory());
                    dataSet._nodeRankedSignatureIndexes = new FixedList <Integer>(dataSet, reader, new IntegerFactory());
                    break;
                }
                dataSet._rankedSignatureIndexes = new FixedCacheList <Integer>(
                    dataSet, reader, new IntegerFactory(), Constants.RankedSignaturesCacheSize);
                switch (dataSet.VersionEnum)
                {
                case BinaryConstants.FormatVersions.PatternV31:
                    dataSet.Nodes = new VariableList <Entities.Node>(dataSet, reader, new NodeStreamFactoryV31(dataSet.Pool), Constants.NodesCacheSize);
                    break;

                case BinaryConstants.FormatVersions.PatternV32:
                    dataSet.Nodes = new VariableList <Entities.Node>(dataSet, reader, new NodeStreamFactoryV32(dataSet.Pool), Constants.NodesCacheSize);
                    break;
                }
                var rootNodes = new MemoryFixedList <Entities.Node>(dataSet, reader, new RootNodeFactory());
                dataSet.RootNodes = rootNodes;
                var profileOffsets = new MemoryFixedList <ProfileOffset>(dataSet, reader, new ProfileOffsetFactory());
                dataSet._profileOffsets = profileOffsets;

                // Read into memory all the small lists which are frequently accessed.
                reader.BaseStream.Position = components.Header.StartPosition;
                components.Read(reader);
                reader.BaseStream.Position = maps.Header.StartPosition;
                maps.Read(reader);
                reader.BaseStream.Position = properties.Header.StartPosition;
                properties.Read(reader);
                reader.BaseStream.Position = rootNodes.Header.StartPosition;
                rootNodes.Read(reader);
                reader.BaseStream.Position = profileOffsets.Header.StartPosition;
                profileOffsets.Read(reader);
            }
            finally
            {
                dataSet.Pool.Release(reader);
            }
        }