/// <summary>
/// Creates a new <see cref="DataSet"/> from the binary reader provided.
/// </summary>
/// <param name="dataSet">The data set to be loaded with data from the reader</param>
/// <param name="reader">
/// Reader connected to the source data structure and positioned to start reading
/// </param>
/// <param name="init">
/// True to indicate that the data set should be fulling initialised
/// </param>
/// <para>
/// A <see cref="DataSet"/> is constructed using the reader to retrieve
/// the header information. This is then passed to the Read methods to create the
/// lists before reading the data into memory. Finally it initialise is required
/// references between entities are worked out and stored.
/// </para>
internal static void Load(DataSet dataSet, Reader reader, bool init)
{
CommonFactory.LoadHeader(dataSet, reader);
var strings = new MemoryVariableList <AsciiString>(dataSet, reader, new AsciiStringFactory());
MemoryFixedList <Component> components = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV32());
break;
}
var maps = new MemoryFixedList <Map>(dataSet, reader, new MapFactory());
var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
var values = new MemoryFixedList <Value>(dataSet, reader, new ValueFactory());
var profiles = new MemoryVariableList <Entities.Profile>(dataSet, reader, new ProfileMemoryFactory());
MemoryFixedList <Signature> signatures = null;
MemoryFixedList <Integer> signatureNodeOffsets = null;
MemoryFixedList <Integer> nodeRankedSignatureIndexes = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
signatures = new MemoryFixedList <Signature>(dataSet, reader, new SignatureFactoryV31(dataSet));
break;
case BinaryConstants.FormatVersions.PatternV32:
signatures = new MemoryFixedList <Signature>(dataSet, reader, new SignatureFactoryV32(dataSet));
signatureNodeOffsets = new MemoryFixedList <Integer>(dataSet, reader, new IntegerFactory());
nodeRankedSignatureIndexes = new MemoryFixedList <Integer>(dataSet, reader, new IntegerFactory());
break;
}
var rankedSignatureIndexes = new MemoryFixedList <Integer>(
dataSet, reader, new IntegerFactory());
MemoryVariableList <Entities.Node> nodes = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
nodes = new MemoryVariableList <Entities.Node>(dataSet, reader, new NodeMemoryFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
nodes = new MemoryVariableList <Entities.Node>(dataSet, reader, new NodeMemoryFactoryV32());
break;
}
var rootNodes = new MemoryFixedList <Entities.Node>(dataSet, reader, new RootNodeFactory());
var profileOffsets = new MemoryFixedList <ProfileOffset>(dataSet, reader, new ProfileOffsetFactory());
dataSet.Strings = strings;
dataSet._components = components;
dataSet._maps = maps;
dataSet._properties = properties;
dataSet._values = values;
dataSet.Profiles = profiles;
dataSet._signatures = signatures;
dataSet._rankedSignatureIndexes = rankedSignatureIndexes;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV32:
dataSet._signatureNodeOffsets = signatureNodeOffsets;
dataSet._nodeRankedSignatureIndexes = nodeRankedSignatureIndexes;
break;
}
dataSet.Nodes = nodes;
dataSet.RootNodes = rootNodes;
dataSet._profileOffsets = profileOffsets;
strings.Read(reader);
components.Read(reader);
maps.Read(reader);
properties.Read(reader);
values.Read(reader);
profiles.Read(reader);
signatures.Read(reader);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV32:
signatureNodeOffsets.Read(reader);
nodeRankedSignatureIndexes.Read(reader);
break;
}
rankedSignatureIndexes.Read(reader);
nodes.Read(reader);
rootNodes.Read(reader);
profileOffsets.Read(reader);
if (init)
{
// Set references between objects.
dataSet.Init();
// The following lists will not be needed anymore
// so they can be freed.
dataSet._signatureNodeOffsets = null;
dataSet._nodeRankedSignatureIndexes = null;
// Force garbage collection as a lot of memory has been freed.
GC.Collect();
}
}
/// <summary>
/// Load the necessary values from the data file in to the DataSet.
/// Initially, this will only load data such as file headers
/// and the smaller lists.
/// </summary>
/// <param name="dataSet">The dataset object to load in to</param>
/// <param name="cacheConfiguration">the cache configuration to use when creating the caches</param>
private static void LoadForStreaming(IndirectDataSet dataSet, Dictionary <CacheType, ICacheOptions> cacheConfiguration)
{
var reader = dataSet.Pool.GetReader();
try
{
CacheMap cacheMap = BuildCaches(cacheConfiguration);
dataSet.CacheMap = cacheMap;
reader.BaseStream.Position = 0;
CommonFactory.LoadHeader(dataSet, reader);
// ---- Create AsciiString list
var stringLoader = EntityLoaderFactory.GetLoaderFor(
new Header(reader),
cacheMap.StringCache,
dataSet,
new StreamAsciiStringFactory());
dataSet.Strings = new StreamList <AsciiString, IStreamDataSet>(stringLoader);
MemoryFixedList <Component, DataSet> components = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
components = new MemoryFixedList <Component, DataSet>(
dataSet,
reader,
new ComponentFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
components = new MemoryFixedList <Component, DataSet>(
dataSet,
reader,
new ComponentFactoryV32());
break;
}
dataSet._components = components;
var maps = new MemoryFixedList <Map, DataSet>(
dataSet,
reader,
new MapFactory());
dataSet._maps = maps;
var properties = new PropertiesList(
dataSet,
reader,
new PropertyFactory());
dataSet._properties = properties;
// ---- Create Value list
var valueLoader = EntityLoaderFactory.GetLoaderFor(new Header(reader),
cacheMap.ValueCache,
dataSet,
new ValueFactory <IStreamDataSet>());
dataSet._values = new StreamList <Value, IStreamDataSet>(valueLoader);
// ---- Create Profile list
var profileLoader = EntityLoaderFactory.GetLoaderFor(new Header(reader),
cacheMap.ProfileCache,
dataSet,
new ProfileStreamFactory(dataSet.Pool));
dataSet.Profiles = new StreamList <Entities.Profile, IStreamDataSet>(profileLoader);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
// ---- Create Signature list for V31
var signatureLoaderV31 = EntityLoaderFactory.GetLoaderFor(
new Header(reader),
cacheMap.SignatureCache,
dataSet,
new SignatureFactoryV31 <IndirectDataSet>(dataSet));
dataSet._signatures = new StreamList <Signature, IndirectDataSet>(
signatureLoaderV31);
break;
case BinaryConstants.FormatVersions.PatternV32:
// ---- Create Signature list for V32
var signatureLoaderV32 = EntityLoaderFactory.GetLoaderFor(
new Header(reader),
cacheMap.SignatureCache,
dataSet,
new SignatureFactoryV32 <IndirectDataSet>(dataSet));
dataSet._signatures = new StreamList <Signature, IndirectDataSet>(
signatureLoaderV32);
dataSet._signatureNodeOffsets = new IntegerList(dataSet, reader);
dataSet._nodeRankedSignatureIndexes = new IntegerList(dataSet, reader);
break;
}
dataSet._rankedSignatureIndexes = new IntegerList(dataSet, reader);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
// ---- Create Nodes list for V31
var nodeLoaderV31 = EntityLoaderFactory.GetLoaderFor(
new Header(reader),
cacheMap.NodeCache,
dataSet,
new NodeStreamFactoryV31(dataSet.Pool));
dataSet.Nodes = new StreamList <Entities.Node, IStreamDataSet>(
nodeLoaderV31);
break;
case BinaryConstants.FormatVersions.PatternV32:
// ---- Create Nodes list for V31
var nodeLoaderV32 = EntityLoaderFactory.GetLoaderFor(
new Header(reader),
cacheMap.NodeCache,
dataSet,
new NodeStreamFactoryV32(dataSet.Pool));
dataSet.Nodes = new StreamList <Entities.Node, IStreamDataSet>(
nodeLoaderV32);
break;
}
var rootNodes = new MemoryFixedList <Entities.Node, DataSet>(
dataSet,
reader,
new RootNodeFactory());
dataSet.RootNodes = rootNodes;
var profileOffsets = new MemoryFixedList <ProfileOffset, DataSet>(
dataSet,
reader,
new ProfileOffsetFactory());
dataSet._profileOffsets = profileOffsets;
// Read into memory all the small lists which are frequently accessed.
reader.BaseStream.Position = components.Header.StartPosition;
components.Read(reader);
reader.BaseStream.Position = maps.Header.StartPosition;
maps.Read(reader);
reader.BaseStream.Position = properties.Header.StartPosition;
properties.Read(reader);
reader.BaseStream.Position = rootNodes.Header.StartPosition;
rootNodes.Read(reader);
reader.BaseStream.Position = profileOffsets.Header.StartPosition;
profileOffsets.Read(reader);
}
finally
{
dataSet.Pool.Release(reader);
}
}
/// <summary>
/// Initialises the dataset <see cref="DataSet"/> using the source of the data set.
/// </summary>
/// <para>
/// A <see cref="DataSet"/> is initialised using the reader to retrieve
/// entity information. The data is only loaded when required by the detection
/// process.
/// </para>
/// <param name="dataSet">A data set to be initialised ready for detection</param>
private static void Load(DataSet dataSet)
{
var reader = dataSet.Pool.GetReader();
try
{
reader.BaseStream.Position = 0;
CommonFactory.LoadHeader(dataSet, reader);
dataSet.Strings = new VariableList<AsciiString<Entities.DataSet>, DataSet>(
dataSet, reader, new StreamAsciiStringFactory(), Constants.StringsCacheSize);
MemoryFixedList<Component, Entities.DataSet> components = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
components = new MemoryFixedList<Component, Entities.DataSet>(dataSet, reader, new ComponentFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
components = new MemoryFixedList<Component, Entities.DataSet>(dataSet, reader, new ComponentFactoryV32());
break;
}
dataSet._components = components;
var maps = new MemoryFixedList<Map, Entities.DataSet>(dataSet, reader, new MapFactory());
dataSet._maps = maps;
var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
dataSet._properties = properties;
dataSet._values = new FixedCacheList<Value, DataSet>(dataSet, reader, new ValueFactory<DataSet>(), Constants.ValuesCacheSize);
dataSet.Profiles = new VariableList<Entities.Profile, DataSet>(dataSet, reader, new ProfileStreamFactory(dataSet.Pool), Constants.ProfilesCacheSize);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
dataSet._signatures = new FixedCacheList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV31<DataSet>(dataSet), Constants.SignaturesCacheSize);
break;
case BinaryConstants.FormatVersions.PatternV32:
dataSet._signatures = new FixedCacheList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV32<DataSet>(dataSet), Constants.SignaturesCacheSize);
dataSet._signatureNodeOffsets = new IntegerList(dataSet, reader);
dataSet._nodeRankedSignatureIndexes = new IntegerList(dataSet, reader);
break;
}
dataSet._rankedSignatureIndexes = new IntegerList(dataSet, reader);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
dataSet.Nodes = new VariableList<Entities.Node, DataSet>(dataSet, reader, new NodeStreamFactoryV31(dataSet.Pool), Constants.NodesCacheSize);
break;
case BinaryConstants.FormatVersions.PatternV32:
dataSet.Nodes = new VariableList<Entities.Node, DataSet>(dataSet, reader, new NodeStreamFactoryV32(dataSet.Pool), Constants.NodesCacheSize);
break;
}
var rootNodes = new MemoryFixedList<Entities.Node, Entities.DataSet>(dataSet, reader, new RootNodeFactory());
dataSet.RootNodes = rootNodes;
var profileOffsets = new MemoryFixedList<ProfileOffset, Entities.DataSet>(dataSet, reader, new ProfileOffsetFactory());
dataSet._profileOffsets = profileOffsets;
// Read into memory all the small lists which are frequently accessed.
reader.BaseStream.Position = components.Header.StartPosition;
components.Read(reader);
reader.BaseStream.Position = maps.Header.StartPosition;
maps.Read(reader);
reader.BaseStream.Position = properties.Header.StartPosition;
properties.Read(reader);
reader.BaseStream.Position = rootNodes.Header.StartPosition;
rootNodes.Read(reader);
reader.BaseStream.Position = profileOffsets.Header.StartPosition;
profileOffsets.Read(reader);
}
finally
{
dataSet.Pool.Release(reader);
}
}
/// <summary>
/// Creates a new <see cref="DataSet"/> from the binary reader provided.
/// </summary>
/// <param name="dataSet">The data set to be loaded with data from the reader</param>
/// <param name="reader">
/// Reader connected to the source data structure and positioned to start reading
/// </param>
/// <param name="init">
/// True to indicate that the data set should be fulling initialised
/// </param>
/// <para>
/// A <see cref="DataSet"/> is constructed using the reader to retrieve
/// the header information. This is then passed to the Read methods to create the
/// lists before reading the data into memory. Finally it initialise is required
/// references between entities are worked out and stored.
/// </para>
internal static void Load(DataSet dataSet, Reader reader, bool init)
{
CommonFactory.LoadHeader(dataSet, reader);
var strings = new MemoryVariableList<AsciiString<DataSet>, DataSet>(dataSet, reader, new MemoryAsciiStringFactory());
MemoryFixedList<Component, DataSet> components = null;
switch(dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
components = new MemoryFixedList<Component, DataSet>(dataSet, reader, new ComponentFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
components = new MemoryFixedList<Component, DataSet>(dataSet, reader, new ComponentFactoryV32());
break;
}
var maps = new MemoryFixedList<Map, DataSet>(dataSet, reader, new MapFactory());
var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
var values = new MemoryFixedList<Value, DataSet>(dataSet, reader, new ValueFactory<DataSet>());
var profiles = new MemoryVariableList<Entities.Profile, DataSet>(dataSet, reader, new ProfileMemoryFactory());
MemoryFixedList<Signature, DataSet> signatures = null;
MemoryIntegerList signatureNodeOffsets = null;
MemoryIntegerList nodeRankedSignatureIndexes = null;
switch(dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
signatures = new MemoryFixedList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV31<DataSet>(dataSet));
break;
case BinaryConstants.FormatVersions.PatternV32:
signatures = new MemoryFixedList<Signature, DataSet>(dataSet, reader, new SignatureFactoryV32<DataSet>(dataSet));
signatureNodeOffsets = new MemoryIntegerList(reader);
nodeRankedSignatureIndexes = new MemoryIntegerList(reader);
break;
}
var rankedSignatureIndexes = new MemoryIntegerList(reader);
MemoryVariableList<Entities.Node, DataSet> nodes = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
nodes = new MemoryVariableList<Entities.Node, DataSet>(dataSet, reader, new NodeMemoryFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
nodes = new MemoryVariableList<Entities.Node, DataSet>(dataSet, reader, new NodeMemoryFactoryV32());
break;
}
var rootNodes = new MemoryFixedList<Entities.Node, DataSet>(dataSet, reader, new RootNodeFactory());
var profileOffsets = new MemoryFixedList<ProfileOffset, DataSet>(dataSet, reader, new ProfileOffsetFactory());
dataSet.Strings = strings;
dataSet._components = components;
dataSet._maps = maps;
dataSet._properties = properties;
dataSet._values = values;
dataSet.Profiles = profiles;
dataSet._signatures = signatures;
dataSet._rankedSignatureIndexes = rankedSignatureIndexes;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV32:
dataSet._signatureNodeOffsets = signatureNodeOffsets;
dataSet._nodeRankedSignatureIndexes = nodeRankedSignatureIndexes;
break;
}
dataSet.Nodes = nodes;
dataSet.RootNodes = rootNodes;
dataSet._profileOffsets = profileOffsets;
strings.Read(reader);
components.Read(reader);
maps.Read(reader);
properties.Read(reader);
values.Read(reader);
profiles.Read(reader);
signatures.Read(reader);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV32:
signatureNodeOffsets.Read(reader);
nodeRankedSignatureIndexes.Read(reader);
break;
}
rankedSignatureIndexes.Read(reader);
nodes.Read(reader);
rootNodes.Read(reader);
profileOffsets.Read(reader);
if (init)
{
// Set references between objects.
dataSet.Init();
// The following lists will not be needed anymore
// so they can be freed.
dataSet._signatureNodeOffsets = null;
dataSet._nodeRankedSignatureIndexes = null;
// Force garbage collection as a lot of memory has been freed.
GC.Collect();
}
}
/// <summary>
/// Initialises the dataset <see cref="DataSet"/> using the source of the data set.
/// </summary>
/// <para>
/// A <see cref="DataSet"/> is initialised using the reader to retrieve
/// entity information. The data is only loaded when required by the detection
/// process.
/// </para>
/// <param name="dataSet">A data set to be initialised ready for detection</param>
private static void Load(DataSet dataSet)
{
var reader = dataSet.Pool.GetReader();
try
{
reader.BaseStream.Position = 0;
CommonFactory.LoadHeader(dataSet, reader);
dataSet.Strings = new VariableList <AsciiString>(dataSet, reader, new AsciiStringFactory(), Constants.StringsCacheSize);
MemoryFixedList <Component> components = null;
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV31());
break;
case BinaryConstants.FormatVersions.PatternV32:
components = new MemoryFixedList <Component>(dataSet, reader, new ComponentFactoryV32());
break;
}
dataSet._components = components;
var maps = new MemoryFixedList <Map>(dataSet, reader, new MapFactory());
dataSet._maps = maps;
var properties = new PropertiesList(dataSet, reader, new PropertyFactory());
dataSet._properties = properties;
dataSet._values = new FixedCacheList <Value>(dataSet, reader, new ValueFactory(), Constants.ValuesCacheSize);
dataSet.Profiles = new VariableList <Entities.Profile>(dataSet, reader, new ProfileStreamFactory(dataSet.Pool), Constants.ProfilesCacheSize);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
dataSet._signatures = new FixedCacheList <Signature>(dataSet, reader, new SignatureFactoryV31(dataSet), Constants.SignaturesCacheSize);
break;
case BinaryConstants.FormatVersions.PatternV32:
dataSet._signatures = new FixedCacheList <Signature>(dataSet, reader, new SignatureFactoryV32(dataSet), Constants.SignaturesCacheSize);
dataSet._signatureNodeOffsets = new FixedList <Integer>(dataSet, reader, new IntegerFactory());
dataSet._nodeRankedSignatureIndexes = new FixedList <Integer>(dataSet, reader, new IntegerFactory());
break;
}
dataSet._rankedSignatureIndexes = new FixedCacheList <Integer>(
dataSet, reader, new IntegerFactory(), Constants.RankedSignaturesCacheSize);
switch (dataSet.VersionEnum)
{
case BinaryConstants.FormatVersions.PatternV31:
dataSet.Nodes = new VariableList <Entities.Node>(dataSet, reader, new NodeStreamFactoryV31(dataSet.Pool), Constants.NodesCacheSize);
break;
case BinaryConstants.FormatVersions.PatternV32:
dataSet.Nodes = new VariableList <Entities.Node>(dataSet, reader, new NodeStreamFactoryV32(dataSet.Pool), Constants.NodesCacheSize);
break;
}
var rootNodes = new MemoryFixedList <Entities.Node>(dataSet, reader, new RootNodeFactory());
dataSet.RootNodes = rootNodes;
var profileOffsets = new MemoryFixedList <ProfileOffset>(dataSet, reader, new ProfileOffsetFactory());
dataSet._profileOffsets = profileOffsets;
// Read into memory all the small lists which are frequently accessed.
reader.BaseStream.Position = components.Header.StartPosition;
components.Read(reader);
reader.BaseStream.Position = maps.Header.StartPosition;
maps.Read(reader);
reader.BaseStream.Position = properties.Header.StartPosition;
properties.Read(reader);
reader.BaseStream.Position = rootNodes.Header.StartPosition;
rootNodes.Read(reader);
reader.BaseStream.Position = profileOffsets.Header.StartPosition;
profileOffsets.Read(reader);
}
finally
{
dataSet.Pool.Release(reader);
}
}
