public void InitOptions(ISynchronizedLinks <TLink> links)
{
if (UseSequenceMarker)
{
if (_equalityComparer.Equals(SequenceMarkerLink, links.Constants.Null))
{
SequenceMarkerLink = links.CreatePoint();
}
else
{
if (!links.Exists(SequenceMarkerLink))
{
var link = links.CreatePoint();
if (!_equalityComparer.Equals(link, SequenceMarkerLink))
{
throw new InvalidOperationException("Cannot recreate sequence marker link.");
}
}
}
if (MarkedSequenceMatcher == null)
{
MarkedSequenceMatcher = new MarkedSequenceCriterionMatcher <TLink>(links, SequenceMarkerLink);
}
}
var balancedVariantConverter = new BalancedVariantConverter <TLink>(links);
if (UseCompression)
{
if (LinksToSequenceConverter == null)
{
ICounter <TLink, TLink> totalSequenceSymbolFrequencyCounter;
if (UseSequenceMarker)
{
totalSequenceSymbolFrequencyCounter = new TotalMarkedSequenceSymbolFrequencyCounter <TLink>(links, MarkedSequenceMatcher);
}
else
{
totalSequenceSymbolFrequencyCounter = new TotalSequenceSymbolFrequencyCounter <TLink>(links);
}
var doubletFrequenciesCache = new LinkFrequenciesCache <TLink>(links, totalSequenceSymbolFrequencyCounter);
var compressingConverter = new CompressingConverter <TLink>(links, balancedVariantConverter, doubletFrequenciesCache);
LinksToSequenceConverter = compressingConverter;
}
}
else
{
if (LinksToSequenceConverter == null)
{
LinksToSequenceConverter = balancedVariantConverter;
}
}
if (UseIndex && Index == null)
{
Index = new SequenceIndex <TLink>(links);
}
if (Walker == null)
{
Walker = new RightSequenceWalker <TLink>(links, new DefaultStack <TLink>());
}
}
/// <summary>
/// <para>
/// Initializes a new <see cref="FileStorage"/> instance.
/// </para>
/// <para></para>
/// </summary>
/// <param name="DBFilename">
/// <para>A db filename.</para>
/// <para></para>
/// </param>
public FileStorage(string DBFilename)
{
var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);
var dataMemory = new FileMappedResizableDirectMemory(DBFilename);
_disposableLinks = new UnitedMemoryLinks <TLinkAddress>(dataMemory, UnitedMemoryLinks <UInt64> .DefaultLinksSizeStep, linksConstants, IndexTreeType.Default);
_synchronizedLinks = new SynchronizedLinks <TLinkAddress>(_disposableLinks);
var link = _synchronizedLinks.Create();
link = _synchronizedLinks.Update(link, newSource: link, newTarget: link);
ushort currentMappingLinkIndex = 1;
Any = _synchronizedLinks.Constants.Any;
_meaningRoot = GetOrCreateMeaningRoot(currentMappingLinkIndex++);
_unicodeSymbolMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_unicodeSequenceMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_setMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_fileMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();
_numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
var balancedVariantConverter = new BalancedVariantConverter <TLinkAddress>(_synchronizedLinks);
var unicodeSymbolCriterionMatcher = new TargetMatcher <TLinkAddress>(_synchronizedLinks, _unicodeSymbolMarker);
var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(_synchronizedLinks, _unicodeSequenceMarker);
var charToUnicodeSymbolConverter = new CharToUnicodeSymbolConverter <TLinkAddress>(_synchronizedLinks, _addressToNumberConverter, _unicodeSymbolMarker);
var unicodeSymbolToCharConverter = new UnicodeSymbolToCharConverter <TLinkAddress>(_synchronizedLinks, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
var sequenceWalker = new RightSequenceWalker <TLinkAddress>(_synchronizedLinks, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);
_stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(_synchronizedLinks, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
_unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(_synchronizedLinks, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
}
public DoubletsDbContext(string dataDBFilename, string indexDBFilename)
{
var dataMemory = new FileMappedResizableDirectMemory(dataDBFilename);
var indexMemory = new FileMappedResizableDirectMemory(indexDBFilename);
var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);
// Init the links storage
_disposableLinks = new UInt32SplitMemoryLinks(dataMemory, indexMemory, UInt32SplitMemoryLinks.DefaultLinksSizeStep, linksConstants); // Low-level logic
_links = new UInt32Links(_disposableLinks); // Main logic in the combined decorator
// Set up constant links (markers, aka mapped links)
TLinkAddress currentMappingLinkIndex = 1;
_meaningRoot = GetOrCreateMeaningRoot(currentMappingLinkIndex++);
_unicodeSymbolMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_unicodeSequenceMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_titlePropertyMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_contentPropertyMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_publicationDateTimePropertyMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
_blogPostMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
// Create properties operator that is able to control reading and writing properties for any link (object)
_defaultLinkPropertyOperator = new PropertiesOperator <TLinkAddress>(_links);
// Create converters that are able to convert link's address (UInt64 value) to a raw number represented with another UInt64 value and back
_numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
_addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();
// Create converters for dates
_longRawNumberToDateTimeConverter = new LongRawNumberSequenceToDateTimeConverter <TLinkAddress>(new LongRawNumberSequenceToNumberConverter <TLinkAddress, long>(_links, _numberToAddressConverter));
_dateTimeToLongRawNumberConverter = new DateTimeToLongRawNumberSequenceConverter <TLinkAddress>(new NumberToLongRawNumberSequenceConverter <long, TLinkAddress>(_links, _addressToNumberConverter));
// Create converters that are able to convert string to unicode sequence stored as link and back
var balancedVariantConverter = new BalancedVariantConverter <TLinkAddress>(_links);
var unicodeSymbolCriterionMatcher = new TargetMatcher <TLinkAddress>(_links, _unicodeSymbolMarker);
var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(_links, _unicodeSequenceMarker);
var charToUnicodeSymbolConverter = new CharToUnicodeSymbolConverter <TLinkAddress>(_links, _addressToNumberConverter, _unicodeSymbolMarker);
var unicodeSymbolToCharConverter = new UnicodeSymbolToCharConverter <TLinkAddress>(_links, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
var sequenceWalker = new RightSequenceWalker <TLinkAddress>(_links, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);
_stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(_links, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
_unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(_links, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
}
public PlatformDataBase(string indexFileName, string dataFileName)
{
this.indexFileName = indexFileName;
this.dataFileName = dataFileName;
var dataMemory = new FileMappedResizableDirectMemory(this.dataFileName);
var indexMemory = new FileMappedResizableDirectMemory(this.indexFileName);
var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);
// Init the links storage
this._disposableLinks = new UInt32SplitMemoryLinks(dataMemory, indexMemory, UInt32SplitMemoryLinks.DefaultLinksSizeStep, linksConstants); // Low-level logic
this.links = new UInt32Links(_disposableLinks); // Main logic in the combined decorator
// Set up constant links (markers, aka mapped links)
TLinkAddress currentMappingLinkIndex = 1;
this._meaningRoot = GerOrCreateMeaningRoot(currentMappingLinkIndex++);
this._unicodeSymbolMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
this._unicodeSequenceMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
this._bookMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
// Create converters that are able to convert link's address (UInt64 value) to a raw number represented with another UInt64 value and back
this._numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
this._addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();
// Create converters that are able to convert string to unicode sequence stored as link and back
var balancedVariantConverter = new BalancedVariantConverter <TLinkAddress>(links);
var unicodeSymbolCriterionMatcher = new TargetMatcher <TLinkAddress>(links, _unicodeSymbolMarker);
var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(links, _unicodeSequenceMarker);
var charToUnicodeSymbolConverter = new CharToUnicodeSymbolConverter <TLinkAddress>(links, _addressToNumberConverter, _unicodeSymbolMarker);
var unicodeSymbolToCharConverter = new UnicodeSymbolToCharConverter <TLinkAddress>(links, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
var sequenceWalker = new RightSequenceWalker <TLinkAddress>(links, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);
this._stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(links, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
this._unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(links, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
}
public static void SavedSequencesOptimizationTest()
{
LinksConstants <ulong> constants = new LinksConstants <ulong>((1, long.MaxValue), (long.MaxValue + 1UL, ulong.MaxValue));
using (var memory = new HeapResizableDirectMemory())
using (var disposableLinks = new UInt64UnitedMemoryLinks(memory, UInt64UnitedMemoryLinks.DefaultLinksSizeStep, constants, IndexTreeType.Default))
{
var links = new UInt64Links(disposableLinks);
var root = links.CreatePoint();
//var numberToAddressConverter = new RawNumberToAddressConverter<ulong>();
var addressToNumberConverter = new AddressToRawNumberConverter <ulong>();
var unicodeSymbolMarker = links.GetOrCreate(root, addressToNumberConverter.Convert(1));
var unicodeSequenceMarker = links.GetOrCreate(root, addressToNumberConverter.Convert(2));
var totalSequenceSymbolFrequencyCounter = new TotalSequenceSymbolFrequencyCounter <ulong>(links);
var linkFrequenciesCache = new LinkFrequenciesCache <ulong>(links, totalSequenceSymbolFrequencyCounter);
var index = new CachedFrequencyIncrementingSequenceIndex <ulong>(linkFrequenciesCache);
var linkToItsFrequencyNumberConverter = new FrequenciesCacheBasedLinkToItsFrequencyNumberConverter <ulong>(linkFrequenciesCache);
var sequenceToItsLocalElementLevelsConverter = new SequenceToItsLocalElementLevelsConverter <ulong>(links, linkToItsFrequencyNumberConverter);
var optimalVariantConverter = new OptimalVariantConverter <ulong>(links, sequenceToItsLocalElementLevelsConverter);
var walker = new RightSequenceWalker <ulong>(links, new DefaultStack <ulong>(), (link) => constants.IsExternalReference(link) || links.IsPartialPoint(link));
var unicodeSequencesOptions = new SequencesOptions <ulong>()
{
UseSequenceMarker = true,
SequenceMarkerLink = unicodeSequenceMarker,
UseIndex = true,
Index = index,
LinksToSequenceConverter = optimalVariantConverter,
Walker = walker,
UseGarbageCollection = true
};
var unicodeSequences = new Sequences.Sequences(new SynchronizedLinks <ulong>(links), unicodeSequencesOptions);
// Create some sequences
var strings = _loremIpsumExample.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
var arrays = strings.Select(x => x.Select(y => addressToNumberConverter.Convert(y)).ToArray()).ToArray();
for (int i = 0; i < arrays.Length; i++)
{
unicodeSequences.Create(arrays[i].ShiftRight());
}
var linksCountAfterCreation = links.Count();
// get list of sequences links
// for each sequence link
// create new sequence version
// if new sequence is not the same as sequence link
// delete sequence link
// collect garbadge
unicodeSequences.CompactAll();
var linksCountAfterCompactification = links.Count();
Assert.True(linksCountAfterCompactification < linksCountAfterCreation);
}
}
