Example #1
0
        public void InitOptions(ISynchronizedLinks <TLink> links)
        {
            if (UseSequenceMarker)
            {
                if (_equalityComparer.Equals(SequenceMarkerLink, links.Constants.Null))
                {
                    SequenceMarkerLink = links.CreatePoint();
                }
                else
                {
                    if (!links.Exists(SequenceMarkerLink))
                    {
                        var link = links.CreatePoint();
                        if (!_equalityComparer.Equals(link, SequenceMarkerLink))
                        {
                            throw new InvalidOperationException("Cannot recreate sequence marker link.");
                        }
                    }
                }
                if (MarkedSequenceMatcher == null)
                {
                    MarkedSequenceMatcher = new MarkedSequenceCriterionMatcher <TLink>(links, SequenceMarkerLink);
                }
            }
            var balancedVariantConverter = new BalancedVariantConverter <TLink>(links);

            if (UseCompression)
            {
                if (LinksToSequenceConverter == null)
                {
                    ICounter <TLink, TLink> totalSequenceSymbolFrequencyCounter;
                    if (UseSequenceMarker)
                    {
                        totalSequenceSymbolFrequencyCounter = new TotalMarkedSequenceSymbolFrequencyCounter <TLink>(links, MarkedSequenceMatcher);
                    }
                    else
                    {
                        totalSequenceSymbolFrequencyCounter = new TotalSequenceSymbolFrequencyCounter <TLink>(links);
                    }
                    var doubletFrequenciesCache = new LinkFrequenciesCache <TLink>(links, totalSequenceSymbolFrequencyCounter);
                    var compressingConverter    = new CompressingConverter <TLink>(links, balancedVariantConverter, doubletFrequenciesCache);
                    LinksToSequenceConverter = compressingConverter;
                }
            }
            else
            {
                if (LinksToSequenceConverter == null)
                {
                    LinksToSequenceConverter = balancedVariantConverter;
                }
            }
            if (UseIndex && Index == null)
            {
                Index = new SequenceIndex <TLink>(links);
            }
            if (Walker == null)
            {
                Walker = new RightSequenceWalker <TLink>(links, new DefaultStack <TLink>());
            }
        }
Example #2
0
        /// <summary>
        /// <para>
        /// Initializes a new <see cref="FileStorage"/> instance.
        /// </para>
        /// <para></para>
        /// </summary>
        /// <param name="DBFilename">
        /// <para>A db filename.</para>
        /// <para></para>
        /// </param>
        public FileStorage(string DBFilename)
        {
            var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);
            var dataMemory     = new FileMappedResizableDirectMemory(DBFilename);

            _disposableLinks   = new UnitedMemoryLinks <TLinkAddress>(dataMemory, UnitedMemoryLinks <UInt64> .DefaultLinksSizeStep, linksConstants, IndexTreeType.Default);
            _synchronizedLinks = new SynchronizedLinks <TLinkAddress>(_disposableLinks);
            var link = _synchronizedLinks.Create();

            link = _synchronizedLinks.Update(link, newSource: link, newTarget: link);
            ushort currentMappingLinkIndex = 1;

            Any                       = _synchronizedLinks.Constants.Any;
            _meaningRoot              = GetOrCreateMeaningRoot(currentMappingLinkIndex++);
            _unicodeSymbolMarker      = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _unicodeSequenceMarker    = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _setMarker                = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _fileMarker               = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();
            _numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
            var balancedVariantConverter        = new BalancedVariantConverter <TLinkAddress>(_synchronizedLinks);
            var unicodeSymbolCriterionMatcher   = new TargetMatcher <TLinkAddress>(_synchronizedLinks, _unicodeSymbolMarker);
            var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(_synchronizedLinks, _unicodeSequenceMarker);
            var charToUnicodeSymbolConverter    = new CharToUnicodeSymbolConverter <TLinkAddress>(_synchronizedLinks, _addressToNumberConverter, _unicodeSymbolMarker);
            var unicodeSymbolToCharConverter    = new UnicodeSymbolToCharConverter <TLinkAddress>(_synchronizedLinks, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
            var sequenceWalker = new RightSequenceWalker <TLinkAddress>(_synchronizedLinks, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);

            _stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(_synchronizedLinks, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
            _unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(_synchronizedLinks, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
        }
Example #3
0
        public DoubletsDbContext(string dataDBFilename, string indexDBFilename)
        {
            var dataMemory  = new FileMappedResizableDirectMemory(dataDBFilename);
            var indexMemory = new FileMappedResizableDirectMemory(indexDBFilename);

            var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);

            // Init the links storage
            _disposableLinks = new UInt32SplitMemoryLinks(dataMemory, indexMemory, UInt32SplitMemoryLinks.DefaultLinksSizeStep, linksConstants); // Low-level logic
            _links           = new UInt32Links(_disposableLinks);                                                                                // Main logic in the combined decorator

            // Set up constant links (markers, aka mapped links)
            TLinkAddress currentMappingLinkIndex = 1;

            _meaningRoot                       = GetOrCreateMeaningRoot(currentMappingLinkIndex++);
            _unicodeSymbolMarker               = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _unicodeSequenceMarker             = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _titlePropertyMarker               = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _contentPropertyMarker             = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _publicationDateTimePropertyMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
            _blogPostMarker                    = GetOrCreateNextMapping(currentMappingLinkIndex++);

            // Create properties operator that is able to control reading and writing properties for any link (object)
            _defaultLinkPropertyOperator = new PropertiesOperator <TLinkAddress>(_links);

            // Create converters that are able to convert link's address (UInt64 value) to a raw number represented with another UInt64 value and back
            _numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
            _addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();

            // Create converters for dates
            _longRawNumberToDateTimeConverter = new LongRawNumberSequenceToDateTimeConverter <TLinkAddress>(new LongRawNumberSequenceToNumberConverter <TLinkAddress, long>(_links, _numberToAddressConverter));
            _dateTimeToLongRawNumberConverter = new DateTimeToLongRawNumberSequenceConverter <TLinkAddress>(new NumberToLongRawNumberSequenceConverter <long, TLinkAddress>(_links, _addressToNumberConverter));

            // Create converters that are able to convert string to unicode sequence stored as link and back
            var balancedVariantConverter        = new BalancedVariantConverter <TLinkAddress>(_links);
            var unicodeSymbolCriterionMatcher   = new TargetMatcher <TLinkAddress>(_links, _unicodeSymbolMarker);
            var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(_links, _unicodeSequenceMarker);
            var charToUnicodeSymbolConverter    = new CharToUnicodeSymbolConverter <TLinkAddress>(_links, _addressToNumberConverter, _unicodeSymbolMarker);
            var unicodeSymbolToCharConverter    = new UnicodeSymbolToCharConverter <TLinkAddress>(_links, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
            var sequenceWalker = new RightSequenceWalker <TLinkAddress>(_links, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);

            _stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(_links, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
            _unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(_links, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
        }
Example #4
0
        public PlatformDataBase(string indexFileName, string dataFileName)
        {
            this.indexFileName = indexFileName;
            this.dataFileName  = dataFileName;

            var dataMemory  = new FileMappedResizableDirectMemory(this.dataFileName);
            var indexMemory = new FileMappedResizableDirectMemory(this.indexFileName);

            var linksConstants = new LinksConstants <TLinkAddress>(enableExternalReferencesSupport: true);

            // Init the links storage
            this._disposableLinks = new UInt32SplitMemoryLinks(dataMemory, indexMemory, UInt32SplitMemoryLinks.DefaultLinksSizeStep, linksConstants); // Low-level logic
            this.links            = new UInt32Links(_disposableLinks);                                                                                // Main logic in the combined decorator

            // Set up constant links (markers, aka mapped links)
            TLinkAddress currentMappingLinkIndex = 1;

            this._meaningRoot           = GerOrCreateMeaningRoot(currentMappingLinkIndex++);
            this._unicodeSymbolMarker   = GetOrCreateNextMapping(currentMappingLinkIndex++);
            this._unicodeSequenceMarker = GetOrCreateNextMapping(currentMappingLinkIndex++);
            this._bookMarker            = GetOrCreateNextMapping(currentMappingLinkIndex++);
            // Create converters that are able to convert link's address (UInt64 value) to a raw number represented with another UInt64 value and back
            this._numberToAddressConverter = new RawNumberToAddressConverter <TLinkAddress>();
            this._addressToNumberConverter = new AddressToRawNumberConverter <TLinkAddress>();

            // Create converters that are able to convert string to unicode sequence stored as link and back
            var balancedVariantConverter        = new BalancedVariantConverter <TLinkAddress>(links);
            var unicodeSymbolCriterionMatcher   = new TargetMatcher <TLinkAddress>(links, _unicodeSymbolMarker);
            var unicodeSequenceCriterionMatcher = new TargetMatcher <TLinkAddress>(links, _unicodeSequenceMarker);
            var charToUnicodeSymbolConverter    = new CharToUnicodeSymbolConverter <TLinkAddress>(links, _addressToNumberConverter, _unicodeSymbolMarker);
            var unicodeSymbolToCharConverter    = new UnicodeSymbolToCharConverter <TLinkAddress>(links, _numberToAddressConverter, unicodeSymbolCriterionMatcher);
            var sequenceWalker = new RightSequenceWalker <TLinkAddress>(links, new DefaultStack <TLinkAddress>(), unicodeSymbolCriterionMatcher.IsMatched);

            this._stringToUnicodeSequenceConverter = new CachingConverterDecorator <string, TLinkAddress>(new StringToUnicodeSequenceConverter <TLinkAddress>(links, charToUnicodeSymbolConverter, balancedVariantConverter, _unicodeSequenceMarker));
            this._unicodeSequenceToStringConverter = new CachingConverterDecorator <TLinkAddress, string>(new UnicodeSequenceToStringConverter <TLinkAddress>(links, unicodeSequenceCriterionMatcher, sequenceWalker, unicodeSymbolToCharConverter));
        }
        public static void SavedSequencesOptimizationTest()
        {
            LinksConstants <ulong> constants = new LinksConstants <ulong>((1, long.MaxValue), (long.MaxValue + 1UL, ulong.MaxValue));

            using (var memory = new HeapResizableDirectMemory())
                using (var disposableLinks = new UInt64UnitedMemoryLinks(memory, UInt64UnitedMemoryLinks.DefaultLinksSizeStep, constants, IndexTreeType.Default))
                {
                    var links = new UInt64Links(disposableLinks);

                    var root = links.CreatePoint();

                    //var numberToAddressConverter = new RawNumberToAddressConverter<ulong>();
                    var addressToNumberConverter = new AddressToRawNumberConverter <ulong>();

                    var unicodeSymbolMarker   = links.GetOrCreate(root, addressToNumberConverter.Convert(1));
                    var unicodeSequenceMarker = links.GetOrCreate(root, addressToNumberConverter.Convert(2));

                    var totalSequenceSymbolFrequencyCounter = new TotalSequenceSymbolFrequencyCounter <ulong>(links);
                    var linkFrequenciesCache = new LinkFrequenciesCache <ulong>(links, totalSequenceSymbolFrequencyCounter);
                    var index = new CachedFrequencyIncrementingSequenceIndex <ulong>(linkFrequenciesCache);
                    var linkToItsFrequencyNumberConverter        = new FrequenciesCacheBasedLinkToItsFrequencyNumberConverter <ulong>(linkFrequenciesCache);
                    var sequenceToItsLocalElementLevelsConverter = new SequenceToItsLocalElementLevelsConverter <ulong>(links, linkToItsFrequencyNumberConverter);
                    var optimalVariantConverter = new OptimalVariantConverter <ulong>(links, sequenceToItsLocalElementLevelsConverter);

                    var walker = new RightSequenceWalker <ulong>(links, new DefaultStack <ulong>(), (link) => constants.IsExternalReference(link) || links.IsPartialPoint(link));

                    var unicodeSequencesOptions = new SequencesOptions <ulong>()
                    {
                        UseSequenceMarker  = true,
                        SequenceMarkerLink = unicodeSequenceMarker,
                        UseIndex           = true,
                        Index = index,
                        LinksToSequenceConverter = optimalVariantConverter,
                        Walker = walker,
                        UseGarbageCollection = true
                    };

                    var unicodeSequences = new Sequences.Sequences(new SynchronizedLinks <ulong>(links), unicodeSequencesOptions);

                    // Create some sequences
                    var strings = _loremIpsumExample.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
                    var arrays  = strings.Select(x => x.Select(y => addressToNumberConverter.Convert(y)).ToArray()).ToArray();
                    for (int i = 0; i < arrays.Length; i++)
                    {
                        unicodeSequences.Create(arrays[i].ShiftRight());
                    }

                    var linksCountAfterCreation = links.Count();

                    // get list of sequences links
                    // for each sequence link
                    //   create new sequence version
                    //   if new sequence is not the same as sequence link
                    //     delete sequence link
                    //     collect garbadge
                    unicodeSequences.CompactAll();

                    var linksCountAfterCompactification = links.Count();

                    Assert.True(linksCountAfterCompactification < linksCountAfterCreation);
                }
        }