示例#1
0
            /// <summary>
            /// Supplied document must already have DocumentSpaceId filled.
            /// Must be Synced tables: DocumentsStorageTablesPrefix + "p", and all DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId
            /// </summary>
            /// <param name="doc"></param>
            public void RemoveDocument(Document doc)
            {
                if (doc == null || (doc.InternalId < 1 && String.IsNullOrEmpty(doc.ExternalId)))
                    throw new Exception("Check supplied Internal or External ID's and Document Space Id");

                if (!h.TryGetValue(doc.DocumentSpaceId, out dhl))
                {
                    dhl = new I1()
                    {
                        DocTableName = DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId.ToString()
                    };

                    //document table
                    dhl.dt = tran.InsertTable<int>(dhl.DocTableName, 1, 0);
                    //Version table Key is composite InitialDocId(int)+VersionNumber(int)+SequentialDocId(int)
                    dhl.vt = tran.InsertTable<int>(dhl.DocTableName, 3, 0);
                    dhl.et = tran.InsertTable<int>(dhl.DocTableName, 2, 0); //ExternalId to InternalId relation
                   // dhl.MaxDocId = tran.Select<int, int>(dhl.DocTableName, 4).Value;
                    h[doc.DocumentSpaceId] = dhl;
                }

                dhl.vt.ValuesLazyLoadingIsOn = false;

                if (doc.InternalId < 1)
                {
                    if (!String.IsNullOrEmpty(doc.ExternalId))  //Getting internalId via external
                    {
                        doc.InternalId = dhl.et.Select<string, int>(doc.ExternalId).Value;
                    }

                    if (doc.InternalId < 1)
                        return;
                }

                //Iterating through all versions of the document
                foreach (var vtRow in dhl.vt.SelectBackwardFromTo<byte[], byte>
                   (
                   doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MaxValue.To_4_bytes_array_BigEndian(), int.MaxValue.To_4_bytes_array_BigEndian()), true,
                   doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MinValue.To_4_bytes_array_BigEndian(), int.MinValue.To_4_bytes_array_BigEndian()), true
                   , true
                   ))
                {
                    if (vtRow.Value == 0)
                    {
                        //Including the last one into processing list with the value 1 (to be deleted)
                        tran.Insert<byte[], byte>(DocumentsStorageTablesPrefix + "p", doc.DocumentSpaceId.To_8_bytes_array_BigEndian().Concat(vtRow.Key.Substring(8, 4)), 1);
                    }
                    break;
                }
            }
示例#2
0
        /// <summary>
        /// Constructor (automatically starts unfinished indexing job)
        /// </summary>
        /// <param name="DBreezeEngine">must be already initialized</param>
        public Storage(DBreezeEngine DBreezeEngine)
        {
            this.OnProcessingStarted += Storage_OnProcessingStarted;
            this.OnProcessingStopped += Storage_OnProcessingStopped;
            if (DBreezeEngine == null)
                throw ThrowException("Storage", "DBreezeEngine must be instantiated");
            //if(SearchWordMinimalLength < 1)
            //    throw ThrowException("Storage", "SearchWordMinimalLength must be > 0");
            //if (DocumentsStorageTablesPrefix.Length < 1)
            //    throw ThrowException("Storage", "DocumentsStorageTablesPrefix.Length must be > 0");

            this.DBreezeEngine = DBreezeEngine;

            //Preparing Protobuf
            ProtoBuf.Serializer.PrepareSerializer<Document>();
            ProtoBuf.Serializer.PrepareSerializer<SearchRequest>();
            ProtoBuf.Serializer.PrepareSerializer<SearchResponse>();
            Document o1 = new Document();
            o1.SerializeProtobuf();
            SearchRequest o2 = new SearchRequest();
            o2.SerializeProtobuf();
            SearchResponse o3 = new SearchResponse();
            o3.SerializeProtobuf();

            //Automatic indexing of unfinished documents
            StartDocumentsIndexing();
        }
示例#3
0
            /// <summary>
            /// Supplied document must already have DocumentSpaceId filled.
            /// Must be Synced tables: DocumentsStorageTablesPrefix + "p", and all DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId
            /// </summary>
            /// <param name="doc"></param>
            public void AppendDocument(Document doc)
            {
                if (!h.TryGetValue(doc.DocumentSpaceId, out dhl))
                {
                    dhl = new I1()
                    {
                        DocTableName = DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId.ToString()
                    };

                    //document table
                    dhl.dt = tran.InsertTable<int>(dhl.DocTableName, 1, 0);
                    //Version table Key is composite InitialDocId(int)+VersionNumber(int)+SequentialDocId(int)
                    dhl.vt = tran.InsertTable<int>(dhl.DocTableName, 3, 0);
                    dhl.et = tran.InsertTable<int>(dhl.DocTableName, 2, 0); //ExternalId to InternalId relation
                    dhl.MaxDocId = tran.Select<int, int>(dhl.DocTableName, 4).Value;

                    h[doc.DocumentSpaceId] = dhl;
                }

                //Increasing maximal docIndex in the docSpace
                dhl.MaxDocId++;
                tran.Insert<int, int>(dhl.DocTableName, 4, dhl.MaxDocId);

                //Saving doc content separately and repack instead SelectDirect link
                if (doc.Content != null)
                {
                    //16 bytes link to Content
                    doc.Content = dhl.dt.InsertDataBlock(null, doc.Content);
                }

                //Extra compressing searchables routine.
                if (!String.IsNullOrEmpty(doc.Searchables))
                {
                    byte[] btSearchables = System.Text.Encoding.UTF8.GetBytes(doc.Searchables);
                    byte[] btSearchablesZipped = btSearchables.CompressGZip();
                    if (btSearchablesZipped.Length < btSearchables.Length)
                        btSearchables = new byte[] { 1 }.Concat(btSearchablesZipped);
                    else
                        btSearchables = new byte[] { 0 }.Concat(btSearchables);

                    doc.InternalStructure = dhl.dt.InsertDataBlock(null, btSearchables);

                    //Now document is lightweight, without real content and searchables
                    doc.Searchables = null;
                }

                if (!String.IsNullOrEmpty(doc.ExternalId))
                {
                    //If externalID is supplied, we use it to retrieve internal id
                    doc.InternalId = dhl.et.Select<string, int>(doc.ExternalId).Value;

                    if (doc.InternalId == 0)
                    {
                        //New doc
                        doc.DocumentSequentialId = dhl.MaxDocId;
                        doc.InternalId = doc.DocumentSequentialId;
                        dhl.et.Insert<string, int>(doc.ExternalId, doc.InternalId);
                        //Inserting into version table
                        //Console.WriteLine("Adding_V_" + doc.InternalId + "_1_" + doc.DocumentSequentialId);
                        dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(((int)1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), new byte[] { 0 });
                    }
                    else
                    {
                        //Updating document (we create new version)
                        doc.DocumentSequentialId = dhl.MaxDocId;
                        //Getting version number
                        foreach (var row in dhl.vt.SelectBackwardFromTo<byte[], byte>(
                            doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MaxValue.To_4_bytes_array_BigEndian(), int.MaxValue.To_4_bytes_array_BigEndian()), true,
                            doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MinValue.To_4_bytes_array_BigEndian(), int.MinValue.To_4_bytes_array_BigEndian()), true
                            ))
                        {
                            //Inserting into version table, new version
                            //Console.WriteLine("Adding_V_" + doc.InternalId + "_" + (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1) + "_" + doc.DocumentSequentialId);
                            dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany
                                (
                                    (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1).To_4_bytes_array_BigEndian(),
                                    doc.DocumentSequentialId.To_4_bytes_array_BigEndian()
                                ), new byte[] { 0 });
                            break;
                        }

                    }
                }
                else
                {
                    if (doc.InternalId < 1)
                    {
                        //New doc
                        doc.DocumentSequentialId = dhl.MaxDocId;
                        doc.InternalId = doc.DocumentSequentialId;
                        //Inserting into version table
                        //Console.WriteLine("Adding_V_" + doc.InternalId + "_1_" + doc.DocumentSequentialId);
                        dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(((int)1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), new byte[] { 0 });
                    }
                    else
                    {
                        //Updating document (we create new version)
                        doc.DocumentSequentialId = dhl.MaxDocId;
                        //Getting version number
                        foreach (var row in
                            dhl.vt.SelectBackwardFromTo<byte[], byte>(
                            doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MaxValue.To_4_bytes_array_BigEndian(), int.MaxValue.To_4_bytes_array_BigEndian()), true,
                            doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MinValue.To_4_bytes_array_BigEndian(), int.MinValue.To_4_bytes_array_BigEndian()), true
                            ))
                        {
                            //Inserting into version table, new version
                            //Console.WriteLine("Adding_V_" + doc.InternalId + "_" + (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1) + "_" + doc.DocumentSequentialId);
                            dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany
                                (
                                    (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1).To_4_bytes_array_BigEndian(),
                                    doc.DocumentSequentialId.To_4_bytes_array_BigEndian()
                                ), new byte[] { 0 });
                            break;
                        }
                    }
                }

                serDoc = doc.SerializeProtobuf();
                btDoc = serDoc.CompressGZip();

                if (serDoc.Length >= btDoc.Length)
                    btDoc = new byte[] { 1 }.Concat(btDoc);
                else
                    btDoc = new byte[] { 0 }.Concat(serDoc);
                dhl.dt.Insert<int, byte[]>(doc.DocumentSequentialId, btDoc);

                //-----------------------------------------------------------------------------------------

                //Adding to processing table
                //Console.WriteLine("Adding_P_" + doc.DocumentSequentialId);
                tran.Insert<byte[], byte>(DocumentsStorageTablesPrefix + "p", doc.DocumentSpaceId.To_8_bytes_array_BigEndian().Concat(doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), 0);
            }