/// <summary> /// Constructor (automatically starts unfinished indexing job) /// </summary> /// <param name="DBreezeEngine">must be already initialized</param> public Storage(DBreezeEngine DBreezeEngine) { this.OnProcessingStarted += Storage_OnProcessingStarted; this.OnProcessingStopped += Storage_OnProcessingStopped; if (DBreezeEngine == null) throw ThrowException("Storage", "DBreezeEngine must be instantiated"); //if(SearchWordMinimalLength < 1) // throw ThrowException("Storage", "SearchWordMinimalLength must be > 0"); //if (DocumentsStorageTablesPrefix.Length < 1) // throw ThrowException("Storage", "DocumentsStorageTablesPrefix.Length must be > 0"); this.DBreezeEngine = DBreezeEngine; //Preparing Protobuf ProtoBuf.Serializer.PrepareSerializer<Document>(); ProtoBuf.Serializer.PrepareSerializer<SearchRequest>(); ProtoBuf.Serializer.PrepareSerializer<SearchResponse>(); Document o1 = new Document(); o1.SerializeProtobuf(); SearchRequest o2 = new SearchRequest(); o2.SerializeProtobuf(); SearchResponse o3 = new SearchResponse(); o3.SerializeProtobuf(); //Automatic indexing of unfinished documents StartDocumentsIndexing(); }
/// <summary> /// Supplied document must already have DocumentSpaceId filled. /// Must be Synced tables: DocumentsStorageTablesPrefix + "p", and all DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId /// </summary> /// <param name="doc"></param> public void AppendDocument(Document doc) { if (!h.TryGetValue(doc.DocumentSpaceId, out dhl)) { dhl = new I1() { DocTableName = DocumentsStorageTablesPrefix + "d" + doc.DocumentSpaceId.ToString() }; //document table dhl.dt = tran.InsertTable<int>(dhl.DocTableName, 1, 0); //Version table Key is composite InitialDocId(int)+VersionNumber(int)+SequentialDocId(int) dhl.vt = tran.InsertTable<int>(dhl.DocTableName, 3, 0); dhl.et = tran.InsertTable<int>(dhl.DocTableName, 2, 0); //ExternalId to InternalId relation dhl.MaxDocId = tran.Select<int, int>(dhl.DocTableName, 4).Value; h[doc.DocumentSpaceId] = dhl; } //Increasing maximal docIndex in the docSpace dhl.MaxDocId++; tran.Insert<int, int>(dhl.DocTableName, 4, dhl.MaxDocId); //Saving doc content separately and repack instead SelectDirect link if (doc.Content != null) { //16 bytes link to Content doc.Content = dhl.dt.InsertDataBlock(null, doc.Content); } //Extra compressing searchables routine. if (!String.IsNullOrEmpty(doc.Searchables)) { byte[] btSearchables = System.Text.Encoding.UTF8.GetBytes(doc.Searchables); byte[] btSearchablesZipped = btSearchables.CompressGZip(); if (btSearchablesZipped.Length < btSearchables.Length) btSearchables = new byte[] { 1 }.Concat(btSearchablesZipped); else btSearchables = new byte[] { 0 }.Concat(btSearchables); doc.InternalStructure = dhl.dt.InsertDataBlock(null, btSearchables); //Now document is lightweight, without real content and searchables doc.Searchables = null; } if (!String.IsNullOrEmpty(doc.ExternalId)) { //If externalID is supplied, we use it to retrieve internal id doc.InternalId = dhl.et.Select<string, int>(doc.ExternalId).Value; if (doc.InternalId == 0) { //New doc doc.DocumentSequentialId = dhl.MaxDocId; doc.InternalId = doc.DocumentSequentialId; dhl.et.Insert<string, int>(doc.ExternalId, doc.InternalId); //Inserting into version table //Console.WriteLine("Adding_V_" + doc.InternalId + "_1_" + doc.DocumentSequentialId); dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(((int)1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), new byte[] { 0 }); } else { //Updating document (we create new version) doc.DocumentSequentialId = dhl.MaxDocId; //Getting version number foreach (var row in dhl.vt.SelectBackwardFromTo<byte[], byte>( doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MaxValue.To_4_bytes_array_BigEndian(), int.MaxValue.To_4_bytes_array_BigEndian()), true, doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MinValue.To_4_bytes_array_BigEndian(), int.MinValue.To_4_bytes_array_BigEndian()), true )) { //Inserting into version table, new version //Console.WriteLine("Adding_V_" + doc.InternalId + "_" + (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1) + "_" + doc.DocumentSequentialId); dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany ( (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian() ), new byte[] { 0 }); break; } } } else { if (doc.InternalId < 1) { //New doc doc.DocumentSequentialId = dhl.MaxDocId; doc.InternalId = doc.DocumentSequentialId; //Inserting into version table //Console.WriteLine("Adding_V_" + doc.InternalId + "_1_" + doc.DocumentSequentialId); dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(((int)1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), new byte[] { 0 }); } else { //Updating document (we create new version) doc.DocumentSequentialId = dhl.MaxDocId; //Getting version number foreach (var row in dhl.vt.SelectBackwardFromTo<byte[], byte>( doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MaxValue.To_4_bytes_array_BigEndian(), int.MaxValue.To_4_bytes_array_BigEndian()), true, doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany(int.MinValue.To_4_bytes_array_BigEndian(), int.MinValue.To_4_bytes_array_BigEndian()), true )) { //Inserting into version table, new version //Console.WriteLine("Adding_V_" + doc.InternalId + "_" + (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1) + "_" + doc.DocumentSequentialId); dhl.vt.Insert<byte[], byte[]>(doc.InternalId.To_4_bytes_array_BigEndian().ConcatMany ( (row.Key.Substring(4, 4).To_Int32_BigEndian() + 1).To_4_bytes_array_BigEndian(), doc.DocumentSequentialId.To_4_bytes_array_BigEndian() ), new byte[] { 0 }); break; } } } serDoc = doc.SerializeProtobuf(); btDoc = serDoc.CompressGZip(); if (serDoc.Length >= btDoc.Length) btDoc = new byte[] { 1 }.Concat(btDoc); else btDoc = new byte[] { 0 }.Concat(serDoc); dhl.dt.Insert<int, byte[]>(doc.DocumentSequentialId, btDoc); //----------------------------------------------------------------------------------------- //Adding to processing table //Console.WriteLine("Adding_P_" + doc.DocumentSequentialId); tran.Insert<byte[], byte>(DocumentsStorageTablesPrefix + "p", doc.DocumentSpaceId.To_8_bytes_array_BigEndian().Concat(doc.DocumentSequentialId.To_4_bytes_array_BigEndian()), 0); }