internal void Create(List <Data.Field> fields)
{
List <Data.Field> tmpFields = new List <Hubble.Core.Data.Field>();
foreach (Data.Field field in fields)
{
if (field.IndexType != Hubble.Core.Data.Field.Index.None)
{
tmpFields.Add(field);
}
}
tmpFields.Sort(new TabCompare());
PayloadFileHead head = new PayloadFileHead();
head.Build(tmpFields);
if (System.IO.File.Exists(_FileName))
{
System.IO.File.Delete(_FileName);
}
using (System.IO.FileStream fs = new System.IO.FileStream(_FileName, System.IO.FileMode.CreateNew, System.IO.FileAccess.ReadWrite))
{
Hubble.Framework.Serialization.BinSerialization.SerializeBinary(head, fs);
fs.SetLength(HeadLength);
}
}
internal PayloadProvider Open(Field docIdReplaceField, List <Data.Field> fields, int payloadLength, out int lastDocId)
{
lastDocId = -1;
List <Data.Field> tmpFields = new List <Hubble.Core.Data.Field>();
PayloadProvider docPayload = new PayloadProvider(docIdReplaceField);
long truncateLength = 0;
int rankTab = -1;
foreach (Data.Field field in fields)
{
if (field.IndexType != Hubble.Core.Data.Field.Index.None)
{
tmpFields.Add(field);
}
if (field.Name.Equals("Rank", StringComparison.CurrentCultureIgnoreCase))
{
if (field.IndexType == Field.Index.Untokenized && field.DataType == DataType.Int)
{
rankTab = field.TabIndex;
}
}
}
tmpFields.Sort(new TabCompare());
PayloadFileHead head = new PayloadFileHead();
head.Build(tmpFields);
if (_StoreLength <= 0)
{
_StoreLength = payloadLength * 4 + sizeof(int);
}
using (System.IO.FileStream fs = new System.IO.FileStream(_FileName, System.IO.FileMode.Open, System.IO.FileAccess.Read))
{
object obj;
Hubble.Framework.Serialization.BinSerialization.DeserializeBinary(fs, out obj);
PayloadFileHead fileHead = (PayloadFileHead)obj;
if (fileHead.IsBigEndian != head.IsBigEndian)
{
throw new Data.DataException(string.Format("Index file IsBigEndian = {0} but current system IsBigEndian = {1}. Can't load payload file",
fileHead.IsBigEndian, head.IsBigEndian));
}
if ((fileHead.Version[0] <= 0 && fileHead.Version[1] < 8) || (fileHead.Version[0] <= 0 && fileHead.Version[1] == 8 && fileHead.Version[2] == 0 && fileHead.Version[3] < 4))
{
throw new Data.DataException("Index file version is less than V0.8.0.4, you have to rebuild the index");
}
for (int i = 0; i < head.FieldMD5.Length; i++)
{
if (head.FieldMD5[i] != fileHead.FieldMD5[i])
{
throw new Data.DataException(string.Format("Payload file name: {0} does not match with the table",
_FileName));
}
}
fs.Seek(HeadLength, System.IO.SeekOrigin.Begin);
if (payloadLength > 0 && InsertProtect.InsertProtectInfo == null)
{
lastDocId = FastLoad(docPayload, fs, rankTab);
_LastStoredId = lastDocId;
_DocumentsCount = (int)((fs.Length - HeadLength) / _StoreLength);
GC.Collect();
return(docPayload);
}
bool breakForProctect = false;
while (fs.Position < fs.Length)
{
if (payloadLength == 0 && InsertProtect.InsertProtectInfo != null)
{
//Exit exception last time
lastDocId = InsertProtect.InsertProtectInfo.LastDocId;
_LastStoredId = lastDocId;
_DocumentsCount = InsertProtect.InsertProtectInfo.DocumentsCount;
break;
}
int fileIndex = (int)((fs.Position - HeadLength) / _StoreLength);
byte[] buf = new byte[sizeof(int)];
fs.Read(buf, 0, buf.Length);
int docidInFile = (int)BitConverter.ToInt32(buf, 0);
if (docidInFile < lastDocId)
{
throw new Data.DataException(string.Format("docid = {0} < last docid ={1}", docidInFile, lastDocId));
}
lastDocId = docidInFile;
_LastStoredId = lastDocId;
if (InsertProtect.InsertProtectInfo != null)
{
if (lastDocId == InsertProtect.InsertProtectInfo.LastDocId)
{
breakForProctect = true;
}
//Exit exception last time
if (lastDocId > InsertProtect.InsertProtectInfo.LastDocId)
{
lastDocId = InsertProtect.InsertProtectInfo.LastDocId;
_LastStoredId = lastDocId;
truncateLength = fs.Position - sizeof(int);
break;
}
}
if (payloadLength == 0)
{
buf = new byte[sizeof(int)];
fs.Read(buf, 0, buf.Length);
_DocumentsCount = (int)BitConverter.ToInt32(buf, 0);
return(docPayload);
}
byte[] byteData = new byte[payloadLength * 4];
fs.Read(byteData, 0, byteData.Length);
Data.Payload payload = new Hubble.Core.Data.Payload(payloadLength);
payload.CopyFrom(byteData);
payload.FileIndex = fileIndex;
if (payloadLength > 0)
{
docPayload.Add(lastDocId, payload);
_DocumentsCount++;
}
if (breakForProctect)
{
truncateLength = fs.Position;
break;
}
}
GC.Collect();
}
if (InsertProtect.InsertProtectInfo != null)
{
//Exit exception last time
if (InsertProtect.InsertProtectInfo.LastDocId < 0)
{
//No Data
lastDocId = -1;
_LastStoredId = lastDocId;
using (System.IO.FileStream fs = new System.IO.FileStream(_FileName, System.IO.FileMode.Open, System.IO.FileAccess.Write))
{
fs.SetLength(HeadLength);
}
}
else
{
//Have data
if (payloadLength == 0)
{
using (System.IO.FileStream fs = new System.IO.FileStream(_FileName, System.IO.FileMode.Open, System.IO.FileAccess.Write))
{
//Seek to end of head
fs.Seek(HeadLength, System.IO.SeekOrigin.Begin);
//Write last doc id
byte[] buf = BitConverter.GetBytes(InsertProtect.InsertProtectInfo.LastDocId);
fs.Write(buf, 0, buf.Length);
//Write documents count
buf = BitConverter.GetBytes(InsertProtect.InsertProtectInfo.DocumentsCount);
fs.Write(buf, 0, buf.Length);
}
}
else
{
using (System.IO.FileStream fs = new System.IO.FileStream(_FileName, System.IO.FileMode.Open, System.IO.FileAccess.Write))
{
if (truncateLength > 0 && truncateLength < fs.Length)
{
fs.SetLength(truncateLength);
}
}
}
}
}
return(docPayload);
}