public void AddTerm(string term, PostingListAddress address)
{
if (!data.TryAdd(term, address))
{
throw new DuplicateTermException(term);
}
}
public ReaderEnumerator(IPersistentStorage storage, PostingListAddress address)
{
this.persistentStorage = storage;
this.address = address;
this.buffer = new byte[ReadBufferSize];
Reset();
}
public void AddTerm(string term, PostingListAddress address, Action <PostingListAddress> onDuplicate)
{
if (!dictionary.AddOrGet(term, address.Offset, out long previosOffset))
{
onDuplicate?.Invoke(new PostingListAddress(previosOffset));
}
}
// TODO: Need unit test for this
public IPostingList GetBasic(PostingListAddress address)
{
var offset = address.Offset;
var buffer = new byte[sizeof(long) + sizeof(int)];
var occurrences = new List <Occurrence>();
while (true)
{
persistentStorage.ReadAll(offset, buffer, 0, buffer.Length);
long continuationOffset = BitConverter.ToInt64(buffer, 0);
int length = BitConverter.ToInt32(buffer, sizeof(long));
var dataBuffer = new byte[length];
persistentStorage.ReadAll(offset + sizeof(long) + sizeof(int), dataBuffer, 0, dataBuffer.Length);
ParseBufferTo(dataBuffer, occurrences);
if (continuationOffset == 0)
{
break;
}
else
{
offset = continuationOffset;
}
}
return(new PostingListArray(occurrences.ToArray()));
}
private long FindContinuationOffset(PostingListAddress address)
{
var offset = address.Offset;
var buffer = new byte[PostingListReader.ReadBufferSize];
while (true)
{
int read = persistentStorage.Read(offset, buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
offset += read;
for (int i = 0; i < read; ++i)
{
if (buffer[i] == EmptyContinuationAddress[0])
{
var readNext = persistentStorage.Read(offset - read + i, buffer, 0, EmptyContinuationAddress.Length);
var nextOffsetText = System.Text.Encoding.UTF8.GetString(buffer, 4, 8);
var nextOffset = Convert.ToInt32(nextOffsetText, 16);
if (nextOffset < 0)
{
return(offset - read + i);
}
offset = nextOffset;
break;
}
}
}
return(-1);
}
public ReaderEnumerator(IPersistentStorage storage, PostingListAddress address, Occurrence firstOccurrence)
{
this.persistentStorage = storage;
this.address = address;
this.state = 0;
this.buffer = new byte[PostingListVarIntDeltaWriter.BlockSize];
this.firstOccurrence = firstOccurrence;
Reset();
}
public PostingListAddress AddOccurrences(string term, IEnumerable <Occurrence> occurrences)
{
var address = new PostingListAddress(postingLists.Count);
if (!postingLists.TryAdd(address, occurrences.ToArray()))
{
throw new InvalidOperationException();
}
return(address);
}
public ReaderEnumerator(IPersistentStorage storage, PostingListAddress address)
{
this.persistentStorage = storage;
this.address = address;
this.buffer = new byte[ReadBufferSize];
this.selectors = new int[4];
this.data = null;
this.dataIndex = 0;
Reset();
}
public void UpdateNextList(PostingListAddress address, PostingListAddress nextList)
{
var offset = FindContinuationOffset(address);
if (offset < 0)
{
throw new InvalidOperationException("Continuation is not found. The list might already have it");
}
var data = System.Text.Encoding.UTF8.GetBytes($" -> {nextList.Offset.ToString("X8")}");
persistentStorage.WriteAll(offset, data, 0, data.Length);
}
protected override void AddTerm(string term, PostingListAddress address)
{
dictionaryWriter.AddTerm(term, address, existingList => occurrenceWriter.UpdateNextList(existingList, address));
++updates;
if (updates > AutoCommitThreshold)
{
dictionaryUpdate.Commit();
dictionaryUpdate = dictionaryWriter.BeginUpdate();
updates = 0;
}
}
public IPostingList Get(PostingListAddress address)
{
var offset = address.Offset;
var line = new StringBuilder();
var buffer = new byte[ReadBufferSize];
var done = false;
while (!done)
{
int read = persistentStorage.Read(offset, buffer, 0, buffer.Length);
if (read == 0)
{
break;
}
offset += read;
for (int i = 0; i < read; ++i)
{
if (buffer[i] == PostingListWriter.EmptyContinuationAddress[0])
{
read = persistentStorage.Read(offset - read + i, buffer, 0, PostingListWriter.EmptyContinuationAddress.Length);
var nextOffsetText = System.Text.Encoding.UTF8.GetString(buffer, 4, 8);
var nextOffset = Convert.ToInt32(nextOffsetText, 16);
if (nextOffset < 0)
{
done = true;
break;
}
line.Append(';');
offset = nextOffset;
break;
}
//if (buffer[i] == '\r' || buffer[i] == '\n')
//{
// done = true;
// break;
//}
line.Append((char)buffer[i]);
}
}
return(new PostingListArray(line.ToString().Split(';').Select(Occurrence.Parse).ToArray()));
}
public void UpdateNextList(PostingListAddress address, PostingListAddress nextList)
{
var buffer = new byte[sizeof(long)];
var offset = address.Offset;
while (true)
{
persistentStorage.ReadAll(offset, buffer, 0, buffer.Length);
long continuationOffset = BitConverter.ToInt64(buffer, 0);
if (continuationOffset == 0)
{
persistentStorage.WriteAll(offset, BitConverter.GetBytes(nextList.Offset), 0, sizeof(long));
break;
}
else
{
offset = continuationOffset;
}
}
}
public DictionaryTerm(string key, PostingListAddress value)
{
this.Key = key;
this.Value = value;
}
public PostingListReaderImpl(IPersistentStorage storage, PostingListAddress address)
{
this.storage = storage;
this.address = address;
}
protected abstract void AddTerm(string term, PostingListAddress address);
protected override void AddTerm(string term, PostingListAddress address)
{
Index.AddTerm(term, address);
}
public void AddTerm(string term, PostingListAddress address, Action <PostingListAddress> onDuplicate)
{
update.AddTerm(term, address, onDuplicate);
}
public IPostingList Get(PostingListAddress address)
{
return(new PostingListReaderImpl(persistentStorage, address));
}
public IPostingList Get(PostingListAddress address)
{
return(new PostingListArray(postingLists[address]));
}
public void AddTerm(string term, PostingListAddress address, Action <PostingListAddress> onDuplicate)
{
input.Add(term);
output.Add((int)address.Offset);
}
public PostingListReaderImpl(IPersistentStorage storage, PostingListAddress address)
{
this.storage = storage;
this.address = address;
this.firstOccurrence = Occurrence.Empty;
}
