public void Flush()
{
_bufferSize = 0;
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
foreach (var kvp in _buffer)
{
var data = _storageEnvironment.CreateTree(tx, "channel:" + kvp.Key);
var buffer = new byte[16];
var key = new Slice(buffer);
var ms = new MemoryStream();
var bw = new BinaryWriter(ms);
foreach (var item in kvp.Value)
{
var date = item.Timestamp;
EndianBitConverter.Big.CopyBytes(date.Ticks, buffer, 0);
EndianBitConverter.Big.CopyBytes(_last++, buffer, 8);
ms.SetLength(0);
bw.Write(item.Value);
ms.Position = 0;
data.Add(tx, key, ms);
}
}
tx.State.Root.Add(tx, _lastKey, new MemoryStream(BitConverter.GetBytes(_last)));
tx.Commit();
}
_buffer.Clear();
}
public override void Accept(Disk d)
{
var ms = new MemoryStream();
_serializer.Serialize(new JsonTextWriter(new StreamWriter(ms)), d);
ms.Position = 0;
var key = new Slice(EndianBitConverter.Big.GetBytes(counter++));
_currentBatch.Add(key, ms, "albums");
foreach (var diskId in d.DiskIds)
{
_currentBatch.MultiAdd(diskId, key, "ix_diskids");
}
if(d.Artist != null)
_currentBatch.MultiAdd(d.Artist.ToLower(), key, "ix_artists");
if (d.Title != null)
_currentBatch.MultiAdd(d.Title.ToLower(), key, "ix_titles");
if (counter%1000 == 0)
{
_storageEnvironment.Writer.Write(_currentBatch);
_currentBatch = new WriteBatch();
}
}
public static void SetInline(Slice slice, NodeHeader* node)
{
slice.Pointer = (byte*)node + Constants.NodeHeaderSize;
slice.Size = node->KeySize;
slice.KeyLength = node->KeySize;
slice.Array = null;
}
public virtual void Add(WriteBatch writeBatch, Slice key, byte[] value, ushort? expectedVersion = null)
{
var stream = new BufferPoolMemoryStream();
stream.Write(value, 0, value.Length);
stream.Position = 0;
writeBatch.Add(key, stream, TableName, expectedVersion);
}
public PrefixedSlice()
{
Options = SliceOptions.Key;
Size = 0;
KeyLength = 0;
Header = new PrefixedSliceHeader();
NonPrefixedData = new Slice(SliceOptions.Key);
}
public virtual void Add(WriteBatch writeBatch, Slice key, RavenJToken value, ushort? expectedVersion = null)
{
var stream = new BufferPoolMemoryStream(BufferPool);
value.WriteTo(stream);
stream.Position = 0;
writeBatch.Add(key, stream, TableName, expectedVersion);
}
public static int Compare(Slice x, PrefixedSlice y, SliceComparer cmp, ushort size)
{
fixed (byte* p1 = x.Array)
fixed (byte* p2 = y.NonPrefixedData.Array)
{
var xPtr = p1 != null ? p1 : x.Pointer;
var yPtr = p2 != null ? p2 : y.NonPrefixedData.Pointer;
if (y.Header.PrefixId == PrefixedSlice.NonPrefixedId)
return Compare(null, 0, null, 0, xPtr, x.KeyLength, yPtr, y.Header.NonPrefixedDataSize, cmp, size);
if (x.PrefixComparisonCache == null)
{
if(y.Prefix == null)
return Compare(null, 0, null, 0, xPtr, x.KeyLength, yPtr, y.Header.NonPrefixedDataSize, cmp, size);
else if (y.Prefix.Value == null)
return Compare(null, 0, y.Prefix.ValuePtr, y.Header.PrefixUsage, xPtr, x.KeyLength, yPtr, y.Header.NonPrefixedDataSize, cmp, size);
else
{
fixed (byte* prefixVal = y.Prefix.Value)
return Compare(null, 0, prefixVal, y.Header.PrefixUsage, xPtr, x.KeyLength, yPtr, y.Header.NonPrefixedDataSize, cmp, size);
}
}
var prefixBytesToCompare = Math.Min(y.Header.PrefixUsage, x.KeyLength);
int r;
if (x.PrefixComparisonCache.TryGetCachedResult(y.Header.PrefixId, y.Prefix.PageNumber, prefixBytesToCompare, out r) == false)
{
if (y.Prefix == null)
r = Compare(null, 0, null, 0, xPtr, x.KeyLength, null, 0, cmp,
prefixBytesToCompare);
else if (y.Prefix.Value == null)
r = Compare(null, 0, y.Prefix.ValuePtr, y.Header.PrefixUsage, xPtr, x.KeyLength, null, 0, cmp,
prefixBytesToCompare);
else
{
fixed (byte* prefixVal = y.Prefix.Value)
r = Compare(null, 0, prefixVal, y.Header.PrefixUsage, xPtr, x.KeyLength, null, 0, cmp,
prefixBytesToCompare);
}
x.PrefixComparisonCache.SetPrefixComparisonResult(y.Header.PrefixId, y.Prefix.PageNumber, prefixBytesToCompare, r);
}
if (r != 0)
return r;
size -= prefixBytesToCompare;
return Compare(null, 0, null, 0, xPtr + prefixBytesToCompare, (ushort)(x.KeyLength - prefixBytesToCompare), yPtr, y.Header.NonPrefixedDataSize, cmp, size);
}
}
public void UpdateSchemaVersion(TableStorage tableStorage, Action<string> output)
{
var schemaVersionSlice = new Slice("schema_version");
using (var tx = tableStorage.Environment.NewTransaction(TransactionFlags.ReadWrite))
{
tx.ReadTree(Tables.Details.TableName).Add(schemaVersionSlice, ToSchemaVersion);
tx.Commit();
}
tableStorage.SetDatabaseIdAndSchemaVersion(tableStorage.Id, ToSchemaVersion);
}
public Slice(Slice other, ushort size)
{
if (other.Array != null)
Array = other.Array;
else
Pointer = other.Pointer;
Options = other.Options;
Size = size;
KeyLength = size;
}
private void Initialize()
{
using (var tx = storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
var serverNamesToIds = storageEnvironment.CreateTree(tx, "serverNames->Ids");
var serverIdsToNames = storageEnvironment.CreateTree(tx, "Ids->serverNames");
storageEnvironment.CreateTree(tx, "servers->lastEtag");
storageEnvironment.CreateTree(tx, "counters");
storageEnvironment.CreateTree(tx, "countersGroups");
var etags = storageEnvironment.CreateTree(tx, "etags->counters");
storageEnvironment.CreateTree(tx, "counters->etags");
var metadata = tx.State.GetTree(tx, "$metadata");
var id = metadata.Read("id");
if (id == null) // new counter db
{
var serverIdBytes = EndianBitConverter.Big.GetBytes(0); // local is always 0
var serverIdSlice = new Slice(serverIdBytes);
serverNamesToIds.Add(CounterStorageUrl, serverIdSlice);
serverIdsToNames.Add(serverIdSlice, CounterStorageUrl);
Id = Guid.NewGuid();
metadata.Add("id", Id.ToByteArray());
metadata.Add("name", Encoding.UTF8.GetBytes(CounterStorageUrl));
tx.Commit();
}
else // existing counter db
{
int used;
Id = new Guid(id.Reader.ReadBytes(16, out used));
var nameResult = metadata.Read("name");
if (nameResult == null)
throw new InvalidOperationException("Could not read name from the store, something bad happened");
var storedName = new StreamReader(nameResult.Reader.AsStream()).ReadToEnd();
if (storedName != CounterStorageUrl)
throw new InvalidOperationException("The stored name " + storedName + " does not match the given name " + CounterStorageUrl);
using (var it = etags.Iterate())
{
if (it.Seek(Slice.AfterAllKeys))
{
LastEtag = it.CurrentKey.CreateReader().ReadBigEndianInt64();
}
}
}
ReplicationTask.StartReplication();
}
}
public DateTimeSeries(string path)
{
_lastKey = "last-key";
_storageEnvironment = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path));
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
var read = tx.State.Root.Read(tx, _lastKey);
_last = read != null ? read.Reader.ReadInt64() : 1;
tx.Commit();
}
}
public PrefixedSlice(byte prefixId, ushort prefixUsage, Slice nonPrefixedValue)
{
Header = new PrefixedSliceHeader
{
PrefixId = prefixId,
PrefixUsage = prefixUsage,
NonPrefixedDataSize = nonPrefixedValue.KeyLength
};
NonPrefixedData = nonPrefixedValue;
Size = (ushort)(Constants.PrefixedSliceHeaderSize + nonPrefixedValue.KeyLength);
KeyLength = (ushort)(prefixUsage + nonPrefixedValue.KeyLength);
Options = nonPrefixedValue.Options;
}
protected override void Init()
{
base.Init();
_positionsTree = Transaction.ReadTree("TermPositions");
_docsTree = Transaction.ReadTree("Docs");
_fieldId = Index.GetFieldNumber(Field);
_prefix = new byte[FullTextIndex.FieldDocumentSize];
_prefixSlice = new Slice(_prefix);
_maxKey = new byte[FullTextIndex.FieldDocumentSize];
_maxKeySlice = new Slice(_maxKey);
}
protected RavenJObject LoadJson(Table table, Slice key, WriteBatch writeBatch, out ushort version)
{
var read = table.Read(Snapshot, key, writeBatch);
if (read == null)
{
version = table.ReadVersion(Snapshot, key, writeBatch) ?? 0;
return null;
}
using (var stream = read.Reader.AsStream())
{
version = read.Version;
return stream.ToJObject();
}
}
public void SetupDatabaseIdAndSchemaVersion()
{
using (var snapshot = storage.CreateSnapshot())
{
var idSlice = new Slice("id");
var schemaVersionSlice = new Slice("schema_version");
Guid id;
string schemaVersion;
var read = storage.Details.Read(snapshot, idSlice, null);
if (read == null) // new db
{
id = Guid.NewGuid();
schemaVersion = SchemaVersion;
using (var writeIdBatch = new WriteBatch())
{
storage.Details.Add(writeIdBatch, idSlice, id.ToByteArray());
storage.Details.Add(writeIdBatch, schemaVersionSlice, schemaVersion);
storage.Write(writeIdBatch);
}
}
else
{
if (read.Reader.Length != 16) //precaution - might prevent NRE in edge cases
throw new InvalidDataException("Failed to initialize Voron transactional storage. Possible data corruption. (no db id)");
using (var stream = read.Reader.AsStream())
using (var reader = new BinaryReader(stream))
{
id = new Guid(reader.ReadBytes((int)stream.Length));
}
var schemaRead = storage.Details.Read(snapshot, schemaVersionSlice, null);
if (schemaRead == null)
throw new InvalidDataException("Failed to initialize Voron transactional storage. Possible data corruption. (no schema version)");
schemaVersion = schemaRead.Reader.ToStringValue();
}
storage.SetDatabaseIdAndSchemaVersion(id, schemaVersion);
}
}
public PrefixedSlice(NodeHeader* node)
{
if (node->KeySize > 0)
{
var prefixHeaderPtr = (PrefixedSliceHeader*)((byte*)node + Constants.NodeHeaderSize);
Header = *prefixHeaderPtr;
NonPrefixedData = new Slice((byte*)prefixHeaderPtr + Constants.PrefixedSliceHeaderSize, Header.NonPrefixedDataSize);
Size = node->KeySize;
KeyLength = (ushort) (Header.PrefixUsage + Header.NonPrefixedDataSize);
}
else
{
Size = 0;
KeyLength = 0;
}
Options = SliceOptions.Key;
}
public override int Accept(string d)
{
var disk = JObject.Parse(d);
var ms = new MemoryStream();
var writer = new StreamWriter(ms);
writer.Write(d);
writer.Flush();
ms.Position = 0;
var key = new Slice(EndianBitConverter.Big.GetBytes(counter++));
_currentBatch.Add(key, ms, "albums");
int count = 1;
foreach (var diskId in disk.Value<JArray>("DiskIds"))
{
count++;
_currentBatch.MultiAdd(diskId.Value<string>(), key, "ix_diskids");
}
var artist = disk.Value<string>("Artist");
if (artist != null)
{
count++;
_currentBatch.MultiAdd(artist.ToLower(), key, "ix_artists");
}
var title = disk.Value<string>("Title");
if (title != null)
{
count++;
_currentBatch.MultiAdd(title.ToLower(), key, "ix_titles");
}
if (counter % 500 == 0)
{
_storageEnvironment.Writer.Write(_currentBatch);
_currentBatch = new WriteBatch();
}
return count;
}
//returns true if it was update operation
private bool WriteDocumentMetadata(JsonDocumentMetadata metadata, Slice key, bool shouldIgnoreConcurrencyExceptions = false)
{
var metadataStream = CreateStream();
metadataStream.Write(metadata.Etag);
metadataStream.Write(metadata.Key);
if (metadata.LastModified.HasValue)
metadataStream.Write(metadata.LastModified.Value.ToBinary());
else
metadataStream.Write((long)0);
metadata.Metadata.WriteTo(metadataStream);
metadataStream.Position = 0;
ushort? existingVersion;
var isUpdate = metadataIndex.Contains(Snapshot, key, writeBatch.Value, out existingVersion);
metadataIndex.Add(writeBatch.Value, key, metadataStream, existingVersion, shouldIgnoreConcurrencyExceptions);
return isUpdate;
}
private bool PutDocumentMetadataInternal(string key, Slice normalizedKey, RavenJObject metadata, Etag newEtag, DateTime savedAt)
{
return WriteDocumentMetadata(new JsonDocumentMetadata
{
Key = key,
Etag = newEtag,
Metadata = metadata,
LastModified = savedAt
}, normalizedKey);
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch, out ushort? version)
{
return snapshot.Contains(TableName, key, out version, writeBatch);
}
public bool Contains(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
ushort? version;
return Contains(snapshot, key, writeBatch, out version);
}
public virtual IIterator MultiRead(SnapshotReader snapshot, Slice key)
{
return snapshot.MultiRead(TableName, key);
}
public virtual ReadResult Read(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return snapshot.Read(TableName, key, writeBatch);
}
public virtual void Increment(WriteBatch writeBatch, Slice key, long delta, ushort? expectedVersion = null)
{
writeBatch.Increment(key, delta, TableName, expectedVersion);
}
public virtual void Add(WriteBatch writeBatch, Slice key, Stream value, ushort? expectedVersion = null, bool shouldIgnoreConcurrencyExceptions = false)
{
writeBatch.Add(key, value, TableName, expectedVersion, shouldIgnoreConcurrencyExceptions);
}
public virtual ushort? ReadVersion(SnapshotReader snapshot, Slice key, WriteBatch writeBatch)
{
return snapshot.ReadVersion(TableName, key, writeBatch);
}
public virtual void Delete(WriteBatch writeBatch, Slice key, ushort? expectedVersion = null)
{
writeBatch.Delete(key, TableName, expectedVersion);
}
private JsonDocumentMetadata ReadDocumentMetadata(string normalizedKey, Slice sliceKey, out int size)
{
try
{
var metadataReadResult = metadataIndex.Read(Snapshot, sliceKey, writeBatch.Value);
size = 0;
if (metadataReadResult == null)
return null;
using (var stream = metadataReadResult.Reader.AsStream())
{
stream.Position = 0;
var etag = stream.ReadEtag();
var originalKey = stream.ReadString();
var lastModifiedDateTimeBinary = stream.ReadInt64();
var existingCachedDocument = documentCacher.GetCachedDocument(normalizedKey, etag);
size = (int)stream.Length;
var metadata = existingCachedDocument != null ? existingCachedDocument.Metadata : stream.ToJObject();
var lastModified = DateTime.FromBinary(lastModifiedDateTimeBinary);
return new JsonDocumentMetadata
{
Key = originalKey,
Etag = etag,
Metadata = metadata,
LastModified = lastModified
};
}
}
catch (Exception e)
{
throw new InvalidDataException("Failed to de-serialize metadata of document " + normalizedKey, e);
}
}
private RavenJObject ReadDocumentData(string normalizedKey, Slice sliceKey, Etag existingEtag, RavenJObject metadata, out int size)
{
try
{
size = -1;
var existingCachedDocument = documentCacher.GetCachedDocument(normalizedKey, existingEtag);
if (existingCachedDocument != null)
{
size = existingCachedDocument.Size;
return existingCachedDocument.Document;
}
var documentReadResult = tableStorage.Documents.Read(Snapshot, sliceKey, writeBatch.Value);
if (documentReadResult == null) //non existing document
return null;
using (var stream = documentReadResult.Reader.AsStream())
{
using (var decodedDocumentStream = documentCodecs.Aggregate(stream,
(current, codec) => codec.Value.Decode(normalizedKey, metadata, current)))
{
var streamToUse = decodedDocumentStream;
if (stream != decodedDocumentStream)
streamToUse = new CountingStream(decodedDocumentStream);
var documentData = decodedDocumentStream.ToJObject();
size = (int)Math.Max(stream.Position, streamToUse.Position);
documentCacher.SetCachedDocument(normalizedKey, existingEtag, documentData, metadata, size);
return documentData;
}
}
}
catch (Exception e)
{
InvalidDataException invalidDataException = null;
try
{
size = -1;
var documentReadResult = tableStorage.Documents.Read(Snapshot, sliceKey, writeBatch.Value);
if (documentReadResult == null) //non existing document
return null;
using (var stream = documentReadResult.Reader.AsStream())
{
using (var reader = new BinaryReader(stream))
{
if (reader.ReadUInt32() == DocumentCompression.CompressFileMagic)
{
invalidDataException = new InvalidDataException(string.Format("Document '{0}' is compressed, but the compression bundle is not enabled.\r\n" +
"You have to enable the compression bundle when dealing with compressed documents.", normalizedKey), e);
}
}
}
}
catch (Exception)
{
// we are already in error handling mode, just ignore this
}
if(invalidDataException != null)
throw invalidDataException;
throw new InvalidDataException("Failed to de-serialize a document: " + normalizedKey, e);
}
}
public virtual void MultiAdd(WriteBatch writeBatch, Slice key, Slice value, ushort? expectedVersion = null)
{
writeBatch.MultiAdd(key, value, TableName, expectedVersion);
}
