void crypto_stream_chacha20_ietf_xor_ic(byte[] c, byte[] m, ulong mlen, byte[] n, ulong ic, byte[] k)
{
Sodium.crypto_stream_chacha20_ietf_xor_ic(c, m, mlen, n, (uint)ic, k);
}
private async Task Stage1(VoiceReadyPayload voiceReady)
{
// IP Discovery
this.UdpClient.Setup(this.UdpEndpoint);
var pck = new byte[70];
PreparePacket(pck);
await this.UdpClient.SendAsync(pck, pck.Length).ConfigureAwait(false);
var ipd = await this.UdpClient.ReceiveAsync().ConfigureAwait(false);
ReadPacket(ipd, out var ip, out var port);
this.DiscoveredEndpoint = new IpEndpoint
{
Address = ip,
Port = port
};
this.Discord.DebugLogger.LogMessage(LogLevel.Debug, "VNext UDP", $"Endpoint discovery resulted in {ip}:{port}", DateTime.Now);
void PreparePacket(byte[] packet)
{
var ssrc = this.SSRC;
var packetSpan = packet.AsSpan();
MemoryMarshal.Write(packetSpan, ref ssrc);
Helpers.ZeroFill(packetSpan);
}
void ReadPacket(byte[] packet, out System.Net.IPAddress decodedIp, out ushort decodedPort)
{
var packetSpan = packet.AsSpan();
var ipString = Utilities.UTF8.GetString(packet, 4, 64 /* 70 - 6 */).TrimEnd('\0');
decodedIp = System.Net.IPAddress.Parse(ipString);
decodedPort = BinaryPrimitives.ReadUInt16LittleEndian(packetSpan.Slice(68 /* 70 - 2 */));
}
// Select voice encryption mode
var selectedEncryptionMode = Sodium.SelectMode(voiceReady.Modes);
this.SelectedEncryptionMode = selectedEncryptionMode.Value;
// Ready
this.Discord.DebugLogger.LogMessage(LogLevel.Debug, "VoiceNext", $"Selected encryption mode: {selectedEncryptionMode.Key}", DateTime.Now);
var vsp = new VoiceDispatch
{
OpCode = 1,
Payload = new VoiceSelectProtocolPayload
{
Protocol = "udp",
Data = new VoiceSelectProtocolPayloadData
{
Address = this.DiscoveredEndpoint.Address.ToString(),
Port = (ushort)this.DiscoveredEndpoint.Port,
Mode = selectedEncryptionMode.Key
}
}
};
var vsj = JsonConvert.SerializeObject(vsp, Formatting.None);
await this.VoiceWs.SendMessageAsync(vsj).ConfigureAwait(false);
this.SenderTokenSource = new CancellationTokenSource();
this.SenderTask = Task.Run(this.VoiceSenderTask, this.SenderToken);
this.ReceiverTokenSource = new CancellationTokenSource();
this.ReceiverTask = Task.Run(this.UdpReceiverTask, this.ReceiverToken);
}
internal static unsafe TimeSeriesRangeResult GetTimeSeriesRange(DocumentsOperationContext context, string docId, string name, DateTime from, DateTime to, ref int start, ref int pageSize,
IncludeDocumentsDuringTimeSeriesLoadingCommand includesCommand = null)
{
if (pageSize == 0)
{
return(null);
}
List <TimeSeriesEntry> values = new List <TimeSeriesEntry>();
var reader = new TimeSeriesReader(context, docId, name, from, to, offset: null);
// init hash
var size = Sodium.crypto_generichash_bytes();
Debug.Assert((int)size == 32);
var cryptoGenerichashStatebytes = (int)Sodium.crypto_generichash_statebytes();
var state = stackalloc byte[cryptoGenerichashStatebytes];
if (Sodium.crypto_generichash_init(state, null, UIntPtr.Zero, size) != 0)
{
ComputeHttpEtags.ThrowFailToInitHash();
}
var initialStart = start;
var hasMore = false;
DateTime lastSeenEntry = from;
includesCommand?.InitializeNewRangeResult(state);
foreach (var(individualValues, segmentResult) in reader.SegmentsOrValues())
{
if (individualValues == null &&
start > segmentResult.Summary.NumberOfLiveEntries)
{
lastSeenEntry = segmentResult.End;
start -= segmentResult.Summary.NumberOfLiveEntries;
continue;
}
var enumerable = individualValues ?? segmentResult.Values;
foreach (var singleResult in enumerable)
{
lastSeenEntry = segmentResult.End;
if (start-- > 0)
{
continue;
}
if (pageSize-- <= 0)
{
hasMore = true;
break;
}
includesCommand?.Fill(singleResult.Tag);
values.Add(new TimeSeriesEntry
{
Timestamp = singleResult.Timestamp,
Tag = singleResult.Tag,
Values = singleResult.Values.ToArray(),
IsRollup = singleResult.Type == SingleResultType.RolledUp
});
}
ComputeHttpEtags.HashChangeVector(state, segmentResult.ChangeVector);
if (pageSize <= 0)
{
break;
}
}
var hash = ComputeHttpEtags.FinalizeHash(size, state);
TimeSeriesRangeResult result;
if (initialStart > 0 && values.Count == 0)
{
// this is a special case, because before the 'start' we might have values
result = new TimeSeriesRangeResult
{
From = lastSeenEntry,
To = to,
Entries = values.ToArray(),
Hash = hash
};
}
else
{
result = new TimeSeriesRangeResult
{
From = (initialStart > 0) ? values[0].Timestamp : from,
To = hasMore ? values.Last().Timestamp : to,
Entries = values.ToArray(),
Hash = hash
};
}
includesCommand?.AddIncludesToResult(result);
return(result);
}
protected override void Configure(StorageEnvironmentOptions options)
{
base.Configure(options);
options.MasterKey = Sodium.GenerateRandomBuffer((int)Sodium.crypto_aead_xchacha20poly1305_ietf_keybytes());
}
private bool ProcessPacket(ReadOnlySpan <byte> data, ref Memory <byte> opus, ref Memory <byte> pcm, IList <ReadOnlyMemory <byte> > pcmPackets, out AudioSender voiceSender, out AudioFormat outputFormat)
{
voiceSender = null;
outputFormat = default;
if (!this.Rtp.IsRtpHeader(data))
{
return(false);
}
this.Rtp.DecodeHeader(data, out var sequence, out var timestamp, out var ssrc, out var hasExtension);
var vtx = this.TransmittingSSRCs[ssrc];
voiceSender = vtx;
if (sequence <= vtx.LastSequence) // out-of-order packet; discard
{
return(false);
}
var gap = vtx.LastSequence != 0 ? sequence - 1 - vtx.LastSequence : 0;
if (gap >= 5)
{
this.Discord.DebugLogger.LogMessage(LogLevel.Warning, "VNext RX", "5 or more voice packets were dropped when receiving", DateTime.Now);
}
Span <byte> nonce = stackalloc byte[Sodium.NonceSize];
this.Sodium.GetNonce(data, nonce, this.SelectedEncryptionMode);
this.Rtp.GetDataFromPacket(data, out var encryptedOpus, this.SelectedEncryptionMode);
var opusSize = Sodium.CalculateSourceSize(encryptedOpus);
opus = opus.Slice(0, opusSize);
var opusSpan = opus.Span;
try
{
this.Sodium.Decrypt(encryptedOpus, opusSpan, nonce);
// Strip extensions, if any
if (hasExtension)
{
// RFC 5285, 4.2 One-Byte header
// http://www.rfcreader.com/#rfc5285_line186
if (opusSpan[0] == 0xBE && opusSpan[1] == 0xDE)
{
var headerLen = opusSpan[2] << 8 | opusSpan[3];
var i = 4;
for (; i < headerLen + 4; i++)
{
var @byte = opusSpan[i];
// ID is currently unused since we skip it anyway
//var id = (byte)(@byte >> 4);
var length = (byte)(@byte & 0x0F) + 1;
i += length;
}
// Strip extension padding too
while (opusSpan[i] == 0)
{
i++;
}
opusSpan = opusSpan.Slice(i);
}
// TODO: consider implementing RFC 5285, 4.3. Two-Byte Header
}
if (gap == 1)
{
var lastSampleCount = this.Opus.GetLastPacketSampleCount(vtx.Decoder);
var fecpcm = new byte[this.AudioFormat.SampleCountToSampleSize(lastSampleCount)];
var fecpcmMem = fecpcm.AsSpan();
this.Opus.Decode(vtx.Decoder, opusSpan, ref fecpcmMem, true, out _);
pcmPackets.Add(fecpcm.AsMemory(0, fecpcmMem.Length));
}
else if (gap > 1)
{
var lastSampleCount = this.Opus.GetLastPacketSampleCount(vtx.Decoder);
for (var i = 0; i < gap; i++)
{
var fecpcm = new byte[this.AudioFormat.SampleCountToSampleSize(lastSampleCount)];
var fecpcmMem = fecpcm.AsSpan();
this.Opus.ProcessPacketLoss(vtx.Decoder, lastSampleCount, ref fecpcmMem);
pcmPackets.Add(fecpcm.AsMemory(0, fecpcmMem.Length));
}
}
var pcmSpan = pcm.Span;
this.Opus.Decode(vtx.Decoder, opusSpan, ref pcmSpan, false, out outputFormat);
pcm = pcm.Slice(0, pcmSpan.Length);
}
finally
{
vtx.LastSequence = sequence;
}
return(true);
}
/// <summary>
/// Initializes a new instance of the <see cref="Device"/> class.
/// </summary>
/// <param name="Socket">The socket.</param>
internal Device(Socket Socket)
{
this.Socket = Socket;
this.Crypto = new Sodium();
}
Int64 decrypt(ArraySegment <byte> src, ArraySegment <byte> dst)
{
Sodium.StreamChacha20XorIc(dst, src, dec_nonce_, enc_key_, dec_idx_);
dec_idx_ += (ulong)src.Count;
return(src.Count);
}
private byte[] LoadMasterKeyWithExecutable()
{
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = _config.MasterKeyExec,
Arguments = _config.MasterKeyExecArguments,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
}
};
var sw = Stopwatch.StartNew();
try
{
process.Start();
}
catch (Exception e)
{
throw new InvalidOperationException($"Unable to get master key by executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments}. Failed to start process.", e);
}
var ms = new MemoryStream();
var readErrors = process.StandardError.ReadToEndAsync();
var readStdOut = process.StandardOutput.BaseStream.CopyToAsync(ms);
string GetStdError()
{
try
{
return(readErrors.Result);
}
catch
{
return("Unable to get stdout");
}
}
if (process.WaitForExit((int)_config.MasterKeyExecTimeout.AsTimeSpan.TotalMilliseconds) == false)
{
process.Kill();
throw new InvalidOperationException($"Unable to get master key by executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments}, waited for {_config.MasterKeyExecTimeout} ms but the process didn't exit. Stderr: {GetStdError()}");
}
try
{
readStdOut.Wait(_config.MasterKeyExecTimeout.AsTimeSpan);
readErrors.Wait(_config.MasterKeyExecTimeout.AsTimeSpan);
}
catch (Exception e)
{
throw new InvalidOperationException($"Unable to get master key by executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments}, waited for {_config.MasterKeyExecTimeout} ms but the process didn't exit. Stderr: {GetStdError()}", e);
}
if (Logger.IsOperationsEnabled)
{
var errors = GetStdError();
Logger.Operations(string.Format($"Executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments} took {sw.ElapsedMilliseconds:#,#;;0} ms. Stderr: {errors}"));
}
if (process.ExitCode != 0)
{
throw new InvalidOperationException(
$"Unable to get master key by executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments}, the exit code was {process.ExitCode}. Stderr: {GetStdError()}");
}
var rawData = ms.ToArray();
var expectedKeySize = (int)Sodium.crypto_aead_xchacha20poly1305_ietf_keybytes();
if (rawData.Length != expectedKeySize)
{
throw new InvalidOperationException(
$"Got wrong master key after executing {_config.MasterKeyExec} {_config.MasterKeyExecArguments}, the size of the key must be {expectedKeySize * 8} bits, but was {rawData.Length * 8} bits.");
}
return(rawData);
}
private static void InitSodium()
{
Sodium.InitNativeLibrary();
}
protected void TryHandleFailureToLockMemory(byte *addressToLock, long sizeToLock)
{
var currentProcess = Process.GetCurrentProcess();
if (PlatformDetails.RunningOnPosix == false)
{
// From: https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234(v=vs.85).aspx
// "The maximum number of pages that a process can lock is equal to the number of pages in its minimum working set minus a small overhead"
// let's increase the max size of memory we can lock by increasing the MinWorkingSet. On Windows, that is available for all users
var nextWorkingSetSize = GetNearestFileSize(currentProcess.MinWorkingSet.ToInt64() + sizeToLock);
if (nextWorkingSetSize > int.MaxValue && IntPtr.Size == sizeof(int))
{
nextWorkingSetSize = int.MaxValue;
}
// Minimum working set size must be less than or equal to the maximum working set size.
// Let's increase the max as well.
if (nextWorkingSetSize > (long)currentProcess.MaxWorkingSet)
{
try
{
currentProcess.MaxWorkingSet = new IntPtr(nextWorkingSetSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Need to increase the min working set size from {(long)currentProcess.MinWorkingSet:#,#;;0} bytes to {nextWorkingSetSize:#,#;;0} bytes but the max working set size was too small: {(long)currentProcess.MaxWorkingSet:#,#;;0}. " +
$"Failed to increase the max working set size so we can lock {sizeToLock:#,#;;0} bytes for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
}
try
{
currentProcess.MinWorkingSet = new IntPtr(nextWorkingSetSize);
}
catch (Exception e)
{
throw new InsufficientMemoryException($"Failed to increase the min working set size to {nextWorkingSetSize:#,#;;0} bytes so we can lock {sizeToLock:#,#;;0} bytes for {FileName}. With encrypted " +
"databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error " +
"and aborting the current operation.", e);
}
// now we can try again, after we raised the limit, we only do so once, though
if (Sodium.sodium_mlock(addressToLock, (UIntPtr)sizeToLock) == 0)
{
return;
}
}
var msg =
$"Unable to lock memory for {FileName} with size {sizeToLock:#,#;;0} bytes), with encrypted databases we lock some memory in order to avoid leaking secrets to disk. Treating this as a catastrophic error and aborting the current operation.{Environment.NewLine}";
if (PlatformDetails.RunningOnPosix)
{
msg +=
$"The admin may configure higher limits using: 'sudo prlimit --pid {currentProcess.Id} --memlock={sizeToLock}' to increase the limit. (It's recommended to do that as part of the startup script){Environment.NewLine}";
}
else
{
msg +=
$"Already tried to raise the the process min working set to {currentProcess.MinWorkingSet.ToInt64():#,#;;0} bytes but still got a failure.{Environment.NewLine}";
}
msg += "This behavior is controlled by the 'Security.DoNotConsiderMemoryLockFailureAsCatastrophicError' setting (expert only, modifications of this setting is not recommended).";
throw new InsufficientMemoryException(msg);
}
private byte[] LoadMasterKey()
{
var debug = "<unknown>";
try
{
if (_config.MasterKeyExec != null)
{
debug = _config.MasterKeyExec + " " + _config.MasterKeyExecArguments;
return(LoadMasterKeyWithExecutable());
}
if (_config.MasterKeyPath != null)
{
debug = _config.MasterKeyPath;
return(LoadMasterKeyFromPath());
}
if (PlatformDetails.RunningOnPosix == false)
{
return(null);
}
var dirpath = Path.Combine(Environment.GetEnvironmentVariable("HOME"), ".ravendb");
dirpath = Path.GetFullPath(dirpath);
var filepath = Path.Combine(dirpath, "secret.key");
debug = filepath;
var buffer = new byte[(int)Sodium.crypto_aead_xchacha20poly1305_ietf_keybytes()];
fixed(byte *pBuf = buffer)
{
if (Directory.Exists(dirpath) == false)
{
Directory.CreateDirectory(dirpath);
}
var fd = Syscall.open(filepath, PerPlatformValues.OpenFlags.O_CREAT | OpenFlags.O_RDWR,
// octal 01600 - Sticky and only user can read it
FilePermissions.S_ISVTX | FilePermissions.S_IRUSR | FilePermissions.S_IWUSR);
if (fd == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"when opening {filepath}");
}
try
{
var ret = Syscall.flock(fd, Syscall.FLockOperations.LOCK_EX);
if (ret != 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"could not lock {filepath}");
}
var size = Syscall.lseek64(fd, 0, WhenceFlags.SEEK_END);
if (size == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"could not get size of {filepath}");
}
if (size == buffer.Length)
{
byte *pos = pBuf;
long amountRead = 0;
while (amountRead < buffer.Length)
{
var read = Syscall.pread(fd, pos, (ulong)(buffer.Length - amountRead), amountRead);
pos += read;
if (read < 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"failed to read {filepath}");
}
if (read == 0)
{
break;
}
amountRead += read;
}
if (amountRead != buffer.Length)
{
throw new FileLoadException($"Failed to read the full key size from {filepath}, expected to read {buffer.Length} but go only {amountRead}");
}
}
else // we assume that if the size isn't a key size, then it was never valid and regenerate the key
{
Sodium.randombytes_buf(pBuf, (UIntPtr)buffer.Length);
if (Syscall.ftruncate(fd, IntPtr.Zero) != 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"Failed to truncate {filepath}");
}
if (Syscall.lseek64(fd, 0, WhenceFlags.SEEK_SET) == -1)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"Failed to seek to beginning of {filepath}");
}
var len = buffer.Length;
while (len > 0)
{
var writeAmount = Syscall.write(fd, pBuf, (ulong)buffer.Length);
if (writeAmount <= 0) // 0 will be considered as error here
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"Failed to write {buffer.Length} bytes into {filepath}, only wrote {len}");
}
len -= (int)writeAmount;
}
if (Syscall.FSync(fd) != 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"Failed to FSync {filepath}");
}
Syscall.FsyncDirectoryFor(filepath);
}
}
finally
{
if (Syscall.close(fd) != 0)
{
var err = Marshal.GetLastWin32Error();
Syscall.ThrowLastError(err, $"Failed to close the secret key file : {filepath}");
}
}
return(buffer);
}
}
catch (Exception e)
{
throw new CryptographicException(
$"Unable to open the master secret key ({debug}), won't proceed because losing this key will lose access to all user encrypted information. Admin assistance required.",
e);
}
}
private int crypto_stream_chacha20_ietf_xor_ic(byte[] c, byte[] m, ulong mlen, byte[] n, ulong ic, byte[] k)
{
return(Sodium.crypto_stream_chacha20_ietf_xor_ic(c, m, mlen, n, (uint)ic, k));
}
protected override void Configure(StorageEnvironmentOptions options)
{
options.Encryption.MasterKey = Sodium.GenerateRandomBuffer((int)Sodium.crypto_aead_xchacha20poly1305_ietf_keybytes());
options.MaxScratchBufferSize = 65536 - 1; // to make ShouldReduceSizeOfCompressionPager() return true
options.Encryption.RegisterForJournalCompressionHandler();
}
public static unsafe string ComputeEtagForDocuments(
List <Document> documents,
List <Document>?includes,
IncludeCountersCommand?includeCounters,
IncludeTimeSeriesCommand?includeTimeSeries,
IncludeCompareExchangeValuesCommand includeCompareExchangeValues)
{
// This method is efficient because we aren't materializing any values
// except the change vector, which we need
if (documents.Count == 1 &&
(includes == null || includes.Count == 0) &&
includeCounters == null &&
includeTimeSeries == null &&
includeCompareExchangeValues == null)
{
return(documents[0]?.ChangeVector ?? string.Empty);
}
var size = Sodium.crypto_generichash_bytes();
Debug.Assert((int)size == 32);
var cryptoGenerichashStatebytes = (int)Sodium.crypto_generichash_statebytes();
byte *state = stackalloc byte[cryptoGenerichashStatebytes];
if (Sodium.crypto_generichash_init(state, null, UIntPtr.Zero, size) != 0)
{
ThrowFailToInitHash();
}
HashNumber(state, documents.Count);
foreach (var doc in documents)
{
HashChangeVector(state, doc?.ChangeVector);
}
if (includes != null)
{
HashNumber(state, includes.Count);
foreach (var doc in includes)
{
if (doc is IncludeDocumentsCommand.ConflictDocument)
{
continue;
}
HashChangeVector(state, doc.ChangeVector);
}
}
if (includeCounters != null)
{
foreach (var countersResult in includeCounters.Results)
{
HashNumber(state, countersResult.Value.Count);
foreach (var counterDetail in countersResult.Value)
{
HashNumber(state, counterDetail?.Etag ?? 0);
}
}
}
if (includeTimeSeries != null)
{
foreach (var tsIncludesForDocument in includeTimeSeries.Results)
{
foreach (var kvp in tsIncludesForDocument.Value)
{
HashNumber(state, tsIncludesForDocument.Value.Count);
foreach (var rangeResult in kvp.Value)
{
HashNumber(state, rangeResult.Entries?.Length ?? 0);
HashChangeVector(state, rangeResult.Hash);
}
}
}
}
if (includeCompareExchangeValues != null)
{
if (includeCompareExchangeValues.Results == null || includeCompareExchangeValues.Results.Count == 0)
{
HashNumber(state, 0);
}
else
{
HashNumber(state, includeCompareExchangeValues.Results.Count);
foreach (var compareExchangeValueInclude in includeCompareExchangeValues.Results)
{
HashNumber(state, compareExchangeValueInclude.Value.Index);
}
}
}
return(FinalizeHash(size, state));
}
public static unsafe string ComputeEtagForDocuments(List <Document> documents, List <Document> includes)
{
// This method is efficient because we aren't materializing any values
// except the change vector, which we need
if (documents.Count == 1 && (includes == null || includes.Count == 0))
{
return(documents[0]?.ChangeVector ?? string.Empty);
}
var size = Sodium.crypto_generichash_bytes();
Debug.Assert((int)size == 32);
var cryptoGenerichashStatebytes = (int)Sodium.crypto_generichash_statebytes();
byte *state = stackalloc byte[cryptoGenerichashStatebytes];
if (Sodium.crypto_generichash_init(state, null, UIntPtr.Zero, size) != 0)
{
ThrowFailToInitHash();
}
foreach (var doc in documents)
{
HashDocumentByChangeVector(state, doc);
}
if (includes != null)
{
foreach (var doc in includes)
{
if (doc is IncludeDocumentsCommand.ConflictDocument)
{
continue;
}
HashDocumentByChangeVector(state, doc);
}
}
byte *final = stackalloc byte[(int)size];
if (Sodium.crypto_generichash_final(state, final, size) != 0)
{
ThrowFailedToFinalizeHash();
}
var str = new string(' ', 49);
fixed(char *p = str)
{
p[0] = 'H';
p[1] = 'a';
p[2] = 's';
p[3] = 'h';
p[4] = '-';
var len = Base64.ConvertToBase64Array(p + 5, final, 0, 32);
Debug.Assert(len == 44);
}
return(str);
}
protected void IncrementNonce(bool isEncrypt)
{
lock (_nonceIncrementLock) {
Sodium.sodium_increment(isEncrypt ? _encNonce : _decNonce, nonceLen);
}
}
private static unsafe void InitComputeHash(JsonOperationContext.ManagedPinnedBuffer cryptoState)
{
var rc = Sodium.crypto_generichash_init(cryptoState.Pointer, null, UIntPtr.Zero, Sodium.crypto_generichash_bytes());
if (rc != 0)
{
throw new InvalidOperationException("Unable to hash attachment: " + rc);
}
}
public unsafe void Dispose()
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
lock (this)
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
//before we dispose of the database we take its latest info to be displayed in the studio
try
{
var databaseInfo = GenerateDatabaseInfo();
if (databaseInfo != null)
{
DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name);
}
}
catch (Exception e)
{
// if we encountered a catastrophic failure we might not be able to retrieve database info
if (_logger.IsInfoEnabled)
{
_logger.Info("Failed to generate and store database info", e);
}
}
_databaseShutdown.Cancel();
// we'll wait for 1 minute to drain all the requests
// from the database
var sp = Stopwatch.StartNew();
while (sp.ElapsedMilliseconds < 60 * 1000)
{
if (Interlocked.Read(ref _usages) == 0)
{
break;
}
if (_waitForUsagesOnDisposal.Wait(1000))
{
_waitForUsagesOnDisposal.Reset();
}
}
var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)} {Name}");
foreach (var connection in RunningTcpConnections)
{
exceptionAggregator.Execute(() =>
{
connection.Dispose();
});
}
exceptionAggregator.Execute(() =>
{
TxMerger?.Dispose();
});
if (_indexStoreTask != null)
{
exceptionAggregator.Execute(() =>
{
_indexStoreTask.Wait(DatabaseShutdown);
_indexStoreTask = null;
});
}
exceptionAggregator.Execute(() =>
{
IndexStore?.Dispose();
IndexStore = null;
});
exceptionAggregator.Execute(() =>
{
ExpiredDocumentsCleaner?.Dispose();
ExpiredDocumentsCleaner = null;
});
exceptionAggregator.Execute(() =>
{
PeriodicBackupRunner?.Dispose();
PeriodicBackupRunner = null;
});
exceptionAggregator.Execute(() =>
{
DocumentTombstoneCleaner?.Dispose();
DocumentTombstoneCleaner = null;
});
exceptionAggregator.Execute(() =>
{
ReplicationLoader?.Dispose();
ReplicationLoader = null;
});
exceptionAggregator.Execute(() =>
{
EtlLoader?.Dispose();
EtlLoader = null;
});
exceptionAggregator.Execute(() =>
{
Operations?.Dispose(exceptionAggregator);
Operations = null;
});
exceptionAggregator.Execute(() =>
{
NotificationCenter?.Dispose();
NotificationCenter = null;
});
exceptionAggregator.Execute(() =>
{
SubscriptionStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ConfigurationStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentsStorage?.Dispose();
DocumentsStorage = null;
});
exceptionAggregator.Execute(() =>
{
_databaseShutdown.Dispose();
});
exceptionAggregator.Execute(() =>
{
if (MasterKey == null)
{
return;
}
fixed(byte *pKey = MasterKey)
{
Sodium.ZeroMemory(pKey, MasterKey.Length);
}
});
exceptionAggregator.ThrowIfNeeded();
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result.Progress);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
string compactTempDirectory = null;
byte[] encryptionKey = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
DatabaseRecord databaseRecord = documentDatabase.ReadDatabaseRecord();
// save the key before unloading the database (it is zeroed when disposing DocumentDatabase).
if (documentDatabase.MasterKey != null)
{
encryptionKey = documentDatabase.MasterKey.ToArray();
}
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((endId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}), StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(src, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
if (configuration.Storage.TempPath != null)
{
compactTempDirectory = configuration.Storage.TempPath.FullPath + "-temp-compacting";
EnsureDirectoriesPermission(compactTempDirectory);
IOExtensions.DeleteDirectory(compactTempDirectory);
configuration.Storage.TempPath = new PathSetting(compactTempDirectory);
}
var revisionsPrefix = CollectionName.GetTablePrefix(CollectionTableType.Revisions);
var compressedCollectionsTableNames = databaseRecord.DocumentsCompression?.Collections
.Select(name => new CollectionName(name).GetTableName(CollectionTableType.Documents))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((envId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}). StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(dst, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result.Progress);
}, (name, schema) =>
{
bool isRevision = name.StartsWith(revisionsPrefix, StringComparison.OrdinalIgnoreCase);
schema.Compressed =
(isRevision && databaseRecord.DocumentsCompression?.CompressRevisions == true) ||
compressedCollectionsTableNames?.Contains(name) == true;
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
if (compactTempDirectory != null)
{
IOExtensions.DeleteDirectory(compactTempDirectory);
}
}
_isCompactionInProgress = false;
if (encryptionKey != null)
{
Sodium.ZeroBuffer(encryptionKey);
}
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
src.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
src.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
src.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
src.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
src.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
src.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out src.MasterKey, documentDatabase.MasterKey);
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
dst.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
dst.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
dst.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
dst.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
dst.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
dst.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out dst.MasterKey, documentDatabase.MasterKey);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
var oldIndexesPath = new PathSetting(basePath + "-old").Combine("Indexes");
var newIndexesPath = new PathSetting(basePath).Combine("Indexes");
IOExtensions.MoveDirectory(oldIndexesPath.FullPath, newIndexesPath.FullPath);
var oldConfigPath = new PathSetting(basePath + "-old").Combine("Configuration");
var newConfigPath = new PathSetting(basePath).Combine("Configuration");
IOExtensions.MoveDirectory(oldConfigPath.FullPath, newConfigPath.FullPath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
_isCompactionInProgress = false;
}
}
}
Int64 decrypt(ArraySegment <byte> src, ArraySegment <byte> dst)
{
Sodium.StreamAes128XorTable(dst, src, dec_nonce_, enc_table_);
Sodium.Increment(dec_nonce_);
return(dst.Count);
}
protected override void cipherUpdate(bool isEncrypt, int length, byte[] buf, byte[] outbuf)
{
// TODO write a unidirection cipher so we don't have to if if if
int bytesRemaining;
ulong ic;
byte[] sodiumBuf;
byte[] iv;
int ret = -1;
if (isEncrypt)
{
bytesRemaining = _encryptBytesRemaining;
ic = _encryptIC;
sodiumBuf = _encryptBuf;
iv = _encryptIV;
}
else
{
bytesRemaining = _decryptBytesRemaining;
ic = _decryptIC;
sodiumBuf = _decryptBuf;
iv = _decryptIV;
}
int padding = bytesRemaining;
Buffer.BlockCopy(buf, 0, sodiumBuf, padding, length);
switch (_cipher)
{
case CIPHER_SALSA20:
ret = Sodium.crypto_stream_salsa20_xor_ic(sodiumBuf, sodiumBuf, (ulong)(padding + length), iv, ic, _key);
break;
case CIPHER_CHACHA20:
ret = Sodium.crypto_stream_chacha20_xor_ic(sodiumBuf, sodiumBuf, (ulong)(padding + length), iv, ic, _key);
break;
case CIPHER_CHACHA20_IETF:
ret = Sodium.crypto_stream_chacha20_ietf_xor_ic(sodiumBuf, sodiumBuf, (ulong)(padding + length), iv, (uint)ic, _key);
break;
}
if (ret != 0)
{
throw new CryptoErrorException();
}
Buffer.BlockCopy(sodiumBuf, padding, outbuf, 0, length);
padding += length;
ic += (ulong)padding / SODIUM_BLOCK_SIZE;
bytesRemaining = padding % SODIUM_BLOCK_SIZE;
if (isEncrypt)
{
_encryptBytesRemaining = bytesRemaining;
_encryptIC = ic;
}
else
{
_decryptBytesRemaining = bytesRemaining;
_decryptIC = ic;
}
}
