private static void ReprocessZipFile()
{
Debug.Assert(_zipFilePath != null);
// Make sure to keep the relative order between the flags, this
// code runs in multiple threads
if (ZipFileProcessingHappening.Raise() == false)
{
return;
}
ZipFileInitialized.Lower();
ZipFileEntries.Clear();
try
{
using (var fileStream = SafeFileStream.Create(_zipFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, false))
{
foreach (var entry in zipArchive.Entries)
{
ZipFileEntries.Add(entry.FullName);
}
}
}
catch
{
// Suppressing this exception is reasonable: there are many
// reasons for which the file may not be available right now.
// The watcher will let us know whenever we can try again.
}
ZipFileInitialized.RaiseOrDie();
ZipFileProcessingHappening.LowerOrDie();
}
private bool TryGetNewStreamAndApplyRetentionPolicies(long maxFileSize, out FileStream fileStream)
{
string[] logFiles;
string[] logGzFiles;
try
{
logFiles = Directory.GetFiles(_path, "*.log");
logGzFiles = Directory.GetFiles(_path, "*.log.gz");
}
catch (Exception)
{
// Something went wrong we will try again later
fileStream = null;
return(false);
}
Array.Sort(logFiles);
Array.Sort(logGzFiles);
if (DateTime.Today != _today)
{
_today = DateTime.Today;
_dateString = LogInfo.DateToLogFormat(DateTime.Today);
}
if (_logNumber < 0)
{
_logNumber = Math.Max(LastLogNumberForToday(logFiles), LastLogNumberForToday(logGzFiles));
}
UpdateLocalDateTimeOffset();
string fileName;
while (true)
{
fileName = Path.Combine(_path, LogInfo.GetNewFileName(_dateString, _logNumber));
if (File.Exists(fileName))
{
if (new FileInfo(fileName).Length < maxFileSize)
{
break;
}
}
else if (File.Exists(LogInfo.AddCompressExtension(fileName)) == false)
{
break;
}
_logNumber++;
}
if (Compressing == false)
{
CleanupOldLogFiles(logFiles);
LimitLogSize(logFiles);
}
fileStream = SafeFileStream.Create(fileName, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false);
fileStream.Write(_headerRow, 0, _headerRow.Length);
return(true);
}
private Stream GetNewStream(long maxFileSize)
{
if (DateTime.Today != _today)
{
lock (this)
{
if (DateTime.Today != _today)
{
_today = DateTime.Today;
_dateString = DateTime.Today.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
_logNumber = 0;
CleanupOldLogFiles();
}
}
}
while (true)
{
var nextLogNumber = Interlocked.Increment(ref _logNumber);
var fileName = Path.Combine(_path, _dateString) + "." +
nextLogNumber.ToString("000", CultureInfo.InvariantCulture) + ".log";
if (File.Exists(fileName) && new FileInfo(fileName).Length >= maxFileSize)
{
continue;
}
var fileStream = SafeFileStream.Create(fileName, FileMode.Create, FileAccess.Write, FileShare.Read, 32 * 1024, false);
fileStream.Write(_headerRow, 0, _headerRow.Length);
return(fileStream);
}
}
public static ClusterTopologyResponse TryLoad(string topologyHash, DocumentConventions conventions, JsonOperationContext context)
{
try
{
var path = GetPath(topologyHash, conventions);
if (File.Exists(path) == false)
{
return(null);
}
using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var json = context.Read(stream, "raven-cluster-topology"))
{
return(JsonDeserializationClient.ClusterTopology(json));
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not understand the persisted cluster topology", e);
}
return(null);
}
}
public static void TrySaving(string topologyHash, ClusterTopologyResponse clusterTopology, DocumentConventions conventions, JsonOperationContext context)
{
try
{
var path = GetPath(topologyHash, conventions);
if (clusterTopology == null)
{
Clear(path);
return;
}
using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
using (var writer = new BlittableJsonTextWriter(context, stream))
{
var json = new DynamicJsonValue
{
[nameof(clusterTopology.Topology)] = clusterTopology.Topology.ToJson(),
[nameof(clusterTopology.Leader)] = clusterTopology.Leader,
[nameof(clusterTopology.NodeTag)] = clusterTopology.NodeTag,
["PersistedAt"] = DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite),
};
context.Write(writer, json);
writer.Flush();
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not persist the cluster topology", e);
}
}
}
/// <summary>
/// Do a incremental backup of a set of environments. Note that the order of the environments matter!
/// </summary>
public long ToFile(IEnumerable <FullBackup.StorageEnvironmentInformation> envs, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null,
Action backupStarted = null)
{
infoNotify = infoNotify ?? (s => { });
long totalNumberOfBackedUpPages = 0;
using (var file = SafeFileStream.Create(backupPath, FileMode.Create))
{
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var e in envs)
{
if (e.Env.Options.IncrementalBackupEnabled == false)
{
throw new InvalidOperationException("Incremental backup is disabled for this storage");
}
infoNotify("Voron backup " + e.Name + "started");
var basePath = Path.Combine(e.Folder, e.Name);
var env = e.Env;
var copier = new DataCopier(Constants.Storage.PageSize * 16);
var numberOfBackedUpPages = Incremental_Backup(env, compression, infoNotify,
backupStarted, package, basePath, copier);
totalNumberOfBackedUpPages += numberOfBackedUpPages;
}
}
file.Flush(true); // make sure that this is actually persisted fully to disk
return(totalNumberOfBackedUpPages);
}
}
public StreamsTempFile(string tempFile, StorageEnvironment environment)
{
_tempFile = tempFile;
_environment = environment;
_file = SafeFileStream.Create(_tempFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose | FileOptions.SequentialScan);
}
public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null,
Action backupStarted = null)
{
infoNotify = infoNotify ?? (s => { });
if (env.Options.IncrementalBackupEnabled == false)
{
throw new InvalidOperationException("Incremental backup is disabled for this storage");
}
var copier = new DataCopier(Constants.Storage.PageSize * 16);
using (var file = SafeFileStream.Create(backupPath, FileMode.Create))
{
long numberOfBackedUpPages;
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
{
numberOfBackedUpPages = Incremental_Backup(env, compression, infoNotify,
backupStarted, package, string.Empty, copier);
}
file.Flush(true); // make sure that this is actually persisted fully to disk
return(numberOfBackedUpPages);
}
}
public StreamsTempFile(string tempFile, DocumentDatabase database)
{
_tempFile = tempFile;
_database = database;
_file = SafeFileStream.Create(_tempFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose | FileOptions.SequentialScan);
}
private void CorruptJournal(long journal, long position, int numberOfCorruptedBytes = Constants.Size.Kilobyte * 4, byte value = 42)
{
Options.Dispose();
Options = StorageEnvironmentOptions.ForPath(DataDir);
Configure(Options);
using (var fileStream = SafeFileStream.Create(Options.GetJournalPath(journal).FullPath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite | FileShare.Delete))
{
fileStream.Position = position;
var buffer = new byte[numberOfCorruptedBytes];
var remaining = buffer.Length;
var start = 0;
while (remaining > 0)
{
var read = fileStream.Read(buffer, start, remaining);
if (read == 0)
{
break;
}
start += read;
remaining -= read;
}
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = value;
}
fileStream.Position = position;
fileStream.Write(buffer, 0, buffer.Length);
}
}
public void ToFile(StorageEnvironment env, string backupPath,
CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null,
Action backupStarted = null)
{
infoNotify = infoNotify ?? (s => { });
infoNotify("Voron backup db started");
using (var file = SafeFileStream.Create(backupPath, FileMode.Create))
{
using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
{
infoNotify("Voron backup started");
var dataPager = env.Options.DataPager;
var copier = new DataCopier(Constants.Storage.PageSize * 16);
Backup(env, compression, infoNotify, backupStarted, dataPager, package, string.Empty,
copier);
file.Flush(true); // make sure that we fully flushed to disk
}
}
infoNotify("Voron backup db finished");
}
public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
if (fs.Length != sizeof(FileHeader))
{
return(false); // wrong file size
}
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int read;
if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to read file " + path);
}
if (read == 0)
{
return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption
}
ptr += read;
remaining -= read;
}
return(true);
}
}
public static void TrySaving(string databaseName, string topologyHash, Topology topology, DocumentConventions conventions, JsonOperationContext context)
{
try
{
var path = GetPath(databaseName, topologyHash, conventions);
if (topology == null)
{
Clear(databaseName);
return;
}
var existingTopology = TryLoad(path, context);
if (existingTopology?.Etag >= topology.Etag)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Skipping save topology with etag {topology.Etag} to cache " +
$"as the cache already have a topology with etag: {existingTopology.Etag}");
}
return;
}
using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
using (var writer = new BlittableJsonTextWriter(context, stream))
{
writer.WriteStartObject();
writer.WritePropertyName(context.GetLazyString(nameof(topology.Nodes)));
writer.WriteStartArray();
for (var i = 0; i < topology.Nodes.Count; i++)
{
var node = topology.Nodes[i];
if (i != 0)
{
writer.WriteComma();
}
WriteNode(writer, node, context);
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName(context.GetLazyString(nameof(topology.Etag)));
writer.WriteInteger(topology.Etag);
writer.WriteComma();
writer.WritePropertyName(context.GetLazyString("PersistedAt"));
writer.WriteString(DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite));
writer.WriteEndObject();
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not persist the database topology", e);
}
}
}
public Stream RentFileStream()
{
if (_files.TryDequeue(out var stream) == false)
{
stream = new TempFileStream(SafeFileStream.Create(
GetTempFileName(_options),
FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, 4096,
FileOptions.DeleteOnClose));
}
public void NonexistentFileReadWorks()
{
var tempFile = new FileInfo(Path.Combine(m_tempDir.FullName, "NonexistentFileReadWorks"));
using (var reader = new SafeFileStream(tempFile.FullName))
{
var result = new byte[10];
var bytesRead = reader.Read(result, 0, result.Length);
Assert.AreEqual(0, bytesRead);
}
}
public static void PerformanceAnalysis(string directory, string outputFile, int size)
{
using (var fileStream = SafeFileStream.Create(outputFile, FileMode.Create))
using (var streamWriter = new StreamWriter(fileStream))
{
var files = Directory.GetFiles(directory, "*.json").OrderBy(f => new FileInfo(f).Length).Take(size);
streamWriter.WriteLine("Name,Json Parse Time,Json Size, Json Time, Blit Parse Time,Blit Size, Blit Time");
using (var blittableContext = JsonOperationContext.ShortTermSingleUse())
{
foreach (var jsonFile in files)
{
Console.Write(string.Format("{0} {1:#,#}", Path.GetFileName(jsonFile), new FileInfo(jsonFile).Length));
streamWriter.Write(Path.GetFileName(jsonFile) + ",");
var sp = Stopwatch.StartNew();
var jsonOjbect = JObject.Load(new JsonTextReader(File.OpenText(jsonFile)));
streamWriter.Write(sp.ElapsedMilliseconds + ",");
using (var stream = SafeFileStream.Create("output.junk", FileMode.Create))
using (var textWriter = new StreamWriter(stream))
{
sp.Restart();
jsonOjbect.WriteTo(new JsonTextWriter(textWriter));
textWriter.Flush();
streamWriter.Write(stream.Length + "," + sp.ElapsedMilliseconds + ",");
}
Console.Write(" json - {0:#,#}ms", sp.ElapsedMilliseconds);
GC.Collect(2);
sp.Restart();
using (var employee = blittableContext.Read(File.OpenRead(jsonFile), "doc1"))
{
streamWriter.Write(sp.ElapsedMilliseconds + ",");
using (var stream = SafeFileStream.Create("output2.junk", FileMode.Create))
{
sp.Restart();
blittableContext.Write(stream, employee);
streamWriter.Write(employee.Size + "," + sp.ElapsedMilliseconds + ",");
}
Console.WriteLine(" blit - {0:#,#} ms, Props: {1}",
sp.ElapsedMilliseconds,
blittableContext.CachedProperties.PropertiesDiscovered);
}
GC.Collect(2);
streamWriter.WriteLine();
}
}
}
}
private bool TryGetNewStreamAndApplyRetentionPolicies(long maxFileSize, out FileStream fileStream)
{
string[] logFiles;
string[] logGzFiles;
try
{
logFiles = Directory.GetFiles(_path, "*.log");
logGzFiles = Directory.GetFiles(_path, "*.log.gz");
}
catch (Exception)
{
// Something went wrong we will try again later
fileStream = null;
return(false);
}
Array.Sort(logFiles);
Array.Sort(logGzFiles);
if (DateTime.Today != _today)
{
_today = DateTime.Today;
_dateString = LogInfo.GetFileName(DateTime.Today);
_logNumber = Math.Max(NextLogNumberForExtension(logFiles, "log"), NextLogNumberForExtension(logGzFiles, "log.gz"));
}
UpdateLocalDateTimeOffset();
string fileName;
while (true)
{
var nextLogNumber = Interlocked.Increment(ref _logNumber);
fileName = Path.Combine(_path, _dateString) + "." + nextLogNumber.ToString("000", CultureInfo.InvariantCulture) + ".log";
if (File.Exists(fileName) == false || new FileInfo(fileName).Length < maxFileSize)
{
break;
}
}
if (Compressing == false)
{
CleanupOldLogFiles(logFiles);
LimitLogSize(logFiles);
}
fileStream = SafeFileStream.Create(fileName, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false);
fileStream.Write(_headerRow, 0, _headerRow.Length);
return(true);
}
/// <summary>
/// HttpWebRequest POST request to send to YouTube to check if the user's is authenticated (signed in) by determining
/// if a generic request is successful given the current authentication cookie value we have stored. ///
/// In this case, we're actually perform a request for personally uploaded music files
/// </summary>
/// <param name="cookieValue">Cookie from a previous YouTube Music sign in via this application (stored in the database)</param>
/// <returns>True if successfully authenticated, false otherwise</returns>
public static bool IsAuthenticated(string cookieValue)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(Global.YTMusicBaseUrl + "browse" + Global.YTMusicParams);
request = AddStandardHeaders(request, cookieValue);
request.ContentType = "application/json; charset=UTF-8";
request.Headers["X-Goog-AuthUser"] = "******";
request.Headers["x-origin"] = "https://music.youtube.com";
request.Headers["X-Goog-Visitor-Id"] = Global.GoogleVisitorId;
request.Headers["Authorization"] = GetAuthorisation(GetSAPISIDFromCookie(cookieValue));
byte[] postBytes = GetPostBytes(
SafeFileStream.ReadAllText(
Path.Combine(Global.WorkingDirectory, @"AppData\check_auth_context.json")));
request.ContentLength = postBytes.Length;
using (var requestStream = request.GetRequestStream())
{
requestStream.Write(postBytes, 0, postBytes.Length);
requestStream.Close();
}
postBytes = null;
using (var response = (HttpWebResponse)request.GetResponse())
{
string result;
using (var brotli = new Brotli.BrotliStream(response.GetResponseStream(),
System.IO.Compression.CompressionMode.Decompress,
true))
{
var streamReader = new StreamReader(brotli);
result = streamReader.ReadToEnd();
}
object json = JsonConvert.DeserializeObject(result);
}
}
catch (Exception ex)
{
Console.Out.WriteLine(ex.Message);
return(false);
}
return(true);
}
public void PositionEqualsToZeroAfterFileRemoval()
{
var tempFile = new FileInfo(Path.Combine(m_tempDir.FullName, "PositionEqualsToZeroAfterFileRemoval"));
var sampleData = new byte[] { 0, 1, 2, 3, 4, 5 };
using (var writer = File.Open(tempFile.FullName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
{
writer.Write(sampleData, 0, sampleData.Length);
}
File.Delete(tempFile.FullName);
using (var reader = new SafeFileStream(tempFile.FullName))
{
var position = reader.Position;
Assert.AreEqual(0, position);
}
}
public void LengthEqualsToZeroAfterFileRemoval()
{
var tempFile = new FileInfo(Path.Combine(m_tempDir.FullName, "LengthEqualsToZeroAfterFileRemoval"));
var sampleData = new byte[] { 0, 1, 2, 3, 4, 5 };
using (var writer = File.OpenWrite(tempFile.FullName))
{
writer.Write(sampleData, 0, sampleData.Length);
}
File.Delete(tempFile.FullName);
using (var reader = new SafeFileStream(tempFile.FullName))
{
var length = reader.Length;
Assert.AreEqual(0, length);
}
}
public VoronIndexOutput(StorageEnvironmentOptions options, string name, Transaction tx, string tree)
{
_name = name;
_tree = tree;
_tx = tx;
_fileTempPath = options.TempPath.Combine(name + "_" + Guid.NewGuid()).FullPath;
if (options.EncryptionEnabled)
{
_file = new TempCryptoStream(_fileTempPath);
}
else
{
_file = SafeFileStream.Create(_fileTempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
}
_tx.ReadTree(_tree).AddStream(name, Stream.Null); // ensure it's visible by LuceneVoronDirectory.FileExists, the actual write is inside Dispose
}
public void Restore(IEnumerable <ZipArchiveEntry> entries,
VoronPathSetting voronDataDir,
VoronPathSetting journalDir = null,
Action <string> onProgress = null,
CancellationToken cancellationToken = default)
{
journalDir = journalDir ?? voronDataDir.Combine("Journals");
if (Directory.Exists(voronDataDir.FullPath) == false)
{
Directory.CreateDirectory(voronDataDir.FullPath);
}
if (Directory.Exists(journalDir.FullPath) == false)
{
Directory.CreateDirectory(journalDir.FullPath);
}
onProgress?.Invoke("Starting snapshot restore");
foreach (var entry in entries)
{
var dst = string.Equals(Path.GetExtension(entry.Name), ".journal", StringComparison.OrdinalIgnoreCase)
? journalDir
: voronDataDir;
var sw = Stopwatch.StartNew();
if (Directory.Exists(dst.FullPath) == false)
{
Directory.CreateDirectory(dst.FullPath);
}
using (var input = entry.Open())
using (var output = SafeFileStream.Create(dst.Combine(entry.Name).FullPath, FileMode.CreateNew))
{
input.CopyTo(output, cancellationToken);
}
onProgress?.Invoke($"Restored file: '{entry.Name}' to: '{dst}', " +
$"size in bytes: {entry.Length:#,#;;0}, " +
$"took: {sw.ElapsedMilliseconds:#,#;;0}ms");
}
}
public unsafe void WriteSeekAndReadInTempCryptoStream(int seed)
{
using (var options = StorageEnvironmentOptions.ForPath(DataDir))
using (var file = SafeFileStream.Create(Path.Combine(DataDir, "EncryptedTempFile"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
using (var stream = new TempCryptoStream(file))
{
var r = new Random(seed);
var bytes = new byte[r.Next(128, 1024 * 1024 * 4)];
fixed(byte *b = bytes)
{
Memory.Set(b, (byte)'I', bytes.Length);
}
var injectionBytes = Encoding.UTF8.GetBytes("XXXXXXX");
stream.Write(bytes, 0, bytes.Length);
var someRandomLocationInTheMiddle = r.Next(0, bytes.Length - 7);
fixed(byte *b = bytes)
{
// injecting 7 'X' characters
Memory.Set(b + someRandomLocationInTheMiddle, (byte)'X', 7);
}
// Writing the same 7 'x's to the stream
stream.Seek(someRandomLocationInTheMiddle, SeekOrigin.Begin);
stream.Write(injectionBytes, 0, injectionBytes.Length);
// Reading the entire stream back.
var readBytes = new byte[bytes.Length];
stream.Seek(0, SeekOrigin.Begin);
var count = readBytes.Length;
var offset = 0;
while (count > 0)
{
var read = stream.Read(readBytes, offset, count);
count -= read;
offset += read;
}
Assert.Equal(bytes, readBytes);
}
}
private Stream InitFileStream(StorageEnvironmentOptions options)
{
try
{
if (options.EncryptionEnabled)
{
return(new TempCryptoStream(_fileTempPath));
}
return(SafeFileStream.Create(_fileTempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose));
}
catch (IOException ioe) when(IsOutOfDiskSpaceException(ioe))
{
ThrowDiskFullException();
// never reached
return(null);
}
}
private void CorruptJournal(long journal, long position, int numberOfCorruptedBytes = Constants.Size.Kilobyte * 4, byte value = 42, bool preserveValue = false)
{
Options.Dispose();
Options = StorageEnvironmentOptions.ForPath(DataDir);
Configure(Options);
using (var fileStream = SafeFileStream.Create(Options.GetJournalPath(journal).FullPath,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite | FileShare.Delete))
{
fileStream.Position = position;
var buffer = new byte[numberOfCorruptedBytes];
var remaining = buffer.Length;
var start = 0;
while (remaining > 0)
{
var read = fileStream.Read(buffer, start, remaining);
if (read == 0)
{
break;
}
start += read;
remaining -= read;
}
for (int i = 0; i < buffer.Length; i++)
{
if (buffer[i] != value || preserveValue)
{
buffer[i] = value;
}
else
{
buffer[i] = (byte)(value + 1); // we really want to change the original value here so it must not stay the same
}
}
fileStream.Position = position;
fileStream.Write(buffer, 0, buffer.Length);
}
}
public static async Task <Stream> CopyRemoteStreamLocally(Stream stream, PathSetting tempPath)
{
if (stream.CanSeek)
{
return(stream);
}
// This is meant to be used by ZipArchive, which will copy the data locally because is *must* be seekable.
// To avoid reading everything to memory, we copy to a local file instead. Note that this also ensure that we
// can process files > 2GB in size. https://github.com/dotnet/runtime/issues/59027
var basePath = tempPath?.FullPath ?? Path.GetTempPath();
var filePath = Path.Combine(basePath, $"{Guid.NewGuid()}.restore-local-file");
var file = SafeFileStream.Create(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read,
32 * 1024, FileOptions.DeleteOnClose);
try
{
await stream.CopyToAsync(file);
file.Seek(0, SeekOrigin.Begin);
return(file);
}
catch
{
try
{
await file.DisposeAsync();
}
catch
{
// nothing we can do
}
finally
{
PosixFile.DeleteOnClose(filePath);
}
throw;
}
}
public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
{
using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
{
var ptr = (byte *)header;
int remaining = sizeof(FileHeader);
while (remaining > 0)
{
int written;
if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out written, null) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to write to file " + path);
}
ptr += written;
remaining -= written;
}
if (Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle) == false)
{
throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to Flush File Buffers (sync) of file " + path);
}
}
}
private static Topology TryLoad(string path, JsonOperationContext context)
{
try
{
if (File.Exists(path) == false)
{
return(null);
}
using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var json = context.Read(stream, "raven-database-topology"))
{
return(JsonDeserializationClient.Topology(json));
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not understand the persisted database topology", e);
}
return(null);
}
}
public unsafe void TempCryptoStream_CanWorkWithFilesGreaterThan2GB()
{
using (StorageEnvironmentOptions.ForPath(DataDir))
using (var file = SafeFileStream.Create(Path.Combine(DataDir, "EncryptedTempFile_RavenDB_10836"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
{
long length = int.MaxValue;
length += 4096;
file.SetLength(length);
using (var stream = new TempCryptoStream(file))
{
var bytes = new byte[4096];
fixed(byte *b = bytes)
{
Memory.Set(b, (byte)'I', bytes.Length);
}
stream.Write(bytes, 0, bytes.Length);
stream.Position = length - 4096 + 1;
stream.Write(bytes, 0, bytes.Length);
stream.Seek(0, SeekOrigin.Begin);
var readBytes = new byte[bytes.Length];
var read = stream.Read(readBytes, 0, readBytes.Length);
Assert.Equal(4096, read);
}
}
}
private void CorruptJournal(long journal, long posOf4KbInJrnl)
{
Options.Dispose();
Options = StorageEnvironmentOptions.ForPath(DataDir);
Configure(Options);
using (var fileStream = SafeFileStream.Create(
Path.Combine(DataDir, StorageEnvironmentOptions.JournalName(journal)),
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite | FileShare.Delete))
{
fileStream.Position = posOf4KbInJrnl * Constants.Size.Kilobyte * 4;
var buffer = new byte[Constants.Size.Kilobyte * 4];
var remaining = buffer.Length;
var start = 0;
while (remaining > 0)
{
var read = fileStream.Read(buffer, start, remaining);
if (read == 0)
{
break;
}
start += read;
remaining -= read;
}
for (int i = 0; i < buffer.Length; i++)
{
buffer[i] = 42;
}
fileStream.Position = posOf4KbInJrnl * Constants.Size.Kilobyte * 4;
fileStream.Write(buffer, 0, buffer.Length);
}
}
public void SimpleReadWorks()
{
var tempFile = new FileInfo(Path.Combine(m_tempDir.FullName, "SimpleReadWorks"));
using (var writer = File.Open(tempFile.FullName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
using (var reader = new SafeFileStream(tempFile.FullName))
{
var sampleData = new byte[] { 0, 1, 2, 3, 4, 5 };
writer.Write(sampleData, 0, sampleData.Length);
writer.Flush();
var result = new byte[sampleData.Length];
var bytesRead = reader.Read(result, 0, result.Length);
Assert.AreEqual(sampleData.Length, bytesRead);
Assert.AreEqual(result, sampleData);
}
}
public WindowsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file,
long?initialFileSize = null,
Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal,
Win32NativeFileAccess access = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite,
bool usePageProtection = false)
: base(options, !fileAttributes.HasFlag(Win32NativeFileAttributes.Temporary), usePageProtection)
{
SYSTEM_INFO systemInfo;
GetSystemInfo(out systemInfo);
FileName = file;
_logger = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");
_access = access;
_copyOnWriteMode = Options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
if (_copyOnWriteMode)
{
_memoryMappedFileAccess = MemoryMappedFileAccess.Read | MemoryMappedFileAccess.CopyOnWrite;
fileAttributes = Win32NativeFileAttributes.Readonly;
_access = Win32NativeFileAccess.GenericRead;
}
else
{
_memoryMappedFileAccess = _access == Win32NativeFileAccess.GenericRead
? MemoryMappedFileAccess.Read
: MemoryMappedFileAccess.ReadWrite;
}
_fileAttributes = fileAttributes;
_handle = Win32NativeFileMethods.CreateFile(file.FullPath, access,
Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways, fileAttributes, IntPtr.Zero);
if (_handle.IsInvalid)
{
int lastWin32ErrorCode = Marshal.GetLastWin32Error();
throw new IOException("Failed to open file storage of WinMemoryMapPager for " + file,
new Win32Exception(lastWin32ErrorCode));
}
_fileInfo = new FileInfo(file.FullPath);
var drive = _fileInfo.Directory.Root.Name.TrimEnd('\\');
try
{
if (PhysicalDrivePerMountCache.TryGetValue(drive, out UniquePhysicalDriveId) == false)
{
UniquePhysicalDriveId = GetPhysicalDriveId(drive);
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Physical drive '{drive}' unique id = '{UniquePhysicalDriveId}' for file '{file}'");
}
}
catch (Exception ex)
{
UniquePhysicalDriveId = 0;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to determine physical drive Id for drive letter '{drive}', file='{file}'", ex);
}
}
var streamAccessType = _access == Win32NativeFileAccess.GenericRead
? FileAccess.Read
: FileAccess.ReadWrite;
_fileStream = SafeFileStream.Create(_handle, streamAccessType);
_totalAllocationSize = _fileInfo.Length;
if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite ||
(access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll ||
(access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE)
{
var fileLength = _fileStream.Length;
if (fileLength == 0 && initialFileSize.HasValue)
{
fileLength = initialFileSize.Value;
}
if (_fileStream.Length == 0 || (fileLength % AllocationGranularity != 0))
{
fileLength = NearestSizeToAllocationGranularity(fileLength);
Win32NativeFileMethods.SetFileLength(_handle, fileLength);
}
_totalAllocationSize = fileLength;
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(CreatePagerState());
}
/// <summary>
/// HttpWebRequest POST request to send to YTM which fetches a list (collection) of Playlists (without playlist tracks).
/// Use the 'Requests.Playlists.GetPlaylist (singular)' method to get an individual playlist complete with track listing.
/// </summary>
/// <param name="cookieValue">Cookie from a previous YouTube Music sign in via this application (stored in the database)</param>
/// <returns>OnlinePlaylistCollection object (list of playlists without tracks)</returns>
public static OnlinePlaylistCollection GetPlaylists(
string cookieValue)
{
var playListCol = new OnlinePlaylistCollection();
try
{
var request = (HttpWebRequest)WebRequest.Create(Global.YTMusicBaseUrl +
"browse?" + Global.YTMusicParams);
request = AddStandardHeaders(request, cookieValue);
request.ContentType = "application/json; charset=UTF-8";
request.Headers["X-Goog-AuthUser"] = "******";
request.Headers["x-origin"] = "https://music.youtube.com";
request.Headers["X-Goog-Visitor-Id"] = Global.GoogleVisitorId;
request.Headers["Authorization"] = GetAuthorisation(GetSAPISIDFromCookie(cookieValue));
byte[] postBytes = GetPostBytes(
SafeFileStream.ReadAllText(
Path.Combine(Global.WorkingDirectory, @"AppData\get_playlists_context.json")));
request.ContentLength = postBytes.Length;
using (var requestStream = request.GetRequestStream())
{
requestStream.Write(postBytes, 0, postBytes.Length);
requestStream.Close();
}
postBytes = null;
using (var response = (HttpWebResponse)request.GetResponse())
{
string result;
using (var brotli = new Brotli.BrotliStream(response.GetResponseStream(),
System.IO.Compression.CompressionMode.Decompress,
true))
{
var streamReader = new StreamReader(brotli);
result = streamReader.ReadToEnd();
var playListsResultContext = JsonConvert.DeserializeObject <BrowsePlaylistsResultsContext>(result);
var playListResults = playListsResultContext.contents
.singleColumnBrowseResultsRenderer
.tabs[0]
.tabRenderer
.content
.sectionListRenderer
.contents[1]
.itemSectionRenderer
.contents[0]
.gridRenderer
.items;
foreach (var item in playListResults)
{
if (item.musicTwoRowItemRenderer.title.runs[0].text != "New playlist" &&
item.musicTwoRowItemRenderer.title.runs[0].text != "Your likes")
{
try
{
playListCol.Add(new OnlinePlaylist
{
Title = item.musicTwoRowItemRenderer.title.runs[0].text,
Subtitle = item.musicTwoRowItemRenderer.subtitle.runs[0].text +
item.musicTwoRowItemRenderer.subtitle.runs[1].text +
item.musicTwoRowItemRenderer.subtitle.runs[2].text,
BrowseId = item.musicTwoRowItemRenderer.navigationEndpoint.browseEndpoint.browseId,
CoverArtUrl = item.musicTwoRowItemRenderer.thumbnailRenderer.musicThumbnailRenderer.thumbnail.thumbnails[0].url
});
}
catch (Exception e)
{
Logger.Log(e, "GetPlaylists - Error fetching a playlist", Log.LogTypeEnum.Error);
}
}
}
}
}
}
catch (Exception e)
{
var _ = e;
#if DEBUG
Console.Out.WriteLine("GetPlaylists: " + e.Message);
#endif
}
return(playListCol);
}
public Windows32BitsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null,
Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal,
Win32NativeFileAccess access = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite,
bool usePageProtection = false)
: base(options, canPrefetchAhead: false, usePageProtection: usePageProtection)
{
_memoryMappedFileAccess = access == Win32NativeFileAccess.GenericRead
? MemoryMappedFileAccess.Read
: MemoryMappedFileAccess.ReadWrite;
_mmFileAccessType = access == Win32NativeFileAccess.GenericRead
? NativeFileMapAccessType.Read
: NativeFileMapAccessType.Read |
NativeFileMapAccessType.Write;
FileName = file;
if (Options.CopyOnWriteMode)
{
ThrowNotSupportedOption(file.FullPath);
}
_fileAttributes = fileAttributes;
_handle = CreateFile(file.FullPath, access,
Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero,
Win32NativeFileCreationDisposition.OpenAlways,
fileAttributes, IntPtr.Zero);
if (_handle.IsInvalid)
{
var lastWin32ErrorCode = Marshal.GetLastWin32Error();
throw new IOException("Failed to open file storage of Windows32BitsMemoryMapPager for " + file,
new Win32Exception(lastWin32ErrorCode));
}
_fileInfo = new FileInfo(file.FullPath);
var streamAccessType = access == Win32NativeFileAccess.GenericRead
? FileAccess.Read
: FileAccess.ReadWrite;
_fileStream = SafeFileStream.Create(_handle, streamAccessType);
_totalAllocationSize = _fileInfo.Length;
if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite ||
(access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll ||
(access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE)
{
var fileLength = _fileStream.Length;
if ((fileLength == 0) && initialFileSize.HasValue)
{
fileLength = initialFileSize.Value;
}
if ((_fileStream.Length == 0) || (fileLength % AllocationGranularity != 0))
{
fileLength = NearestSizeToAllocationGranularity(fileLength);
SetFileLength(_handle, fileLength, file.FullPath);
}
_totalAllocationSize = fileLength;
}
NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
SetPagerState(CreatePagerState());
}
public void ReadAfterRealtimeDeleteWorks()
{
var tempFile = new FileInfo(Path.Combine(m_tempDir.FullName, "ReadAfterRealtimeDeleteWorks"));
var sampleData = new byte[] { 0, 1, 2, 3, 4, 5 };
using (var writer = File.Open(tempFile.FullName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
{
writer.Write(sampleData, 0, sampleData.Length);
}
using (var reader = new SafeFileStream(tempFile.FullName))
{
var result = new byte[sampleData.Length];
// 1st step
var bytesRead = reader.Read(result, 0, result.Length);
Assert.AreEqual(sampleData.Length, bytesRead);
Assert.AreEqual(result, sampleData);
File.Delete(tempFile.FullName);
//2nd step
bytesRead = reader.Read(result, 0, result.Length);
Assert.AreEqual(0, bytesRead);
using (var writer = File.Open(tempFile.FullName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete))
{
writer.Write(sampleData, 0, sampleData.Length);
}
// 3rd step
bytesRead = reader.Read(result, 0, result.Length);
Assert.AreEqual(sampleData.Length, bytesRead);
Assert.AreEqual(result, sampleData);
}
}
