Exemplo n.º 1
0
        public void ToFile(StorageEnvironment env, VoronPathSetting backupPath,
                           CompressionLevel compression = CompressionLevel.Optimal,
                           Action <string> infoNotify   = null,
                           Action backupStarted         = null)
        {
            infoNotify = infoNotify ?? (s => { });

            infoNotify("Voron backup db started");

            using (var file = SafeFileStream.Create(backupPath.FullPath, FileMode.Create))
            {
                using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                {
                    infoNotify("Voron backup started");
                    var dataPager = env.Options.DataPager;
                    var copier    = new DataCopier(Constants.Storage.PageSize * 16);
                    Backup(env, compression, infoNotify, backupStarted, dataPager, package, string.Empty,
                           copier);

                    file.Flush(true); // make sure that we fully flushed to disk
                }
            }

            infoNotify("Voron backup db finished");
        }
Exemplo n.º 2
0
        public static ClusterTopologyResponse TryLoad(string topologyHash, JsonOperationContext context)
        {
            try
            {
                var path = GetPath(topologyHash);
                if (File.Exists(path) == false)
                {
                    return(null);
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    using (var blittableJsonReaderObject = context.Read(stream, "raven-cluster-topology"))
                    {
                        return(JsonDeserializationClient.ClusterTopology(blittableJsonReaderObject));
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not understand the persisted cluster topology", e);
                }
                return(null);
            }
        }
Exemplo n.º 3
0
        private static void ReprocessZipFile()
        {
            Debug.Assert(_zipFilePath != null);

            // Make sure to keep the relative order between the flags,  this
            // code runs in multiple threads
            if (ZipFileProcessingHappening.Raise() == false)
            {
                return;
            }

            ZipFileInitialized.Lower();
            ZipFileEntries.Clear();

            try
            {
                using (var fileStream = SafeFileStream.Create(_zipFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
                    using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, false))
                    {
                        foreach (var entry in zipArchive.Entries)
                        {
                            ZipFileEntries.Add(entry.FullName);
                        }
                    }
            }
            catch (Exception)
            {
                // Suppressing this exception is reasonable: there are many
                // reasons for which the file may not be available right now.
                // The watcher will let us know whenever we can try again.
            }

            ZipFileInitialized.RaiseOrDie();
            ZipFileProcessingHappening.LowerOrDie();
        }
Exemplo n.º 4
0
 public static unsafe bool TryReadFileHeader(FileHeader *header, VoronPathSetting path)
 {
     using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
     {
         if (fs.Length != sizeof(FileHeader))
         {
             return(false); // wrong file size
         }
         var ptr       = (byte *)header;
         int remaining = sizeof(FileHeader);
         while (remaining > 0)
         {
             int read;
             if (Win32NativeFileMethods.ReadFile(fs.SafeFileHandle, ptr, remaining, out read, null) == false)
             {
                 throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to read file " + path);
             }
             if (read == 0)
             {
                 return(false); // we should be reading _something_ here, if we can't, then it is an error and we assume corruption
             }
             ptr       += read;
             remaining -= read;
         }
         return(true);
     }
 }
Exemplo n.º 5
0
        public StreamsTempFile(string tempFile, StorageEnvironment environment)
        {
            _tempFile    = tempFile;
            _environment = environment;

            _file = new TempFileStream(SafeFileStream.Create(_tempFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose | FileOptions.SequentialScan));
        }
Exemplo n.º 6
0
        public static void TrySaving(string topologyHash, ClusterTopologyResponse clusterTopology, JsonOperationContext context)
        {
            try
            {
                var path = GetPath(topologyHash);
                if (clusterTopology == null)
                {
                    Clear(path);
                    return;
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
                    using (var writer = new BlittableJsonTextWriter(context, stream))
                    {
                        var json = new DynamicJsonValue
                        {
                            [nameof(clusterTopology.Topology)] = clusterTopology.Topology.ToJson(),
                            [nameof(clusterTopology.Leader)]   = clusterTopology.Leader,
                            [nameof(clusterTopology.NodeTag)]  = clusterTopology.NodeTag,
                            ["PersistedAt"] = DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite),
                        };

                        context.Write(writer, json);
                        writer.Flush();
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not persist the cluster topology", e);
                }
            }
        }
Exemplo n.º 7
0
        public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
                           Action <string> infoNotify = null,
                           Action backupStarted       = null)
        {
            infoNotify = infoNotify ?? (s => { });

            if (env.Options.IncrementalBackupEnabled == false)
            {
                throw new InvalidOperationException("Incremental backup is disabled for this storage");
            }

            var copier = new DataCopier(Constants.Storage.PageSize * 16);

            using (var file = SafeFileStream.Create(backupPath, FileMode.Create))
            {
                long numberOfBackedUpPages;
                using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                {
                    numberOfBackedUpPages = Incremental_Backup(env, compression, infoNotify,
                                                               backupStarted, package, string.Empty, copier);
                }
                file.Flush(true); // make sure that this is actually persisted fully to disk
                return(numberOfBackedUpPages);
            }
        }
Exemplo n.º 8
0
        private Stream GetNewStream(long maxFileSize)
        {
            if (DateTime.Today != _today)
            {
                lock (this)
                {
                    if (DateTime.Today != _today)
                    {
                        _today      = DateTime.Today;
                        _dateString = DateTime.Today.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
                        _logNumber  = 0;

                        CleanupOldLogFiles();
                    }
                }
            }
            while (true)
            {
                var nextLogNumber = Interlocked.Increment(ref _logNumber);
                var fileName      = Path.Combine(_path, _dateString) + "." +
                                    nextLogNumber.ToString("000", CultureInfo.InvariantCulture) + ".log";
                if (File.Exists(fileName) && new FileInfo(fileName).Length >= maxFileSize)
                {
                    continue;
                }
                var fileStream = SafeFileStream.Create(fileName, FileMode.Create, FileAccess.Write, FileShare.Read, 32 * 1024, false);
                fileStream.Write(_headerRow, 0, _headerRow.Length);
                return(fileStream);
            }
        }
Exemplo n.º 9
0
        private bool TryGetNewStreamAndApplyRetentionPolicies(long maxFileSize, out FileStream fileStream)
        {
            string[] logFiles;
            string[] logGzFiles;
            try
            {
                logFiles   = Directory.GetFiles(_path, "*.log");
                logGzFiles = Directory.GetFiles(_path, "*.log.gz");
            }
            catch (Exception)
            {
                // Something went wrong we will try again later
                fileStream = null;
                return(false);
            }
            Array.Sort(logFiles);
            Array.Sort(logGzFiles);

            if (DateTime.Today != _today)
            {
                _today      = DateTime.Today;
                _dateString = LogInfo.DateToLogFormat(DateTime.Today);
            }

            if (_logNumber < 0)
            {
                _logNumber = Math.Max(LastLogNumberForToday(logFiles), LastLogNumberForToday(logGzFiles));
            }

            UpdateLocalDateTimeOffset();

            string fileName;

            while (true)
            {
                fileName = Path.Combine(_path, LogInfo.GetNewFileName(_dateString, _logNumber));
                if (File.Exists(fileName))
                {
                    if (new FileInfo(fileName).Length < maxFileSize)
                    {
                        break;
                    }
                }
                else if (File.Exists(LogInfo.AddCompressExtension(fileName)) == false)
                {
                    break;
                }
                _logNumber++;
            }

            if (Compressing == false)
            {
                CleanupOldLogFiles(logFiles);
                LimitLogSize(logFiles);
            }

            fileStream = SafeFileStream.Create(fileName, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false);
            fileStream.Write(_headerRow, 0, _headerRow.Length);
            return(true);
        }
Exemplo n.º 10
0
        /// <summary>
        /// Do a incremental backup of a set of environments. Note that the order of the environments matter!
        /// </summary>
        public long ToFile(IEnumerable <FullBackup.StorageEnvironmentInformation> envs, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
                           Action <string> infoNotify = null,
                           Action backupStarted       = null)
        {
            infoNotify = infoNotify ?? (s => { });

            long totalNumberOfBackedUpPages = 0;

            using (var file = SafeFileStream.Create(backupPath, FileMode.Create))
            {
                using (var package = new ZipArchive(file, ZipArchiveMode.Create, leaveOpen: true))
                {
                    foreach (var e in envs)
                    {
                        if (e.Env.Options.IncrementalBackupEnabled == false)
                        {
                            throw new InvalidOperationException("Incremental backup is disabled for this storage");
                        }
                        infoNotify("Voron backup " + e.Name + "started");
                        var basePath = Path.Combine(e.Folder, e.Name);
                        var env      = e.Env;
                        var copier   = new DataCopier(Constants.Storage.PageSize * 16);
                        var numberOfBackedUpPages = Incremental_Backup(env, compression, infoNotify,
                                                                       backupStarted, package, basePath, copier);
                        totalNumberOfBackedUpPages += numberOfBackedUpPages;
                    }
                }
                file.Flush(true); // make sure that this is actually persisted fully to disk

                return(totalNumberOfBackedUpPages);
            }
        }
        public static async Task <ClusterTopologyResponse> TryLoadAsync(string topologyHash, DocumentConventions conventions, JsonOperationContext context)
        {
            try
            {
                if (conventions.DisableTopologyCache)
                {
                    return(null);
                }

                var path = GetPath(topologyHash, conventions);
                if (File.Exists(path) == false)
                {
                    return(null);
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    using (var json = await context.ReadForMemoryAsync(stream, "raven-cluster-topology").ConfigureAwait(false))
                    {
                        return(JsonDeserializationClient.ClusterTopology(json));
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not understand the persisted cluster topology", e);
                }
                return(null);
            }
        }
Exemplo n.º 12
0
        private void CorruptJournal(long journal, long position, int numberOfCorruptedBytes = Constants.Size.Kilobyte * 4, byte value = 42)
        {
            Options.Dispose();
            Options = StorageEnvironmentOptions.ForPath(DataDir);
            Configure(Options);
            using (var fileStream = SafeFileStream.Create(Options.GetJournalPath(journal).FullPath,
                                                          FileMode.Open,
                                                          FileAccess.ReadWrite,
                                                          FileShare.ReadWrite | FileShare.Delete))
            {
                fileStream.Position = position;

                var buffer = new byte[numberOfCorruptedBytes];

                var remaining = buffer.Length;
                var start     = 0;
                while (remaining > 0)
                {
                    var read = fileStream.Read(buffer, start, remaining);
                    if (read == 0)
                    {
                        break;
                    }
                    start     += read;
                    remaining -= read;
                }

                for (int i = 0; i < buffer.Length; i++)
                {
                    buffer[i] = value;
                }
                fileStream.Position = position;
                fileStream.Write(buffer, 0, buffer.Length);
            }
        }
Exemplo n.º 13
0
        public StreamsTempFile(string tempFile, DocumentDatabase database)
        {
            _tempFile = tempFile;
            _database = database;

            _file = SafeFileStream.Create(_tempFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose | FileOptions.SequentialScan);
        }
Exemplo n.º 14
0
        public static void TrySaving(string databaseName, string topologyHash, Topology topology, DocumentConventions conventions, JsonOperationContext context)
        {
            try
            {
                var path = GetPath(databaseName, topologyHash, conventions);
                if (topology == null)
                {
                    Clear(databaseName);
                    return;
                }

                var existingTopology = TryLoad(path, context);
                if (existingTopology?.Etag >= topology.Etag)
                {
                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info($"Skipping save topology with etag {topology.Etag} to cache " +
                                     $"as the cache already have a topology with etag: {existingTopology.Etag}");
                    }
                    return;
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
                    using (var writer = new BlittableJsonTextWriter(context, stream))
                    {
                        writer.WriteStartObject();

                        writer.WritePropertyName(context.GetLazyString(nameof(topology.Nodes)));
                        writer.WriteStartArray();
                        for (var i = 0; i < topology.Nodes.Count; i++)
                        {
                            var node = topology.Nodes[i];
                            if (i != 0)
                            {
                                writer.WriteComma();
                            }
                            WriteNode(writer, node, context);
                        }
                        writer.WriteEndArray();

                        writer.WriteComma();
                        writer.WritePropertyName(context.GetLazyString(nameof(topology.Etag)));
                        writer.WriteInteger(topology.Etag);

                        writer.WriteComma();
                        writer.WritePropertyName(context.GetLazyString("PersistedAt"));
                        writer.WriteString(DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite));

                        writer.WriteEndObject();
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not persist the database topology", e);
                }
            }
        }
Exemplo n.º 15
0
 public Stream RentFileStream()
 {
     if (_files.TryDequeue(out var stream) == false)
     {
         stream = new TempFileStream(SafeFileStream.Create(
                                         GetTempFileName(_options),
                                         FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, 4096,
                                         FileOptions.DeleteOnClose));
     }
Exemplo n.º 16
0
        public static void PerformanceAnalysis(string directory, string outputFile, int size)
        {
            using (var fileStream = SafeFileStream.Create(outputFile, FileMode.Create))
                using (var streamWriter = new StreamWriter(fileStream))
                {
                    var files = Directory.GetFiles(directory, "*.json").OrderBy(f => new FileInfo(f).Length).Take(size);

                    streamWriter.WriteLine("Name,Json Parse Time,Json Size, Json Time, Blit Parse Time,Blit Size, Blit Time");
                    using (var blittableContext = JsonOperationContext.ShortTermSingleUse())
                    {
                        foreach (var jsonFile in files)
                        {
                            Console.Write(string.Format("{0} {1:#,#}", Path.GetFileName(jsonFile), new FileInfo(jsonFile).Length));
                            streamWriter.Write(Path.GetFileName(jsonFile) + ",");
                            var sp         = Stopwatch.StartNew();
                            var jsonOjbect = JObject.Load(new JsonTextReader(File.OpenText(jsonFile)));
                            streamWriter.Write(sp.ElapsedMilliseconds + ",");

                            using (var stream = SafeFileStream.Create("output.junk", FileMode.Create))
                                using (var textWriter = new StreamWriter(stream))
                                {
                                    sp.Restart();
                                    jsonOjbect.WriteTo(new JsonTextWriter(textWriter));
                                    textWriter.Flush();
                                    streamWriter.Write(stream.Length + "," + sp.ElapsedMilliseconds + ",");
                                }


                            Console.Write(" json - {0:#,#}ms", sp.ElapsedMilliseconds);
                            GC.Collect(2);

                            sp.Restart();
                            using (var employee = blittableContext.Read(File.OpenRead(jsonFile), "doc1"))
                            {
                                streamWriter.Write(sp.ElapsedMilliseconds + ",");
                                using (var stream = SafeFileStream.Create("output2.junk", FileMode.Create))
                                {
                                    sp.Restart();
                                    blittableContext.Write(stream, employee);
                                    streamWriter.Write(employee.Size + "," + sp.ElapsedMilliseconds + ",");
                                }
                                Console.WriteLine(" blit - {0:#,#} ms, Props: {1}",
                                                  sp.ElapsedMilliseconds,
                                                  blittableContext.CachedProperties.PropertiesDiscovered);
                            }
                            GC.Collect(2);

                            streamWriter.WriteLine();
                        }
                    }
                }
        }
Exemplo n.º 17
0
        private bool TryGetNewStreamAndApplyRetentionPolicies(long maxFileSize, out FileStream fileStream)
        {
            string[] logFiles;
            string[] logGzFiles;
            try
            {
                logFiles   = Directory.GetFiles(_path, "*.log");
                logGzFiles = Directory.GetFiles(_path, "*.log.gz");
            }
            catch (Exception)
            {
                // Something went wrong we will try again later
                fileStream = null;
                return(false);
            }
            Array.Sort(logFiles);
            Array.Sort(logGzFiles);

            if (DateTime.Today != _today)
            {
                _today      = DateTime.Today;
                _dateString = LogInfo.GetFileName(DateTime.Today);
                _logNumber  = Math.Max(NextLogNumberForExtension(logFiles, "log"), NextLogNumberForExtension(logGzFiles, "log.gz"));
            }

            UpdateLocalDateTimeOffset();

            string fileName;

            while (true)
            {
                var nextLogNumber = Interlocked.Increment(ref _logNumber);
                fileName = Path.Combine(_path, _dateString) + "." + nextLogNumber.ToString("000", CultureInfo.InvariantCulture) + ".log";
                if (File.Exists(fileName) == false || new FileInfo(fileName).Length < maxFileSize)
                {
                    break;
                }
            }

            if (Compressing == false)
            {
                CleanupOldLogFiles(logFiles);
                LimitLogSize(logFiles);
            }

            fileStream = SafeFileStream.Create(fileName, FileMode.Append, FileAccess.Write, FileShare.Read, 32 * 1024, false);
            fileStream.Write(_headerRow, 0, _headerRow.Length);
            return(true);
        }
Exemplo n.º 18
0
        public VoronIndexOutput(StorageEnvironmentOptions options, string name, Transaction tx, string tree)
        {
            _name         = name;
            _tree         = tree;
            _tx           = tx;
            _fileTempPath = options.TempPath.Combine(name + "_" + Guid.NewGuid()).FullPath;

            if (options.EncryptionEnabled)
            {
                _file = new TempCryptoStream(_fileTempPath);
            }
            else
            {
                _file = SafeFileStream.Create(_fileTempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
            }

            _tx.ReadTree(_tree).AddStream(name, Stream.Null); // ensure it's visible by LuceneVoronDirectory.FileExists, the actual write is inside Dispose
        }
Exemplo n.º 19
0
        public void Restore(IEnumerable <ZipArchiveEntry> entries,
                            VoronPathSetting voronDataDir,
                            VoronPathSetting journalDir         = null,
                            Action <string> onProgress          = null,
                            CancellationToken cancellationToken = default)
        {
            journalDir = journalDir ?? voronDataDir.Combine("Journals");

            if (Directory.Exists(voronDataDir.FullPath) == false)
            {
                Directory.CreateDirectory(voronDataDir.FullPath);
            }

            if (Directory.Exists(journalDir.FullPath) == false)
            {
                Directory.CreateDirectory(journalDir.FullPath);
            }

            onProgress?.Invoke("Starting snapshot restore");

            foreach (var entry in entries)
            {
                var dst = string.Equals(Path.GetExtension(entry.Name), ".journal", StringComparison.OrdinalIgnoreCase)
                    ? journalDir
                    : voronDataDir;

                var sw = Stopwatch.StartNew();

                if (Directory.Exists(dst.FullPath) == false)
                {
                    Directory.CreateDirectory(dst.FullPath);
                }

                using (var input = entry.Open())
                    using (var output = SafeFileStream.Create(dst.Combine(entry.Name).FullPath, FileMode.CreateNew))
                    {
                        input.CopyTo(output, cancellationToken);
                    }

                onProgress?.Invoke($"Restored file: '{entry.Name}' to: '{dst}', " +
                                   $"size in bytes: {entry.Length:#,#;;0}, " +
                                   $"took: {sw.ElapsedMilliseconds:#,#;;0}ms");
            }
        }
Exemplo n.º 20
0
        public unsafe void WriteSeekAndReadInTempCryptoStream(int seed)
        {
            using (var options = StorageEnvironmentOptions.ForPath(DataDir))
                using (var file = SafeFileStream.Create(Path.Combine(DataDir, "EncryptedTempFile"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
                    using (var stream = new TempCryptoStream(file))
                    {
                        var r = new Random(seed);

                        var bytes = new byte[r.Next(128, 1024 * 1024 * 4)];
                        fixed(byte *b = bytes)
                        {
                            Memory.Set(b, (byte)'I', bytes.Length);
                        }

                        var injectionBytes = Encoding.UTF8.GetBytes("XXXXXXX");

                        stream.Write(bytes, 0, bytes.Length);

                        var someRandomLocationInTheMiddle = r.Next(0, bytes.Length - 7);
                        fixed(byte *b = bytes)
                        {
                            // injecting 7 'X' characters
                            Memory.Set(b + someRandomLocationInTheMiddle, (byte)'X', 7);
                        }

                        // Writing the same 7 'x's to the stream
                        stream.Seek(someRandomLocationInTheMiddle, SeekOrigin.Begin);
                        stream.Write(injectionBytes, 0, injectionBytes.Length);

                        // Reading the entire stream back.
                        var readBytes = new byte[bytes.Length];
                        stream.Seek(0, SeekOrigin.Begin);

                        var count  = readBytes.Length;
                        var offset = 0;
                        while (count > 0)
                        {
                            var read = stream.Read(readBytes, offset, count);
                            count  -= read;
                            offset += read;
                        }
                        Assert.Equal(bytes, readBytes);
                    }
        }
Exemplo n.º 21
0
        private Stream InitFileStream(StorageEnvironmentOptions options)
        {
            try
            {
                if (options.EncryptionEnabled)
                {
                    return(new TempCryptoStream(_fileTempPath));
                }

                return(SafeFileStream.Create(_fileTempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose));
            }
            catch (IOException ioe) when(IsOutOfDiskSpaceException(ioe))
            {
                ThrowDiskFullException();

                // never reached
                return(null);
            }
        }
Exemplo n.º 22
0
        private void CorruptJournal(long journal, long position, int numberOfCorruptedBytes = Constants.Size.Kilobyte * 4, byte value = 42, bool preserveValue = false)
        {
            Options.Dispose();
            Options = StorageEnvironmentOptions.ForPath(DataDir);
            Configure(Options);
            using (var fileStream = SafeFileStream.Create(Options.GetJournalPath(journal).FullPath,
                                                          FileMode.Open,
                                                          FileAccess.ReadWrite,
                                                          FileShare.ReadWrite | FileShare.Delete))
            {
                fileStream.Position = position;

                var buffer = new byte[numberOfCorruptedBytes];

                var remaining = buffer.Length;
                var start     = 0;
                while (remaining > 0)
                {
                    var read = fileStream.Read(buffer, start, remaining);
                    if (read == 0)
                    {
                        break;
                    }
                    start     += read;
                    remaining -= read;
                }

                for (int i = 0; i < buffer.Length; i++)
                {
                    if (buffer[i] != value || preserveValue)
                    {
                        buffer[i] = value;
                    }
                    else
                    {
                        buffer[i] = (byte)(value + 1); // we really want to change the original value here so it must not stay the same
                    }
                }
                fileStream.Position = position;
                fileStream.Write(buffer, 0, buffer.Length);
            }
        }
Exemplo n.º 23
0
        public static async Task <Stream> CopyRemoteStreamLocally(Stream stream, PathSetting tempPath)
        {
            if (stream.CanSeek)
            {
                return(stream);
            }

            // This is meant to be used by ZipArchive, which will copy the data locally because is *must* be seekable.
            // To avoid reading everything to memory, we copy to a local file instead. Note that this also ensure that we
            // can process files > 2GB in size. https://github.com/dotnet/runtime/issues/59027
            var basePath = tempPath?.FullPath ?? Path.GetTempPath();
            var filePath = Path.Combine(basePath, $"{Guid.NewGuid()}.restore-local-file");
            var file     = SafeFileStream.Create(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read,
                                                 32 * 1024, FileOptions.DeleteOnClose);

            try
            {
                await stream.CopyToAsync(file);

                file.Seek(0, SeekOrigin.Begin);

                return(file);
            }
            catch
            {
                try
                {
                    await file.DisposeAsync();
                }
                catch
                {
                    // nothing we can do
                }
                finally
                {
                    PosixFile.DeleteOnClose(filePath);
                }

                throw;
            }
        }
        public static async Task TrySavingAsync(string topologyHash, ClusterTopologyResponse clusterTopology, DocumentConventions conventions, JsonOperationContext context, CancellationToken token)
        {
            try
            {
                if (conventions.DisableTopologyCache)
                {
                    return;
                }

                var path = GetPath(topologyHash, conventions);
                if (clusterTopology == null)
                {
                    Clear(path);
                    return;
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Create, FileAccess.Write, FileShare.Read))
                    await using (var writer = new AsyncBlittableJsonTextWriter(context, stream))
                    {
                        var json = new DynamicJsonValue
                        {
                            [nameof(clusterTopology.Topology)] = clusterTopology.Topology.ToJson(),
                            [nameof(clusterTopology.Leader)]   = clusterTopology.Leader,
                            [nameof(clusterTopology.NodeTag)]  = clusterTopology.NodeTag,
                            [nameof(clusterTopology.Etag)]     = clusterTopology.Etag,
                            ["PersistedAt"] = DateTimeOffset.UtcNow.ToString(DefaultFormat.DateTimeOffsetFormatsToWrite),
                        };

                        context.Write(writer, json);
                        await writer.FlushAsync(token).ConfigureAwait(false);
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not persist the cluster topology", e);
                }
            }
        }
Exemplo n.º 25
0
 public static unsafe void WriteFileHeader(FileHeader *header, VoronPathSetting path)
 {
     using (var fs = SafeFileStream.Create(path.FullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.None))
     {
         var ptr       = (byte *)header;
         int remaining = sizeof(FileHeader);
         while (remaining > 0)
         {
             int written;
             if (Win32NativeFileMethods.WriteFile(fs.SafeFileHandle, ptr, remaining, out written, null) == false)
             {
                 throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to write to file " + path);
             }
             ptr       += written;
             remaining -= written;
         }
         if (Win32NativeFileMethods.FlushFileBuffers(fs.SafeFileHandle) == false)
         {
             throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to Flush File Buffers (sync) of file " + path);
         }
     }
 }
Exemplo n.º 26
0
        private void CorruptJournal(long journal, long posOf4KbInJrnl)
        {
            Options.Dispose();
            Options = StorageEnvironmentOptions.ForPath(DataDir);
            Configure(Options);
            using (var fileStream = SafeFileStream.Create(
                       Path.Combine(DataDir, StorageEnvironmentOptions.JournalName(journal)),
                       FileMode.Open,
                       FileAccess.ReadWrite,
                       FileShare.ReadWrite | FileShare.Delete))
            {
                fileStream.Position = posOf4KbInJrnl * Constants.Size.Kilobyte * 4;

                var buffer = new byte[Constants.Size.Kilobyte * 4];

                var remaining = buffer.Length;
                var start     = 0;
                while (remaining > 0)
                {
                    var read = fileStream.Read(buffer, start, remaining);
                    if (read == 0)
                    {
                        break;
                    }
                    start     += read;
                    remaining -= read;
                }

                for (int i = 0; i < buffer.Length; i++)
                {
                    buffer[i] = 42;
                }
                fileStream.Position = posOf4KbInJrnl * Constants.Size.Kilobyte * 4;
                fileStream.Write(buffer, 0, buffer.Length);
            }
        }
Exemplo n.º 27
0
        private static Topology TryLoad(string path, JsonOperationContext context)
        {
            try
            {
                if (File.Exists(path) == false)
                {
                    return(null);
                }

                using (var stream = SafeFileStream.Create(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    using (var json = context.Read(stream, "raven-database-topology"))
                    {
                        return(JsonDeserializationClient.Topology(json));
                    }
            }
            catch (Exception e)
            {
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Could not understand the persisted database topology", e);
                }
                return(null);
            }
        }
Exemplo n.º 28
0
        public unsafe void TempCryptoStream_CanWorkWithFilesGreaterThan2GB()
        {
            using (StorageEnvironmentOptions.ForPath(DataDir))
                using (var file = SafeFileStream.Create(Path.Combine(DataDir, "EncryptedTempFile_RavenDB_10836"), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
                {
                    long length = int.MaxValue;

                    length += 4096;

                    file.SetLength(length);

                    using (var stream = new TempCryptoStream(file))
                    {
                        var bytes = new byte[4096];

                        fixed(byte *b = bytes)
                        {
                            Memory.Set(b, (byte)'I', bytes.Length);
                        }

                        stream.Write(bytes, 0, bytes.Length);

                        stream.Position = length - 4096 + 1;

                        stream.Write(bytes, 0, bytes.Length);

                        stream.Seek(0, SeekOrigin.Begin);

                        var readBytes = new byte[bytes.Length];

                        var read = stream.Read(readBytes, 0, readBytes.Length);

                        Assert.Equal(4096, read);
                    }
                }
        }
Exemplo n.º 29
0
        public WindowsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file,
                                     long?initialFileSize = null,
                                     Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal,
                                     Win32NativeFileAccess access             = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite,
                                     bool usePageProtection = false)
            : base(options, !fileAttributes.HasFlag(Win32NativeFileAttributes.Temporary), usePageProtection)
        {
            SYSTEM_INFO systemInfo;

            GetSystemInfo(out systemInfo);
            FileName = file;
            _logger  = LoggingSource.Instance.GetLogger <StorageEnvironment>($"Pager-{file}");

            _access          = access;
            _copyOnWriteMode = Options.CopyOnWriteMode && FileName.FullPath.EndsWith(Constants.DatabaseFilename);
            if (_copyOnWriteMode)
            {
                _memoryMappedFileAccess = MemoryMappedFileAccess.Read | MemoryMappedFileAccess.CopyOnWrite;
                fileAttributes          = Win32NativeFileAttributes.Readonly;
                _access = Win32NativeFileAccess.GenericRead;
            }
            else
            {
                _memoryMappedFileAccess = _access == Win32NativeFileAccess.GenericRead
                ? MemoryMappedFileAccess.Read
                : MemoryMappedFileAccess.ReadWrite;
            }
            _fileAttributes = fileAttributes;

            _handle = Win32NativeFileMethods.CreateFile(file.FullPath, access,
                                                        Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero,
                                                        Win32NativeFileCreationDisposition.OpenAlways, fileAttributes, IntPtr.Zero);
            if (_handle.IsInvalid)
            {
                int lastWin32ErrorCode = Marshal.GetLastWin32Error();
                throw new IOException("Failed to open file storage of WinMemoryMapPager for " + file,
                                      new Win32Exception(lastWin32ErrorCode));
            }

            _fileInfo = new FileInfo(file.FullPath);
            var drive = _fileInfo.Directory.Root.Name.TrimEnd('\\');

            try
            {
                if (PhysicalDrivePerMountCache.TryGetValue(drive, out UniquePhysicalDriveId) == false)
                {
                    UniquePhysicalDriveId = GetPhysicalDriveId(drive);
                }

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Physical drive '{drive}' unique id = '{UniquePhysicalDriveId}' for file '{file}'");
                }
            }
            catch (Exception ex)
            {
                UniquePhysicalDriveId = 0;
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Failed to determine physical drive Id for drive letter '{drive}', file='{file}'", ex);
                }
            }

            var streamAccessType = _access == Win32NativeFileAccess.GenericRead
                ? FileAccess.Read
                : FileAccess.ReadWrite;

            _fileStream = SafeFileStream.Create(_handle, streamAccessType);

            _totalAllocationSize = _fileInfo.Length;

            if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite ||
                (access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll ||
                (access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE)
            {
                var fileLength = _fileStream.Length;
                if (fileLength == 0 && initialFileSize.HasValue)
                {
                    fileLength = initialFileSize.Value;
                }

                if (_fileStream.Length == 0 || (fileLength % AllocationGranularity != 0))
                {
                    fileLength = NearestSizeToAllocationGranularity(fileLength);

                    Win32NativeFileMethods.SetFileLength(_handle, fileLength);
                }

                _totalAllocationSize = fileLength;
            }

            NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;

            SetPagerState(CreatePagerState());
        }
Exemplo n.º 30
0
        public Windows32BitsMemoryMapPager(StorageEnvironmentOptions options, VoronPathSetting file, long?initialFileSize = null,
                                           Win32NativeFileAttributes fileAttributes = Win32NativeFileAttributes.Normal,
                                           Win32NativeFileAccess access             = Win32NativeFileAccess.GenericRead | Win32NativeFileAccess.GenericWrite,
                                           bool usePageProtection = false)
            : base(options, canPrefetchAhead: false, usePageProtection: usePageProtection)
        {
            _memoryMappedFileAccess = access == Win32NativeFileAccess.GenericRead
              ? MemoryMappedFileAccess.Read
              : MemoryMappedFileAccess.ReadWrite;

            _mmFileAccessType = access == Win32NativeFileAccess.GenericRead
                ? NativeFileMapAccessType.Read
                : NativeFileMapAccessType.Read |
                                NativeFileMapAccessType.Write;

            FileName = file;

            if (Options.CopyOnWriteMode)
            {
                ThrowNotSupportedOption(file.FullPath);
            }

            _fileAttributes = fileAttributes;
            _handle         = CreateFile(file.FullPath, access,
                                         Win32NativeFileShare.Read | Win32NativeFileShare.Write | Win32NativeFileShare.Delete, IntPtr.Zero,
                                         Win32NativeFileCreationDisposition.OpenAlways,
                                         fileAttributes, IntPtr.Zero);


            if (_handle.IsInvalid)
            {
                var lastWin32ErrorCode = Marshal.GetLastWin32Error();
                throw new IOException("Failed to open file storage of Windows32BitsMemoryMapPager for " + file,
                                      new Win32Exception(lastWin32ErrorCode));
            }

            _fileInfo = new FileInfo(file.FullPath);

            var streamAccessType = access == Win32NativeFileAccess.GenericRead
                 ? FileAccess.Read
                 : FileAccess.ReadWrite;

            _fileStream = SafeFileStream.Create(_handle, streamAccessType);

            _totalAllocationSize = _fileInfo.Length;

            if ((access & Win32NativeFileAccess.GenericWrite) == Win32NativeFileAccess.GenericWrite ||
                (access & Win32NativeFileAccess.GenericAll) == Win32NativeFileAccess.GenericAll ||
                (access & Win32NativeFileAccess.FILE_GENERIC_WRITE) == Win32NativeFileAccess.FILE_GENERIC_WRITE)
            {
                var fileLength = _fileStream.Length;
                if ((fileLength == 0) && initialFileSize.HasValue)
                {
                    fileLength = initialFileSize.Value;
                }

                if ((_fileStream.Length == 0) || (fileLength % AllocationGranularity != 0))
                {
                    fileLength = NearestSizeToAllocationGranularity(fileLength);

                    SetFileLength(_handle, fileLength, file.FullPath);
                }
                _totalAllocationSize = fileLength;
            }

            NumberOfAllocatedPages = _totalAllocationSize / Constants.Storage.PageSize;
            SetPagerState(CreatePagerState());
        }