コード例 #1
0
		public void WriteSnapshot(LogWriter logWriter, VersionSet versionSet)
		{
			var edit = new VersionEdit();
			AddMetadata(edit, storageContext.Options);
			AddCompactionPointers(edit, versionSet);
			AddFiles(edit, versionSet);

			edit.EncodeTo(logWriter);
		}
コード例 #2
0
		private static void AddCompactionPointers(VersionEdit edit, VersionSet versionSet)
		{
			for (int level = 0; level < Config.NumberOfLevels; level++)
			{
				var compactionPointer = versionSet.CompactionPointers[level];
				if (!compactionPointer.IsEmpty())
				{
					edit.SetCompactionPointer(level, new InternalKey(compactionPointer));
				}
			}
		}
コード例 #3
0
		private static void AddFiles(VersionEdit edit, VersionSet versionSet)
		{
			for (int level = 0; level < Config.NumberOfLevels; level++)
			{
				var files = versionSet.Current.Files[level];
				foreach (var file in files)
				{
					edit.AddFile(level, file);
				}
			}
		}
コード例 #4
0
        public static VersionEdit DecodeFrom(Stream stream)
        {
            var result = new VersionEdit();

            while (true)
            {
                Tag tag;
                try
                {
                    tag = (Tag)stream.Read7BitEncodedInt();
                }
                catch (EndOfStreamException)
                {
                    break;
                }

                int level;
                switch (tag)
                {
                    case Tag.Comparator:
                        var slice = stream.ReadLengthPrefixedSlice();
                        result.SetComparatorName(slice);
                        break;
                    case Tag.LogNumber:
                        result.SetLogNumber((ulong)stream.Read7BitEncodedLong());
                        break;
                    case Tag.PrevLogNumber:
                        result.SetPrevLogNumber((ulong)stream.Read7BitEncodedLong());
                        break;
                    case Tag.NextFileNumber:
                        result.SetNextFile((ulong)stream.Read7BitEncodedLong());
                        break;
                    case Tag.LastSequence:
                        result.SetLastSequence((ulong)stream.Read7BitEncodedLong());
                        break;
                    case Tag.CompactPointer:
                        level = stream.Read7BitEncodedInt();
                        var compactionPointer = stream.ReadLengthPrefixedInternalKey();

                        result.SetCompactionPointer(level, compactionPointer);
                        break;
                    case Tag.DeletedFile:
                        level = stream.Read7BitEncodedInt();
                        var fileNumber = (ulong)stream.Read7BitEncodedLong();

                        result.DeleteFile(level, fileNumber);
                        break;
                    case Tag.NewFile:
                        level = stream.Read7BitEncodedInt();
                        var fileMetadata = new FileMetadata
                                               {
                                                   FileNumber = (ulong)stream.Read7BitEncodedLong(),
                                                   FileSize = stream.Read7BitEncodedLong(),
                                                   SmallestKey = stream.ReadLengthPrefixedInternalKey(),
                                                   LargestKey = stream.ReadLengthPrefixedInternalKey()
                                               };

                        result.AddFile(level, fileMetadata);
                        break;
                }
            }

            return result;
        }
コード例 #5
0
		public void ShouldEncodeAndDecode()
		{
			var versionEdit = new VersionEdit();

			versionEdit.SetComparatorName("testComparator");
			versionEdit.SetLastSequence(10);
			versionEdit.SetLogNumber(2);
			versionEdit.SetNextFile(3);
			versionEdit.SetPrevLogNumber(1);
			versionEdit.SetLastSequence(1);

			versionEdit.AddFile(0, new FileMetadata()
				{
					FileNumber = 0,
					FileSize = 128,
					LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
					SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
				});
			versionEdit.AddFile(1, new FileMetadata()
			{
				FileNumber = 1,
				FileSize = 128,
				LargestKey = new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value),
				SmallestKey = new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value),
			});

			versionEdit.DeleteFile(0, 3);
			versionEdit.DeleteFile(1, 4);

			versionEdit.SetCompactionPointer(0, new InternalKey("begin", Format.MaxSequenceNumber, ItemType.Value));
			versionEdit.SetCompactionPointer(1, new InternalKey("end", Format.MaxSequenceNumber, ItemType.Value));

			var memoryStream = new MemoryStream();
			var logWriter = new LogWriter(new InMemoryFileSystem("test"),memoryStream, new BufferPool());
			versionEdit.EncodeTo(logWriter);

			memoryStream.Position = 0;


			var afterDecode = VersionEdit.DecodeFrom(new LogRecordStream(memoryStream, true, new BufferPool()));

			Assert.Equal(versionEdit.Comparator, afterDecode.Comparator);
			Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
			Assert.Equal(versionEdit.NextFileNumber, afterDecode.NextFileNumber);
			Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
			Assert.Equal(versionEdit.LogNumber, afterDecode.LogNumber);
			Assert.Equal(versionEdit.PrevLogNumber, afterDecode.PrevLogNumber);
			Assert.Equal(versionEdit.LastSequence, afterDecode.LastSequence);

			Assert.Equal(versionEdit.NewFiles[0][0].FileNumber, afterDecode.NewFiles[0][0].FileNumber);
			Assert.Equal(versionEdit.NewFiles[0][0].FileSize, afterDecode.NewFiles[0][0].FileSize);
			Assert.Equal(versionEdit.NewFiles[0][0].LargestKey, afterDecode.NewFiles[0][0].LargestKey);
			Assert.Equal(versionEdit.NewFiles[0][0].SmallestKey, afterDecode.NewFiles[0][0].SmallestKey);

			Assert.Equal(versionEdit.NewFiles[1][0].FileNumber, afterDecode.NewFiles[1][0].FileNumber);
			Assert.Equal(versionEdit.NewFiles[1][0].FileSize, afterDecode.NewFiles[1][0].FileSize);
			Assert.Equal(versionEdit.NewFiles[1][0].LargestKey, afterDecode.NewFiles[1][0].LargestKey);
			Assert.Equal(versionEdit.NewFiles[1][0].SmallestKey, afterDecode.NewFiles[1][0].SmallestKey);

			Assert.Equal(versionEdit.DeletedFiles[0][0], afterDecode.DeletedFiles[0][0]);
			Assert.Equal(versionEdit.DeletedFiles[1][0], afterDecode.DeletedFiles[1][0]);

			Assert.Equal(versionEdit.CompactionPointers[0][0], afterDecode.CompactionPointers[0][0]);
			Assert.Equal(versionEdit.CompactionPointers[1][0], afterDecode.CompactionPointers[1][0]);
		} 
コード例 #6
0
		private static void AddMetadata(VersionEdit edit, StorageOptions options)
		{
			edit.SetComparatorName(options.Comparator.Name);
		}
コード例 #7
0
		public async Task LogAndApplyAsync(VersionEdit edit, AsyncLock.LockScope locker)
		{
			string newManifestFile = null;

			try
			{
				Version version;

				using (await locker.LockAsync())
				{
					if (!edit.LogNumber.HasValue) edit.SetLogNumber(VersionSet.LogNumber);
					else if (edit.LogNumber < VersionSet.LogNumber || edit.LogNumber >= VersionSet.NextFileNumber)
						throw new InvalidOperationException("LogNumber");

					if (!edit.PrevLogNumber.HasValue) edit.SetPrevLogNumber(VersionSet.PrevLogNumber);

					edit.SetNextFile(VersionSet.NextFileNumber);
					edit.SetLastSequence(VersionSet.LastSequence);

					version = new Version(this, VersionSet);

					var builder = new Builder(this, VersionSet, VersionSet.Current);
					builder.Apply(edit);
					builder.SaveTo(version);

					Version.Finalize(version);

					// Initialize new descriptor log file if necessary by creating
					// a temporary file that contains a snapshot of the current version.

					if (DescriptorLogWriter == null)
					{
						// No reason to unlock *mu here since we only hit this path in the
						// first call to LogAndApply (when opening the database).

						newManifestFile = FileSystem.DescriptorFileName(VersionSet.ManifestFileNumber);
						edit.SetNextFile(VersionSet.NextFileNumber);
						var descriptorFile = FileSystem.NewWritable(newManifestFile);

						DescriptorLogWriter = new LogWriter(FileSystem, descriptorFile, Options.BufferPool);

						Snapshooter.WriteSnapshot(DescriptorLogWriter, VersionSet);
					}
				}

				// Write new record to MANIFEST log

				edit.EncodeTo(DescriptorLogWriter);

				// If we just created a new descriptor file, install it by writing a
				// new CURRENT file that points to it.
				if (!string.IsNullOrEmpty(newManifestFile))
				{
					SetCurrentFile(VersionSet.ManifestFileNumber);
					// No need to double-check MANIFEST in case of error since it
					// will be discarded below.
				}

				using (await locker.LockAsync())
				{
					// Install the new version
					VersionSet.AppendVersion(version);
					VersionSet.SetLogNumber(edit.LogNumber.Value);
					VersionSet.SetPrevLogNumber(edit.PrevLogNumber.Value);
				}
			}
			catch (Exception)
			{
				if (!string.IsNullOrEmpty(newManifestFile))
				{
					if (DescriptorLogWriter != null)
					{
						DescriptorLogWriter.Dispose();
						DescriptorLogWriter = null;
					}

					FileSystem.DeleteFile(newManifestFile);
				}

				throw;
			}
		}
コード例 #8
0
		public void CreateNewDatabase()
		{
			var newDb = new VersionEdit();

			newDb.SetComparatorName(Options.Comparator.Name);
			newDb.SetLogNumber(0);
			newDb.SetNextFile(2);
			newDb.SetLastSequence(0);

			var manifest = FileSystem.DescriptorFileName(1);

			try
			{
				using (var file = FileSystem.NewWritable(manifest))
				{
					using (var logWriter = new LogWriter(FileSystem, file, Options.BufferPool))
					{
						newDb.EncodeTo(logWriter);
					}
				}

				SetCurrentFile(1);
			}
			catch (Exception)
			{
				FileSystem.DeleteFile(manifest);

				throw;
			}
		}
コード例 #9
0
		private ulong RecoverLogFile(ulong logNumber, VersionEdit edit, ulong maxSequence)
		{
			var logFileName = FileSystem.GetLogFileName(logNumber);

			Log.Info("Starting to recover from log: {0}", logFileName);

			MemTable mem = null;
			using (var logFile = FileSystem.OpenForReading(logFileName))
			{
				foreach (var item in WriteBatch.ReadFromLog(logFile, Options.BufferPool))
				{
					var lastSequence = item.WriteSequence + (ulong)item.WriteBatch.OperationCount - 1;

					if (lastSequence > maxSequence)
					{
						maxSequence = lastSequence;
					}

					if (mem == null)
					{
						mem = new MemTable(this);
					}

					item.WriteBatch.Prepare(mem);
					item.WriteBatch.Apply(mem, new Reference<ulong> { Value = item.WriteSequence });

					if (mem.ApproximateMemoryUsage > Options.WriteBatchSize)
					{
						Compactor.WriteLevel0Table(mem, null, edit);
						mem = null;
					}
				}
			}

			if (mem != null)
			{
				Compactor.WriteLevel0Table(mem, null, edit);
			}

			return maxSequence;
		}
コード例 #10
0
		public VersionEdit Recover()
		{
			FileSystem.EnsureDatabaseDirectoryExists();
			FileSystem.Lock();

			if (FileSystem.Exists(FileSystem.GetCurrentFileName()) == false)
			{
				if (Options.CreateIfMissing)
				{
					CreateNewDatabase();
				}
				else
				{
					throw new InvalidDataException(DatabaseName + " does not exist. Storage option CreateIfMissing is set to false.");
				}
				Log.Info("Creating db {0} from scratch", DatabaseName);
			}
			else
			{
				if (Options.ErrorIfExists)
				{
					throw new InvalidDataException(DatabaseName + " exists, while the ErrorIfExists option is set to true.");
				}
				Log.Info("Loading db {0} from existing source", DatabaseName);
			}

			VersionSet.Recover();
			var minLog = VersionSet.LogNumber;
			var prevLog = VersionSet.PrevLogNumber;

			var databaseFiles = FileSystem.GetFiles();

			var expected = VersionSet.GetLiveFiles();

			var logNumbers = new List<ulong>();

			foreach (var databaseFile in databaseFiles)
			{
				ulong number;
				FileType fileType;
				if (FileSystem.TryParseDatabaseFile(databaseFile, out number, out fileType))
				{
					expected.Remove(number);

					if (fileType == FileType.LogFile && ((number >= minLog) || (number == prevLog)))
					{
						logNumbers.Add(number);
					}
				}
			}

			if (expected.Count > 0)
			{
				throw new CorruptedDataException(string.Format("Cannot recover because there are {0} missing files", expected.Count));
			}

			logNumbers.Sort();

			ulong maxSequence = 0;
			var edit = new VersionEdit();

			foreach (var logNumber in logNumbers)
			{
				maxSequence = RecoverLogFile(logNumber, edit, maxSequence);
				VersionSet.MarkFileNumberUsed(logNumber);
			}

			if (VersionSet.LastSequence < maxSequence)
			{
				VersionSet.LastSequence = maxSequence;
			}

			return edit;
		}