Example #1
0
		public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
		{
			using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
			{
				try
				{
					var textReader = new JsonTextReader(new StreamReader(commitPointFile));
					var jsonCommitPoint = RavenJObject.Load(textReader);
					var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag");

					Etag recoveredEtag = null;
					if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1}
					{
						jsonCommitPoint.Remove("HighestCommitedETag");
						recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes"));
					}

					indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>();

					if (indexCommit == null)
						return false;

					if (recoveredEtag != null)
						indexCommit.HighestCommitedETag = recoveredEtag;

					if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0)
						return false;

					return true;
				}
				catch (Exception e)
				{
					log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e);

					indexCommit = null;
					return false;
				}
			}
		}
Example #2
0
		private bool TryReusePreviousCommitPointsToRecoverIndex(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete)
		{
			indexCommit = null;
			keysToDelete = null;

			if (indexDefinition.IsMapReduce)
				return false;

			var indexFullPath = Path.Combine(indexStoragePath, indexDefinition.IndexId.ToString());



			var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath);

			if (Directory.Exists(allCommitPointsFullPath) == false)
				return false;

			var filesInIndexDirectory = Directory.GetFiles(indexFullPath).Select(Path.GetFileName);

			var existingCommitPoints =
				IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);

			Array.Reverse(existingCommitPoints); // start from the highest generation

			foreach (var commitPointDirectoryName in existingCommitPoints)
			{
				try
				{
					var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.IndexId.ToString(),
																				commitPointDirectoryName);

					if (TryGetCommitPoint(commitPointDirectory, out indexCommit) == false)
					{
						IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
						continue; // checksum is invalid, try another commit point
					}

					var missingFile =
						indexCommit.SegmentsInfo.ReferencedFiles.Any(
							referencedFile => filesInIndexDirectory.Contains(referencedFile) == false);

					if (missingFile)
					{
						IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
						continue; // there are some missing files, try another commit point
					}

					var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName;

					// here there should be only one segments_N file, however remove all if there is more
					foreach (var currentSegmentsFile in Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*"))
					{
						File.Delete(currentSegmentsFile);
					}

					// copy old segments_N file
					File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile),
							  Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true);

					try
					{
						// update segments.gen file
						using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN))
						{
							genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS);
							genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
							genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
						}
					}
					catch (Exception)
					{
						// here we can ignore, segments.gen is used only as fallback
					}

					if (File.Exists(commitPointDirectory.DeletedKeysFile))
						keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray();

					return true;
				}
				catch (Exception ex)
				{
					startupLog.WarnException("Could not recover an index named '" + indexDefinition.IndexId +
									   "'from segments of the following generation " + commitPointDirectoryName, ex);
				}
			}

			return false;
		}
Example #3
0
		private static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
		{
			using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
			{
				var jsonSerializer = new JsonSerializer();
				var textReader = new JsonTextReader(new StreamReader(commitPointFile));

				indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader);

				return indexCommit != null;
			}
		}
Example #4
0
		public void StoreCommitPoint(string indexName, IndexCommitPoint indexCommit)
		{
			if (indexCommit.SegmentsInfo == null || indexCommit.SegmentsInfo.IsIndexCorrupted)
				return;

			var directoryName = indexCommit.SegmentsInfo.Generation.ToString("0000000000000000000", CultureInfo.InvariantCulture);
			var commitPointDirectory = new IndexCommitPointDirectory(path, indexName, directoryName);

			if (Directory.Exists(commitPointDirectory.AllCommitPointsFullPath) == false)
			{
				Directory.CreateDirectory(commitPointDirectory.AllCommitPointsFullPath);
			}

			Directory.CreateDirectory(commitPointDirectory.FullPath);

			using (var commitPointFile = File.Create(commitPointDirectory.FileFullPath))
			using (var sw = new StreamWriter(commitPointFile))
			{
				var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
				var textWriter = new JsonTextWriter(sw);

				jsonSerializer.Serialize(textWriter, indexCommit);
				sw.Flush();
			}

			var currentSegmentsFileName = indexCommit.SegmentsInfo.SegmentsFileName;

			File.Copy(Path.Combine(commitPointDirectory.IndexFullPath, currentSegmentsFileName),
						Path.Combine(commitPointDirectory.FullPath, currentSegmentsFileName),
						overwrite: true);

			var storedCommitPoints = Directory.GetDirectories(commitPointDirectory.AllCommitPointsFullPath);

			if (storedCommitPoints.Length > configuration.MaxNumberOfStoredCommitPoints)
			{
				foreach (var toDelete in storedCommitPoints.Take(storedCommitPoints.Length - configuration.MaxNumberOfStoredCommitPoints))
				{
					IOExtensions.DeleteDirectory(toDelete);
				}
			}
		}
Example #5
0
		private void ResetLastIndexedEtagAccordingToRestoredCommitPoint(IndexDefinition indexDefinition,
																		IndexCommitPoint lastCommitPoint)
		{
			documentDatabase.TransactionalStorage.Batch(
				accessor =>
				accessor.Indexing.UpdateLastIndexed(indexDefinition.Name, lastCommitPoint.HighestCommitedETag,
													lastCommitPoint.TimeStamp));
		}