public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit) { using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath)) { try { var textReader = new JsonTextReader(new StreamReader(commitPointFile)); var jsonCommitPoint = RavenJObject.Load(textReader); var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag"); Etag recoveredEtag = null; if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1} { jsonCommitPoint.Remove("HighestCommitedETag"); recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes")); } indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>(); if (indexCommit == null) return false; if (recoveredEtag != null) indexCommit.HighestCommitedETag = recoveredEtag; if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0) return false; return true; } catch (Exception e) { log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e); indexCommit = null; return false; } } }
public void StoreCommitPoint(string indexName, IndexCommitPoint indexCommit) { if (indexCommit.SegmentsInfo == null || indexCommit.SegmentsInfo.IsIndexCorrupted) return; var directoryName = indexCommit.SegmentsInfo.Generation.ToString("0000000000000000000", CultureInfo.InvariantCulture); var commitPointDirectory = new IndexCommitPointDirectory(path, indexName, directoryName); if (Directory.Exists(commitPointDirectory.AllCommitPointsFullPath) == false) { Directory.CreateDirectory(commitPointDirectory.AllCommitPointsFullPath); } Directory.CreateDirectory(commitPointDirectory.FullPath); using (var commitPointFile = File.Create(commitPointDirectory.FileFullPath)) using (var sw = new StreamWriter(commitPointFile)) { var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer(); var textWriter = new JsonTextWriter(sw); jsonSerializer.Serialize(textWriter, indexCommit); sw.Flush(); } var currentSegmentsFileName = indexCommit.SegmentsInfo.SegmentsFileName; File.Copy(Path.Combine(commitPointDirectory.IndexFullPath, currentSegmentsFileName), Path.Combine(commitPointDirectory.FullPath, currentSegmentsFileName), overwrite: true); var storedCommitPoints = Directory.GetDirectories(commitPointDirectory.AllCommitPointsFullPath); if (storedCommitPoints.Length > configuration.MaxNumberOfStoredCommitPoints) { foreach (var toDelete in storedCommitPoints.Take(storedCommitPoints.Length - configuration.MaxNumberOfStoredCommitPoints)) { IOExtensions.DeleteDirectory(toDelete); } } }
private bool TryReusePreviousCommitPointsToRecoverIndex(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete) { indexCommit = null; keysToDelete = null; if (indexDefinition.IsMapReduce) return false; var indexFullPath = Path.Combine(indexStoragePath, indexDefinition.IndexId.ToString()); var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath); if (Directory.Exists(allCommitPointsFullPath) == false) return false; var filesInIndexDirectory = Directory.GetFiles(indexFullPath).Select(Path.GetFileName); var existingCommitPoints = IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath); Array.Reverse(existingCommitPoints); // start from the highest generation foreach (var commitPointDirectoryName in existingCommitPoints) { try { var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.IndexId.ToString(), commitPointDirectoryName); if (TryGetCommitPoint(commitPointDirectory, out indexCommit) == false) { IOExtensions.DeleteDirectory(commitPointDirectory.FullPath); continue; // checksum is invalid, try another commit point } var missingFile = indexCommit.SegmentsInfo.ReferencedFiles.Any( referencedFile => filesInIndexDirectory.Contains(referencedFile) == false); if (missingFile) { IOExtensions.DeleteDirectory(commitPointDirectory.FullPath); continue; // there are some missing files, try another commit point } var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName; // here there should be only one segments_N file, however remove all if there is more foreach (var currentSegmentsFile in Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*")) { File.Delete(currentSegmentsFile); } // copy old segments_N file File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile), Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true); try { // update segments.gen file using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN)) { genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS); genOutput.WriteLong(indexCommit.SegmentsInfo.Generation); genOutput.WriteLong(indexCommit.SegmentsInfo.Generation); } } catch (Exception) { // here we can ignore, segments.gen is used only as fallback } if (File.Exists(commitPointDirectory.DeletedKeysFile)) keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray(); return true; } catch (Exception ex) { startupLog.WarnException("Could not recover an index named '" + indexDefinition.IndexId + "'from segments of the following generation " + commitPointDirectoryName, ex); } } return false; }
private static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit) { using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath)) { var jsonSerializer = new JsonSerializer(); var textReader = new JsonTextReader(new StreamReader(commitPointFile)); indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader); return indexCommit != null; } }