private static void ParseAndValidate(string text, int expectedLength) { var reader = new StringReader(text); var jsonTextReader = new JsonTextReader(reader); Assert.True(jsonTextReader.Read()); Assert.Equal(JsonToken.StartObject, jsonTextReader.TokenType); Assert.True(jsonTextReader.Read()); Assert.Equal(JsonToken.PropertyName, jsonTextReader.TokenType); Assert.Equal("Data", jsonTextReader.Value); var memoryStream = jsonTextReader.ReadBytesAsStream(); Assert.Equal(expectedLength, memoryStream.Length); Assert.True(jsonTextReader.Read()); Assert.Equal(JsonToken.PropertyName, jsonTextReader.TokenType); Assert.Equal("Metadata", jsonTextReader.Value); Assert.True(jsonTextReader.Read()); Assert.Equal(JsonToken.Null, jsonTextReader.TokenType); Assert.True(jsonTextReader.Read()); Assert.Equal(JsonToken.EndObject, jsonTextReader.TokenType); Assert.False(jsonTextReader.Read()); }
public void FloatParseHandling() { string json = "[1.0,1,9.9,1E-06]"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); reader.FloatParseHandling = Json.FloatParseHandling.Decimal; Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.StartArray, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(1.0m, reader.Value); Assert.AreEqual(typeof(decimal), reader.ValueType); Assert.AreEqual(JsonToken.Float, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(1L, reader.Value); Assert.AreEqual(typeof(long), reader.ValueType); Assert.AreEqual(JsonToken.Integer, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(9.9m, reader.Value); Assert.AreEqual(typeof(decimal), reader.ValueType); Assert.AreEqual(JsonToken.Float, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(Convert.ToDecimal(1E-06), reader.Value); Assert.AreEqual(typeof(decimal), reader.ValueType); Assert.AreEqual(JsonToken.Float, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.EndArray, reader.TokenType); }
public virtual object Deserialize(Stream stream, System.Type type) { using (StreamReader r = new StreamReader(stream)) using (var jsonReader = new JsonTextReader(r)) using (RavenJTokenReader tokenReader = new RavenJTokenReader(RavenJToken.ReadFrom(jsonReader))) { return serializer.Deserialize(tokenReader, type); } }
public void CanDeserialiseNullableStruct() { var reader = new JsonTextReader(new StringReader("{\"Value\":{\"X\":\"Bob\"}}")); var i2 = _serialiser.Deserialize<C>(reader); Assert.That(i2, Is.Not.Null); Assert.That(i2.Value, Is.Not.Null); Assert.That(((D)i2.Value).X, Is.EqualTo("Bob")); }
public DumperStats Import(string file) { var stopwatch = Stopwatch.StartNew(); using (var streamReader = new StreamReader(new GZipStream(File.OpenRead(file), CompressionMode.Decompress))) { var jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return new DumperStats(); if (jsonReader.TokenType != JsonToken.StartObject) throw new InvalidDataException("StartObject was expected"); var indexes = new Dictionary<string, IndexDefinition>(); var indexCount = 0; Read(jsonReader, "Indexes", index => { indexCount++; var indexName = index.Value<string>("name"); if (!indexName.StartsWith("Raven/")) { indexes[indexName] = index.Value<RavenJObject>("definition").JsonDeserialization<IndexDefinition>(); } }); foreach (var index in indexes) { _documentDatabase.PutIndex(index.Key, index.Value); } var total = 0; var batch = new List<RavenJObject>(); Read(jsonReader, "Docs", document => { total++; batch.Add((RavenJObject)document); if (batch.Count >= 128) { FlushBatch(batch); } }); FlushBatch(batch); stopwatch.Stop(); return new DumperStats { Attachments = 0, Documents = total, Indexes = indexCount, Elapsed = stopwatch.Elapsed }; } }
public void ImportJobs(string jobsJson) { var serializer = this.DocumentSession.Advanced.DocumentStore.Conventions.CreateSerializer(); using (var reader = new StringReader(jobsJson)) using (var jsonReader = new JsonTextReader(reader)) { var data = serializer.Deserialize<List<JobDefinition>>(jsonReader); this.JobManager.ImportJobs(data); } }
public void Can_read_date_time_offset_from_lucene_query() { var jsonSerializer = new DocumentConvention().CreateSerializer(); using (var reader = new JsonTextReader(new StringReader(@"{""Item"": ""20090402193554412""}"))) { var deserialize = jsonSerializer.Deserialize<Test>(reader); Assert.Equal(2009, deserialize.Item.Year); } }
public void ReadSingleQuoteInsideDoubleQuoteString() { string json = @"{""NameOfStore"":""Forest's Bakery And Cafe""}"; JsonTextReader jsonTextReader = new JsonTextReader(new StringReader(json)); jsonTextReader.Read(); jsonTextReader.Read(); jsonTextReader.Read(); Assert.AreEqual(@"Forest's Bakery And Cafe", jsonTextReader.Value); }
public void CanDeserialiseInheritedObject() { var item = new A() { Name = "B", Children = new List<A>() { new B() { Id = 10, Name = "Child B", }, new A { Name = "Child A" } } }; var json = new StringWriter(); _serialiser.Serialize(json, item); var reader = new JsonTextReader(new StringReader(json.ToString())); var b = _serialiser.Deserialize<A>(reader); Assert.That(b.Children.First(), Is.InstanceOf<B>()); Assert.That(((B)b.Children.First()).Id, Is.EqualTo(10)); }
public void ReadMultilineString() { string json = @"""first line second line third line"""; JsonTextReader jsonTextReader = new JsonTextReader(new StringReader(json)); Assert.IsTrue(jsonTextReader.Read()); Assert.AreEqual(JsonToken.String, jsonTextReader.TokenType); Assert.AreEqual(@"first line second line third line", jsonTextReader.Value); }
public ActionResult Import(HttpPostedFileBase fileData) { string fileString = StreamHelper.ReadAll(fileData.InputStream); var serializer = this.DocumentSession.Advanced.DocumentStore.Conventions.CreateSerializer(); using (var streamReader = new StreamReader(fileData.InputStream)) { using (var jsonReader = new JsonTextReader(streamReader)) { var data = serializer.Deserialize<ImportExportFile>(jsonReader); this.JobManager.ImportJobs(data.JobDefinitionList); this.SettingsManager.ImportSettings(data.SettingsContainerList); return RedirectToAction("ImportComplete"); } } }
public void SurrogatePairValid() { string json = @"{ ""MATHEMATICAL ITALIC CAPITAL ALPHA"": ""\uD835\uDEE2"" }"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); Assert.IsTrue(reader.Read()); Assert.IsTrue(reader.Read()); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.String, reader.TokenType); string s = reader.Value.ToString(); Assert.AreEqual(2, s.Length); StringInfo stringInfo = new StringInfo(s); Assert.AreEqual(1, stringInfo.LengthInTextElements); }
public async Task Non_existing_data_folder_should_throw_proper_exception() { using (var store = NewStore(requestedStorage: "voron", runInMemory: true)) { using (var session = store.OpenAsyncSession()) { session.RegisterUpload("test.file", new MemoryStream(new byte[] { 1, 2, 3 })); await session.SaveChangesAsync(); } await store.AsyncFilesCommands.Admin.StartBackup(backupLocation, null, false, store.DefaultFileSystem); WaitForBackup(store.AsyncFilesCommands, true); FileSystemDocument document; using (var file = File.OpenText(Path.Combine(backupLocation, Constants.FilesystemDocumentFilename))) using (var reader = new JsonTextReader(file)) document = (new JsonSerializer()).Deserialize<FileSystemDocument>(reader); var drives = DriveInfo.GetDrives().Select(x => x.Name.ToLower()[0]).ToArray(); var lastDriveLetter = 'a'; while (lastDriveLetter != 'z') { if (drives.Contains(lastDriveLetter) == false) break; lastDriveLetter++; } document.Settings[Constants.FileSystem.DataDirectory] = string.Format("{0}:\\", (char)(lastDriveLetter )); //on purpose, non existing path using (var file = File.CreateText(Path.Combine(backupLocation, Constants.FilesystemDocumentFilename))) using (var writer = new JsonTextWriter(file)) (new JsonSerializer()).Serialize(writer, document); Assert.Throws<BadRequestException>(() => AsyncHelpers.RunSync(() => store.AsyncFilesCommands.Admin.StartRestore(new FilesystemRestoreRequest { BackupLocation = backupLocation }))); } }
public override void Execute(object parameter) { if (string.IsNullOrWhiteSpace(queryModel.Address)) return; var url = "http://dev.virtualearth.net/REST/v1/Locations?q=" + Uri.EscapeUriString(queryModel.Address) + "&key=Anlj2YMQu676uXmSj1QTSni66f8DjuBGToZ21t5z9E__lL8IHRhFP8LtF7umitL6"; var webRequest = WebRequest.Create(new Uri(url, UriKind.Absolute)); webRequest.GetResponseAsync().ContinueOnSuccessInTheUIThread(doc => { RavenJObject jsonData; using (var stream = doc.GetResponseStream()) using (var reader = new StreamReader(stream)) using (var jsonReader = new JsonTextReader(reader)) jsonData = RavenJObject.Load(jsonReader); var set = jsonData["resourceSets"]; var item = set.Values().First().Values().ToList()[1].Values().ToList(); if (item.Count == 0) { ApplicationModel.Current.AddInfoNotification("Could not calculate the given address"); return; } var result = item.First().SelectToken("point").SelectToken("coordinates").Values().ToList(); if (result != null) { var latitude = double.Parse(result[0].ToString()); var longitude = double.Parse(result[1].ToString()); var addressData = new AddressData { Address = queryModel.Address, Latitude = latitude, Longitude = longitude }; queryModel.UpdateResultsFromCalculate(addressData); } }).Catch(); }
public void ReadSingleValueAsStreamShouldWork(int size) { var random = new Random(size); var buffer = new byte[size]; random.NextBytes(buffer); var d = new Data { Foo = buffer }; var jsonObj = RavenJToken.FromObject(d); var jsonObjAsString = jsonObj.ToString(); using (var textReader = new StringReader(jsonObjAsString)) using (var jsonReader = new JsonTextReader(textReader)) { jsonReader.Read(); // start object jsonReader.Read(); // property name var actual = new byte[buffer.Length]; using (var objectStream = jsonReader.ReadBytesAsStream()) { objectStream.Read(actual, 0, actual.Length); for (int a = 0; a < actual.Length; a++) { if (buffer[a] != actual[a]) { Assert.True(false, "not equal on byte " + a); } } } } }
public void ThrowErrorWhenParsingUnquoteStringThatStartsWithNE() { const string json = @"{ ""ItemName"": ""value"", ""u"":netanelsalinger,""r"":9 }"; JsonTextReader reader = new JsonTextReader(new StringReader(json)); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.StartObject, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.PropertyName, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.String, reader.TokenType); Assert.IsTrue(reader.Read()); Assert.AreEqual(JsonToken.PropertyName, reader.TokenType); ExceptionAssert.Throws<JsonReaderException>("Unexpected content while parsing JSON. Path 'u', line 1, position 27.", () => { reader.Read(); }); }
private static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit) { using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath)) { var jsonSerializer = new JsonSerializer(); var textReader = new JsonTextReader(new StreamReader(commitPointFile)); indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader); return indexCommit != null; } }
public static IndexDefinition CreateIndexDefinition() { // this is how the index looks like in JSON var jsonIndexDefinition = @" { ""Map"" : "" from e in docs.Events select new { Tag = \""Event\"", _lat = SpatialIndex.Lat(e.Latitude), _lng = SpatialIndex.Lng(e.Longitude), _tier_2 = SpatialIndex.Tier(2, e.Latitude, e.Longitude), _tier_3 = SpatialIndex.Tier(3, e.Latitude, e.Longitude), _tier_4 = SpatialIndex.Tier(4, e.Latitude, e.Longitude), _tier_5 = SpatialIndex.Tier(5, e.Latitude, e.Longitude), _tier_6 = SpatialIndex.Tier(6, e.Latitude, e.Longitude), _tier_7 = SpatialIndex.Tier(7, e.Latitude, e.Longitude), _tier_8 = SpatialIndex.Tier(8, e.Latitude, e.Longitude), _tier_9 = SpatialIndex.Tier(9, e.Latitude, e.Longitude), _tier_10 = SpatialIndex.Tier(10, e.Latitude, e.Longitude), _tier_11 = SpatialIndex.Tier(11, e.Latitude, e.Longitude), _tier_12 = SpatialIndex.Tier(12, e.Latitude, e.Longitude), _tier_13 = SpatialIndex.Tier(13, e.Latitude, e.Longitude), _tier_14 = SpatialIndex.Tier(14, e.Latitude, e.Longitude), _tier_15 = SpatialIndex.Tier(15, e.Latitude, e.Longitude) }"", ""Stores"" :{ ""latitude"" : ""Yes"", ""longitude"" : ""Yes"", ""_tier_2"" : ""Yes"", ""_tier_3"" : ""Yes"", ""_tier_4"" : ""Yes"", ""_tier_5"" : ""Yes"", ""_tier_6"" : ""Yes"", ""_tier_7"" : ""Yes"", ""_tier_8"" : ""Yes"", ""_tier_9"" : ""Yes"", ""_tier_10"" : ""Yes"", ""_tier_11"" : ""Yes"", ""_tier_12"" : ""Yes"", ""_tier_13"" : ""Yes"", ""_tier_14"" : ""Yes"", ""_tier_15"" : ""Yes"" }, ""Indexes"" :{ ""Tag"" : ""NotAnalyzed"", ""latitude"" : ""NotAnalyzed"", ""longitude"" : ""NotAnalyzed"", ""_tier_2"" : ""NotAnalyzedNoNorms"", ""_tier_3"" : ""NotAnalyzedNoNorms"", ""_tier_4"" : ""NotAnalyzedNoNorms"", ""_tier_5"" : ""NotAnalyzedNoNorms"", ""_tier_6"" : ""NotAnalyzedNoNorms"", ""_tier_7"" : ""NotAnalyzedNoNorms"", ""_tier_8"" : ""NotAnalyzedNoNorms"", ""_tier_9"" : ""NotAnalyzedNoNorms"", ""_tier_10"" : ""NotAnalyzedNoNorms"", ""_tier_11"" : ""NotAnalyzedNoNorms"", ""_tier_12"" : ""NotAnalyzedNoNorms"", ""_tier_13"" : ""NotAnalyzedNoNorms"", ""_tier_14"" : ""NotAnalyzedNoNorms"", ""_tier_15"" : ""NotAnalyzedNoNorms"" } }"; using (var stringReader = new StringReader(jsonIndexDefinition)) using (var jsonReader = new JsonTextReader(stringReader)) { return JsonExtensions.CreateDefaultJsonSerializer().Deserialize<IndexDefinition>(jsonReader); } }
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options) { var count = 0; if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Docs", jsonReader.Value) == false) throw new InvalidDataException("Docs property was expected"); if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var size = GetRoughSize(document); if (size > 1024 * 1024) { Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}", (double)size / 1024, document["@metadata"].Value<string>("@id")); } if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents) continue; if (options.MatchFilters(document) == false) continue; if (!string.IsNullOrEmpty(options.TransformScript)) document = await TransformDocument(document, options.TransformScript); if (document == null) continue; await PutDocument(document); count++; if (count % options.BatchSize == 0) { ShowProgress("Read {0} documents", count); } } await PutDocument(null); // force flush return count; }
private async Task<int> ImportAttachments(JsonTextReader jsonReader, SmugglerOptions options) { var count = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) return count; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Attachments", jsonReader.Value) == false) throw new InvalidDataException("Attachment property was expected"); if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var item = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments) continue; var attachmentExportInfo = new JsonSerializer { Converters = { new JsonToJsonConverter() } }.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item)); ShowProgress("Importing attachment {0}", attachmentExportInfo.Key); await PutAttachment(attachmentExportInfo); count++; } await PutAttachment(null); // force flush return count; }
private async Task<int> ImportTransformers(JsonTextReader jsonReader, SmugglerOptions options) { var count = 0; if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject) return count; if (jsonReader.TokenType != JsonToken.PropertyName) throw new InvalidDataException("PropertyName was expected"); if (Equals("Transformers", jsonReader.Value) == false) throw new InvalidDataException("Transformers property was expected"); if (jsonReader.Read() == false) return count; if (jsonReader.TokenType != JsonToken.StartArray) throw new InvalidDataException("StartArray was expected"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { var transformer = RavenJToken.ReadFrom(jsonReader); if ((options.OperateOnTypes & ItemType.Transformers) != ItemType.Transformers) continue; var transformerName = transformer.Value<string>("name"); await PutTransformer(transformerName, transformer); count++; } await PutTransformer(null, null); // force flush return count; }
public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit) { using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath)) { try { var textReader = new JsonTextReader(new StreamReader(commitPointFile)); var jsonCommitPoint = RavenJObject.Load(textReader); var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag"); Etag recoveredEtag = null; if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1} { jsonCommitPoint.Remove("HighestCommitedETag"); recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes")); } indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>(); if (indexCommit == null) return false; if (recoveredEtag != null) indexCommit.HighestCommitedETag = recoveredEtag; if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0) return false; return true; } catch (Exception e) { log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e); indexCommit = null; return false; } } }
private static void JsonBodyToSoapXml(Stream json, Stream xml) { Raven.Imports.Newtonsoft.Json.JsonSerializerSettings settings = new Raven.Imports.Newtonsoft.Json.JsonSerializerSettings(); settings.Converters.Add(new Raven.Imports.Newtonsoft.Json.Converters.XmlNodeConverter()); Raven.Imports.Newtonsoft.Json.JsonSerializer serializer = Raven.Imports.Newtonsoft.Json.JsonSerializer.Create(settings); using (Newtonsoft.Json.JsonTextReader reader = new Raven.Imports.Newtonsoft.Json.JsonTextReader(new System.IO.StreamReader(json))) { XmlDocument doc = (XmlDocument)serializer.Deserialize(reader, typeof(XmlDocument)); if (reader.Read() && reader.TokenType != JsonToken.Comment) throw new JsonSerializationException("Additional text found in JSON string after finishing deserializing object."); using (XmlWriter writer = XmlWriter.Create(xml)) { doc.Save(writer); } } }
public void UnexpectedEndWhenParsingUnquotedProperty() { JsonReader reader = new JsonTextReader(new StringReader(@"{aww")); Assert.IsTrue(reader.Read()); ExceptionAssert.Throws<JsonReaderException>( "Unexpected end while parsing unquoted property name. Path '', line 1, position 4.", () => { reader.Read(); }); }
public void ShouldDeleteCommitPointIfCouldNotRecoverFromIt() { var dataDir = NewDataPath("ShouldDeleteCommitPointIfCouldNotRecoverFromIt"); string indexFullPath; string commitPointsDirectory; var index = new MapRecoveryTestIndex(); using (var server = GetNewServer(runInMemory: false, dataDirectory: dataDir)) { CommitPointAfterEachCommit(server.Database.Configuration); indexFullPath = Path.Combine(server.Database.Configuration.IndexStoragePath, MonoHttpUtility.UrlEncode(index.IndexName)); commitPointsDirectory = Path.Combine(server.Database.Configuration.IndexStoragePath, MonoHttpUtility.UrlEncode(index.IndexName) + "\\CommitPoints"); using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { index.Execute(store); using (var session = store.OpenSession()) { session.Store(new Recovery { Name = "One", Number = 1 }); session.SaveChanges(); // first commit point WaitForIndexing(store); session.Store(new Recovery { Name = "Two", Number = 2 }); session.SaveChanges(); // second commit point WaitForIndexing(store); session.Store(new Recovery { Name = "Three", Number = 3 }); session.SaveChanges(); // second commit point WaitForIndexing(store, timeout: TimeSpan.FromSeconds(60)); } } } // make sure that there are 3 commit points var directories = Directory.GetDirectories(commitPointsDirectory); Assert.Equal(3, directories.Length); // mess "index.CommitPoint" file in the SECOND commit point by adding additional files required to recover from it using (var commitPointFile = File.Open(Path.Combine(directories[1], "index.CommitPoint"), FileMode.Open)) { var jsonSerializer = new JsonSerializer(); var textReader = new JsonTextReader(new StreamReader(commitPointFile)); var indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader); indexCommit.SegmentsInfo.ReferencedFiles.Add("file-that-doesnt-exist"); commitPointFile.Position = 0; using (var sw = new StreamWriter(commitPointFile)) { var textWriter = new JsonTextWriter(sw); jsonSerializer.Serialize(textWriter, indexCommit); sw.Flush(); } } // mess "index.CommitPoint" file in the THIRD commit point by adding additional files required to recover from it using (var commitPointFile = File.Open(Path.Combine(directories[2], "index.CommitPoint"), FileMode.Open)) { var jsonSerializer = new JsonSerializer(); var textReader = new JsonTextReader(new StreamReader(commitPointFile)); var indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader); indexCommit.SegmentsInfo.ReferencedFiles.Add("file-that-doesnt-exist"); commitPointFile.Position = 0; using (var sw = new StreamWriter(commitPointFile)) { var textWriter = new JsonTextWriter(sw); jsonSerializer.Serialize(textWriter, indexCommit); sw.Flush(); } } IndexMessing.MessSegmentsFile(indexFullPath); using (GetNewServer(runInMemory: false, dataDirectory: dataDir)) // do not delete previous directory { using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize()) { using (var session = store.OpenSession()) { var result = session.Query<Recovery, MapRecoveryTestIndex>().Customize(x => x.WaitForNonStaleResults()).ToList(); Assert.Equal(3, result.Count); } } } // there should be exactly 2 commit points: // the first one which we used to recover // and the second one created because of indexing after recovery Assert.Equal(2, Directory.GetDirectories(commitPointsDirectory).Length); }
private static bool TryReusePreviousCommitPointsToRecoverIndex(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete) { indexCommit = null; keysToDelete = null; if (indexDefinition.IsMapReduce) return false; var indexFullPath = Path.Combine(indexStoragePath, MonoHttpUtility.UrlEncode(indexDefinition.Name)); var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath); if (Directory.Exists(allCommitPointsFullPath) == false) return false; var filesInIndexDirectory = Directory.GetFiles(indexFullPath).Select(Path.GetFileName); var existingCommitPoints = IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath); Array.Reverse(existingCommitPoints); // start from the highest generation foreach (var commitPointDirectoryName in existingCommitPoints) { try { var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.Name, commitPointDirectoryName); using (var commitPointFile = File.Open(commitPointDirectory.FileFullPath, FileMode.Open)) { var jsonSerializer = new JsonSerializer(); var textReader = new JsonTextReader(new StreamReader(commitPointFile)); indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader); } var missingFile = indexCommit.SegmentsInfo.ReferencedFiles.Any( referencedFile => filesInIndexDirectory.Contains(referencedFile) == false); if (missingFile) { IOExtensions.DeleteDirectory(commitPointDirectory.FullPath); continue; // there are some missing files, try another commit point } var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName; // here there should be only one segments_N file, however remove all if there is more foreach (var currentSegmentsFile in Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*")) { File.Delete(currentSegmentsFile); } // copy old segments_N file File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile), Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true); try { // update segments.gen file using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN)) { genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS); genOutput.WriteLong(indexCommit.SegmentsInfo.Generation); genOutput.WriteLong(indexCommit.SegmentsInfo.Generation); } } catch (Exception) { // here we can ignore, segments.gen is used only as fallback } if (File.Exists(commitPointDirectory.DeletedKeysFile)) keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray(); return true; } catch (Exception ex) { startupLog.WarnException("Could not recover an index named '" + indexDefinition.Name + "'from segments of the following generation " + commitPointDirectoryName, ex); } } return false; }
private XmlNode DeserializeXmlNode(string json, string deserializeRootElementName) { JsonTextReader reader; reader = new JsonTextReader(new StringReader(json)); reader.Read(); XmlNodeConverter converter = new XmlNodeConverter(); if (deserializeRootElementName != null) converter.DeserializeRootElementName = deserializeRootElementName; XmlNode node = (XmlNode)converter.ReadJson(reader, typeof (XmlDocument), null, new JsonSerializer()); #if !NET20 string xmlText = node.OuterXml; reader = new JsonTextReader(new StringReader(json)); reader.Read(); XDocument d = (XDocument) converter.ReadJson(reader, typeof (XDocument), null, new JsonSerializer()); string linqXmlText = d.ToString(SaveOptions.DisableFormatting); if (d.Declaration != null) linqXmlText = d.Declaration + linqXmlText; Assert.AreEqual(xmlText, linqXmlText); #endif return node; }
public static void ReadLastEtagsFromFile(SmugglerOptions options) { var log = LogManager.GetCurrentClassLogger(); var etagFileLocation = Path.Combine(options.BackupPath, IncrementalExportStateFile); if (!File.Exists(etagFileLocation)) return; using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open))) using (var jsonReader = new JsonTextReader(streamReader)) { RavenJObject ravenJObject; try { ravenJObject = RavenJObject.Load(jsonReader); } catch (Exception e) { log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e); return; } options.LastDocsEtag = Etag.Parse(ravenJObject.Value<string>("LastDocEtag")); options.LastAttachmentEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentEtag")); } }
public void ReadBytesFromEmptyString() { var bytes = new HasBytes { Bytes = new byte[0] }; var json = JsonConvert.SerializeObject(bytes); TextReader textReader = new StringReader(json); JsonReader jsonReader = new JsonTextReader(textReader); var jToken = JToken.ReadFrom(jsonReader); jsonReader = new JTokenReader(jToken); var result2 = (HasBytes)JsonSerializer.Create(null) .Deserialize(jsonReader, typeof(HasBytes)); CollectionAssert.AreEquivalent(new byte[0], result2.Bytes); }
public async virtual Task ImportData(Stream stream, SmugglerOptions options) { options = options ?? SmugglerOptions; if (options == null) throw new ArgumentNullException("options"); await DetectServerSupportedFeatures(); await EnsureDatabaseExists(); Stream sizeStream; var sw = Stopwatch.StartNew(); // Try to read the stream compressed, otherwise continue uncompressed. JsonTextReader jsonReader; try { sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress)); var streamReader = new StreamReader(sizeStream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } catch (Exception e) { if (e is InvalidDataException == false #if SILVERLIGHT && e is ZlibException == false #endif ) throw; stream.Seek(0, SeekOrigin.Begin); sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress)); var streamReader = new StreamReader(stream); jsonReader = new JsonTextReader(streamReader); if (jsonReader.Read() == false) return; } if (jsonReader.TokenType != JsonToken.StartObject) throw new InvalidDataException("StartObject was expected"); ShowProgress("Begin reading indexes"); var indexCount = await ImportIndexes(jsonReader, options); ShowProgress(string.Format("Done with reading indexes, total: {0}", indexCount)); ShowProgress("Begin reading documents"); var documentCount = await ImportDocuments(jsonReader, options); ShowProgress(string.Format("Done with reading documents, total: {0}", documentCount)); ShowProgress("Begin reading attachments"); var attachmentCount = await ImportAttachments(jsonReader, options); ShowProgress(string.Format("Done with reading attachments, total: {0}", attachmentCount)); ShowProgress("Begin reading transformers"); var transformersCount = await ImportTransformers(jsonReader, options); ShowProgress(string.Format("Done with reading transformers, total: {0}", transformersCount)); sw.Stop(); ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", documentCount, attachmentCount, sw.ElapsedMilliseconds); }