private static void ParseAndValidate(string text, int expectedLength)
{
var reader = new StringReader(text);
var jsonTextReader = new JsonTextReader(reader);
Assert.True(jsonTextReader.Read());
Assert.Equal(JsonToken.StartObject, jsonTextReader.TokenType);
Assert.True(jsonTextReader.Read());
Assert.Equal(JsonToken.PropertyName, jsonTextReader.TokenType);
Assert.Equal("Data", jsonTextReader.Value);
var memoryStream = jsonTextReader.ReadBytesAsStream();
Assert.Equal(expectedLength, memoryStream.Length);
Assert.True(jsonTextReader.Read());
Assert.Equal(JsonToken.PropertyName, jsonTextReader.TokenType);
Assert.Equal("Metadata", jsonTextReader.Value);
Assert.True(jsonTextReader.Read());
Assert.Equal(JsonToken.Null, jsonTextReader.TokenType);
Assert.True(jsonTextReader.Read());
Assert.Equal(JsonToken.EndObject, jsonTextReader.TokenType);
Assert.False(jsonTextReader.Read());
}
public void FloatParseHandling()
{
string json = "[1.0,1,9.9,1E-06]";
JsonTextReader reader = new JsonTextReader(new StringReader(json));
reader.FloatParseHandling = Json.FloatParseHandling.Decimal;
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.StartArray, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(1.0m, reader.Value);
Assert.AreEqual(typeof(decimal), reader.ValueType);
Assert.AreEqual(JsonToken.Float, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(1L, reader.Value);
Assert.AreEqual(typeof(long), reader.ValueType);
Assert.AreEqual(JsonToken.Integer, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(9.9m, reader.Value);
Assert.AreEqual(typeof(decimal), reader.ValueType);
Assert.AreEqual(JsonToken.Float, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(Convert.ToDecimal(1E-06), reader.Value);
Assert.AreEqual(typeof(decimal), reader.ValueType);
Assert.AreEqual(JsonToken.Float, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.EndArray, reader.TokenType);
}
public virtual object Deserialize(Stream stream, System.Type type)
{
using (StreamReader r = new StreamReader(stream))
using (var jsonReader = new JsonTextReader(r))
using (RavenJTokenReader tokenReader = new RavenJTokenReader(RavenJToken.ReadFrom(jsonReader)))
{
return serializer.Deserialize(tokenReader, type);
}
}
public void CanDeserialiseNullableStruct()
{
var reader = new JsonTextReader(new StringReader("{\"Value\":{\"X\":\"Bob\"}}"));
var i2 = _serialiser.Deserialize<C>(reader);
Assert.That(i2, Is.Not.Null);
Assert.That(i2.Value, Is.Not.Null);
Assert.That(((D)i2.Value).X, Is.EqualTo("Bob"));
}
public DumperStats Import(string file)
{
var stopwatch = Stopwatch.StartNew();
using (var streamReader =
new StreamReader(new GZipStream(File.OpenRead(file), CompressionMode.Decompress)))
{
var jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return new DumperStats();
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
var indexes = new Dictionary<string, IndexDefinition>();
var indexCount = 0;
Read(jsonReader,
"Indexes",
index =>
{
indexCount++;
var indexName = index.Value<string>("name");
if (!indexName.StartsWith("Raven/"))
{
indexes[indexName] = index.Value<RavenJObject>("definition").JsonDeserialization<IndexDefinition>();
}
});
foreach (var index in indexes)
{
_documentDatabase.PutIndex(index.Key, index.Value);
}
var total = 0;
var batch = new List<RavenJObject>();
Read(jsonReader,
"Docs",
document =>
{
total++;
batch.Add((RavenJObject)document);
if (batch.Count >= 128)
{
FlushBatch(batch);
}
});
FlushBatch(batch);
stopwatch.Stop();
return new DumperStats
{
Attachments = 0,
Documents = total,
Indexes = indexCount,
Elapsed = stopwatch.Elapsed
};
}
}
public void ImportJobs(string jobsJson)
{
var serializer = this.DocumentSession.Advanced.DocumentStore.Conventions.CreateSerializer();
using (var reader = new StringReader(jobsJson))
using (var jsonReader = new JsonTextReader(reader))
{
var data = serializer.Deserialize<List<JobDefinition>>(jsonReader);
this.JobManager.ImportJobs(data);
}
}
public void Can_read_date_time_offset_from_lucene_query()
{
var jsonSerializer = new DocumentConvention().CreateSerializer();
using (var reader = new JsonTextReader(new StringReader(@"{""Item"": ""20090402193554412""}")))
{
var deserialize = jsonSerializer.Deserialize<Test>(reader);
Assert.Equal(2009, deserialize.Item.Year);
}
}
public void ReadSingleQuoteInsideDoubleQuoteString()
{
string json = @"{""NameOfStore"":""Forest's Bakery And Cafe""}";
JsonTextReader jsonTextReader = new JsonTextReader(new StringReader(json));
jsonTextReader.Read();
jsonTextReader.Read();
jsonTextReader.Read();
Assert.AreEqual(@"Forest's Bakery And Cafe", jsonTextReader.Value);
}
public void CanDeserialiseInheritedObject()
{
var item = new A() { Name = "B", Children = new List<A>() { new B() { Id = 10, Name = "Child B", }, new A { Name = "Child A" } } };
var json = new StringWriter();
_serialiser.Serialize(json, item);
var reader = new JsonTextReader(new StringReader(json.ToString()));
var b = _serialiser.Deserialize<A>(reader);
Assert.That(b.Children.First(), Is.InstanceOf<B>());
Assert.That(((B)b.Children.First()).Id, Is.EqualTo(10));
}
public void ReadMultilineString()
{
string json = @"""first line
second line
third line""";
JsonTextReader jsonTextReader = new JsonTextReader(new StringReader(json));
Assert.IsTrue(jsonTextReader.Read());
Assert.AreEqual(JsonToken.String, jsonTextReader.TokenType);
Assert.AreEqual(@"first line
second line
third line", jsonTextReader.Value);
}
public ActionResult Import(HttpPostedFileBase fileData)
{
string fileString = StreamHelper.ReadAll(fileData.InputStream);
var serializer = this.DocumentSession.Advanced.DocumentStore.Conventions.CreateSerializer();
using (var streamReader = new StreamReader(fileData.InputStream))
{
using (var jsonReader = new JsonTextReader(streamReader))
{
var data = serializer.Deserialize<ImportExportFile>(jsonReader);
this.JobManager.ImportJobs(data.JobDefinitionList);
this.SettingsManager.ImportSettings(data.SettingsContainerList);
return RedirectToAction("ImportComplete");
}
}
}
public void SurrogatePairValid()
{
string json = @"{ ""MATHEMATICAL ITALIC CAPITAL ALPHA"": ""\uD835\uDEE2"" }";
JsonTextReader reader = new JsonTextReader(new StringReader(json));
Assert.IsTrue(reader.Read());
Assert.IsTrue(reader.Read());
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.String, reader.TokenType);
string s = reader.Value.ToString();
Assert.AreEqual(2, s.Length);
StringInfo stringInfo = new StringInfo(s);
Assert.AreEqual(1, stringInfo.LengthInTextElements);
}
public async Task Non_existing_data_folder_should_throw_proper_exception()
{
using (var store = NewStore(requestedStorage: "voron", runInMemory: true))
{
using (var session = store.OpenAsyncSession())
{
session.RegisterUpload("test.file", new MemoryStream(new byte[] { 1, 2, 3 }));
await session.SaveChangesAsync();
}
await store.AsyncFilesCommands.Admin.StartBackup(backupLocation,
null, false, store.DefaultFileSystem);
WaitForBackup(store.AsyncFilesCommands, true);
FileSystemDocument document;
using (var file = File.OpenText(Path.Combine(backupLocation, Constants.FilesystemDocumentFilename)))
using (var reader = new JsonTextReader(file))
document = (new JsonSerializer()).Deserialize<FileSystemDocument>(reader);
var drives = DriveInfo.GetDrives().Select(x => x.Name.ToLower()[0]).ToArray();
var lastDriveLetter = 'a';
while (lastDriveLetter != 'z')
{
if (drives.Contains(lastDriveLetter) == false)
break;
lastDriveLetter++;
}
document.Settings[Constants.FileSystem.DataDirectory] = string.Format("{0}:\\", (char)(lastDriveLetter )); //on purpose, non existing path
using (var file = File.CreateText(Path.Combine(backupLocation, Constants.FilesystemDocumentFilename)))
using (var writer = new JsonTextWriter(file))
(new JsonSerializer()).Serialize(writer, document);
Assert.Throws<BadRequestException>(() =>
AsyncHelpers.RunSync(() => store.AsyncFilesCommands.Admin.StartRestore(new FilesystemRestoreRequest
{
BackupLocation = backupLocation
})));
}
}
public override void Execute(object parameter)
{
if (string.IsNullOrWhiteSpace(queryModel.Address))
return;
var url = "http://dev.virtualearth.net/REST/v1/Locations?q=" + Uri.EscapeUriString(queryModel.Address) +
"&key=Anlj2YMQu676uXmSj1QTSni66f8DjuBGToZ21t5z9E__lL8IHRhFP8LtF7umitL6";
var webRequest = WebRequest.Create(new Uri(url, UriKind.Absolute));
webRequest.GetResponseAsync().ContinueOnSuccessInTheUIThread(doc =>
{
RavenJObject jsonData;
using (var stream = doc.GetResponseStream())
using (var reader = new StreamReader(stream))
using (var jsonReader = new JsonTextReader(reader))
jsonData = RavenJObject.Load(jsonReader);
var set = jsonData["resourceSets"];
var item = set.Values().First().Values().ToList()[1].Values().ToList();
if (item.Count == 0)
{
ApplicationModel.Current.AddInfoNotification("Could not calculate the given address");
return;
}
var result = item.First().SelectToken("point").SelectToken("coordinates").Values().ToList();
if (result != null)
{
var latitude = double.Parse(result[0].ToString());
var longitude = double.Parse(result[1].ToString());
var addressData = new AddressData { Address = queryModel.Address, Latitude = latitude, Longitude = longitude };
queryModel.UpdateResultsFromCalculate(addressData);
}
}).Catch();
}
public void ReadSingleValueAsStreamShouldWork(int size)
{
var random = new Random(size);
var buffer = new byte[size];
random.NextBytes(buffer);
var d = new Data
{
Foo = buffer
};
var jsonObj = RavenJToken.FromObject(d);
var jsonObjAsString = jsonObj.ToString();
using (var textReader = new StringReader(jsonObjAsString))
using (var jsonReader = new JsonTextReader(textReader))
{
jsonReader.Read(); // start object
jsonReader.Read(); // property name
var actual = new byte[buffer.Length];
using (var objectStream = jsonReader.ReadBytesAsStream())
{
objectStream.Read(actual, 0, actual.Length);
for (int a = 0; a < actual.Length; a++)
{
if (buffer[a] != actual[a])
{
Assert.True(false, "not equal on byte " + a);
}
}
}
}
}
public void ThrowErrorWhenParsingUnquoteStringThatStartsWithNE()
{
const string json = @"{ ""ItemName"": ""value"", ""u"":netanelsalinger,""r"":9 }";
JsonTextReader reader = new JsonTextReader(new StringReader(json));
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.StartObject, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.String, reader.TokenType);
Assert.IsTrue(reader.Read());
Assert.AreEqual(JsonToken.PropertyName, reader.TokenType);
ExceptionAssert.Throws<JsonReaderException>("Unexpected content while parsing JSON. Path 'u', line 1, position 27.",
() =>
{
reader.Read();
});
}
private static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
{
using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
{
var jsonSerializer = new JsonSerializer();
var textReader = new JsonTextReader(new StreamReader(commitPointFile));
indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader);
return indexCommit != null;
}
}
public static IndexDefinition CreateIndexDefinition()
{
// this is how the index looks like in JSON
var jsonIndexDefinition = @"
{
""Map"" : ""
from e in docs.Events
select new {
Tag = \""Event\"",
_lat = SpatialIndex.Lat(e.Latitude),
_lng = SpatialIndex.Lng(e.Longitude),
_tier_2 = SpatialIndex.Tier(2, e.Latitude, e.Longitude),
_tier_3 = SpatialIndex.Tier(3, e.Latitude, e.Longitude),
_tier_4 = SpatialIndex.Tier(4, e.Latitude, e.Longitude),
_tier_5 = SpatialIndex.Tier(5, e.Latitude, e.Longitude),
_tier_6 = SpatialIndex.Tier(6, e.Latitude, e.Longitude),
_tier_7 = SpatialIndex.Tier(7, e.Latitude, e.Longitude),
_tier_8 = SpatialIndex.Tier(8, e.Latitude, e.Longitude),
_tier_9 = SpatialIndex.Tier(9, e.Latitude, e.Longitude),
_tier_10 = SpatialIndex.Tier(10, e.Latitude, e.Longitude),
_tier_11 = SpatialIndex.Tier(11, e.Latitude, e.Longitude),
_tier_12 = SpatialIndex.Tier(12, e.Latitude, e.Longitude),
_tier_13 = SpatialIndex.Tier(13, e.Latitude, e.Longitude),
_tier_14 = SpatialIndex.Tier(14, e.Latitude, e.Longitude),
_tier_15 = SpatialIndex.Tier(15, e.Latitude, e.Longitude)
}"",
""Stores"" :{
""latitude"" : ""Yes"",
""longitude"" : ""Yes"",
""_tier_2"" : ""Yes"",
""_tier_3"" : ""Yes"",
""_tier_4"" : ""Yes"",
""_tier_5"" : ""Yes"",
""_tier_6"" : ""Yes"",
""_tier_7"" : ""Yes"",
""_tier_8"" : ""Yes"",
""_tier_9"" : ""Yes"",
""_tier_10"" : ""Yes"",
""_tier_11"" : ""Yes"",
""_tier_12"" : ""Yes"",
""_tier_13"" : ""Yes"",
""_tier_14"" : ""Yes"",
""_tier_15"" : ""Yes""
},
""Indexes"" :{
""Tag"" : ""NotAnalyzed"",
""latitude"" : ""NotAnalyzed"",
""longitude"" : ""NotAnalyzed"",
""_tier_2"" : ""NotAnalyzedNoNorms"",
""_tier_3"" : ""NotAnalyzedNoNorms"",
""_tier_4"" : ""NotAnalyzedNoNorms"",
""_tier_5"" : ""NotAnalyzedNoNorms"",
""_tier_6"" : ""NotAnalyzedNoNorms"",
""_tier_7"" : ""NotAnalyzedNoNorms"",
""_tier_8"" : ""NotAnalyzedNoNorms"",
""_tier_9"" : ""NotAnalyzedNoNorms"",
""_tier_10"" : ""NotAnalyzedNoNorms"",
""_tier_11"" : ""NotAnalyzedNoNorms"",
""_tier_12"" : ""NotAnalyzedNoNorms"",
""_tier_13"" : ""NotAnalyzedNoNorms"",
""_tier_14"" : ""NotAnalyzedNoNorms"",
""_tier_15"" : ""NotAnalyzedNoNorms""
}
}";
using (var stringReader = new StringReader(jsonIndexDefinition))
using (var jsonReader = new JsonTextReader(stringReader))
{
return JsonExtensions.CreateDefaultJsonSerializer().Deserialize<IndexDefinition>(jsonReader);
}
}
private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
{
var count = 0;
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var size = GetRoughSize(document);
if (size > 1024 * 1024)
{
Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
(double)size / 1024,
document["@metadata"].Value<string>("@id"));
}
if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
continue;
if (options.MatchFilters(document) == false)
continue;
if (!string.IsNullOrEmpty(options.TransformScript))
document = await TransformDocument(document, options.TransformScript);
if (document == null)
continue;
await PutDocument(document);
count++;
if (count % options.BatchSize == 0)
{
ShowProgress("Read {0} documents", count);
}
}
await PutDocument(null); // force flush
return count;
}
private async Task<int> ImportAttachments(JsonTextReader jsonReader, SmugglerOptions options)
{
var count = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
return count;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Attachments", jsonReader.Value) == false)
throw new InvalidDataException("Attachment property was expected");
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var item = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Attachments) != ItemType.Attachments)
continue;
var attachmentExportInfo =
new JsonSerializer
{
Converters =
{
new JsonToJsonConverter()
}
}.Deserialize<AttachmentExportInfo>(new RavenJTokenReader(item));
ShowProgress("Importing attachment {0}", attachmentExportInfo.Key);
await PutAttachment(attachmentExportInfo);
count++;
}
await PutAttachment(null); // force flush
return count;
}
private async Task<int> ImportTransformers(JsonTextReader jsonReader, SmugglerOptions options)
{
var count = 0;
if (jsonReader.Read() == false || jsonReader.TokenType == JsonToken.EndObject)
return count;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Transformers", jsonReader.Value) == false)
throw new InvalidDataException("Transformers property was expected");
if (jsonReader.Read() == false)
return count;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var transformer = RavenJToken.ReadFrom(jsonReader);
if ((options.OperateOnTypes & ItemType.Transformers) != ItemType.Transformers)
continue;
var transformerName = transformer.Value<string>("name");
await PutTransformer(transformerName, transformer);
count++;
}
await PutTransformer(null, null); // force flush
return count;
}
public static bool TryGetCommitPoint(IndexCommitPointDirectory commitPointDirectory, out IndexCommitPoint indexCommit)
{
using (var commitPointFile = File.OpenRead(commitPointDirectory.FileFullPath))
{
try
{
var textReader = new JsonTextReader(new StreamReader(commitPointFile));
var jsonCommitPoint = RavenJObject.Load(textReader);
var jsonEtag = jsonCommitPoint.Value<RavenJToken>("HighestCommitedETag");
Etag recoveredEtag = null;
if (jsonEtag.Type == JTokenType.Object) // backward compatibility - HighestCommitedETag is written as {"Restarts":123,"Changes":1}
{
jsonCommitPoint.Remove("HighestCommitedETag");
recoveredEtag = new Etag(UuidType.Documents, jsonEtag.Value<long>("Restarts"), jsonEtag.Value<long>("Changes"));
}
indexCommit = jsonCommitPoint.JsonDeserialization<IndexCommitPoint>();
if (indexCommit == null)
return false;
if (recoveredEtag != null)
indexCommit.HighestCommitedETag = recoveredEtag;
if (indexCommit.HighestCommitedETag == null || indexCommit.HighestCommitedETag.CompareTo(Etag.Empty) == 0)
return false;
return true;
}
catch (Exception e)
{
log.Warn("Could not get commit point from the following location {0}. Exception {1}", commitPointDirectory.FileFullPath, e);
indexCommit = null;
return false;
}
}
}
private static void JsonBodyToSoapXml(Stream json, Stream xml)
{
Raven.Imports.Newtonsoft.Json.JsonSerializerSettings settings = new Raven.Imports.Newtonsoft.Json.JsonSerializerSettings();
settings.Converters.Add(new Raven.Imports.Newtonsoft.Json.Converters.XmlNodeConverter());
Raven.Imports.Newtonsoft.Json.JsonSerializer serializer = Raven.Imports.Newtonsoft.Json.JsonSerializer.Create(settings);
using (Newtonsoft.Json.JsonTextReader reader = new Raven.Imports.Newtonsoft.Json.JsonTextReader(new System.IO.StreamReader(json)))
{
XmlDocument doc = (XmlDocument)serializer.Deserialize(reader, typeof(XmlDocument));
if (reader.Read() && reader.TokenType != JsonToken.Comment)
throw new JsonSerializationException("Additional text found in JSON string after finishing deserializing object.");
using (XmlWriter writer = XmlWriter.Create(xml))
{
doc.Save(writer);
}
}
}
public void UnexpectedEndWhenParsingUnquotedProperty()
{
JsonReader reader = new JsonTextReader(new StringReader(@"{aww"));
Assert.IsTrue(reader.Read());
ExceptionAssert.Throws<JsonReaderException>(
"Unexpected end while parsing unquoted property name. Path '', line 1, position 4.",
() =>
{
reader.Read();
});
}
public void ShouldDeleteCommitPointIfCouldNotRecoverFromIt()
{
var dataDir = NewDataPath("ShouldDeleteCommitPointIfCouldNotRecoverFromIt");
string indexFullPath;
string commitPointsDirectory;
var index = new MapRecoveryTestIndex();
using (var server = GetNewServer(runInMemory: false, dataDirectory: dataDir))
{
CommitPointAfterEachCommit(server.Database.Configuration);
indexFullPath = Path.Combine(server.Database.Configuration.IndexStoragePath,
MonoHttpUtility.UrlEncode(index.IndexName));
commitPointsDirectory = Path.Combine(server.Database.Configuration.IndexStoragePath,
MonoHttpUtility.UrlEncode(index.IndexName) + "\\CommitPoints");
using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
index.Execute(store);
using (var session = store.OpenSession())
{
session.Store(new Recovery
{
Name = "One",
Number = 1
});
session.SaveChanges(); // first commit point
WaitForIndexing(store);
session.Store(new Recovery
{
Name = "Two",
Number = 2
});
session.SaveChanges(); // second commit point
WaitForIndexing(store);
session.Store(new Recovery
{
Name = "Three",
Number = 3
});
session.SaveChanges(); // second commit point
WaitForIndexing(store, timeout: TimeSpan.FromSeconds(60));
}
}
}
// make sure that there are 3 commit points
var directories = Directory.GetDirectories(commitPointsDirectory);
Assert.Equal(3, directories.Length);
// mess "index.CommitPoint" file in the SECOND commit point by adding additional files required to recover from it
using (var commitPointFile = File.Open(Path.Combine(directories[1], "index.CommitPoint"), FileMode.Open))
{
var jsonSerializer = new JsonSerializer();
var textReader = new JsonTextReader(new StreamReader(commitPointFile));
var indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader);
indexCommit.SegmentsInfo.ReferencedFiles.Add("file-that-doesnt-exist");
commitPointFile.Position = 0;
using (var sw = new StreamWriter(commitPointFile))
{
var textWriter = new JsonTextWriter(sw);
jsonSerializer.Serialize(textWriter, indexCommit);
sw.Flush();
}
}
// mess "index.CommitPoint" file in the THIRD commit point by adding additional files required to recover from it
using (var commitPointFile = File.Open(Path.Combine(directories[2], "index.CommitPoint"), FileMode.Open))
{
var jsonSerializer = new JsonSerializer();
var textReader = new JsonTextReader(new StreamReader(commitPointFile));
var indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader);
indexCommit.SegmentsInfo.ReferencedFiles.Add("file-that-doesnt-exist");
commitPointFile.Position = 0;
using (var sw = new StreamWriter(commitPointFile))
{
var textWriter = new JsonTextWriter(sw);
jsonSerializer.Serialize(textWriter, indexCommit);
sw.Flush();
}
}
IndexMessing.MessSegmentsFile(indexFullPath);
using (GetNewServer(runInMemory: false, dataDirectory: dataDir)) // do not delete previous directory
{
using (var store = new DocumentStore { Url = "http://localhost:8079" }.Initialize())
{
using (var session = store.OpenSession())
{
var result =
session.Query<Recovery, MapRecoveryTestIndex>().Customize(x => x.WaitForNonStaleResults()).ToList();
Assert.Equal(3, result.Count);
}
}
}
// there should be exactly 2 commit points:
// the first one which we used to recover
// and the second one created because of indexing after recovery
Assert.Equal(2, Directory.GetDirectories(commitPointsDirectory).Length);
}
private static bool TryReusePreviousCommitPointsToRecoverIndex(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition, string indexStoragePath, out IndexCommitPoint indexCommit, out string[] keysToDelete)
{
indexCommit = null;
keysToDelete = null;
if (indexDefinition.IsMapReduce)
return false;
var indexFullPath = Path.Combine(indexStoragePath, MonoHttpUtility.UrlEncode(indexDefinition.Name));
var allCommitPointsFullPath = IndexCommitPointDirectory.GetAllCommitPointsFullPath(indexFullPath);
if (Directory.Exists(allCommitPointsFullPath) == false)
return false;
var filesInIndexDirectory = Directory.GetFiles(indexFullPath).Select(Path.GetFileName);
var existingCommitPoints =
IndexCommitPointDirectory.ScanAllCommitPointsDirectory(indexFullPath);
Array.Reverse(existingCommitPoints); // start from the highest generation
foreach (var commitPointDirectoryName in existingCommitPoints)
{
try
{
var commitPointDirectory = new IndexCommitPointDirectory(indexStoragePath, indexDefinition.Name,
commitPointDirectoryName);
using (var commitPointFile = File.Open(commitPointDirectory.FileFullPath, FileMode.Open))
{
var jsonSerializer = new JsonSerializer();
var textReader = new JsonTextReader(new StreamReader(commitPointFile));
indexCommit = jsonSerializer.Deserialize<IndexCommitPoint>(textReader);
}
var missingFile =
indexCommit.SegmentsInfo.ReferencedFiles.Any(
referencedFile => filesInIndexDirectory.Contains(referencedFile) == false);
if (missingFile)
{
IOExtensions.DeleteDirectory(commitPointDirectory.FullPath);
continue; // there are some missing files, try another commit point
}
var storedSegmentsFile = indexCommit.SegmentsInfo.SegmentsFileName;
// here there should be only one segments_N file, however remove all if there is more
foreach (var currentSegmentsFile in Directory.GetFiles(commitPointDirectory.IndexFullPath, "segments_*"))
{
File.Delete(currentSegmentsFile);
}
// copy old segments_N file
File.Copy(Path.Combine(commitPointDirectory.FullPath, storedSegmentsFile),
Path.Combine(commitPointDirectory.IndexFullPath, storedSegmentsFile), true);
try
{
// update segments.gen file
using (var genOutput = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN))
{
genOutput.WriteInt(SegmentInfos.FORMAT_LOCKLESS);
genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
genOutput.WriteLong(indexCommit.SegmentsInfo.Generation);
}
}
catch (Exception)
{
// here we can ignore, segments.gen is used only as fallback
}
if (File.Exists(commitPointDirectory.DeletedKeysFile))
keysToDelete = File.ReadLines(commitPointDirectory.DeletedKeysFile).ToArray();
return true;
}
catch (Exception ex)
{
startupLog.WarnException("Could not recover an index named '" + indexDefinition.Name +
"'from segments of the following generation " + commitPointDirectoryName, ex);
}
}
return false;
}
private XmlNode DeserializeXmlNode(string json, string deserializeRootElementName)
{
JsonTextReader reader;
reader = new JsonTextReader(new StringReader(json));
reader.Read();
XmlNodeConverter converter = new XmlNodeConverter();
if (deserializeRootElementName != null)
converter.DeserializeRootElementName = deserializeRootElementName;
XmlNode node = (XmlNode)converter.ReadJson(reader, typeof (XmlDocument), null, new JsonSerializer());
#if !NET20
string xmlText = node.OuterXml;
reader = new JsonTextReader(new StringReader(json));
reader.Read();
XDocument d = (XDocument) converter.ReadJson(reader, typeof (XDocument), null, new JsonSerializer());
string linqXmlText = d.ToString(SaveOptions.DisableFormatting);
if (d.Declaration != null)
linqXmlText = d.Declaration + linqXmlText;
Assert.AreEqual(xmlText, linqXmlText);
#endif
return node;
}
public static void ReadLastEtagsFromFile(SmugglerOptions options)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(options.BackupPath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
return;
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
options.LastDocsEtag = Etag.Parse(ravenJObject.Value<string>("LastDocEtag"));
options.LastAttachmentEtag = Etag.Parse(ravenJObject.Value<string>("LastAttachmentEtag"));
}
}
public void ReadBytesFromEmptyString()
{
var bytes = new HasBytes { Bytes = new byte[0] };
var json = JsonConvert.SerializeObject(bytes);
TextReader textReader = new StringReader(json);
JsonReader jsonReader = new JsonTextReader(textReader);
var jToken = JToken.ReadFrom(jsonReader);
jsonReader = new JTokenReader(jToken);
var result2 = (HasBytes)JsonSerializer.Create(null)
.Deserialize(jsonReader, typeof(HasBytes));
CollectionAssert.AreEquivalent(new byte[0], result2.Bytes);
}
public async virtual Task ImportData(Stream stream, SmugglerOptions options)
{
options = options ?? SmugglerOptions;
if (options == null)
throw new ArgumentNullException("options");
await DetectServerSupportedFeatures();
await EnsureDatabaseExists();
Stream sizeStream;
var sw = Stopwatch.StartNew();
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
var streamReader = new StreamReader(sizeStream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
catch (Exception e)
{
if (e is InvalidDataException == false
#if SILVERLIGHT
&& e is ZlibException == false
#endif
)
throw;
stream.Seek(0, SeekOrigin.Begin);
sizeStream = new CountingStream(new GZipStream(stream, CompressionMode.Decompress));
var streamReader = new StreamReader(stream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
}
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
ShowProgress("Begin reading indexes");
var indexCount = await ImportIndexes(jsonReader, options);
ShowProgress(string.Format("Done with reading indexes, total: {0}", indexCount));
ShowProgress("Begin reading documents");
var documentCount = await ImportDocuments(jsonReader, options);
ShowProgress(string.Format("Done with reading documents, total: {0}", documentCount));
ShowProgress("Begin reading attachments");
var attachmentCount = await ImportAttachments(jsonReader, options);
ShowProgress(string.Format("Done with reading attachments, total: {0}", attachmentCount));
ShowProgress("Begin reading transformers");
var transformersCount = await ImportTransformers(jsonReader, options);
ShowProgress(string.Format("Done with reading transformers, total: {0}", transformersCount));
sw.Stop();
ShowProgress("Imported {0:#,#;;0} documents and {1:#,#;;0} attachments in {2:#,#;;0} ms", documentCount, attachmentCount, sw.ElapsedMilliseconds);
}