private static LastFilesEtagsInfo ReadLastEtagsFromIncrementalExportFile(string path)
{
var etagFileLocation = Path.Combine(path, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
{
return(null);
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
//TODO arek log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return(null);
}
var result = new LastFilesEtagsInfo
{
LastFileEtag = Etag.Parse(ravenJObject.Value <string>("LastFileEtag")),
LastDeletedFileEtag = Etag.Parse(ravenJObject.Value <string>("LastDeletedFileEtag") ?? Etag.Empty.ToString())
};
return(result);
}
}
public void AddDocumentsToDb(string path)
{
int count = 1;
using (var stream = string.IsNullOrWhiteSpace(path) ? GetEmbeddedLastFmSubset() : File.OpenRead(path))
using (var zip = new ZipArchive(stream, ZipArchiveMode.Read))
using (var bulkInsert = DocumentStoreHolder.MediaStore.BulkInsert(options: new BulkInsertOptions {
OverwriteExisting = true, BatchSize = 256
}))
{
foreach (var entry in zip.Entries)
{
if (entry.Length == 0)
{
continue;
}
using (var entryStream = entry.Open())
{
var docAsJson = RavenJObject.Load(new JsonTextReader(new StreamReader(entryStream)));
var doc = new LastFm
{
Artist = docAsJson.Value <string>("artist"),
TimeStamp = DateTime.Parse(docAsJson.Value <string>("timestamp")),
Title = docAsJson.Value <string>("title"),
TrackId = docAsJson.Value <string>("track_id"),
Tags =
docAsJson.Value <RavenJArray>("tags")
.Select(x => ((RavenJArray)x)[0].Value <string>())
.ToList()
};
bulkInsert.Store(doc, "lastfm/" + (count++));
}
}
}
}
private IndexDefinition DirectGetIndex(string indexName, string operationUrl)
{
var httpJsonRequest = jsonRequestFactory.CreateHttpJsonRequest(this, operationUrl + "/indexes/" + indexName + "?definition=yes", "GET", credentials, convention);
httpJsonRequest.AddOperationHeaders(OperationsHeaders);
string indexDefAsString;
try
{
indexDefAsString = httpJsonRequest.ReadResponseString();
}
catch (WebException e)
{
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse != null &&
httpWebResponse.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
throw;
}
var indexDefResultAsJson = RavenJObject.Load(new JsonTextReader(new StringReader(indexDefAsString)));
return(convention.CreateSerializer().Deserialize <IndexDefinition>(
new RavenJTokenReader(indexDefResultAsJson["Index"])
));
}
private static bool HandleWebExceptionForGetAsync(string key, WebException e)
{
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
return(false);
}
if (httpWebResponse.StatusCode == HttpStatusCode.NotFound)
{
return(true);
}
if (httpWebResponse.StatusCode == HttpStatusCode.Conflict)
{
var conflicts = new StreamReader(httpWebResponse.GetResponseStream());
var conflictsDoc = RavenJObject.Load(new JsonTextReader(conflicts));
var conflictIds = conflictsDoc.Value <RavenJArray>("Conflicts").Select(x => x.Value <string>()).ToArray();
throw new ConflictException("Conflict detected on " + key +
", conflict must be resolved before the document will be accessible")
{
ConflictedVersionIds = conflictIds
};
}
return(false);
}
internal DatabaseSmugglerOperationState ReadLastEtagsFromFile(string etagFileLocation)
{
if (File.Exists(etagFileLocation) == false)
{
return(null);
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
_log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return(null);
}
return(new DatabaseSmugglerOperationState
{
LastDocsEtag = Etag.Parse(ravenJObject.Value <string>("LastDocEtag")),
LastDocDeleteEtag = Etag.Parse(ravenJObject.Value <string>("LastDocDeleteEtag") ?? Etag.Empty.ToString())
});
}
}
private async Task <RavenJObject> ReadJsonAsync()
{
using (var stream = await Request.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream))
using (var jsonReader = new RavenJsonTextReader(streamReader))
return(RavenJObject.Load(jsonReader));
}
public static void ReadLastEtagsFromFile(OperationState result, string etagFileLocation)
{
var log = LogManager.GetCurrentClassLogger();
if (!File.Exists(etagFileLocation))
{
return;
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
result.LastDocsEtag = Etag.Parse(ravenJObject.Value <string>("LastDocEtag"));
result.LastAttachmentsEtag = Etag.Parse(ravenJObject.Value <string>("LastAttachmentEtag"));
result.LastDocDeleteEtag = Etag.Parse(ravenJObject.Value <string>("LastDocDeleteEtag") ?? Etag.Empty.ToString());
result.LastAttachmentsDeleteEtag = Etag.Parse(ravenJObject.Value <string>("LastAttachmentsDeleteEtag") ?? Etag.Empty.ToString());
}
}
public async Task <HttpResponseMessage> ApplyConflict(string filename, long remoteVersion, string remoteServerId, string remoteServerUrl)
{
var canonicalFilename = FileHeader.Canonize(filename);
var localMetadata = Synchronizations.GetLocalMetadata(canonicalFilename);
if (localMetadata == null)
{
throw new HttpResponseException(HttpStatusCode.NotFound);
}
var contentStream = await Request.Content.ReadAsStreamAsync().ConfigureAwait(false);
var current = new HistoryItem
{
ServerId = Storage.Id.ToString(),
Version = localMetadata.Value <long>(SynchronizationConstants.RavenSynchronizationVersion)
};
var currentConflictHistory = Historian.DeserializeHistory(localMetadata);
currentConflictHistory.Add(current);
var remote = new HistoryItem
{
ServerId = remoteServerId,
Version = remoteVersion
};
var remoteMetadata = RavenJObject.Load(new JsonTextReader(new StreamReader(contentStream)));
var remoteConflictHistory = Historian.DeserializeHistory(remoteMetadata);
remoteConflictHistory.Add(remote);
var conflict = new ConflictItem
{
CurrentHistory = currentConflictHistory,
RemoteHistory = remoteConflictHistory,
FileName = canonicalFilename,
RemoteServerUrl = Uri.UnescapeDataString(remoteServerUrl)
};
ConflictArtifactManager.Create(canonicalFilename, conflict);
Publisher.Publish(new ConflictNotification
{
FileName = filename,
SourceServerUrl = remoteServerUrl,
Status = ConflictStatus.Detected,
RemoteFileHeader = new FileHeader(canonicalFilename, remoteMetadata)
});
if (Log.IsDebugEnabled)
{
Log.Debug("Conflict applied for a file '{0}' (remote version: {1}, remote server id: {2}).", filename, remoteVersion, remoteServerId);
}
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
private IEnumerable <NamedApiKeyDefinition> YieldResults(Stream stream, HttpJsonRequest request)
{
using (request)
using (stream)
using (var jtr = new JsonTextReader(new StreamReader(stream)))
{
if (jtr.Read() == false || jtr.TokenType != JsonToken.StartArray)
{
throw new InvalidOperationException("Expected start array");
}
while (true)
{
if (jtr.Read() == false)
{
throw new InvalidOperationException("Unexpected EOF");
}
if (jtr.TokenType == JsonToken.EndArray)
{
break;
}
var ravenJObject = RavenJObject.Load(jtr);
yield return(ravenJObject.Deserialize <NamedApiKeyDefinition>(new DocumentConvention()));
}
}
}
private static RavenJObject ReadJObject(Stream log)
{
return(RavenJObject.Load(new BsonReader(log)
{
DateTimeKindHandling = DateTimeKind.Utc,
}));
}
private async Task <bool> IsOperationCompleted(long operationId)
{
ErrorResponseException errorResponse;
try
{
var status = await GetOperationStatus(operationId);
if (status == null)
{
return(true);
}
if (status.Value <bool>("Completed"))
{
return(true);
}
return(false);
}
catch (ErrorResponseException e)
{
if (e.StatusCode != HttpStatusCode.Conflict)
{
throw;
}
errorResponse = e;
}
var conflictsDocument = RavenJObject.Load(new RavenJsonTextReader(new StringReader(errorResponse.ResponseString)));
throw new ConcurrencyException(conflictsDocument.Value <string>("Error"));
}
/// <summary>
/// Convert a byte array to a RavenJObject
/// </summary>
public static RavenJObject ToJObject(this Stream self)
{
return(RavenJObject.Load(new BsonReader(self)
{
DateTimeKindHandling = DateTimeKind.Utc,
}));
}
public static void ReadLastEtagsFromFile(SmugglerOptions options)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(options.BackupPath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
{
return;
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
options.LastDocsEtag = Etag.Parse(ravenJObject.Value <string>("LastDocEtag"));
options.LastAttachmentEtag = Etag.Parse(ravenJObject.Value <string>("LastAttachmentEtag"));
}
}
public async Task <RavenJObject> ReadJsonAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync())
using (var streamReader = new StreamReader(stream, GetRequestEncoding()))
using (var jsonReader = new RavenJsonTextReader(streamReader))
return(RavenJObject.Load(jsonReader));
}
/// <summary>
/// Convert a byte array to a RavenJObject
/// </summary>
public static RavenJObject ToJObject(this byte [] self)
{
return(RavenJObject.Load(new BsonReader(new MemoryStream(self))
{
DateTimeKindHandling = DateTimeKind.Utc,
}));
}
private static void ReadLastEtagsFromFile(ExportFilesResult result)
{
var log = LogManager.GetCurrentClassLogger();
var etagFileLocation = Path.Combine(result.FilePath, IncrementalExportStateFile);
if (!File.Exists(etagFileLocation))
{
return;
}
using (var streamReader = new StreamReader(new FileStream(etagFileLocation, FileMode.Open)))
using (var jsonReader = new JsonTextReader(streamReader))
{
RavenJObject ravenJObject;
try
{
ravenJObject = RavenJObject.Load(jsonReader);
}
catch (Exception e)
{
log.WarnException("Could not parse etag document from file : " + etagFileLocation + ", ignoring, will start from scratch", e);
return;
}
result.LastFileEtag = Etag.Parse(ravenJObject.Value <string>("LastFileEtag"));
result.LastDeletedFileEtag = Etag.Parse(ravenJObject.Value <string>("LastDeletedFileEtag") ?? Etag.Empty.ToString());
}
}
public static RavenJArray ReadBsonArray(this IHttpContext context)
{
using (var jsonReader = new BsonReader(context.Request.InputStream))
{
var jObject = RavenJObject.Load(jsonReader);
return(new RavenJArray(jObject.Values <RavenJToken>()));
}
}
public static RavenJObject DeserializeToRavenJObject(this string input)
{
var stringReader = new StringReader(input);
var textReader = new JsonTextReader(stringReader);
var obj = RavenJObject.Load(textReader);
return(obj);
}
protected async Task <RavenJObject> ReadJsonAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var buffered = new BufferedStream(stream))
using (var streamReader = new StreamReader(buffered, GetRequestEncoding()))
using (var jsonReader = new RavenJsonTextReader(streamReader))
return(RavenJObject.Load(jsonReader));
}
/// <summary>
/// Convert a byte array to a RavenJObject
/// </summary>
public static RavenJObject ToJObject(this Stream self)
{
var streamWithCachedHeader = new StreamWithCachedHeader(self, 5);
// note that we intentionally don't close it here
var jsonReader = new JsonTextReader(new StreamReader(streamWithCachedHeader));
return(RavenJObject.Load(jsonReader));
}
public async Task <RavenJArray> ReadBsonArrayAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync())
using (var jsonReader = new BsonReader(stream))
{
var jObject = RavenJObject.Load(jsonReader);
return(new RavenJArray(jObject.Values <RavenJToken>()));
}
}
private async Task <RavenJObject> Recieve(RavenClientWebSocket webSocket)
{
try
{
if (webSocket.State != WebSocketState.Open)
{
throw new InvalidOperationException(
$"Trying to 'ReceiveAsync' WebSocket while not in Open state. State is {webSocket.State}");
}
using (var ms = new MemoryStream())
{
ArraySegment <byte> bytes;
ms.SetLength(MaxWebSocketRecvSize);
ms.TryGetBuffer(out bytes);
var arraySegment = new ArraySegment <byte>(bytes.Array, 0, MaxWebSocketRecvSize);
var result = await webSocket.ReceiveAsync(arraySegment, disposedToken.Token);
if (result.MessageType == WebSocketMessageType.Close)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug("Client got close message from server and is closing connection");
}
// actual socket close from dispose
return(null);
}
if (result.EndOfMessage == false)
{
var err = $"In Recieve got response longer then {MaxWebSocketRecvSize}";
var ex = new InvalidOperationException(err);
Logger.DebugException(err, ex);
throw ex;
}
using (var reader = new StreamReader(ms, Encoding.UTF8, true, MaxWebSocketRecvSize, true))
using (var jsonReader = new RavenJsonTextReader(reader)
{
SupportMultipleContent = true
})
{
if (jsonReader.Read() == false)
{
throw new InvalidDataException("Couldn't read recieved websocket json message");
}
return(RavenJObject.Load(jsonReader));
}
}
}
catch (WebSocketException ex)
{
Logger.DebugException("Failed to receive a message, client was probably disconnected", ex);
throw;
}
}
protected async Task <RavenJArray> ReadBsonArrayAsync()
{
using (var stream = await InnerRequest.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var buffered = new BufferedStream(stream))
using (var jsonReader = new BsonReader(buffered))
{
var jObject = RavenJObject.Load(jsonReader);
return(new RavenJArray(jObject.Values <RavenJToken>()));
}
}
/// <summary>
/// Begins an async get operation
/// </summary>
/// <param name="key">The key.</param>
/// <returns></returns>
public Task <JsonDocument> GetAsync(string key)
{
EnsureIsNotNullOrEmpty(key, "key");
var metadata = new RavenJObject();
AddTransactionInformation(metadata);
var request = jsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(this, url + "/docs/" + key, "GET", metadata, credentials, convention));
return(request.ReadResponseJsonAsync()
.ContinueWith(task =>
{
try
{
var requestJson = task.Result;
var docKey = request.ResponseHeaders[Constants.DocumentIdFieldName] ?? key;
request.ResponseHeaders.Remove(Constants.DocumentIdFieldName);
return SerializationHelper.DeserializeJsonDocument(docKey, requestJson, request.ResponseHeaders, request.ResponseStatusCode);
}
catch (AggregateException e)
{
var we = e.ExtractSingleInnerException() as WebException;
if (we == null)
{
throw;
}
var httpWebResponse = we.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
if (httpWebResponse.StatusCode == HttpStatusCode.NotFound)
{
return null;
}
if (httpWebResponse.StatusCode == HttpStatusCode.Conflict)
{
var conflicts = new StreamReader(httpWebResponse.GetResponseStreamWithHttpDecompression());
var conflictsDoc = RavenJObject.Load(new JsonTextReader(conflicts));
var conflictIds = conflictsDoc.Value <RavenJArray>("Conflicts").Select(x => x.Value <string>()).ToArray();
throw new ConflictException("Conflict detected on " + key +
", conflict must be resolved before the document will be accessible")
{
ConflictedVersionIds = conflictIds,
Etag = new Guid(httpWebResponse.GetResponseHeader("ETag"))
};
}
throw;
}
}));
}
/// <summary>
/// Begins an async get operation
/// </summary>
/// <param name="key">The key.</param>
/// <returns></returns>
public Task <JsonDocument> GetAsync(string key)
{
EnsureIsNotNullOrEmpty(key, "key");
var metadata = new RavenJObject();
AddTransactionInformation(metadata);
var request = jsonRequestFactory.CreateHttpJsonRequest(this, url + "/docs/" + key, "GET", metadata, credentials, convention);
return(Task.Factory.FromAsync <string>(request.BeginReadResponseString, request.EndReadResponseString, null)
.ContinueWith(task =>
{
try
{
var responseString = task.Result;
return new JsonDocument
{
DataAsJson = RavenJObject.Parse(responseString),
NonAuthoritiveInformation = request.ResponseStatusCode == HttpStatusCode.NonAuthoritativeInformation,
Key = key,
LastModified = DateTime.ParseExact(request.ResponseHeaders["Last-Modified"], "r", CultureInfo.InvariantCulture).ToLocalTime(),
Etag = new Guid(request.ResponseHeaders["ETag"]),
Metadata = request.ResponseHeaders.FilterHeaders(isServerDocument: false)
};
}
catch (WebException e)
{
var httpWebResponse = e.Response as HttpWebResponse;
if (httpWebResponse == null)
{
throw;
}
if (httpWebResponse.StatusCode == HttpStatusCode.NotFound)
{
return null;
}
if (httpWebResponse.StatusCode == HttpStatusCode.Conflict)
{
var conflicts = new StreamReader(httpWebResponse.GetResponseStreamWithHttpDecompression());
var conflictsDoc = RavenJObject.Load(new JsonTextReader(conflicts));
var conflictIds = conflictsDoc.Value <RavenJArray>("Conflicts").Select(x => x.Value <string>()).ToArray();
throw new ConflictException("Conflict detected on " + key +
", conflict must be resolved before the document will be accessible")
{
ConflictedVersionIds = conflictIds
};
}
throw;
}
}));
}
public static RavenJObject ExecuteManyFileReads2(string filePath)
{
string text = File.ReadAllText(filePath);
RavenJObject temp = null;
for (int i = 0; i < IterationsCount; i++)
{
using (var streamReader = new StringReader(text))
using (var jsonReader = new JsonTextReader(streamReader))
temp = RavenJObject.Load(jsonReader);
}
return(temp);
}
private async Task StartBulkInsertAsync(BulkInsertOptions options)
{
using (ConnectionOptions.Expect100Continue(operationClient.Url))
{
var operationUrl = CreateOperationUrl(options);
var token = await GetToken().ConfigureAwait(false);
try
{
token = await ValidateThatWeCanUseAuthenticateTokens(token).ConfigureAwait(false);
}
catch (Exception e)
{
throw new InvalidOperationException("Could not authenticate token for bulk insert, if you are using ravendb in IIS make sure you have Anonymous Authentication enabled in the IIS configuration", e);
}
using (operationRequest = CreateOperationRequest(operationUrl, token))
{
var cancellationToken = CreateCancellationToken();
var response = await operationRequest.ExecuteRawRequestAsync((stream, source) => Task.Factory.StartNew(() =>
{
try
{
WriteQueueToServer(stream, options, cancellationToken);
var x = source.TrySetResult(null);
}
catch (Exception e)
{
source.TrySetException(e);
}
}, TaskCreationOptions.LongRunning)).ConfigureAwait(false);
await response.AssertNotFailingResponse();
long operationId;
using (response)
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var streamReader = new StreamReader(stream))
{
var result = RavenJObject.Load(new JsonTextReader(streamReader));
operationId = result.Value <long>("OperationId");
}
if (await IsOperationCompleted(operationId).ConfigureAwait(false))
{
responseOperationId = operationId;
}
}
}
}
/// <summary>
/// Convert a byte array to a RavenJObject
/// </summary>
public static RavenJObject ToJObject(this Stream self)
{
var streamWithCachedHeader = new StreamWithCachedHeader(self, 5);
if (IsJson(streamWithCachedHeader))
{
// note that we intentionally don't close it here
var jsonReader = new JsonTextReader(new StreamReader(streamWithCachedHeader));
return RavenJObject.Load(jsonReader);
}
return RavenJObject.Load(new BsonReader(streamWithCachedHeader)
{
DateTimeKindHandling = DateTimeKind.Utc,
});
}
///<summary>
/// Get the possible terms for the specified field in the index asynchronously
/// You can page through the results by use fromValue parameter as the
/// starting point for the next query
///</summary>
///<returns></returns>
public Task <string[]> GetTermsAsync(string index, string field, string fromValue, int pageSize)
{
return(url.Terms(index, field, fromValue, pageSize)
.NoCache()
.ToJsonRequest(this, credentials, convention)
.ReadResponseStringAsync()
.ContinueWith(task =>
{
using (var reader = new JsonTextReader(new StringReader(task.Result)))
{
var json = RavenJObject.Load(reader);
return json.Select(x => x.Value.Value <string>()).ToArray();
}
}));
}
/// <summary>
/// Reads the JSON representation of the object.
/// </summary>
/// <param name="reader">The <see cref="T:Raven.Imports.Newtonsoft.Json.JsonReader"/> to read from.</param><param name="objectType">Type of the object.</param><param name="existingValue">The existing value of object being read.</param><param name="serializer">The calling serializer.</param>
/// <returns>
/// The object value.
/// </returns>
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType != JsonToken.StartObject)
{
return(DeferReadToNextConverter(reader, objectType, serializer, existingValue));
}
var jObject = RavenJObject.Load(reader);
var dateTime = jObject.Value <DateTime>("DateTime");
return(new DateTimeOffset(
dateTime.Year, dateTime.Month, dateTime.Day, dateTime.Hour, dateTime.Minute, dateTime.Second, dateTime.Millisecond,
TimeSpan.FromMilliseconds(jObject.Value <double>("Offset"))
));
}