public void Dispose()
{
_writer?.WriteEndObject();
_writer?.Flush();
_streamWriter?.Flush();
_streamWriter?.Dispose();
_gZipStream?.Dispose();
if (_leaveOpen == false)
{
_stream?.Flush();
_stream?.Dispose();
}
}
public override void ExecuteResult(ControllerContext context)
{
if (context == null) throw new ArgumentNullException("context");
HttpResponseBase response = context.HttpContext.Response;
response.ContentType = !string.IsNullOrEmpty(ContentType)
? ContentType
: "application/json";
if (ContentEncoding != null) response.ContentEncoding = ContentEncoding;
if (Data != null)
{
using (var writer = new JsonTextWriter(response.Output) { Formatting = Formatting })
{
var serializer = JsonSerializer.Create(SerializerSettings);
serializer.Serialize(writer, Data);
writer.Flush();
}
}
}
private async Task InvokeTokenEndpointAsync()
{
DateTimeOffset currentUtc = Options.SystemClock.UtcNow;
// remove milliseconds in case they don't round-trip
currentUtc = currentUtc.Subtract(TimeSpan.FromMilliseconds(currentUtc.Millisecond));
IFormCollection form = await Request.ReadFormAsync();
var clientContext = new OAuthValidateClientAuthenticationContext(
Context,
Options,
form);
await Options.Provider.ValidateClientAuthentication(clientContext);
if (!clientContext.IsValidated)
{
_logger.WriteError("clientID is not valid.");
if (!clientContext.HasError)
{
clientContext.SetError(Constants.Errors.InvalidClient);
}
await SendErrorAsJsonAsync(clientContext);
return;
}
var tokenEndpointRequest = new TokenEndpointRequest(form);
var validatingContext = new OAuthValidateTokenRequestContext(Context, Options, tokenEndpointRequest, clientContext);
AuthenticationTicket ticket = null;
if (tokenEndpointRequest.IsAuthorizationCodeGrantType)
{
// Authorization Code Grant http://tools.ietf.org/html/rfc6749#section-4.1
// Access Token Request http://tools.ietf.org/html/rfc6749#section-4.1.3
ticket = await InvokeTokenEndpointAuthorizationCodeGrantAsync(validatingContext, currentUtc);
}
else if (tokenEndpointRequest.IsResourceOwnerPasswordCredentialsGrantType)
{
// Resource Owner Password Credentials Grant http://tools.ietf.org/html/rfc6749#section-4.3
// Access Token Request http://tools.ietf.org/html/rfc6749#section-4.3.2
ticket = await InvokeTokenEndpointResourceOwnerPasswordCredentialsGrantAsync(validatingContext, currentUtc);
}
else if (tokenEndpointRequest.IsClientCredentialsGrantType)
{
// Client Credentials Grant http://tools.ietf.org/html/rfc6749#section-4.4
// Access Token Request http://tools.ietf.org/html/rfc6749#section-4.4.2
ticket = await InvokeTokenEndpointClientCredentialsGrantAsync(validatingContext, currentUtc);
}
else if (tokenEndpointRequest.IsRefreshTokenGrantType)
{
// Refreshing an Access Token
// http://tools.ietf.org/html/rfc6749#section-6
ticket = await InvokeTokenEndpointRefreshTokenGrantAsync(validatingContext, currentUtc);
}
else if (tokenEndpointRequest.IsCustomExtensionGrantType)
{
// Defining New Authorization Grant Types
// http://tools.ietf.org/html/rfc6749#section-8.3
ticket = await InvokeTokenEndpointCustomGrantAsync(validatingContext, currentUtc);
}
else
{
// Error Response http://tools.ietf.org/html/rfc6749#section-5.2
// The authorization grant type is not supported by the
// authorization server.
_logger.WriteError("grant type is not recognized");
validatingContext.SetError(Constants.Errors.UnsupportedGrantType);
}
if (ticket == null)
{
await SendErrorAsJsonAsync(validatingContext);
return;
}
ticket.Properties.IssuedUtc = currentUtc;
ticket.Properties.ExpiresUtc = currentUtc.Add(Options.AccessTokenExpireTimeSpan);
var tokenEndpointContext = new OAuthTokenEndpointContext(
Context,
Options,
ticket,
tokenEndpointRequest);
await Options.Provider.TokenEndpoint(tokenEndpointContext);
if (tokenEndpointContext.TokenIssued)
{
ticket = new AuthenticationTicket(
tokenEndpointContext.Identity,
tokenEndpointContext.Properties);
}
else
{
_logger.WriteError("Token was not issued to tokenEndpointContext");
validatingContext.SetError(Constants.Errors.InvalidGrant);
await SendErrorAsJsonAsync(validatingContext);
return;
}
var accessTokenContext = new AuthenticationTokenCreateContext(
Context,
Options.AccessTokenFormat,
ticket);
await Options.AccessTokenProvider.CreateAsync(accessTokenContext);
string accessToken = accessTokenContext.Token;
if (string.IsNullOrEmpty(accessToken))
{
accessToken = accessTokenContext.SerializeTicket();
}
DateTimeOffset?accessTokenExpiresUtc = ticket.Properties.ExpiresUtc;
var refreshTokenCreateContext = new AuthenticationTokenCreateContext(
Context,
Options.RefreshTokenFormat,
accessTokenContext.Ticket);
await Options.RefreshTokenProvider.CreateAsync(refreshTokenCreateContext);
string refreshToken = refreshTokenCreateContext.Token;
var tokenEndpointResponseContext = new OAuthTokenEndpointResponseContext(
Context,
Options,
ticket,
tokenEndpointRequest,
accessToken,
tokenEndpointContext.AdditionalResponseParameters);
await Options.Provider.TokenEndpointResponse(tokenEndpointResponseContext);
var memory = new MemoryStream();
byte[] body;
using (var writer = new JsonTextWriter(new StreamWriter(memory)))
{
writer.WriteStartObject();
writer.WritePropertyName(Constants.Parameters.AccessToken);
writer.WriteValue(accessToken);
writer.WritePropertyName(Constants.Parameters.TokenType);
writer.WriteValue(Constants.TokenTypes.Bearer);
if (accessTokenExpiresUtc.HasValue)
{
TimeSpan?expiresTimeSpan = accessTokenExpiresUtc - currentUtc;
var expiresIn = (long)expiresTimeSpan.Value.TotalSeconds;
if (expiresIn > 0)
{
writer.WritePropertyName(Constants.Parameters.ExpiresIn);
writer.WriteValue(expiresIn);
}
}
if (!String.IsNullOrEmpty(refreshToken))
{
writer.WritePropertyName(Constants.Parameters.RefreshToken);
writer.WriteValue(refreshToken);
}
foreach (var additionalResponseParameter in tokenEndpointResponseContext.AdditionalResponseParameters)
{
writer.WritePropertyName(additionalResponseParameter.Key);
writer.WriteValue(additionalResponseParameter.Value);
}
writer.WriteEndObject();
writer.Flush();
body = memory.ToArray();
}
Response.ContentType = "application/json;charset=UTF-8";
Response.Headers.Set("Cache-Control", "no-cache");
Response.Headers.Set("Pragma", "no-cache");
Response.Headers.Set("Expires", "-1");
Response.ContentLength = body.Length;
await Response.WriteAsync(body, Request.CallCancelled);
}
private TelekomJsonWebRequest <SmsResponse> CreateAuthenticatedJsonRequest <ResponseType>(string uri, SendSmsRequest request)
{
TelekomJsonWebRequest <SmsResponse> webRequest = CreateAuthenticatedRequest <SmsResponse>(uri, HttpMethod.POST);
Dictionary <string, Dictionary <string, object> > jsonRequest = new Dictionary <string, Dictionary <string, object> >();
//---------------------------------------------------------------------------------------------------------------
Dictionary <string, object> val = new Dictionary <string, object>();
val.Add("address", request.Address);
Dictionary <string, object> helper = new Dictionary <string, object>();
String key = "";
String key2 = "";
if (request.SMSType.Equals(OutboundSMSType.TEXT))
{
key = "outboundSMSTextMessage";
key2 = "message";
}
else if (request.SMSType.Equals(OutboundSMSType.BINARY))
{
key = "outboundSMSBinaryMessage";
key2 = "message";
}
else if (request.SMSType.Equals(OutboundSMSType.FLASH))
{
key = "outboundSMSFlashMessage";
key2 = "flashMessage";
}
helper.Add(key2, request.Message);
val.Add(key, helper);
val.Add("senderAddress", request.SenderAddress);
if (request.SenderName != null)
{
val.Add("senderName", request.SenderName);
}
if (request.Account != null)
{
val.Add("account", request.Account);
}
helper = new Dictionary <string, object>();
if (request.CallbackData != null || request.NotifyURL != null)
{
if (request.CallbackData != null)
{
helper.Add("callbackData", request.CallbackData);
}
if (request.NotifyURL != null)
{
helper.Add("notifyURL", request.NotifyURL);
}
val.Add("receiptRequest", helper);
}
if (request.Encoding != null)
{
if (request.Encoding.Equals(OutboundEncoding.GSM))
{
val.Add("outboundEncoding", "7bitGSM");
}
else if (request.Encoding.Equals(OutboundEncoding.UCS))
{
val.Add("outboundEncoding", "16bitUCS2");
}
}
if (request.ClientCorrelator != null)
{
val.Add("clientCorrelator", request.ClientCorrelator);
}
jsonRequest.Add("outboundSMSMessageRequest", val);
JsonSerializer serializer = new JsonSerializer();
MemoryStream ms = new MemoryStream();
StreamWriter sw = new StreamWriter(ms);
JsonWriter writer = new JsonTextWriter(sw);
serializer.Serialize(writer, jsonRequest);
writer.Flush();
sw.Flush();
ms.Position = 0;
ms.Flush();
webRequest.SetRawContent(ms, "application/json");
//var sr = new StreamReader(ms);
//var myStr = sr.ReadToEnd();
//Console.WriteLine(myStr);
return(webRequest);
}
protected override async Task <int> Run()
{
var payload = new JObject();
payload["@t"] = string.IsNullOrWhiteSpace(_timestamp) ?
DateTimeOffset.Now.ToString("o") :
_timestamp;
if (_level != null && _level != "Information")
{
payload["@l"] = _level;
}
if (!string.IsNullOrWhiteSpace(_message))
{
payload["@mt"] = _message;
}
if (!string.IsNullOrWhiteSpace(_exception))
{
payload["@x"] = _exception;
}
foreach (var property in _properties.Properties)
{
if (string.IsNullOrWhiteSpace(property.Key))
{
continue;
}
var name = property.Key.Trim();
if (name.StartsWith("@"))
{
name = $"@{name}";
}
payload[name] = new JValue(property.Value);
}
StringContent content;
using (var builder = new StringWriter())
using (var jsonWriter = new JsonTextWriter(builder))
{
payload.WriteTo(jsonWriter);
jsonWriter.Flush();
builder.WriteLine();
content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediatType);
}
var connection = _connectionFactory.Connect(_connection);
var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint)
{
Content = content
};
if (_connection.IsApiKeySpecified)
{
request.Headers.Add("X-Seq-ApiKey", _connection.ApiKey);
}
var result = await connection.Client.HttpClient.SendAsync(request);
if (result.IsSuccessStatusCode)
{
return(0);
}
var resultJson = await result.Content.ReadAsStringAsync();
if (!string.IsNullOrWhiteSpace(resultJson))
{
try
{
var error = JsonConvert.DeserializeObject <dynamic>(resultJson);
Log.Error("Failed with status code {StatusCode}: {ErrorMessage}",
result.StatusCode,
(string)error.ErrorMessage);
}
catch
{
// ignored
}
}
Log.Error("Failed with status code {StatusCode} ({ReasonPhrase})", result.StatusCode, result.ReasonPhrase);
return(1);
}
private async Task TransformToSDataPayload(Stream targetStream)
{
using (var oContentStream = await originalContent.ReadAsStreamAsync())
{
JsonReader reader = null;
JsonWriter outStream = null;
int level = -1;
try
{
reader = new JsonTextReader(new StreamReader(oContentStream));
outStream = new JsonTextWriter(new StreamWriter(targetStream));
int respContentType = GetContentType();
bool searchForCollection = respContentType == ContentOdataCollection ||
respContentType == ContentApiCollection;
while (reader.Read())
{
if (searchForCollection && level == 0 && reader.TokenType == JsonToken.PropertyName)
{
if ((string)reader.Value == ParamCollectionNameApi || (string)reader.Value == ParamCollectionNameOdata)
{
WritePagingProperties(outStream);
outStream.WritePropertyName(ParamCollectionNameSdata);
}
else if ((string)reader.Value == ParamTotalresultOdata)
{
outStream.WritePropertyName(ParamTotalresultSdata);
}
else
{
outStream.WriteToken(reader, false);
}
}
else if (reader.TokenType == JsonToken.PropertyName)
{
outStream.WritePropertyName(CallOptionalMaps((string)reader.Value));
}
else
{
if (reader.TokenType == JsonToken.StartObject)
{
level++;
}
else if (reader.TokenType == JsonToken.EndObject)
{
level--;
}
outStream.WriteToken(reader, false);
}
}
outStream.Flush();
}
catch (Exception e)
{
// TODO can we get the user and tenant context from here?
}
finally
{
if (reader != null)
{
reader.Close();
}
if (outStream != null)
{
outStream.Close();
}
}
}
}
private static void Go(Engine engine)
{
dynamic outpostLookupClass = engine.GetClass("WillowGame.EmergencyTeleportOutpostLookup");
dynamic willowGlobalsClass = engine.GetClass("WillowGame.WillowGlobals");
dynamic dlcPackageClass = engine.GetClass("WillowGame.DLCPackageDefinition");
if (outpostLookupClass == null ||
willowGlobalsClass == null ||
dlcPackageClass == null)
{
throw new InvalidOperationException();
}
Directory.CreateDirectory("dumps");
var sourceLookup = new Dictionary <string, string>();
dynamic willowGlobals = engine.Objects.Single(
o => o.IsA(willowGlobalsClass) == true &&
o.GetName().StartsWith("Default__") == false);
dynamic masterLookup = willowGlobals.MasterRegistrationStationList;
var outpostOrder = new List <string>();
foreach (dynamic lookupObject in masterLookup.OutpostLookupList)
{
outpostOrder.Add(lookupObject.OutpostName);
}
foreach (dynamic dlcPackage in engine.Objects.Where(
o => o.IsA(dlcPackageClass) == true &&
o.GetName().StartsWith("Default__") == false))
{
sourceLookup.Add(dlcPackage.TeleportLookupObject.GetPath(), dlcPackage.GetPath());
}
using (var output = new StreamWriter(Path.Combine("dumps", "Emergency Teleport Outposts.json"), false, Encoding.Unicode))
using (var writer = new JsonTextWriter(output))
{
writer.Indentation = 2;
writer.IndentChar = ' ';
writer.Formatting = Formatting.Indented;
var outpostNames = new List <string>();
writer.WriteStartObject();
foreach (dynamic lookup in engine.Objects
.Where(
o => o.IsA(outpostLookupClass) &&
o.GetName().StartsWith("Default__") == false)
.OrderBy(o => o.GetPath())
.Distinct())
{
// The master list has every single station, we don't want to write it too.
if (lookup == masterLookup)
{
continue;
}
writer.WritePropertyName(lookup.GetPath());
writer.WriteStartObject();
string source;
if (sourceLookup.TryGetValue((string)lookup.GetPath(), out source) == true &&
source != "None")
{
writer.WritePropertyName("dlc_package");
writer.WriteValue(source);
}
writer.WritePropertyName("outposts");
writer.WriteStartArray();
foreach (dynamic lookupObject in ((UnrealObject[])lookup.OutpostLookupList)
.Cast <dynamic>()
.OrderBy(l => outpostOrder.IndexOf(l.OutpostName)))
{
var outpostName = (string)lookupObject.OutpostName;
if (outpostNames.Contains(outpostName) == true)
{
Console.WriteLine($"Skipping duplicate outpost {outpostName}.");
continue;
}
outpostNames.Add(outpostName);
writer.WriteStartObject();
writer.WritePropertyName("name");
writer.WriteValue(outpostName);
writer.WritePropertyName("path");
writer.WriteValue(lookupObject.OutpostPathName);
var isInitiallyActive = (bool)lookupObject.bInitiallyActive;
if (isInitiallyActive == true)
{
writer.WritePropertyName("is_initially_active");
writer.WriteValue(true);
}
var isCheckpointOnly = (bool)lookupObject.bCheckpointOnly;
if (isCheckpointOnly == true)
{
writer.WritePropertyName("is_checkpoint_only");
writer.WriteValue(true);
}
var displayName = (string)lookupObject.OutpostDisplayName;
if (string.IsNullOrEmpty(displayName) == false)
{
writer.WritePropertyName("display_name");
writer.WriteValue(displayName);
}
var description = (string)lookupObject.OutpostDescription;
if (string.IsNullOrEmpty(description) == false &&
description != "No Description")
{
writer.WritePropertyName("description");
writer.WriteValue(description);
}
var previousOutpost = (string)lookupObject.PreviousOutpost;
if (string.IsNullOrEmpty(previousOutpost) == false &&
previousOutpost != "None")
{
writer.WritePropertyName("previous_outpost");
writer.WriteValue(previousOutpost);
}
if (lookupObject.MissionDependencies != null &&
lookupObject.MissionDependencies.Length > 0)
{
writer.WritePropertyName("mission_dependencies");
writer.WriteStartObject();
foreach (var missionDependency in lookupObject.MissionDependencies)
{
writer.WritePropertyName(missionDependency.MissionDefinition.GetPath());
writer.WriteValue(((MissionStatus)missionDependency.MissionStatus).ToString());
}
writer.WriteEndObject();
}
var outpostIndex = outpostOrder.IndexOf(outpostName);
if (outpostIndex < 0)
{
throw new InvalidOperationException();
}
writer.WritePropertyName("sort_order");
writer.WriteValue(outpostIndex);
writer.WriteEndObject();
}
writer.WriteEndArray();
writer.WriteEndObject();
}
writer.WriteEndObject();
writer.Flush();
}
}
public static void ImportData(string instanceUrl, string file)
{
var sw = Stopwatch.StartNew();
using (FileStream fileStream = File.OpenRead(file))
{
// Try to read the stream compressed, otherwise continue uncompressed.
JsonTextReader jsonReader;
try
{
StreamReader streamReader = new StreamReader(new GZipStream(fileStream, CompressionMode.Decompress));
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
catch (InvalidDataException)
{
fileStream.Seek(0, SeekOrigin.Begin);
StreamReader streamReader = new StreamReader(fileStream);
jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
{
return;
}
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
// should read indexes now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Indexes", jsonReader.Value) == false)
{
throw new InvalidDataException("Indexes property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
using (var webClient = new WebClient())
{
webClient.UseDefaultCredentials = true;
webClient.Headers.Add("Content-Type", "application/json; charset=utf-8");
webClient.Credentials = CredentialCache.DefaultNetworkCredentials;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = RavenJToken.ReadFrom(jsonReader);
var indexName = index.Value <string>("name");
if (indexName.StartsWith("Raven/") || indexName.StartsWith("Temp/"))
{
continue;
}
using (var streamWriter = new StreamWriter(webClient.OpenWrite(instanceUrl + "indexes/" + indexName, "PUT")))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
index.Value <RavenJObject>("definition").WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
streamWriter.Flush();
}
}
}
// should read documents now
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Docs", jsonReader.Value) == false)
{
throw new InvalidDataException("Docs property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
var batch = new List <RavenJObject>();
int totalCount = 0;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
totalCount += 1;
var document = RavenJToken.ReadFrom(jsonReader);
batch.Add((RavenJObject)document);
if (batch.Count >= 128)
{
FlushBatch(instanceUrl, batch);
}
}
FlushBatch(instanceUrl, batch);
var attachmentCount = 0;
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new InvalidDataException("PropertyName was expected");
}
if (Equals("Attachments", jsonReader.Value) == false)
{
throw new InvalidDataException("Attachment property was expected");
}
if (jsonReader.Read() == false)
{
return;
}
if (jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
using (var client = new WebClient())
{
attachmentCount += 1;
var item = RavenJToken.ReadFrom(jsonReader);
var attachmentExportInfo =
new JsonSerializer
{
Converters = { new TrivialJsonToJsonJsonConverter() }
}.Deserialize <AttachmentExportInfo>(new RavenJTokenReader(item));
Console.WriteLine("Importing attachment {0}", attachmentExportInfo.Key);
if (attachmentExportInfo.Metadata != null)
{
foreach (var header in attachmentExportInfo.Metadata)
{
client.Headers.Add(header.Key, StripQuotesIfNeeded(header.Value));
}
}
using (var writer = client.OpenWrite(instanceUrl + "static/" + attachmentExportInfo.Key, "PUT"))
{
writer.Write(attachmentExportInfo.Data, 0, attachmentExportInfo.Data.Length);
writer.Flush();
}
}
}
Console.WriteLine("Imported {0:#,#} documents and {1:#,#} attachments in {2:#,#} ms", totalCount, attachmentCount, sw.ElapsedMilliseconds);
}
}
public BulkBatchResult Bulk(IEnumerable <BulkOperation> operations, Action <string> logger)
{
var bulkBatchResult = new BulkBatchResult();
if (operations == null)
{
return(bulkBatchResult);
}
var uri = $"{_settings.Host}/_bulk?pipeline={Pipelines.Attachment.Name}";
var operationList = operations.ToList();
operationList.ForEach(operation =>
{
if (operation.MetaData.Index == null)
{
operation.MetaData.Index = operation.MetaData.IndexCandidate;
}
if (operation.MetaData.Index == null)
{
throw new InvalidOperationException("Index missing");
}
});
var indexes = operationList
.Select(o => o.MetaData.Index.ToLower())
.Distinct()
.ToList();
var totalCount = operationList.Count;
var counter = 0;
var size = _settings.BulkSize;
while (operationList.Count > 0)
{
var batch = operationList.Take(size).ToList();
try
{
var ms = new MemoryStream();
using (var sw = new StreamWriter(ms, Encoding.UTF8, 1024, true))
{
counter += size;
var from = counter - size;
var to = Math.Min(counter, totalCount);
if (totalCount > 1)
{
_logger.Information($"Processing batch {from}-{to} of {totalCount}");
}
if (_logger.IsDebugEnabled())
{
logger("WARNING: Debug logging is enabled, this will have a huge impact on indexing-time for large structures.");
}
foreach (BulkOperation operation in batch)
{
using (var jsonTextWriter = new JsonTextWriter(sw))
{
var serializer = GetSerializer();
serializer.Serialize(jsonTextWriter, operation.MetaData);
jsonTextWriter.WriteWhitespace("\n");
try
{
serializer.Serialize(jsonTextWriter, operation.Data);
}
catch (OutOfMemoryException)
{
_logger.Warning($"Failed to process operation {operation.MetaData.Id}. Too much data.");
}
jsonTextWriter.WriteWhitespace("\n");
jsonTextWriter.Flush();
}
}
}
if (ms.CanSeek)
{
ms.Seek(0, SeekOrigin.Begin);
}
var results = _httpClientHelper.Post(new Uri(uri), ms);
var stringReader = new StringReader(Encoding.UTF8.GetString(results));
var bulkResult = GetSerializer().Deserialize <BulkResult>(new JsonTextReader(stringReader));
bulkBatchResult.Batches.Add(bulkResult);
}
catch (Exception e)
{
_logger.Error("Batch failed", e);
logger("Batch failed: " + e.Message);
}
finally
{
operationList.RemoveAll(o => batch.Contains(o));
}
}
indexes.ForEach(RefreshIndex);
return(bulkBatchResult);
}
public void ProcessRequest(HttpContext context)
{
NameObjectCollectionBase.KeysCollection
keys = context.Request.Form.Keys;
string
tmpString = string.Empty;
foreach (string key in keys)
{
if (tmpString != string.Empty)
{
tmpString += ", ";
}
tmpString += "Form[\"" + key + "\"]=\"" + context.Request.Form[key] + "\"";
}
context.Response.CacheControl = "no-cache";
context.Response.Expires = -1;
context.Response.ContentType = "application/json";
/*
* JsonObject
* RootJsonObject = new JsonObject(new Dictionary<string, object> {
* { "success", false },
* { "errorMessage", "errorMessage" }
* });
*/
JsonObject
RootJsonObject = new JsonObject(new Dictionary <string, object> {
{ "success", true },
{ "data", new JsonObject(new Dictionary <string, object> {
{ "HiddenField1", 123456 },
{ "NumberField", "123456" },
{ "TextField", "dleiFtxeT" },
{ "DateField", "17.06.2010" },
{ "TimeField", "13:00" },
{ "DateTimeField", new DateTime(2010, 06, 17, 13, 0, 13).ToString("o") },
{ "Radio", "Radio2" },
{ "CheckBox", "1" },
{ "TextArea", "TextArea" },
{ "HtmlEditor", "HtmlEditor" },
//{ "ComboBox1", "778" },
//{ "ComboBox1", 778 },
{ "ComboBox1HN", "FOTO ГРAЧИ УЛЕТЕЛИ" }, // 778
//{ "ComboBox2", "800" },
{ "ComboBox2HN", "FOTO ГРАЧИ БЕЗНАЛИЧНЫЕ" }, // 800
{ "ComboBox3HN", "9" },
//{ "ComboBox4HN", "11" }
{ "ComboBox4", 11 }
}) }
});
JsonTextWriter
tmpJsonTextWriter = new JsonTextWriter(context.Response.Output);
RootJsonObject.Export(tmpJsonTextWriter);
tmpJsonTextWriter.Flush();
tmpJsonTextWriter.Close();
}
private bool SaveInternal(string filePath)
{
string typeName = GetType().Name;
DebugHelper.WriteLine("{0} save started: {1}", typeName, filePath);
bool isSuccess = false;
try
{
if (!string.IsNullOrEmpty(filePath))
{
lock (this)
{
Helpers.CreateDirectoryFromFilePath(filePath);
string tempFilePath = filePath + ".temp";
using (FileStream fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.Read))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (JsonTextWriter jsonWriter = new JsonTextWriter(streamWriter))
{
jsonWriter.DateTimeZoneHandling = DateTimeZoneHandling.Utc;
jsonWriter.Formatting = Formatting.Indented;
JsonSerializer serializer = new JsonSerializer();
serializer.ContractResolver = new WritablePropertiesOnlyResolver();
serializer.Converters.Add(new StringEnumConverter());
serializer.Serialize(jsonWriter, this);
jsonWriter.Flush();
}
if (File.Exists(filePath))
{
if (CreateBackup)
{
Helpers.CopyFile(filePath, BackupFolder);
}
File.Delete(filePath);
}
File.Move(tempFilePath, filePath);
if (CreateWeeklyBackup && !string.IsNullOrEmpty(BackupFolder))
{
Helpers.BackupFileWeekly(filePath, BackupFolder);
}
isSuccess = true;
}
}
}
catch (Exception e)
{
DebugHelper.WriteException(e);
OnSettingsSaveFailed(e);
}
finally
{
DebugHelper.WriteLine("{0} save {1}: {2}", typeName, isSuccess ? "successful" : "failed", filePath);
}
return(isSuccess);
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
using (var streamWriter = new StreamWriter(stream))
using (var writer = new JsonTextWriter(streamWriter))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.DatabaseOperationTimeout))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
Action <JsonDocument> addDocument = doc =>
{
timeout.Delay();
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return;
}
if (batchDocCount >= options.MaxDocCount)
{
return;
}
lastProcessedDocEtag = doc.Etag;
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
if (MatchCriteria(criteria, doc) == false)
{
return;
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
};
int nextStart = 0;
do
{
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSettingDocumentsInDocumentCache())
{
if (!string.IsNullOrWhiteSpace(criteria.KeyStartsWith))
{
Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
}
else
{
Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
}
}
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
});
} while (hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
public static void Pack(string inputFilePath, string outputFilePath)
{
var inputDirectoryPath = Path.GetDirectoryName(inputFilePath);//文件根目录
JObject json;
using (var jsonStream = File.OpenRead(inputFilePath))
using (var jsonStreamReader = new StreamReader(jsonStream))
using (var jsonTextReader = new JsonTextReader(jsonStreamReader))
{
json = (JObject)JToken.ReadFrom(jsonTextReader);
}
var position = 0;
List <string> views = new List <string>();
var buffers = (JArray)json["buffers"];
var bufferViews = (JArray)json["bufferViews"];
var images = (JArray)json["images"];
if (buffers != null)
{
for (var index = buffers.Count - 1; index >= 0; index--)
{
var buffer = (JObject)buffers[index];
var uri = (string)buffer["uri"];
if (uri != null && !Tools.IsBase64(uri))
{
foreach (JObject bufferView in bufferViews)
{
var bufferIndex = (int)bufferView["buffer"];
if (bufferIndex == index)
{
bufferView["buffer"] = -1;
var byteOffset = (int?)bufferView["byteOffset"] ?? 0;
bufferView.SetValue("byteOffset", position + byteOffset, 0);
}
}
var filePath = Path.Combine(inputDirectoryPath, uri);
views.Add(filePath);
var fileLength = Tools.GetFileLength(filePath);
position += fileLength;
position = Tools.Align(position);
buffers.RemoveAt(index);
}
}
}
if (images != null)
{
foreach (JObject image in images)
{
var uri = (string)image["uri"];
if (uri != null && !Tools.IsBase64(uri))
{
var filePath = Path.Combine(inputDirectoryPath, uri);
views.Add(filePath);
var fileLength = Tools.GetFileLength(filePath);
image.Remove("uri");
image["bufferView"] = bufferViews.Count;
image["mimeType"] = MimeType.FromFileExtension(Path.GetExtension(uri));
position = Tools.Align(position);
JObject bufferView = new JObject();
bufferView["buffer"] = -1;
bufferView["byteLength"] = fileLength;
bufferView.SetValue("byteOffset", position, 0);
bufferViews.Add(bufferView);
position += fileLength;
}
}
}
if (views.Count != 0)
{
if (buffers == null)
{
json["buffers"] = new JArray();
}
JObject item = new JObject();
item["byteLength"] = position;
buffers.Insert(0, item);
foreach (var bufferView in bufferViews)
{
var bufferIndex = (int)bufferView["buffer"];
bufferView["buffer"] = bufferIndex + 1;
}
}
using (var fileStream = File.Create(outputFilePath))
using (var binaryWriter = new BinaryWriter(fileStream))
{
binaryWriter.Write(Binary.Magic);
binaryWriter.Write(Binary.Version);
var chunksPosition = binaryWriter.BaseStream.Position;
binaryWriter.Write(0U); // length
var jsonChunkPosition = binaryWriter.BaseStream.Position;
binaryWriter.Write(0U); // json chunk length
binaryWriter.Write(Binary.ChunkFormatJson);
var streamWriter = new StreamWriter(binaryWriter.BaseStream, new UTF8Encoding(false, true), 1024);
var jsonTextWriter = new JsonTextWriter(streamWriter);
json.WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
binaryWriter.BaseStream.Align(0x20);
var jsonChunkLength = checked ((uint)(binaryWriter.BaseStream.Length - jsonChunkPosition)) - Binary.ChunkHeaderLength;
binaryWriter.BaseStream.Seek(jsonChunkPosition, SeekOrigin.Begin);
binaryWriter.Write(jsonChunkLength);
if (views.Count != 0)
{
binaryWriter.BaseStream.Seek(0, SeekOrigin.End);
var binChunkPosition = binaryWriter.BaseStream.Position;
binaryWriter.Write(0); // bin chunk length
binaryWriter.Write(Binary.ChunkFormatBin);
foreach (var fileName in views)
{
var viewBytes = File.ReadAllBytes(fileName);
binaryWriter.BaseStream.Align();
binaryWriter.Write(viewBytes);
}
binaryWriter.BaseStream.Align(0x20);
var binChunkLength = checked ((uint)(binaryWriter.BaseStream.Length - binChunkPosition)) - Binary.ChunkHeaderLength;
binaryWriter.BaseStream.Seek(binChunkPosition, SeekOrigin.Begin);
binaryWriter.Write(binChunkLength);
}
var length = checked ((uint)binaryWriter.BaseStream.Length);
binaryWriter.BaseStream.Seek(chunksPosition, SeekOrigin.Begin);
binaryWriter.Write(length);
jsonTextWriter.Close();
streamWriter.Dispose();
}
}
private async Task <bool> InvokeIntrospectionEndpointAsync()
{
OpenIdConnectMessage request;
// See https://tools.ietf.org/html/rfc7662#section-2.1
// and https://tools.ietf.org/html/rfc7662#section-4
if (string.Equals(Request.Method, "GET", StringComparison.OrdinalIgnoreCase))
{
request = new OpenIdConnectMessage(Request.Query)
{
RequestType = OpenIdConnectRequestType.AuthenticationRequest
};
}
else if (string.Equals(Request.Method, "POST", StringComparison.OrdinalIgnoreCase))
{
// See http://openid.net/specs/openid-connect-core-1_0.html#FormSerialization
if (string.IsNullOrEmpty(Request.ContentType))
{
Options.Logger.LogError("The introspection request was rejected because " +
"the mandatory 'Content-Type' header was missing.");
return(await SendErrorPayloadAsync(new OpenIdConnectMessage {
Error = OpenIdConnectConstants.Errors.InvalidRequest,
ErrorDescription = "A malformed introspection request has been received: " +
"the mandatory 'Content-Type' header was missing from the POST request."
}));
}
// May have media/type; charset=utf-8, allow partial match.
if (!Request.ContentType.StartsWith("application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase))
{
Options.Logger.LogError("The introspection request was rejected because an invalid 'Content-Type' " +
"header was received: {ContentType}.", Request.ContentType);
return(await SendErrorPayloadAsync(new OpenIdConnectMessage {
Error = OpenIdConnectConstants.Errors.InvalidRequest,
ErrorDescription = "A malformed introspection request has been received: " +
"the 'Content-Type' header contained an unexcepted value. " +
"Make sure to use 'application/x-www-form-urlencoded'."
}));
}
request = new OpenIdConnectMessage(await Request.ReadFormAsync())
{
RequestType = OpenIdConnectRequestType.AuthenticationRequest
};
}
else
{
Options.Logger.LogError("The introspection request was rejected because an invalid " +
"HTTP method was received: {Method}.", Request.Method);
return(await SendErrorPageAsync(new OpenIdConnectMessage {
Error = OpenIdConnectConstants.Errors.InvalidRequest,
ErrorDescription = "A malformed introspection request has been received: " +
"make sure to use either GET or POST."
}));
}
if (string.IsNullOrWhiteSpace(request.Token))
{
return(await SendErrorPayloadAsync(new OpenIdConnectMessage {
Error = OpenIdConnectConstants.Errors.InvalidRequest,
ErrorDescription = "A malformed introspection request has been received: " +
"a 'token' parameter with an access, refresh, or identity token is required."
}));
}
// Insert the introspection request in the OWIN context.
Context.SetOpenIdConnectRequest(request);
// When client_id and client_secret are both null, try to extract them from the Authorization header.
// See http://tools.ietf.org/html/rfc6749#section-2.3.1 and
// http://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication
if (string.IsNullOrEmpty(request.ClientId) && string.IsNullOrEmpty(request.ClientSecret))
{
var header = Request.Headers.Get("Authorization");
if (!string.IsNullOrEmpty(header) && header.StartsWith("Basic ", StringComparison.OrdinalIgnoreCase))
{
try {
var value = header.Substring("Basic ".Length).Trim();
var data = Encoding.UTF8.GetString(Convert.FromBase64String(value));
var index = data.IndexOf(':');
if (index >= 0)
{
request.ClientId = data.Substring(0, index);
request.ClientSecret = data.Substring(index + 1);
}
}
catch (FormatException) { }
catch (ArgumentException) { }
}
}
var validatingContext = new ValidateIntrospectionRequestContext(Context, Options, request);
await Options.Provider.ValidateIntrospectionRequest(validatingContext);
if (validatingContext.IsRejected)
{
Options.Logger.LogInformation("The introspection request was rejected by application code.");
return(await SendErrorPayloadAsync(new OpenIdConnectMessage {
Error = validatingContext.Error ?? OpenIdConnectConstants.Errors.InvalidRequest,
ErrorDescription = validatingContext.ErrorDescription,
ErrorUri = validatingContext.ErrorUri
}));
}
// Ensure that the client_id has been set from the ValidateIntrospectionRequest event.
else if (validatingContext.IsValidated && string.IsNullOrEmpty(request.ClientId))
{
Options.Logger.LogError("The introspection request was validated but the client_id was not set.");
return(await SendErrorPayloadAsync(new OpenIdConnectMessage {
Error = OpenIdConnectConstants.Errors.ServerError,
ErrorDescription = "An internal server error occurred."
}));
}
AuthenticationTicket ticket = null;
// Note: use the "token_type_hint" parameter to determine
// the type of the token sent by the client application.
// See https://tools.ietf.org/html/rfc7662#section-2.1
switch (request.GetTokenTypeHint())
{
case OpenIdConnectConstants.Usages.AccessToken:
ticket = await DeserializeAccessTokenAsync(request.Token, request);
break;
case OpenIdConnectConstants.Usages.RefreshToken:
ticket = await DeserializeRefreshTokenAsync(request.Token, request);
break;
case OpenIdConnectConstants.Usages.IdToken:
ticket = await DeserializeIdentityTokenAsync(request.Token, request);
break;
}
// Note: if the token can't be found using "token_type_hint",
// the search must be extended to all supported token types.
// See https://tools.ietf.org/html/rfc7662#section-2.1
if (ticket == null)
{
ticket = await DeserializeAccessTokenAsync(request.Token, request) ??
await DeserializeIdentityTokenAsync(request.Token, request) ??
await DeserializeRefreshTokenAsync(request.Token, request);
}
if (ticket == null)
{
Options.Logger.LogInformation("The introspection request was rejected because the token was invalid.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
// Note: unlike refresh or identity tokens that can only be validated by client applications,
// access tokens can be validated by either resource servers or client applications:
// in both cases, the caller must be authenticated if the ticket is marked as confidential.
if (validatingContext.IsSkipped && ticket.IsConfidential())
{
Options.Logger.LogWarning("The introspection request was rejected because the caller was not authenticated.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
// If the ticket is already expired, directly return active=false.
if (ticket.Properties.ExpiresUtc.HasValue &&
ticket.Properties.ExpiresUtc < Options.SystemClock.UtcNow)
{
Options.Logger.LogInformation("The introspection request was rejected because the token was expired.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
// When a client_id can be inferred from the introspection request,
// ensure that the client application is a valid audience/presenter.
if (!string.IsNullOrEmpty(request.ClientId))
{
// Ensure the caller is listed as a valid audience or authorized presenter.
if (ticket.IsAccessToken() && ticket.HasAudience() && !ticket.HasAudience(request.ClientId) &&
ticket.HasPresenter() && !ticket.HasPresenter(request.ClientId))
{
Options.Logger.LogWarning("The introspection request was rejected because the access token " +
"was issued to a different client or for another resource server.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
// Reject the request if the caller is not listed as a valid audience.
else if (ticket.IsIdentityToken() && ticket.HasAudience() && !ticket.HasAudience(request.ClientId))
{
Options.Logger.LogWarning("The introspection request was rejected because the " +
"identity token was issued to a different client.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
// Reject the introspection request if the caller doesn't
// correspond to the client application the token was issued to.
else if (ticket.IsRefreshToken() && ticket.HasPresenter() && !ticket.HasPresenter(request.ClientId))
{
Options.Logger.LogWarning("The introspection request was rejected because the " +
"refresh token was issued to a different client.");
return(await SendPayloadAsync(new JObject {
[OpenIdConnectConstants.Claims.Active] = false
}));
}
}
var notification = new HandleIntrospectionRequestContext(Context, Options, request, ticket);
notification.Active = true;
// Use the unique ticket identifier to populate the "jti" claim.
notification.TokenId = ticket.GetTicketId();
// Note: only set "token_type" when the received token is an access token.
// See https://tools.ietf.org/html/rfc7662#section-2.2
// and https://tools.ietf.org/html/rfc6749#section-5.1
if (ticket.IsAccessToken())
{
notification.TokenType = OpenIdConnectConstants.TokenTypes.Bearer;
}
notification.Issuer = Context.GetIssuer(Options);
notification.Subject = ticket.Identity.GetClaim(ClaimTypes.NameIdentifier);
notification.IssuedAt = ticket.Properties.IssuedUtc;
notification.ExpiresAt = ticket.Properties.ExpiresUtc;
// Copy the audiences extracted from the "aud" claim.
foreach (var audience in ticket.GetAudiences())
{
notification.Audiences.Add(audience);
}
// Note: non-metadata claims are only added if the caller is authenticated
// AND is in the specified audiences, unless there's so explicit audience.
if (!ticket.HasAudience() || (!string.IsNullOrEmpty(request.ClientId) && ticket.HasAudience(request.ClientId)))
{
notification.Username = ticket.Identity.Name;
notification.Scope = ticket.GetProperty(OpenIdConnectConstants.Properties.Scopes);
// Potentially sensitive claims are only exposed to trusted callers
// if the ticket corresponds to an access or identity token.
if (ticket.IsAccessToken() || ticket.IsIdentityToken())
{
foreach (var claim in ticket.Identity.Claims)
{
// Exclude standard claims, that are already handled via strongly-typed properties.
// Make sure to always update this list when adding new built-in claim properties.
if (string.Equals(claim.Type, ticket.Identity.NameClaimType, StringComparison.Ordinal) ||
string.Equals(claim.Type, ClaimTypes.NameIdentifier, StringComparison.Ordinal))
{
continue;
}
if (string.Equals(claim.Type, OpenIdConnectConstants.Claims.Audience, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.ExpiresAt, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.IssuedAt, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.Issuer, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.NotBefore, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.Scope, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.Subject, StringComparison.Ordinal) ||
string.Equals(claim.Type, OpenIdConnectConstants.Claims.TokenType, StringComparison.Ordinal))
{
continue;
}
string type;
// Try to resolve the short name associated with the claim type:
// if none can be found, the claim type is used as-is.
if (!JwtSecurityTokenHandler.OutboundClaimTypeMap.TryGetValue(claim.Type, out type))
{
type = claim.Type;
}
// Note: make sure to use the indexer
// syntax to avoid duplicate properties.
notification.Claims[type] = claim.Value;
}
}
}
await Options.Provider.HandleIntrospectionRequest(notification);
// Flow the changes made to the authentication ticket.
ticket = notification.Ticket;
if (notification.HandledResponse)
{
return(true);
}
else if (notification.Skipped)
{
return(false);
}
var payload = new JObject();
payload.Add(OpenIdConnectConstants.Claims.Active, notification.Active);
// Only add the other properties if
// the token is considered as active.
if (notification.Active)
{
if (!string.IsNullOrEmpty(notification.Issuer))
{
payload.Add(OpenIdConnectConstants.Claims.Issuer, notification.Issuer);
}
if (!string.IsNullOrEmpty(notification.Username))
{
payload.Add(OpenIdConnectConstants.Claims.Username, notification.Username);
}
if (!string.IsNullOrEmpty(notification.Subject))
{
payload.Add(OpenIdConnectConstants.Claims.Subject, notification.Subject);
}
if (!string.IsNullOrEmpty(notification.Scope))
{
payload.Add(OpenIdConnectConstants.Claims.Scope, notification.Scope);
}
if (notification.IssuedAt.HasValue)
{
payload.Add(OpenIdConnectConstants.Claims.IssuedAt,
EpochTime.GetIntDate(notification.IssuedAt.Value.UtcDateTime));
payload.Add(OpenIdConnectConstants.Claims.NotBefore,
EpochTime.GetIntDate(notification.IssuedAt.Value.UtcDateTime));
}
if (notification.ExpiresAt.HasValue)
{
payload.Add(OpenIdConnectConstants.Claims.ExpiresAt,
EpochTime.GetIntDate(notification.ExpiresAt.Value.UtcDateTime));
}
if (!string.IsNullOrEmpty(notification.TokenId))
{
payload.Add(OpenIdConnectConstants.Claims.JwtId, notification.TokenId);
}
if (!string.IsNullOrEmpty(notification.TokenType))
{
payload.Add(OpenIdConnectConstants.Claims.TokenType, notification.TokenType);
}
switch (notification.Audiences.Count)
{
case 0: break;
case 1:
payload.Add(OpenIdConnectConstants.Claims.Audience, notification.Audiences[0]);
break;
default:
payload.Add(OpenIdConnectConstants.Claims.Audience, JArray.FromObject(notification.Audiences));
break;
}
foreach (var claim in notification.Claims)
{
// Ignore claims whose value is null.
if (claim.Value == null)
{
continue;
}
// Note: make sure to use the indexer
// syntax to avoid duplicate properties.
payload[claim.Key] = claim.Value;
}
}
var context = new ApplyIntrospectionResponseContext(Context, Options, payload);
await Options.Provider.ApplyIntrospectionResponse(context);
if (context.HandledResponse)
{
return(true);
}
else if (context.Skipped)
{
return(false);
}
using (var buffer = new MemoryStream())
using (var writer = new JsonTextWriter(new StreamWriter(buffer))) {
payload.WriteTo(writer);
writer.Flush();
Response.ContentLength = buffer.Length;
Response.ContentType = "application/json;charset=UTF-8";
Response.Headers.Set("Cache-Control", "no-cache");
Response.Headers.Set("Pragma", "no-cache");
Response.Headers.Set("Expires", "-1");
buffer.Seek(offset: 0, loc: SeekOrigin.Begin);
await buffer.CopyToAsync(Response.Body, 4096, Request.CallCancelled);
return(true);
}
}
public static void CreateInfoPackageForDatabase(ZipArchive package, DocumentDatabase database, RequestManager requestManager, ClusterManager clusterManager, string zipEntryPrefix = null)
{
zipEntryPrefix = zipEntryPrefix ?? string.Empty;
var databaseName = database.Name;
if (string.IsNullOrWhiteSpace(databaseName))
{
databaseName = Constants.SystemDatabase;
}
var jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
jsonSerializer.Formatting = Formatting.Indented;
if (database.StartupTasks.OfType <ReplicationTask>().Any())
{
var replication = package.CreateEntry(zipEntryPrefix + "replication.json", CompressionLevel);
using (var statsStream = replication.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, ReplicationUtils.GetReplicationInformation(database));
streamWriter.Flush();
}
}
var sqlReplicationTask = database.StartupTasks.OfType <SqlReplicationTask>().FirstOrDefault();
if (sqlReplicationTask != null)
{
var replication = package.CreateEntry(zipEntryPrefix + "sql_replication.json", CompressionLevel);
using (var statsStream = replication.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, sqlReplicationTask.Statistics);
streamWriter.Flush();
}
}
var stats = package.CreateEntry(zipEntryPrefix + "stats.json", CompressionLevel);
using (var statsStream = stats.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, database.Statistics);
streamWriter.Flush();
}
var indexingPerformanceStats = package.CreateEntry(zipEntryPrefix + "indexing_performance_stats.json", CompressionLevel);
using (var statsStream = indexingPerformanceStats.Open())
using (var streamWriter = new StreamWriter(statsStream))
{
jsonSerializer.Serialize(streamWriter, database.IndexingPerformanceStatistics);
streamWriter.Flush();
}
var metrics = package.CreateEntry(zipEntryPrefix + "metrics.json", CompressionLevel);
using (var metricsStream = metrics.Open())
using (var streamWriter = new StreamWriter(metricsStream))
{
jsonSerializer.Serialize(streamWriter, database.CreateMetrics());
streamWriter.Flush();
}
var logs = package.CreateEntry(zipEntryPrefix + "logs.csv", CompressionLevel);
using (var logsStream = logs.Open())
using (var streamWriter = new StreamWriter(logsStream))
{
var target = LogManager.GetTarget <DatabaseMemoryTarget>();
if (target == null)
{
streamWriter.WriteLine("DatabaseMemoryTarget was not registered in the log manager, logs are not available");
}
else
{
var boundedMemoryTarget = target[databaseName];
var log = boundedMemoryTarget.GeneralLog;
streamWriter.WriteLine("time,logger,level,message,exception");
foreach (var logEvent in log)
{
streamWriter.WriteLine("{0:O},{1},{2},{3},{4}", logEvent.TimeStamp, logEvent.LoggerName, logEvent.Level, logEvent.FormattedMessage, logEvent.Exception);
}
}
streamWriter.Flush();
}
var config = package.CreateEntry(zipEntryPrefix + "config.json", CompressionLevel);
using (var configStream = config.Open())
using (var streamWriter = new StreamWriter(configStream))
using (var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
})
{
GetConfigForDebug(database).WriteTo(jsonWriter, new EtagJsonConverter());
jsonWriter.Flush();
}
var indexes = package.CreateEntry(zipEntryPrefix + "indexes.json", CompressionLevel);
using (var indexesStream = indexes.Open())
using (var streamWriter = new StreamWriter(indexesStream))
{
jsonSerializer.Serialize(streamWriter, database.IndexDefinitionStorage.IndexDefinitions.ToDictionary(x => x.Key, x => x.Value));
streamWriter.Flush();
}
var currentlyIndexing = package.CreateEntry(zipEntryPrefix + "currently-indexing.json", CompressionLevel);
using (var currentlyIndexingStream = currentlyIndexing.Open())
using (var streamWriter = new StreamWriter(currentlyIndexingStream))
{
jsonSerializer.Serialize(streamWriter, GetCurrentlyIndexingForDebug(database));
streamWriter.Flush();
}
var queries = package.CreateEntry(zipEntryPrefix + "queries.json", CompressionLevel);
using (var queriesStream = queries.Open())
using (var streamWriter = new StreamWriter(queriesStream))
{
jsonSerializer.Serialize(streamWriter, database.WorkContext.CurrentlyRunningQueries);
streamWriter.Flush();
}
var version = package.CreateEntry(zipEntryPrefix + "version.json", CompressionLevel);
using (var versionStream = version.Open())
using (var streamWriter = new StreamWriter(versionStream))
{
jsonSerializer.Serialize(streamWriter, new
{
DocumentDatabase.ProductVersion,
DocumentDatabase.BuildVersion
});
streamWriter.Flush();
}
var prefetchStatus = package.CreateEntry(zipEntryPrefix + "prefetch-status.json", CompressionLevel);
using (var prefetchStatusStream = prefetchStatus.Open())
using (var streamWriter = new StreamWriter(prefetchStatusStream))
{
jsonSerializer.Serialize(streamWriter, GetPrefetchingQueueStatusForDebug(database));
streamWriter.Flush();
}
var requestTracking = package.CreateEntry(zipEntryPrefix + "request-tracking.json", CompressionLevel);
using (var requestTrackingStream = requestTracking.Open())
using (var streamWriter = new StreamWriter(requestTrackingStream))
{
jsonSerializer.Serialize(streamWriter, GetRequestTrackingForDebug(requestManager, databaseName));
streamWriter.Flush();
}
var tasks = package.CreateEntry(zipEntryPrefix + "tasks.json", CompressionLevel);
using (var tasksStream = tasks.Open())
using (var streamWriter = new StreamWriter(tasksStream))
{
jsonSerializer.Serialize(streamWriter, GetTasksForDebug(database));
streamWriter.Flush();
}
var systemUtilization = package.CreateEntry(zipEntryPrefix + "system-utilization.json", CompressionLevel);
using (var systemUtilizationStream = systemUtilization.Open())
using (var streamWriter = new StreamWriter(systemUtilizationStream))
{
Size? totalPhysicalMemory = null;
Size? availableMemory = null;
object cpuTimes;
try
{
totalPhysicalMemory = MemoryStatistics.TotalPhysicalMemory;
availableMemory = MemoryStatistics.AvailableMemory;
using (var searcher = new ManagementObjectSearcher("select * from Win32_PerfFormattedData_PerfOS_Processor"))
{
cpuTimes = searcher.Get()
.Cast <ManagementObject>()
.Select(mo => new
{
Name = mo["Name"],
Usage = string.Format("{0} %", mo["PercentProcessorTime"])
}).ToArray();
}
}
catch (Exception e)
{
cpuTimes = "Could not get CPU times" + Environment.NewLine + e;
}
jsonSerializer.Serialize(streamWriter, new
{
TotalPhysicalMemory = totalPhysicalMemory?.ToString(),
AvailableMemory = availableMemory?.ToString(),
CurrentCpuUsage = cpuTimes
});
streamWriter.Flush();
}
if (clusterManager != null && database.IsSystemDatabase())
{
var clusterTopology = package.CreateEntry(zipEntryPrefix + "cluster-topology.json", CompressionLevel);
using (var stream = clusterTopology.Open())
using (var streamWriter = new StreamWriter(stream))
{
jsonSerializer.Serialize(streamWriter, clusterManager.GetTopology());
streamWriter.Flush();
}
var configurationJson = database.Documents.Get(Constants.Cluster.ClusterConfigurationDocumentKey);
if (configurationJson == null)
{
return;
}
var configuration = configurationJson.DataAsJson.JsonDeserialization <ClusterConfiguration>();
var clusterConfiguration = package.CreateEntry(zipEntryPrefix + "cluster-configuration.json", CompressionLevel);
using (var stream = clusterConfiguration.Open())
using (var streamWriter = new StreamWriter(stream))
{
jsonSerializer.Serialize(streamWriter, configuration);
streamWriter.Flush();
}
}
}
private void StreamToClient(long id, SubscriptionActions subscriptions, Stream stream)
{
var sentDocuments = false;
var bufferStream = new BufferedStream(stream, 1024 * 64);
using (var writer = new JsonTextWriter(new StreamWriter(bufferStream)))
{
var options = subscriptions.GetBatchOptions(id);
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
using (var cts = new CancellationTokenSource())
using (var timeout = cts.TimeoutAfter(DatabasesLandlord.SystemConfiguration.Core.DatabaseOperationTimeout.AsTimeSpan))
{
Etag lastProcessedDocEtag = null;
var batchSize = 0;
var batchDocCount = 0;
var processedDocuments = 0;
var hasMoreDocs = false;
var config = subscriptions.GetSubscriptionConfig(id);
var startEtag = config.AckEtag;
var criteria = config.Criteria;
bool isPrefixCriteria = !string.IsNullOrWhiteSpace(criteria.KeyStartsWith);
Func <JsonDocument, bool> addDocument = doc =>
{
timeout.Delay();
if (doc == null)
{
// we only have this heartbeat when the streaming has gone on for a long time
// and we haven't send anything to the user in a while (because of filtering, skipping, etc).
writer.WriteRaw(Environment.NewLine);
writer.Flush();
return(true);
}
processedDocuments++;
// We cant continue because we have already maxed out the batch bytes size.
if (options.MaxSize.HasValue && batchSize >= options.MaxSize)
{
return(false);
}
// We cant continue because we have already maxed out the amount of documents to send.
if (batchDocCount >= options.MaxDocCount)
{
return(false);
}
// We can continue because we are ignoring system documents.
if (doc.Key.StartsWith("Raven/", StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
// We can continue because we are ignoring the document as it doesn't fit the criteria.
if (MatchCriteria(criteria, doc) == false)
{
return(true);
}
doc.ToJson().WriteTo(writer);
writer.WriteRaw(Environment.NewLine);
batchSize += doc.SerializedSizeOnDisk;
batchDocCount++;
return(true); // We get the next document
};
int retries = 0;
do
{
int lastIndex = processedDocuments;
Database.TransactionalStorage.Batch(accessor =>
{
// we may be sending a LOT of documents to the user, and most
// of them aren't going to be relevant for other ops, so we are going to skip
// the cache for that, to avoid filling it up very quickly
using (DocumentCacher.SkipSetAndGetDocumentsInDocumentCache())
{
if (isPrefixCriteria)
{
// If we don't get any document from GetDocumentsWithIdStartingWith it could be that we are in presence of a lagoon of uninteresting documents, so we are hitting a timeout.
lastProcessedDocEtag = Database.Documents.GetDocumentsWithIdStartingWith(criteria.KeyStartsWith, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
hasMoreDocs = false;
}
else
{
// It doesn't matter if we match the criteria or not, the document has been already processed.
lastProcessedDocEtag = Database.Documents.GetDocuments(-1, options.MaxDocCount - batchDocCount, startEtag, cts.Token, addDocument);
// If we don't get any document from GetDocuments it may be a signal that something is wrong.
if (lastProcessedDocEtag == null)
{
hasMoreDocs = false;
}
else
{
var lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
hasMoreDocs = EtagUtil.IsGreaterThan(lastDocEtag, lastProcessedDocEtag);
startEtag = lastProcessedDocEtag;
}
retries = lastIndex == batchDocCount ? retries : 0;
}
}
});
if (lastIndex == processedDocuments)
{
if (retries == 3)
{
log.Warn("Subscription processing did not end up replicating any documents for 3 times in a row, stopping operation", retries);
}
else
{
log.Warn("Subscription processing did not end up replicating any documents, due to possible storage error, retry number: {0}", retries);
}
retries++;
}
}while (retries < 3 && hasMoreDocs && batchDocCount < options.MaxDocCount && (options.MaxSize.HasValue == false || batchSize < options.MaxSize));
writer.WriteEndArray();
if (batchDocCount > 0 || isPrefixCriteria)
{
writer.WritePropertyName("LastProcessedEtag");
writer.WriteValue(lastProcessedDocEtag.ToString());
sentDocuments = true;
}
writer.WriteEndObject();
writer.Flush();
bufferStream.Flush();
}
}
if (sentDocuments)
{
subscriptions.UpdateBatchSentTime(id);
}
}
public async Task ExportAsync(Stream outStream, ExportImportOptions options, Action <ExportImportProgressInfo> progressCallback,
ICancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var progressInfo = new ExportImportProgressInfo {
Description = "loading data..."
};
progressCallback(progressInfo);
using (var sw = new StreamWriter(outStream, System.Text.Encoding.UTF8))
using (var writer = new JsonTextWriter(sw))
{
writer.WriteStartObject();
progressInfo.Description = "Currencies exporting...";
progressCallback(progressInfo);
var currencyResult = await _currencyService.GetAllCurrenciesAsync();
writer.WritePropertyName("CurrencyTotalCount");
writer.WriteValue(currencyResult.Count());
writer.WritePropertyName("Currencies");
writer.WriteStartArray();
foreach (var currency in currencyResult)
{
_serializer.Serialize(writer, currency);
}
writer.Flush();
progressInfo.Description = $"{currencyResult.Count()} currencies exported";
progressCallback(progressInfo);
writer.WriteEndArray();
var packageTypesResult = await _packageTypesService.GetAllPackageTypesAsync();
writer.WritePropertyName("PackageTypeTotalCount");
writer.WriteValue(packageTypesResult.Count());
writer.WritePropertyName("PackageTypes");
writer.WriteStartArray();
foreach (var packageType in packageTypesResult)
{
_serializer.Serialize(writer, packageType);
}
writer.Flush();
progressInfo.Description = $"{packageTypesResult.Count()} package types exported";
progressCallback(progressInfo);
writer.WriteEndArray();
var seoResult = await _seoService.GetAllSeoDuplicatesAsync();
writer.WritePropertyName("SeoInfoTotalCount");
writer.WriteValue(seoResult.Count());
writer.WritePropertyName("SeoInfos");
writer.WriteStartArray();
foreach (var seoInfo in seoResult)
{
_serializer.Serialize(writer, seoInfo);
}
writer.Flush();
progressInfo.Description = $"{seoResult.Count()} seo info exported";
progressCallback(progressInfo);
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
}
}
public void Write(IEnumerable <IRow> rows)
{
var textWriter = new StreamWriter(_stream);
var jsonWriter = new JsonTextWriter(textWriter);
jsonWriter.WriteStartObject(); //root
jsonWriter.WritePropertyName("type");
jsonWriter.WriteValue("FeatureCollection");
jsonWriter.WritePropertyName("features");
jsonWriter.WriteStartArray(); //features
var tableBuilder = new StringBuilder();
foreach (var row in rows)
{
jsonWriter.WriteStartObject(); //feature
jsonWriter.WritePropertyName("type");
jsonWriter.WriteValue("Feature");
jsonWriter.WritePropertyName("geometry");
jsonWriter.WriteStartObject(); //geometry
jsonWriter.WritePropertyName("type");
jsonWriter.WriteValue("Point");
jsonWriter.WritePropertyName("coordinates");
jsonWriter.WriteStartArray();
jsonWriter.WriteValue(row[_longitudeField]);
jsonWriter.WriteValue(row[_latitudeField]);
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject(); //geometry
jsonWriter.WritePropertyName("properties");
jsonWriter.WriteStartObject(); //properties
foreach (var field in _propertyFields)
{
jsonWriter.WritePropertyName(field.Label);
jsonWriter.WriteValue(field.Format == string.Empty ? row[field] : string.Format(string.Concat("{0:", field.Format, "}"), row[field]));
}
jsonWriter.WritePropertyName("description");
tableBuilder.Clear();
tableBuilder.AppendLine("<table class=\"table table-striped table-condensed\">");
foreach (var field in _propertyFields.Where(f => f.Alias != "BatchValue"))
{
tableBuilder.AppendLine("<tr>");
tableBuilder.AppendLine("<td><strong>");
tableBuilder.AppendLine(field.Label);
tableBuilder.AppendLine(":</strong></td>");
tableBuilder.AppendLine("<td>");
tableBuilder.AppendLine(field.Raw ? row[field].ToString() : System.Security.SecurityElement.Escape(row[field].ToString()));
tableBuilder.AppendLine("</td>");
tableBuilder.AppendLine("</tr>");
}
tableBuilder.AppendLine("</table>");
jsonWriter.WriteValue(tableBuilder.ToString());
if (_hasStyle)
{
if (_colorField != null)
{
jsonWriter.WritePropertyName("marker-color");
var color = row[_colorField].ToString().TrimStart('#').Right(6);
jsonWriter.WriteValue("#" + (color.Length == 6 ? color : "0080ff"));
}
if (_sizeField != null)
{
jsonWriter.WritePropertyName("marker-size");
var size = row[_sizeField].ToString().ToLower();
if (_sizes.Contains(size))
{
jsonWriter.WriteValue(size);
}
else
{
jsonWriter.WriteValue(_scales.ContainsKey(size) ? _scales[size] : "medium");
}
}
if (_symbolField != null)
{
var symbol = row[_symbolField].ToString();
if (symbol.StartsWith("http"))
{
symbol = "marker";
}
jsonWriter.WritePropertyName("marker-symbol");
jsonWriter.WriteValue(symbol);
}
}
jsonWriter.WriteEndObject(); //properties
jsonWriter.WriteEndObject(); //feature
}
jsonWriter.WriteEndArray(); //features
jsonWriter.WriteEndObject(); //root
jsonWriter.Flush();
}
private static void GenerateMetaData(List <DiscData> discs, Stream outputStream)
{
var uniqueId = 0L;
// Record each file name, and if it has a JSON file, also record the title, when it was taken (photoTakenTime/creationTime), where it was taken (geoData/geoDataExif), its description, and which people are in it
using (var wtr = new StreamWriter(outputStream, leaveOpen: true))
{
wtr.Write("window.archiverMetaData = ");
using (var json = new JsonTextWriter(wtr))
{
json.WriteStartObject();
json.WritePropertyName("fieldMeaning");
json.WriteStartObject();
json.WritePropertyName("u");
json.WriteValue("Unique ID");
json.WritePropertyName("n");
json.WriteValue("Name");
json.WritePropertyName("r");
json.WriteValue("Relative Path");
json.WritePropertyName("t");
json.WriteValue("Title");
json.WritePropertyName("d");
json.WriteValue("Date Taken");
json.WritePropertyName("l");
json.WriteValue("Location Coordinates");
json.WritePropertyName("x");
json.WriteValue("Description/Notes");
json.WritePropertyName("p");
json.WriteValue("People");
json.WritePropertyName("c");
json.WriteValue("Archival Disc Number");
json.WritePropertyName("h");
json.WriteValue("SHA1 Hash");
json.WriteEndObject();
json.WritePropertyName("files");
json.WriteStartArray();
foreach (var discFile in discs.SelectMany(x => x.Files.Select(y => (Disc: x, File: y))))
{
json.WriteStartObject();
json.WritePropertyName("u");
json.WriteValue(++uniqueId);
json.WritePropertyName("n");
json.WriteValue(Path.GetFileName(discFile.File.PrincipalPath));
json.WritePropertyName("r");
json.WriteValue(discFile.File.PrincipalRelativePath.Replace('\\', '/'));
if (!string.IsNullOrWhiteSpace(discFile.File.MetaDataPath))
{
var metaDataText = File.ReadAllText(discFile.File.MetaDataPath);
var metaData = JsonConvert.DeserializeObject <GoogleMetaData>(metaDataText);
if (!string.IsNullOrWhiteSpace(metaData.Title))
{
json.WritePropertyName("t"); json.WriteValue(metaData.Title);
}
var date = metaData.PhotoTakenTime ?? metaData.CreationTime;
if (long.TryParse(date?.Timestamp, out var ts))
{
json.WritePropertyName("d"); json.WriteValue(ts > 0 && ts < 10000000000 ? ts * 1000 : ts);
}
var lat = metaData.GeoData?.Latitude ?? metaData.GeoDataExif?.Latitude;
var lng = metaData.GeoData?.Longitude ?? metaData.GeoDataExif?.Longitude;
if (lat != null && double.IsFinite(lat.Value) && lat != 0.0 && lng != null && double.IsFinite(lng.Value) && lng != 0.0)
{
json.WritePropertyName("l"); json.WriteValue($"{lat},{lng}");
}
if (!string.IsNullOrWhiteSpace(metaData.Description))
{
json.WritePropertyName("x"); json.WriteValue(metaData.Description);
}
var people = metaData.People?.Where(x => !string.IsNullOrWhiteSpace(x?.Name)).ToArray();
if (people != null && people.Length > 0)
{
json.WritePropertyName("p");
json.WriteStartArray();
foreach (var person in people)
{
json.WriteValue(person.Name);
}
json.WriteEndArray();
}
}
json.WritePropertyName("c");
json.WriteValue(discFile.Disc.DiscNumber);
json.WritePropertyName("h");
json.WriteValue(discFile.File.Hash);
json.WriteEndObject();
}
json.WriteEndArray();
json.WriteEndObject();
json.Flush();
}
}
}
/// <summary>
/// Save the graph. This method produces the whole file content. It must be called only once.
/// </summary>
public void SaveGraphData(
IPipScheduleTraversal graph,
PipExecutionContext context,
IEnumerable <PipExecutionPerformanceEventData> executionData,
IEnumerable <string> workers,
IEnumerable <KeyValuePair <FileArtifact, FileContentInfo> > fileData,
Dictionary <PipId, ProcessExecutionMonitoringReportedEventData> directoryInputContent,
Dictionary <PipId, PipExecutionDirectoryOutputs> directoryOutputContent)
{
Contract.Requires(graph != null);
Contract.Requires(context != null);
Contract.Assume(m_fileIds.Count == 0);
// We have transitioned to the Exporting state.
// Observed execution info may get queued up for when we begin draining the queue (below).
// Don't overlap pip ids with file/directory ids
int nextId = graph.PipCount + 1;
var directoryIds = new Dictionary <DirectoryArtifact, int>(capacity: 1000);
m_writer.WriteStartObject(); // Outermost object
WritePreamble();
m_writer.WritePropertyName("artifacts");
m_writer.WriteStartArray();
// To save some work on pip deserialization (we are pathologically visiting every pip in the pip table),
// we hold pips alive between the various passes.
var pips = new List <Pip>();
{
var directoryPips = new List <SealDirectory>();
// TODO: This is pretty expensive, as it loads all pips in memory
foreach (Pip pip in graph.RetrieveScheduledPips())
{
pips.Add(pip);
// We retrieve outputs rather than inputs since we do not expect duplicate outputs among pips
// (not true with inputs). This means the ID assignment work is linear in the number of artifacts.
PipArtifacts.ForEachOutput(pip, output =>
{
if (output.IsFile)
{
AssignFileIdAndWriteFileEntry(output.FileArtifact, context, ref nextId);
}
return(true);
},
includeUncacheable: true);
// SealDirectory pips are the only ones with directory outputs. As part of the artifact entry for the directory output,
// we want to capture the membership of the directory. This means we want to have assigned IDs to all member files before
// writing out that list (for easier parsing). So, we defer writing out seal directory pips until all file artifacts have been visited.
if (pip.PipType == PipType.SealDirectory)
{
var directoryPip = (SealDirectory)pip;
Contract.Assume(directoryPip.IsInitialized);
directoryPips.Add(directoryPip);
}
// Record files that are written into SharedOpaqueDirectories
if (directoryOutputContent.TryGetValue(pip.PipId, out var pipOutput))
{
foreach (var output in pipOutput.DirectoryOutputs)
{
DirectoryArtifact dir = output.directoryArtifact;
var content = output.fileArtifactArray;
foreach (var file in content)
{
AssignFileIdAndWriteFileEntry(file, context, ref nextId);
}
}
}
}
foreach (SealDirectory directoryPip in directoryPips)
{
AssignDirectoryIdAndWriteDirectoryEntry(directoryPip.Directory, context, directoryPip.Contents, directoryIds, ref nextId);
}
}
m_writer.WriteEndArray();
m_writer.WritePropertyName("graph");
m_writer.WriteStartArray();
{
// Note we are using the 'pips' list captured above rather than possibly deserializing pips again.
foreach (Pip pip in pips)
{
if (!IncludePip(pip.PipType))
{
continue;
}
WriteGraphNodeEntry(pip, graph, context, directoryIds, directoryInputContent, directoryOutputContent);
}
}
m_writer.WriteEndArray();
// Avoid holding a reference to this map if it isn't later needed by the special file details exporter
if (fileData == null)
{
m_fileIds = null;
}
if (executionData != null)
{
m_writer.WritePropertyName("execution");
m_writer.WriteStartArray();
// Begin draining the execution entry queue, now that we are in an allowed state for that.
// Iteration will complete only after Finish() is called, which marks the queue as complete (no new items).
foreach (var pipWithPerf in executionData)
{
WriteExecutionEntry(pipWithPerf.PipId, pipWithPerf.ExecutionPerformance);
}
// End the execution: array
m_writer.WriteEndArray();
}
if (workers != null)
{
ExportWorkerDetails(workers);
}
if (fileData != null)
{
ExportFileDetails(fileData);
}
// End the outermost object
m_writer.WriteEndObject();
m_writer.Flush();
}
public void ProcessRequest(HttpContext context)
{
string
startStr = context.Request.Form["start"],
limitStr = context.Request.Form["limit"];
int
start,
limit;
if (!int.TryParse(startStr, out start))
{
start = 0;
}
if (!int.TryParse(limitStr, out limit))
{
limit = 2;
}
context.Response.CacheControl = "no-cache";
context.Response.Expires = -1;
context.Response.ContentType = "application/json";
JsonObject
RootJsonObject = null;
JsonTextWriter
tmpJsonTextWriter = new JsonTextWriter(context.Response.Output);
string
Action = context.Request.Form["xaction"];
switch (Action)
{
case "read":
{
RootJsonObject = DataTableToJson(GetData(context), start, limit);
break;
}
case "update":
{
DataTable
tmpDataTable = GetData(context);
object
tmpObject = JsonConvert.Import(context.Request.Form["rows"]);
if (tmpObject is JsonArray)
{
JsonArray
tmpJsonArray = (JsonArray)tmpObject;
for (int i = 0; i < tmpJsonArray.Count; ++i)
{
UpdateRow((JsonObject)tmpJsonArray[i], IdPropertyName, tmpDataTable);
}
}
else if (tmpObject is JsonObject)
{
UpdateRow((JsonObject)tmpObject, IdPropertyName, tmpDataTable);
}
RootJsonObject = new JsonObject(new Dictionary <string, object> {
{ "success", true }, { "message", "200OK" }, { "msg", "200OK" }
});
break;
}
}
RootJsonObject.Export(tmpJsonTextWriter);
tmpJsonTextWriter.Flush();
tmpJsonTextWriter.Close();
}
public override string ToJSON()
{
MemoryStream str = new MemoryStream();
StreamWriter tw = new StreamWriter(str);
JsonTextWriter w = new JsonTextWriter(tw);
StreamReader rdr;
List <string> dateResult = Values.SelectMany(v => v.Value.Select(v2 => v2.TimeStamp.ToString())).ToList();
dateResult = dateResult.OrderBy(d => d).Distinct().ToList();
w.WriteStartObject();
w.WritePropertyName("labels");
w.WriteStartArray();
foreach (string dateVal in dateResult)
{
DateTimeStamp dtstamp = DateTimeStamp.Parse(dateVal);
w.WriteValue(string.Format("{0}", dtstamp.ToStandardString()));
}
w.WriteEndArray();
w.WritePropertyName("datasets");
w.WriteStartArray();
int colorIncrementer = 0;
foreach (var key in Values.Keys)
{
w.WriteStartObject();
w.WritePropertyName("label");
w.WriteValue(string.Format("{0}", key.Name));
w.WritePropertyName("fillColor");
w.WriteValue(GetArgb(colorIncrementer, 0.2f));
w.WritePropertyName("strokeColor");
w.WriteValue(GetArgb(colorIncrementer, 0.7f));
w.WritePropertyName("pointColor");
w.WriteValue(GetArgb(colorIncrementer, 1.0f));
w.WritePropertyName("data");
w.WriteStartArray();
foreach (string dt in dateResult)
{
var DevLog = Values[key].Where(v => v.TimeStamp.ToString() == dt).FirstOrDefault();
string value = null;
if (DevLog != null)
{
value = DevLog.Value;
}
w.WriteValue(value);
}
w.WriteEndArray();
w.WriteEndObject();
colorIncrementer++;
}
w.WriteEndArray();
w.WriteEndObject();
w.Flush();
str.Flush();
str.Position = 0;
rdr = new StreamReader(str);
return(rdr.ReadToEnd());
}
private async Task PostAndReadPosterWithStreams()
{
// generate a movie poster of 500KB
var random = new Random();
var generatedBytes = new byte[524288];
random.NextBytes(generatedBytes);
var posterForCreation = new PosterForCreation()
{
Name = "A new poster for The Big Lebowski",
Bytes = generatedBytes
};
var memoryContentStream = new MemoryStream();
using (var streamWriter = new StreamWriter(memoryContentStream,
new UTF8Encoding(), 1024, true))
{
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
var jsonSerializer = new JsonSerializer();
jsonSerializer.Serialize(jsonTextWriter, posterForCreation);
jsonTextWriter.Flush();
}
}
memoryContentStream.Seek(0, SeekOrigin.Begin);
using (var request = new HttpRequestMessage(
HttpMethod.Post,
$"api/movies/d8663e5e-7494-4f81-8739-6e0de1bea7ee/posters"))
{
request.Headers.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
using (var streamContent = new StreamContent(memoryContentStream))
{
request.Content = streamContent;
request.Content.Headers.ContentType =
new MediaTypeHeaderValue("application/json");
using (var response = await _httpClient
.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
{
response.EnsureSuccessStatusCode();
var stream = await response.Content.ReadAsStreamAsync();
using (var streamReader = new StreamReader(stream))
{
using (var jsonTextReader = new JsonTextReader(streamReader))
{
var jsonSerializer = new JsonSerializer();
var poster = jsonSerializer.Deserialize <Poster>(jsonTextReader);
// do something with the poster
}
}
}
}
}
}
internal static void ProcessResults(JProperty[] results, IConfiguration configuration, LoggingContext loggingContext)
{
int maxLogSize = configuration.Logging.AriaIndividualMessageSizeLimitBytes;
using (var sbPool = Pools.GetStringBuilder())
{
var sb = sbPool.Instance;
using var sw = new StringWriter(sb);
using var writer = new JsonTextWriter(sw);
var logStarted = false;
var hasProperty = false;
var lenSum = 0;
for (int i = 0; i < results.Length; i++)
{
startLoggingIfNot();
var name = results[i].Name.ToString();
var value = results[i].Value.ToString();
lenSum += name.Length + value.Length;
if (lenSum < maxLogSize)
{
writeProperty(name, value);
}
else
{
// End the current batch before start a new one.
endLoggingIfStarted();
// Log a single event, if this single result itself is too big.
if ((name.Length + value.Length) >= maxLogSize)
{
// Have to shorten the result to fit the telemetry.
var marker = "[...]";
var prefix = value.Substring(0, maxLogSize / 2);
var suffix = value.Substring(value.Length - maxLogSize / 2);
logAsSingle(name, prefix + marker + suffix);
}
else
{
// Start a new batch.
startLoggingIfNot();
writeProperty(name, value);
lenSum = name.Length + value.Length;
}
}
}
endLoggingIfStarted();
void writeProperty(string name, string value)
{
writer.WritePropertyName(name);
writer.WriteRawValue(value);
hasProperty = true;
}
void endLogging()
{
writer.WriteEndObject();
writer.WriteEndObject();
// Only log when has result in it.
if (hasProperty)
{
Logger.Log.CacheMissAnalysisBatchResults(loggingContext, sw.ToString());
}
logStarted = false;
hasProperty = false;
lenSum = 0;
writer.Flush();
sb.Clear();
}
void endLoggingIfStarted()
{
// Only log when at least one result has been written to the Json string
if (logStarted)
{
endLogging();
}
}
void startLogging()
{
writer.Flush();
sb.Clear();
writer.WriteStartObject();
writer.WritePropertyName("CacheMissAnalysisResults");
writer.WriteStartObject();
logStarted = true;
}
void startLoggingIfNot()
{
// Only log when at least one result has been written to the Json string
if (!logStarted)
{
startLogging();
}
}
void logAsSingle(string name, string value)
{
startLogging();
writeProperty(name, value);
endLogging();
}
}
}
public static void JsonBufferedSerializeEnd(JsonTextWriter writer)
{
writer.WriteEndArray();
writer.Flush();
}
public override void Flush()
{
_textWriter.Flush();
_innerWriter.Flush();
}
public void Serialize(object value, Stream s, bool minimal)
{
TextWriter sw;
var cms = s as ChunkedMemoryStream;
char[] buffer;
if (cms != null)
{
sw = cms.GetWriter();
buffer = cms.SmallBuffer;
}
else
{
sw = new InvariantWriter(s);
buffer = new char[38];
}
var jo = value as IJsonObject;
if (jo != null)
{
jo.Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
var array = value as IJsonObject[];
if (array != null)
{
sw.Write('[');
if (array.Length > 0)
{
if (array[0] != null)
{
array[0].Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
for (int i = 1; i < array.Length; i++)
{
sw.Write(',');
if (array[i] != null)
{
array[i].Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
}
}
sw.Write(']');
}
else if (value is IList <IJsonObject> )
{
var list = value as IList <IJsonObject>;
sw.Write('[');
if (list.Count > 0)
{
if (list[0] != null)
{
list[0].Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
for (int i = 1; i < list.Count; i++)
{
sw.Write(',');
if (list[i] != null)
{
list[i].Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
}
}
sw.Write(']');
}
else if (value is IEnumerable <IJsonObject> )
{
var col = value as IEnumerable <IJsonObject>;
sw.Write('[');
var total = col.Count() - 1;
if (total >= 0)
{
var enumerator = col.GetEnumerator();
IJsonObject item;
for (var i = 0; enumerator.MoveNext() && i < total; i++)
{
item = enumerator.Current;
if (item != null)
{
item.Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
sw.Write(',');
}
item = enumerator.Current;
if (item != null)
{
item.Serialize(sw, buffer, minimal, SharedSerializerDelegate);
}
else
{
sw.Write("null");
}
}
sw.Write(']');
}
else
{
var jw = new JsonTextWriter(sw);
SharedSerializer.Serialize(jw, value);
jw.Flush();
}
}
sw.Flush();
}
private static void Go(Engine engine)
{
var travelStationDefinitionClass = engine.GetClass("WillowGame.TravelStationDefinition");
var fastTravelStationDefinitionClass = engine.GetClass("WillowGame.FastTravelStationDefinition");
var levelTravelStationDefinitionClass = engine.GetClass("WillowGame.LevelTravelStationDefinition");
var fastTravelStationsListOrderClass = engine.GetClass("WillowGame.FastTravelStationsListOrder");
if (travelStationDefinitionClass == null ||
fastTravelStationDefinitionClass == null ||
levelTravelStationDefinitionClass == null ||
fastTravelStationsListOrderClass == null)
{
throw new System.InvalidOperationException();
}
Directory.CreateDirectory("dumps");
using (var output = new StreamWriter(Path.Combine("dumps", "Travel Stations.json"), false, Encoding.Unicode))
using (var writer = new JsonTextWriter(output))
{
writer.Indentation = 2;
writer.IndentChar = ' ';
writer.Formatting = Formatting.Indented;
writer.WriteStartObject();
var travelStationDefinitions = engine.Objects
.Where(o =>
(o.IsA(travelStationDefinitionClass) == true ||
o.IsA(fastTravelStationDefinitionClass) == true ||
o.IsA(levelTravelStationDefinitionClass) == true) &&
o.GetName().StartsWith("Default__") == false)
.OrderBy(o => o.GetPath());
foreach (dynamic travelStationDefinition in travelStationDefinitions)
{
UnrealClass uclass = travelStationDefinition.GetClass();
if (uclass.Path != "WillowGame.FastTravelStationDefinition" &&
uclass.Path != "WillowGame.LevelTravelStationDefinition")
{
throw new System.InvalidOperationException();
}
writer.WritePropertyName(travelStationDefinition.GetPath());
writer.WriteStartObject();
if (uclass.Path != "WillowGame.TravelStationDefinition")
{
writer.WritePropertyName("$type");
writer.WriteValue(uclass.Name);
}
writer.WritePropertyName("resource_name");
writer.WriteValue(travelStationDefinition.GetName());
string stationLevelName = travelStationDefinition.StationLevelName;
if (string.IsNullOrEmpty(stationLevelName) == false)
{
writer.WritePropertyName("level_name");
writer.WriteValue(stationLevelName);
}
var dlcExpansion = travelStationDefinition.DlcExpansion;
if (dlcExpansion != null)
{
writer.WritePropertyName("dlc_expansion");
writer.WriteValue(dlcExpansion.GetPath());
}
if (travelStationDefinition.PreviousStation != null)
{
writer.WritePropertyName("previous_station");
writer.WriteValue(travelStationDefinition.PreviousStation.GetPath());
}
string stationDisplayName = travelStationDefinition.StationDisplayName;
if (string.IsNullOrEmpty(stationDisplayName) == false)
{
writer.WritePropertyName("station_display_name");
writer.WriteValue(stationDisplayName);
}
var missionDependencies = ((IEnumerable <dynamic>)travelStationDefinition.MissionDependencies)
.Where(md => md.MissionDefinition != null)
.OrderBy(md => md.MissionDefinition.GetPath())
.ToArray();
if (missionDependencies.Length > 0)
{
writer.WritePropertyName("mission_dependencies");
writer.WriteStartArray();
foreach (var missionDependency in missionDependencies)
{
writer.WriteStartObject();
writer.WritePropertyName("mission_definition");
writer.WriteValue(missionDependency.MissionDefinition.GetPath());
writer.WritePropertyName("mission_status");
writer.WriteValue(((MissionStatus)missionDependency.MissionStatus).ToString());
if ((bool)missionDependency.bIsObjectiveSpecific == true)
{
writer.WritePropertyName("is_objective_specific");
writer.WriteValue(true);
if (missionDependency.MissionObjective != null)
{
writer.WritePropertyName("objective_definition");
writer.WriteValue(missionDependency.MissionObjective.GetPath());
}
writer.WritePropertyName("objective_status");
writer.WriteValue(
((ObjectiveDependencyStatus)missionDependency.ObjectiveStatus).ToString());
}
writer.WriteEndObject();
}
writer.WriteEndArray();
}
if (uclass == fastTravelStationDefinitionClass)
{
writer.WritePropertyName("initially_active");
writer.WriteValue((bool)travelStationDefinition.bInitiallyActive);
writer.WritePropertyName("send_only");
writer.WriteValue((bool)travelStationDefinition.bSendOnly);
string stationDescription = travelStationDefinition.StationDescription;
if (string.IsNullOrEmpty(stationDescription) == false &&
stationDescription != "No Description" &&
stationDescription != stationDisplayName)
{
writer.WritePropertyName("description");
writer.WriteValue(stationDescription);
}
string stationSign = travelStationDefinition.StationSign;
if (string.IsNullOrEmpty(stationSign) == false &&
stationSign != stationDisplayName)
{
writer.WritePropertyName("sign");
writer.WriteValue(stationSign);
}
if (travelStationDefinition.InaccessibleObjective != null)
{
writer.WritePropertyName("inaccessible_objective");
writer.WriteValue(travelStationDefinition.InaccessibleObjective.GetPath());
}
if (travelStationDefinition.AccessibleObjective != null)
{
writer.WritePropertyName("accessible_objective");
writer.WriteValue(travelStationDefinition.AccessibleObjective.GetPath());
}
}
else if (uclass == levelTravelStationDefinitionClass)
{
if (travelStationDefinition.DestinationStationDefinition != null)
{
writer.WritePropertyName("destination_station");
writer.WriteValue(travelStationDefinition.DestinationStationDefinition.GetPath());
}
string displayName = travelStationDefinition.DisplayName;
if (string.IsNullOrEmpty(displayName) == false &&
displayName != "No Description" &&
displayName != stationDisplayName)
{
writer.WritePropertyName("display_name");
writer.WriteValue(displayName);
}
}
writer.WriteEndObject();
}
writer.WriteEndObject();
writer.Flush();
}
var fastTravelStationsListOrders = engine.Objects
.Where(o =>
(o.IsA(fastTravelStationsListOrderClass) == true) &&
o.GetName().StartsWith("Default__") == false)
.OrderBy(o => o.GetPath());
using (var output = new StreamWriter(Path.Combine("dumps", "Fast Travel Station Ordering.json"), false, Encoding.Unicode))
using (var writer = new JsonTextWriter(output))
{
writer.Indentation = 2;
writer.IndentChar = ' ';
writer.Formatting = Formatting.Indented;
writer.WriteStartObject();
foreach (dynamic fastTravelStationsListOrder in fastTravelStationsListOrders)
{
writer.WritePropertyName(fastTravelStationsListOrder.GetPath());
writer.WriteStartObject();
writer.WritePropertyName("stations");
writer.WriteStartArray();
foreach (var fastTravelStationDefinition in fastTravelStationsListOrder.FastTravelStationOrderList)
{
writer.WriteValue(fastTravelStationDefinition.GetPath());
}
writer.WriteEndArray();
var dlcExpansion = fastTravelStationsListOrder.DlcExpansion;
if (dlcExpansion != null)
{
writer.WritePropertyName("dlc_expansion");
writer.WriteValue(dlcExpansion.GetPath());
}
writer.WriteEndObject();
}
writer.WriteEndObject();
}
}
private bool TryApplyContentType(string acceptType, ControllerContext context)
{
var response = context.HttpContext.Response;
int semi = acceptType.IndexOf(';');
if (semi > 0)
{
acceptType = acceptType.Substring(0, semi);
}
acceptType = acceptType.ToLower();
// Look for specific content types
if (acceptType == "application/json")
{
response.ContentType = "application/json";
using (JsonTextWriter writer = new JsonTextWriter(response.Output)
{
Formatting =
FormatOutput ? Newtonsoft.Json.Formatting.Indented : Newtonsoft.Json.Formatting.None
})
{
JsonSerializer serializer = JsonSerializer.Create();
serializer.Serialize(writer, Data);
writer.Flush();
}
return(true);
}
if (acceptType == "text/xml" || acceptType == "application/xml")
{
response.ContentType = "text/xml";
if (Data != null)
{
using (var writer = new XmlTextWriter(response.OutputStream, new UTF8Encoding()))
{
if (FormatOutput)
{
writer.Formatting = System.Xml.Formatting.Indented;
}
XmlSerializer serializer = new XmlSerializer(Data.GetType());
serializer.Serialize(writer, Data);
writer.Flush();
}
}
return(true);
}
if (!string.IsNullOrEmpty(ViewName) && acceptType == "text/html")
{
response.ContentType = "text/html";
var viewData = context.Controller.ViewData;
viewData.Model = Data;
var viewResult = new ViewResult
{
ViewName = ViewName,
MasterName = null,
ViewData = viewData,
TempData = context.Controller.TempData,
ViewEngineCollection = ((Controller)context.Controller).ViewEngineCollection
};
viewResult.ExecuteResult(context.Controller.ControllerContext);
return(true);
}
if (acceptType == "text/plain")
{
response.ContentType = "text/plain";
response.Write(Data);
return(true);
}
return(false);
}
public void Dispose()
{
_jsonTextWriter?.WriteEndArray();
_jsonTextWriter?.Flush();
_jsonTextWriter?.Close();
}
public override string ToString()
{
var strWriter = new StringWriter();
var jsonWriter = new JsonTextWriter(strWriter);
jsonWriter.Formatting = Formatting.Indented;
jsonWriter.WriteStartObject();
if (TryGetFee(out var fee))
{
jsonWriter.WritePropertyValue("fee", $"{fee} BTC");
}
else
{
jsonWriter.WritePropertyName("fee");
jsonWriter.WriteToken(JsonToken.Null);
}
if (TryGetEstimatedFeeRate(out var feeRate))
{
jsonWriter.WritePropertyValue("feeRate", $"{feeRate}");
}
else
{
jsonWriter.WritePropertyName("feeRate");
jsonWriter.WriteToken(JsonToken.Null);
}
jsonWriter.WritePropertyName("tx");
jsonWriter.WriteStartObject();
var formatter = new RPC.BlockExplorerFormatter();
formatter.WriteTransaction2(jsonWriter, tx);
jsonWriter.WriteEndObject();
if (GlobalXPubs.Count != 0)
{
jsonWriter.WritePropertyName("xpubs");
jsonWriter.WriteStartArray();
foreach (var xpub in GlobalXPubs)
{
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyValue("key", xpub.Key.ToString());
jsonWriter.WritePropertyValue("value", xpub.Value.ToString());
jsonWriter.WriteEndObject();
}
jsonWriter.WriteEndArray();
}
if (unknown.Count != 0)
{
jsonWriter.WritePropertyName("unknown");
jsonWriter.WriteStartObject();
foreach (var el in unknown)
{
jsonWriter.WritePropertyValue(Encoders.Hex.EncodeData(el.Key), Encoders.Hex.EncodeData(el.Value));
}
jsonWriter.WriteEndObject();
}
jsonWriter.WritePropertyName("inputs");
jsonWriter.WriteStartArray();
foreach (var input in this.Inputs)
{
input.Write(jsonWriter);
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("outputs");
jsonWriter.WriteStartArray();
foreach (var output in this.Outputs)
{
output.Write(jsonWriter);
}
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject();
jsonWriter.Flush();
return(strWriter.ToString());
}
