protected override void LoadToFunction(string collectionName, ScriptRunnerResult document)
{
if (collectionName == null)
{
ThrowLoadParameterIsMandatory(nameof(collectionName));
}
string id;
if (_script.IsLoadedToDefaultCollection(Current, collectionName))
{
id = Current.DocumentId;
}
else
{
id = GetPrefixedId(Current.DocumentId, collectionName, OperationType.Put);
var metadata = document.GetOrCreate(Constants.Documents.Metadata.Key);
metadata.Put(Constants.Documents.Metadata.Collection, collectionName, false);
metadata.Put(Constants.Documents.Metadata.Id, id, false);
}
var transformed = document.TranslateToObject(Context);
var transformResult = Context.ReadObject(transformed, id);
_commands.Add(new PutCommandDataWithBlittableJson(id, null, transformResult));
}
private void LoadToFunction(string tableName, string key, ScriptRunnerResult runnerResult, List <string> partitions = null)
{
if (key == null)
{
ThrowLoadParameterIsMandatory(nameof(key));
}
var result = runnerResult.TranslateToObject(Context);
var props = new List <OlapColumn>(result.Count);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < result.Count; i++)
{
result.GetPropertyByIndex(i, ref prop);
props.Add(new OlapColumn
{
Name = prop.Name,
Value = prop.Value,
Type = prop.Token
});
}
var olapItem = new ToOlapItem(Current)
{
Properties = props
};
var transformed = GetOrAdd(tableName, key, partitions);
transformed.AddItem(olapItem);
_stats.IncrementBatchSize(result.Size);
}
protected override void LoadToFunction(string indexName, ScriptRunnerResult document)
{
if (indexName == null)
{
ThrowLoadParameterIsMandatory(nameof(indexName));
}
var result = document.TranslateToObject(Context);
var index = GetOrAdd(indexName);
if (result.TryGet(index.DocumentIdProperty, out object _) == false)
{
result.Modifications = new DynamicJsonValue
{
[index.DocumentIdProperty] = ElasticSearchEtl.LowerCaseDocumentIdProperty(Current.Document.Id)
};
}
index.Inserts.Add(new ElasticSearchItem(Current)
{
TransformationResult = result
});
}
protected override void LoadToFunction(string collectionName, ScriptRunnerResult document)
{
if (collectionName == null)
{
ThrowLoadParameterIsMandatory(nameof(collectionName));
}
string id;
var loadedToDifferentCollection = false;
if (_script.IsLoadedToDefaultCollection(Current, collectionName))
{
id = Current.DocumentId;
}
else
{
id = GetPrefixedId(Current.DocumentId, collectionName);
loadedToDifferentCollection = true;
}
var metadata = document.GetOrCreate(Constants.Documents.Metadata.Key);
if (loadedToDifferentCollection || metadata.HasProperty(Constants.Documents.Metadata.Collection) == false)
{
metadata.Put(Constants.Documents.Metadata.Collection, collectionName, throwOnError: true);
}
if (metadata.HasProperty(Constants.Documents.Metadata.Attachments))
{
metadata.Delete(Constants.Documents.Metadata.Attachments, throwOnError: true);
}
if (metadata.HasProperty(Constants.Documents.Metadata.Counters))
{
metadata.Delete(Constants.Documents.Metadata.Counters, throwOnError: true);
}
var transformed = document.TranslateToObject(Context);
var transformResult = Context.ReadObject(transformed, id);
_currentRun.Put(id, document.Instance, transformResult);
if (_transformation.IsAddingAttachments)
{
var docInstance = (ObjectInstance)document.Instance;
docInstance.DefineOwnProperty(Transformation.AddAttachment, _addAttachmentMethod, throwOnError: true);
}
if (_transformation.IsAddingCounters)
{
var docInstance = (ObjectInstance)document.Instance;
docInstance.DefineOwnProperty(Transformation.AddCounter, _addCounterMethod, throwOnError: true);
}
}
private JsValue LoadToFunctionTranslatorInternal(string name, ObjectInstance key, ObjectInstance obj, string methodSignature)
{
var objectInstance = key;
if (objectInstance.HasOwnProperty(PartitionKeys) == false)
{
ThrowInvalidScriptMethodCall(
$"{methodSignature} argument 'key' must have {PartitionKeys} property. Did you forget to use 'partitionBy(p)' / 'noPartition()' ? ");
}
var partitionBy = objectInstance.GetOwnProperty(PartitionKeys).Value;
var result = new ScriptRunnerResult(DocumentScript, obj);
if (partitionBy.IsNull())
{
// no partition
LoadToFunction(name, key: name, result);
return(result.Instance);
}
if (partitionBy.IsArray() == false)
{
ThrowInvalidScriptMethodCall($"{methodSignature} property {PartitionKeys} of argument 'key' must be an array instance");
}
var sb = new StringBuilder(name);
var arr = partitionBy.AsArray();
var partitions = new List <string>((int)arr.Length);
foreach (var item in arr)
{
if (item.IsArray() == false)
{
ThrowInvalidScriptMethodCall($"{methodSignature} items in array {PartitionKeys} of argument 'key' must be array instances");
}
var tuple = item.AsArray();
if (tuple.Length != 2)
{
ThrowInvalidScriptMethodCall(
$"{methodSignature} items in array {PartitionKeys} of argument 'key' must be array instances of size 2, but got '{tuple.Length}'");
}
sb.Append('/');
string val = tuple[1].IsDate()
? tuple[1].AsDate().ToDateTime().ToString(DateFormat)
: tuple[1].ToString();
var partition = $"{tuple[0]}={val}";
sb.Append(partition);
partitions.Add(partition);
}
LoadToFunction(name, sb.ToString(), result, partitions);
return(result.Instance);
}
protected override void LoadToFunction(string tableName, ScriptRunnerResult cols)
{
if (tableName == null)
{
ThrowLoadParameterIsMandatory(nameof(tableName));
}
var result = cols.TranslateToObject(Context);
var columns = new List <SqlColumn>(result.Count);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < result.Count; i++)
{
result.GetPropertyByIndex(i, ref prop);
var sqlColumn = new SqlColumn
{
Id = prop.Name,
Value = prop.Value,
Type = prop.Token
};
if (_transformation.HasLoadAttachment &&
prop.Token == BlittableJsonToken.String && IsLoadAttachment(prop.Value as LazyStringValue, out var attachmentName))
{
Stream attachmentStream;
using (Slice.From(Context.Allocator, Current.Document.ChangeVector, out var cv))
{
attachmentStream = Database.DocumentsStorage.AttachmentsStorage.GetAttachment(
Context,
Current.DocumentId,
attachmentName,
AttachmentType.Document,
cv)
?.Stream ?? Stream.Null;
}
sqlColumn.Type = 0;
sqlColumn.Value = attachmentStream;
}
columns.Add(sqlColumn);
}
GetOrAdd(tableName).Inserts.Add(new ToSqlItem(Current)
{
Columns = columns
});
}
public static string ConvertResultToString(ScriptRunnerResult result)
{
var ms = new MemoryStream();
using (var ctx = JsonOperationContext.ShortTermSingleUse())
using (var writer = new BlittableJsonTextWriter(ctx, ms))
{
writer.WriteStartObject();
writer.WritePropertyName("Result");
if (result.IsNull)
{
writer.WriteNull();
}
else if (result.RawJsValue.IsBoolean())
{
writer.WriteBool(result.RawJsValue.AsBoolean());
}
else if (result.RawJsValue.IsString())
{
writer.WriteString(result.RawJsValue.AsString());
}
else if (result.RawJsValue.IsDate())
{
var date = result.RawJsValue.AsDate();
writer.WriteString(date.ToDateTime().ToString("O"));
}
else if (result.RawJsValue.IsNumber())
{
writer.WriteDouble(result.RawJsValue.AsNumber());
}
else
{
writer.WriteObject(result.TranslateToObject(ctx));
}
writer.WriteEndObject();
writer.Flush();
}
var str = Encoding.UTF8.GetString(ms.ToArray());
return(str);
}
private JsValue LoadToFunctionTranslator(string name, JsValue self, JsValue[] args)
{
if (args.Length != 1)
{
throw new InvalidOperationException($"loadTo{name}(, obj) must be called with exactly 1 parameter");
}
if (args[0].IsObject() == false)
{
throw new InvalidOperationException($"loadTo{name} argument must be an object");
}
using (var result = new ScriptRunnerResult(SingleRun, args[0].AsObject()))
LoadToFunction(name, result);
return(self);
}
private JsValue LoadToFunctionTranslator(string name, JsValue self, JsValue[] args)
{
if (args.Length != 1)
{
ThrowInvalidScriptMethodCall($"loadTo{name}(obj) must be called with exactly 1 parameter");
}
if (args[0].IsObject() == false)
{
ThrowInvalidScriptMethodCall($"loadTo{name}(obj) argument must be an object");
}
using (var result = new ScriptRunnerResult(DocumentScript, args[0].AsObject()))
{
LoadToFunction(name, result);
return(result.Instance);
}
}
protected override void LoadToFunction(string tableName, ScriptRunnerResult cols)
{
if (tableName == null)
{
ThrowLoadParameterIsMandatory(nameof(tableName));
}
var result = cols.TranslateToObject(Context);
var columns = new List <SqlColumn>(result.Count);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < result.Count; i++)
{
result.GetPropertyByIndex(i, ref prop);
var sqlColumn = new SqlColumn
{
Id = prop.Name,
Value = prop.Value,
Type = prop.Token
};
if (_transformation.IsLoadingAttachments &&
prop.Token == BlittableJsonToken.String && IsLoadAttachment(prop.Value as LazyStringValue, out var attachmentName))
{
var attachment = _loadedAttachments[attachmentName].Dequeue();
sqlColumn.Type = 0;
sqlColumn.Value = attachment.Stream;
_stats.IncrementBatchSize(attachment.Stream.Length);
}
columns.Add(sqlColumn);
}
GetOrAdd(tableName).Inserts.Add(new ToSqlItem(Current)
{
Columns = columns
});
_stats.IncrementBatchSize(result.Size);
}
protected override void LoadToFunction(string collectionName, ScriptRunnerResult document)
{
if (collectionName == null)
{
ThrowLoadParameterIsMandatory(nameof(collectionName));
}
string id;
var loadedToDifferentCollection = false;
if (_script.IsLoadedToDefaultCollection(Current, collectionName))
{
id = Current.DocumentId;
}
else
{
id = GetPrefixedId(Current.DocumentId, collectionName);
loadedToDifferentCollection = true;
}
var metadata = document.GetOrCreate(Constants.Documents.Metadata.Key);
if (loadedToDifferentCollection || metadata.HasProperty(Constants.Documents.Metadata.Collection) == false)
{
metadata.Put(Constants.Documents.Metadata.Collection, collectionName, throwOnError: true);
}
if (metadata.HasProperty(Constants.Documents.Metadata.Id) == false)
{
metadata.Put(Constants.Documents.Metadata.Id, id, throwOnError: true);
}
if (metadata.HasProperty(Constants.Documents.Metadata.Attachments))
{
metadata.Delete(Constants.Documents.Metadata.Attachments, throwOnError: true);
}
var transformed = document.TranslateToObject(Context);
var transformResult = Context.ReadObject(transformed, id);
_commands.Add(new PutCommandDataWithBlittableJson(id, null, transformResult));
}
private JsValue LoadToFunctionTranslator(string name, JsValue self, JsValue[] args)
{
if (args.Length != 1)
{
ThrowInvalidScriptMethodCall($"loadTo{name}(obj) must be called with exactly 1 parameter");
}
if (args[0].IsObject() == false)
{
ThrowInvalidScriptMethodCall($"loadTo{name}(obj) argument must be an object");
}
// explicitly not disposing here, this will clear the context from the JavaScriptUtils, but this is
// called _midway_ through the script, so that is not something that we want to do. The caller will
// already be calling that.
var result = new ScriptRunnerResult(DocumentScript, args[0].AsObject());
LoadToFunction(name, result);
return(result.Instance);
}
public Document Transform(Document document)
{
var ctx = document.Data._context;
object translatedResult;
using (document)
{
using (_run.ScriptEngine.ChangeMaxStatements(_options.MaxStepsForTransformScript))
{
try
{
using (ScriptRunnerResult result = _run.Run(ctx, null, "execute", new object[] { document }))
{
translatedResult = _run.Translate(result, ctx, usageMode: BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
}
catch (Client.Exceptions.Documents.Patching.JavaScriptException e)
{
if (e.InnerException is Jint.Runtime.JavaScriptException innerException && string.Equals(innerException.Message, "skip", StringComparison.OrdinalIgnoreCase))
{
return(null);
}
throw;
}
}
if (!(translatedResult is BlittableJsonReaderObject bjro))
{
return(null);
}
var cloned = document.Clone(ctx);
using (cloned.Data)
{
cloned.Data = bjro;
}
return(cloned);
}
}
private JsValue LoadToFunctionTranslator(JsValue self, JsValue[] args)
{
if (args.Length != 2)
{
ThrowInvalidScriptMethodCall("loadTo(name, obj) must be called with exactly 2 parameters");
}
if (args[0].IsString() == false)
{
ThrowInvalidScriptMethodCall("loadTo(name, obj) first argument must be a string");
}
if (args[1].IsObject() == false)
{
ThrowInvalidScriptMethodCall("loadTo(name, obj) second argument must be an object");
}
using (var result = new ScriptRunnerResult(DocumentScript, args[1].AsObject()))
{
LoadToFunction(args[0].AsString(), result);
return(result.Instance);
}
}
private JsValue LoadToFunctionTranslator(JsValue self, JsValue[] args)
{
if (args.Length != 2)
{
throw new InvalidOperationException("loadTo(name, obj) must be called with exactly 2 parameters");
}
if (args[0].IsString() == false)
{
throw new InvalidOperationException("loadTo(name, obj) first argument must be a string");
}
if (args[1].IsObject() == false)
{
throw new InvalidOperationException("loadTo(name, obj) second argument must be an object");
}
using (var result = new ScriptRunnerResult(SingleRun, args[1].AsObject()))
LoadToFunction(args[0].AsString(), result);
return(self);
}
protected abstract void LoadToFunction(string tableName, ScriptRunnerResult colsAsObject);
protected override void LoadToFunction(string tableName, ScriptRunnerResult colsAsObject)
{
}
