public override void Transform(ToSqlItem item, EtlStatsScope stats, EtlProcessState state)
{
_stats = stats;
if (item.IsDelete == false)
{
Current = item;
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
}
// ReSharper disable once ForCanBeConvertedToForeach
for (int i = 0; i < _tablesForScript.Count; i++)
{
// delete all the rows that might already exist there
var sqlTable = _tablesForScript[i];
if (sqlTable.InsertOnlyMode)
{
continue;
}
GetOrAdd(sqlTable.TableName).Deletes.Add(item);
}
}
public override void Transform(ToOlapItem item, OlapEtlStatsScope stats, EtlProcessState state)
{
// Tombstones extraction is skipped by OLAP ETL. This must never happen
Debug.Assert(item.IsDelete == false,
$"Invalid item '{item.DocumentId}', OLAP ETL shouldn't handle tombstones");
_stats = stats;
Current = item;
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, RawDatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue)
{
EtlProcessState etlState;
if (existingValue != null)
{
etlState = JsonDeserializationClient.EtlProcessState(existingValue);
var databaseTask = GetMatchingConfiguration(record);
if (databaseTask == null)
{
throw new RachisApplyException($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's configuration can't be found");
}
var topology = record.Topology;
var lastResponsibleNode = GetLastResponsibleNode(HasHighlyAvailableTasks, topology, NodeTag);
if (topology.WhoseTaskIsIt(RachisState.Follower, databaseTask, lastResponsibleNode) != NodeTag)
{
throw new RachisApplyException($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's not its task to update this ETL");
}
}
else
{
etlState = new EtlProcessState
{
ConfigurationName = ConfigurationName,
TransformationName = TransformationName
};
}
if (DbId != null)
{
etlState.LastProcessedEtagPerDbId[DbId] = LastProcessedEtag;
}
#pragma warning disable 618
if (etlState.LastProcessedEtagPerNode?.Count > 0)
{
etlState.LastProcessedEtagPerNode[NodeTag] = LastProcessedEtag;
}
#pragma warning restore 618
etlState.ChangeVector = ChangeVector;
etlState.NodeTag = NodeTag;
etlState.SkippedTimeSeriesDocs = SkippedTimeSeriesDocs;
etlState.LastBatchTime = LastBatchTime;
return(context.ReadObject(etlState.ToJson(), GetItemId()));
}
public static EtlProcessState GetProcessState(DocumentDatabase database, string configurationName, string transformationName)
{
using (database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var stateBlittable = database.ServerStore.Cluster.Read(context, EtlProcessState.GenerateItemName(database.Name, configurationName, transformationName));
if (stateBlittable != null)
{
return JsonDeserializationClient.EtlProcessState(stateBlittable);
}
return new EtlProcessState();
}
}
public void HandleDatabaseValueChanged(DatabaseRecord record)
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
foreach (var process in _processes)
{
var state = _serverStore.Cluster.Read(context, EtlProcessState.GenerateItemName(record.DatabaseName, process.ConfigurationName, process.TransformationName));
if (state == null)
{
process.Reset();
}
}
}
}
public override void Transform(ElasticSearchItem item, EtlStatsScope stats, EtlProcessState state)
{
if (item.IsDelete == false)
{
Current = item;
DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }).Dispose();
}
for (int i = 0; i < _indexesForScript.Count; i++)
{
// delete all the rows that might already exist there
var elasticIndex = _indexesForScript[i];
GetOrAdd(elasticIndex.IndexName).Deletes.Add(item);
}
}
public static EtlProcessState GetProcessState(List <Transformation> configTransforms, DocumentDatabase database, string configurationName)
{
EtlProcessState processState = null;
foreach (var transform in configTransforms)
{
if (transform.Name == null)
{
continue;
}
processState = EtlProcess.GetProcessState(database, configurationName, transform.Name);
if (processState.NodeTag != null)
{
break;
}
}
return(processState ?? new EtlProcessState());
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, RachisState state)
{
EtlProcessState etlState;
if (existingValue != null)
{
etlState = JsonDeserializationClient.EtlProcessState(existingValue);
var databaseTask = GetMatchingConfiguration(record);
if (databaseTask == null)
{
throw new Exception($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's configuration can't be found");
}
var lastResponsibleNode = GetLastResponsibleNode(HasHighlyAvailableTasks, record.Topology, NodeTag);
if (record.Topology.WhoseTaskIsIt(state, databaseTask, lastResponsibleNode) != NodeTag)
{
throw new Exception($"Can't update progress of ETL {ConfigurationName} by node {NodeTag}, because it's not its task to update this ETL");
}
}
else
{
etlState = new EtlProcessState
{
ConfigurationName = ConfigurationName,
TransformationName = TransformationName
};
}
etlState.LastProcessedEtagPerNode[NodeTag] = LastProcessedEtag;
etlState.ChangeVector = ChangeVector;
etlState.NodeTag = NodeTag;
return(context.ReadObject(etlState.ToJson(), GetItemId()));
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, bool isPassive)
{
EtlProcessState state;
if (existingValue != null)
{
state = JsonDeserializationClient.EtlProcessState(existingValue);
}
else
{
state = new EtlProcessState
{
ConfigurationName = ConfigurationName,
TransformationName = TransformationName
};
}
state.LastProcessedEtagPerNode[NodeTag] = LastProcessedEtag;
state.ChangeVector = ChangeVector;
return(context.ReadObject(state.ToJson(), GetItemId()));
}
public override string GetItemId()
{
return(EtlProcessState.GenerateItemName(DatabaseName, ConfigurationName, TransformationName));
}
private void HandleSingleTimeSeriesSegment(string loadBehaviorFunction, EtlStatsScope stats, EtlProcessState state)
{
var docId = Current.DocumentId;
var segmentEntry = Current.TimeSeriesSegmentEntry;
var doc = Database.DocumentsStorage.Get(Context, docId, DocumentFields.Default);
if (doc == null)
{
//Through replication the Etl source database can have time-series without its document.
//This is a rare situation and we will skip Etl this time-series and will mark the document so when it will be Etl we will send all its time-series with it
(state.SkippedTimeSeriesDocs ??= new HashSet <string>()).Add(docId);
return;
}
var timeSeriesEntries = segmentEntry.Segment.YieldAllValues(Context, segmentEntry.Start, false);
if (loadBehaviorFunction != null && FilterSingleTimeSeriesSegmentByLoadBehaviorScript(ref timeSeriesEntries, docId, segmentEntry, loadBehaviorFunction))
{
return;
}
if (doc.Etag > segmentEntry.Etag)
{
//There is a chance that the document didn't Etl yet so we push it with the time-series to be sure
doc = Database.DocumentsStorage.Get(Context, docId);
if (DocumentScript != null)
{
Current.Document = doc;
DocumentScript.Run(Context, Context, "execute", new object[] { doc }).Dispose();
if (_currentRun.IsDocumentLoadedToSameCollection(docId) == false)
{
return;
}
}
else
{
_currentRun.PutFullDocument(docId, doc.Data);
}
}
var timeSeriesName = Database.DocumentsStorage.TimeSeriesStorage.GetTimeSeriesNameOriginalCasing(Context, docId, segmentEntry.Name);
foreach (var entry in timeSeriesEntries)
{
_currentRun.AddTimeSeries(docId, timeSeriesName, entry);
}
}
private bool ShouldLoadTimeSeriesWithDoc(RavenEtlItem item, EtlProcessState state)
{
//If an Etag of time-series is lower then its document Etag then replication can send the time-series before the document.
//In this situation Etl process will skip the time-series, mark it as skipped and will send all of it here with the document
return(state.SkippedTimeSeriesDocs != null && state.SkippedTimeSeriesDocs.Remove(item.DocumentId));
}
public override void Transform(RavenEtlItem item, EtlStatsScope stats, EtlProcessState state)
{
Current = item;
_currentRun ??= new RavenEtlScriptRun(stats);
if (item.IsDelete == false)
{
switch (item.Type)
{
case EtlItemType.Document:
if (_script.HasTransformation)
{
using (DocumentScript.Run(Context, Context, "execute", new object[] { Current.Document }))
{
ApplyDeleteCommands(item, OperationType.Put, out var isLoadedToDefaultCollectionDeleted);
if (_currentRun.IsDocumentLoadedToSameCollection(item.DocumentId) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out var counterFunction))
{
var counterGroups = GetCounterGroupsFor(item);
if (counterGroups != null)
{
AddCounters(item.DocumentId, counterGroups, counterFunction);
}
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out var timeSeriesLoadBehaviorFunc))
{
if (isLoadedToDefaultCollectionDeleted || ShouldLoadTimeSeriesWithDoc(item, state))
{
var timeSeriesReaders = GetTimeSeriesFor(item, timeSeriesLoadBehaviorFunc);
if (timeSeriesReaders != null)
{
AddAndRemoveTimeSeries(item.DocumentId, timeSeriesReaders);
}
}
}
}
}
else
{
var attachments = GetAttachmentsFor(item);
var counterOperations = GetCounterOperationsFor(item);
var timeSeriesOperations = ShouldLoadTimeSeriesWithDoc(item, state) ? GetTimeSeriesOperationsFor(item) : null;
_currentRun.PutFullDocument(item.DocumentId, item.Document.Data, attachments, counterOperations, timeSeriesOperations);
}
break;
case EtlItemType.CounterGroup:
string cFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadCounterBehaviorFunctionFor(item.Collection, out cFunction) == false)
{
break;
}
}
AddSingleCounterGroup(item.DocumentId, item.CounterGroupDocument, cFunction);
break;
case EtlItemType.TimeSeries:
string tsFunction = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out tsFunction) == false)
{
break;
}
}
HandleSingleTimeSeriesSegment(tsFunction, stats, state);
break;
}
}
else
{
switch (item.Type)
{
case EtlItemType.Document:
if (ShouldFilterOutDeletion(item))
{
break;
}
if (_script.HasTransformation)
{
Debug.Assert(item.IsAttachmentTombstone == false, "attachment tombstones are tracked only if script is empty");
ApplyDeleteCommands(item, OperationType.Delete, out _);
}
else
{
if (item.IsAttachmentTombstone == false)
{
_currentRun.Delete(new DeleteCommandData(item.DocumentId, null, null));
}
else
{
var(doc, attachmentName) = AttachmentsStorage.ExtractDocIdAndAttachmentNameFromTombstone(Context, item.AttachmentTombstoneId);
_currentRun.DeleteAttachment(doc, attachmentName);
}
}
break;
case EtlItemType.TimeSeries:
string function = null;
if (_script.HasTransformation)
{
if (_script.MayLoadToDefaultCollection(item) == false)
{
break;
}
if (_script.TryGetLoadTimeSeriesBehaviorFunctionFor(item.Collection, out function) == false)
{
break;
}
}
HandleSingleTimeSeriesDeletedRangeItem(item.TimeSeriesDeletedRangeItem, function);
break;
default:
throw new InvalidOperationException($"Dead Etl item can be of type {EtlItemType.Document} or {EtlItemType.TimeSeries} but got {item.Type}");
}
}
}
public abstract void Transform(TExtracted item, EtlStatsScope stats, EtlProcessState state);
