protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
foreach (var ReportFile in Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString());
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
//checksum init
foreach (var m in ImportManager.Measures.Values)
{
if (m.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum[m.Name] = 0;
}
}
using (ReportReader)
{
dynamic readerHelper;
while (ReportReader.Read())
{
if (isAttribute)
{
readerHelper = ReportReader.Current.Attributes;
}
else
{
readerHelper = ReportReader.Current;
}
this.Mappings.OnFieldRequired = field => readerHelper[field];
GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(genericMetricsUnit);
foreach (var m in genericMetricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
genericMetricsUnit.Output = currentOutput;
ImportManager.ImportMetrics(genericMetricsUnit);
}
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
foreach (var ReportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*");
using (reportReader)
{
int colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
///sssss
}
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
// Checksums
_isChecksum = true;
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current[field];
if (reportReader.Read())
{
GenericMetricsUnit checksumUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(checksumUnit);
foreach (var m in checksumUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
}
}
}
_isChecksum = false;
//Get Valuees
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
GenericMetricsUnit tempUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(tempUnit);
SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit existingUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out existingUnit))
{
tempUnit.Output = currentOutput;
data.Add(tracker.Value, tempUnit);
}
else
{
// if tracker already exists, merge with existing values
foreach (var m in tempUnit.MeasureValues)
{
if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
existingUnit.MeasureValues[m.Key] += m.Value;
}
}
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
foreach (var ReportFile in Delivery.Files)
{
//check number of recordes
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize");
int numOfRecordes = 0;
if (reportReader.Read())
{
numOfRecordes = int.Parse(reportReader.Current.totalSize);
}
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
if (reportReader.Read())
{
Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning);
}
if (numOfRecordes > 0)
{
//Get Values
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field =>
{
object value = new object();
string[] nestedFields;
try
{
if (field.Contains('.'))
{
nestedFields = field.Split('.');
value = reportReader.Current[nestedFields[0]];
foreach (var item in nestedFields)
{
if (item.Equals(nestedFields[0]))
{
continue;
}
value = ((Dictionary <string, object>)value)[item];
}
value = reportReader.Current[nestedFields[0]][nestedFields[1]];
Console.Write(value.ToString());
return(value);
}
else
{
Console.Write(value.ToString());
return(reportReader.Current[field]);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex);
}
};
while (reportReader.Read())
{
GenericMetricsUnit metricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(metricsUnit);
if (metricsUnit.MeasureValues != null)
{
SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit importedUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out importedUnit))
{
metricsUnit.Output = currentOutput;
data.Add(tracker.Value, metricsUnit);
}
else //Tracker already exists
{
// Merge captured measure with existing measures
foreach (var capturedMeasure in metricsUnit.MeasureValues)
{
if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
//Measure already exists per tracker than aggregate:
if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key))
{
importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value;
}
else
{
//Captured Measure doest exists with this tracker:
importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value);
}
}
}
#region Validation
// For validations
foreach (var m in metricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
#endregion
}
}
}
}
else
{
Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information);
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}