protected override void OnRead()
{
var metrics = new GenericMetricsUnit();
_metricsMappings.Apply(metrics);
//Writing to Log
//if (metrics.Output.Checksum.Count() == 0)
//{
// Edge.Core.Utilities.Log("Output checksum is empty",Core.Utilities.LogMessageType.Information);
//}
var signature = new Signature();
_signatureMappings.Apply(signature);
//checking if signature is already exists in delivery outputs
var outputs = from output in this.Delivery.Outputs
where output.Signature.Equals(signature.ToString())
select output;
DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>();
if (op != null)
{
//Attaching output to Metrics
(metrics as GenericMetricsUnit).Output = op;
}
else
{
DeliveryOutput deliveryOutput = new DeliveryOutput()
{
Signature = signature.Value,
TimePeriodStart = metrics.TimePeriodStart,
TimePeriodEnd = metrics.TimePeriodEnd,
Account = metrics.Account,
Channel = metrics.Channel
};
this.Delivery.Outputs.Add(deliveryOutput);
//Attaching output to Metrics
(metrics as GenericMetricsUnit).Output = deliveryOutput;
}
this.ImportManager.ImportMetrics(metrics);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
foreach (var ReportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*");
using (reportReader)
{
int colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
///sssss
}
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
// Checksums
_isChecksum = true;
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current[field];
if (reportReader.Read())
{
GenericMetricsUnit checksumUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(checksumUnit);
foreach (var m in checksumUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
}
}
}
_isChecksum = false;
//Get Valuees
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
GenericMetricsUnit tempUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(tempUnit);
SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit existingUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out existingUnit))
{
tempUnit.Output = currentOutput;
data.Add(tracker.Value, tempUnit);
}
else
{
// if tracker already exists, merge with existing values
foreach (var m in tempUnit.MeasureValues)
{
if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
existingUnit.MeasureValues[m.Key] += m.Value;
}
}
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
foreach (var ReportFile in Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString());
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
//checksum init
foreach (var m in ImportManager.Measures.Values)
{
if (m.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum[m.Name] = 0;
}
}
using (ReportReader)
{
dynamic readerHelper;
while (ReportReader.Read())
{
if (isAttribute)
{
readerHelper = ReportReader.Current.Attributes;
}
else
{
readerHelper = ReportReader.Current;
}
this.Mappings.OnFieldRequired = field => readerHelper[field];
GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(genericMetricsUnit);
foreach (var m in genericMetricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
genericMetricsUnit.Output = currentOutput;
ImportManager.ImportMetrics(genericMetricsUnit);
}
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader };
DeliveryOutput currentOutput = Delivery.Outputs.First();
//Open Auto Plac file
DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]];
var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
Dictionary <string, double> _totals = new Dictionary <string, double>();
this.ImportManager.BeginImport(this.Delivery);
//Intializing totals for validation
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_autoPlacReader)
{
this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field];
while (_autoPlacReader.Read())
{
#region Setting Totals
/*==================================================================================================================*/
// If end of file
if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF)
{
//Setting totals for validation from totals line in adowrds file
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break;
}
}
break;
}
/*==================================================================================================================*/
#endregion
GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit();
autoPlacMetricsUnit.Output = currentOutput;
autoPlacMetricsUnit.Channel = new Channel()
{
ID = 1
};
autoPlacMetricsUnit.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"]
};
autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>();
//ADDING CAMPAIGN
Campaign campaign = new Campaign()
{
OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName],
Name = _autoPlacReader.Current[Const.CampaignFieldName],
Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign);
//ADDING ADGROUP
AdGroup adgroup = new AdGroup()
{
Campaign = campaign,
Value = _autoPlacReader.Current[Const.AdGroupFieldName],
OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup);
//INSERTING METRICS DATA
autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000);
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName]));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName]));
//CREATING PLACEMENT
autoPlacMetricsUnit.TargetDimensions = new List <Target>();
PlacementTarget placement = new PlacementTarget()
{
Placement = _autoPlacReader.Current[Const.DomainFieldName],
PlacementType = PlacementType.Automatic
// Add status !!!
};
autoPlacMetricsUnit.TargetDimensions.Add(placement);
//SETTING TIME PERIOD
autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
this.ImportManager.ImportMetrics(autoPlacMetricsUnit);
}
}
this.Delivery.Outputs.First().Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
foreach (var ReportFile in Delivery.Files)
{
//check number of recordes
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize");
int numOfRecordes = 0;
if (reportReader.Read())
{
numOfRecordes = int.Parse(reportReader.Current.totalSize);
}
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
if (reportReader.Read())
{
Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning);
}
if (numOfRecordes > 0)
{
//Get Values
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field =>
{
object value = new object();
string[] nestedFields;
try
{
if (field.Contains('.'))
{
nestedFields = field.Split('.');
value = reportReader.Current[nestedFields[0]];
foreach (var item in nestedFields)
{
if (item.Equals(nestedFields[0]))
{
continue;
}
value = ((Dictionary <string, object>)value)[item];
}
value = reportReader.Current[nestedFields[0]][nestedFields[1]];
Console.Write(value.ToString());
return(value);
}
else
{
Console.Write(value.ToString());
return(reportReader.Current[field]);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex);
}
};
while (reportReader.Read())
{
GenericMetricsUnit metricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(metricsUnit);
if (metricsUnit.MeasureValues != null)
{
SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit importedUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out importedUnit))
{
metricsUnit.Output = currentOutput;
data.Add(tracker.Value, metricsUnit);
}
else //Tracker already exists
{
// Merge captured measure with existing measures
foreach (var capturedMeasure in metricsUnit.MeasureValues)
{
if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
//Measure already exists per tracker than aggregate:
if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key))
{
importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value;
}
else
{
//Captured Measure doest exists with this tracker:
importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value);
}
}
}
#region Validation
// For validations
foreach (var m in metricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
#endregion
}
}
}
}
else
{
Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information);
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
