protected void LoadCurrentMetrics()
{
// fill the metrics using mapping
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
MetricsMappings.Apply(CurrentMetricsUnit);
var signature = new Signature();
SignatureMappings.Apply(signature);
// check if signature is already exists in delivery outputs
var output = Delivery.Outputs.FirstOrDefault(x => x.Signature == signature.Value);
// attach output to Metrics: take existing or create new
if (output != null)
{
CurrentMetricsUnit.Output = output;
}
else
{
var deliveryOutput = new DeliveryOutput
{
Signature = signature.Value,
TimePeriodStart = CurrentMetricsUnit.TimePeriodStart,
TimePeriodEnd = CurrentMetricsUnit.TimePeriodEnd,
Account = CurrentMetricsUnit.Account,
Channel = CurrentMetricsUnit.Channel
};
Delivery.Outputs.Add(deliveryOutput);
CurrentMetricsUnit.Output = deliveryOutput;
}
}
public void ImportMetrics(DeliveryOutput targetOutput, MetricsUnit metrics)
{
EnsureBeginImport();
foreach (EdgeObject obj in metrics.GetObjectDimensions())
{
}
}
protected override void OnRead()
{
var ad = new Ad();
_adMappings.Apply(ad);
this.ImportManager.ImportAd(ad);
var metrics = new AdMetricsUnit();
metrics.Ad = ad;
_metricsMappings.Apply(metrics);
var signature = new Signature();
_signatureMappings.Apply(signature);
//checking if signature is already exists in delivery outputs
var outputs = from output in this.Delivery.Outputs
where output.Signature.Equals(signature.ToString())
select output;
DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>();
if (op != null)
{
//Attaching output to Metrics
(metrics as AdMetricsUnit).Output = op;
}
else
{
DeliveryOutput deliveryOutput = new DeliveryOutput()
{
Signature = signature.Value,
TimePeriodStart = metrics.TimePeriodStart,
TimePeriodEnd = metrics.TimePeriodEnd,
Account = metrics.Ad.Account,
Channel = metrics.Ad.Channel
};
this.Delivery.Outputs.Add(deliveryOutput);
//Attaching output to Metrics
(metrics as AdMetricsUnit).Output = deliveryOutput;
}
this.ImportManager.ImportMetrics(metrics);
}
protected override void OnRead()
{
var metrics = new GenericMetricsUnit();
_metricsMappings.Apply(metrics);
//Writing to Log
//if (metrics.Output.Checksum.Count() == 0)
//{
// Edge.Core.Utilities.Log("Output checksum is empty",Core.Utilities.LogMessageType.Information);
//}
var signature = new Signature();
_signatureMappings.Apply(signature);
//checking if signature is already exists in delivery outputs
var outputs = from output in this.Delivery.Outputs
where output.Signature.Equals(signature.ToString())
select output;
DeliveryOutput op = outputs.FirstOrDefault <DeliveryOutput>();
if (op != null)
{
//Attaching output to Metrics
(metrics as GenericMetricsUnit).Output = op;
}
else
{
DeliveryOutput deliveryOutput = new DeliveryOutput()
{
Signature = signature.Value,
TimePeriodStart = metrics.TimePeriodStart,
TimePeriodEnd = metrics.TimePeriodEnd,
Account = metrics.Account,
Channel = metrics.Channel
};
this.Delivery.Outputs.Add(deliveryOutput);
//Attaching output to Metrics
(metrics as GenericMetricsUnit).Output = deliveryOutput;
}
this.ImportManager.ImportMetrics(metrics);
}
protected override void OnRollbackOutput(DeliveryOutput output, int pass)
{
string guid = output.OutputID.ToString("N");
_rollbackCommand = _rollbackCommand ?? DataManager.CreateCommand(this.Options.SqlRollbackCommand, CommandType.StoredProcedure);
_rollbackCommand.Connection = _sqlConnection;
_rollbackCommand.Transaction = _rollbackTransaction;
_rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid;
_rollbackCommand.Parameters["@TableName"].Value = output.Parameters[Consts.DeliveryOutputParameters.CommitTableName];
_rollbackCommand.ExecuteNonQuery();
// This is redundant (SP already does this) but to sync our objects in memory we do it here also
output.Status = DeliveryOutputStatus.RolledBack;
//For new db
/*string guid = output.OutputID.ToString("N");
* if (output.Status == DeliveryOutputStatus.Staged)
* {
* _rollbackCommand = _rollbackCommand ?? DataManager.CreateCommand(this.Options.SqlRollbackCommand, CommandType.StoredProcedure);
* _rollbackCommand.Connection = _sqlConnection;
* _rollbackCommand.Transaction = _rollbackTransaction;
*
* _rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid;
* _rollbackCommand.Parameters["@TableName"].Value = output.Parameters[Consts.DeliveryHistoryParameters.CommitTableName];
*
* _rollbackCommand.ExecuteNonQuery();
* output.Status=DeliveryOutputStatus.Canceled;
* }
* else if (output.Status == DeliveryOutputStatus.Committed)
* {
* output.Status = DeliveryOutputStatus.PendingRoleBack;
* }
* else
* {
* throw new Exception("It should not happend");
* } */
}
protected override void OnRollbackOutput(DeliveryOutput output, int pass)
{
string guid = output.OutputID.ToString("N");
if (output.Status == DeliveryOutputStatus.Staged)
{
_rollbackCommand = _rollbackCommand ?? SqlUtility.CreateCommand(Options.SqlRollbackCommand, CommandType.StoredProcedure);
_rollbackCommand.Connection = _deliverySqlConnection;
_rollbackCommand.Transaction = _rollbackTransaction;
_rollbackCommand.Parameters["@DeliveryOutputID"].Value = guid;
_rollbackCommand.Parameters["@StagingTable"].Value = output.Parameters[Consts.DeliveryHistoryParameters.StagingMetricsTableName];
_rollbackCommand.ExecuteNonQuery();
}
else if (output.Status == DeliveryOutputStatus.Committed)
{
output.Status = DeliveryOutputStatus.RollbackPending;
}
else
{
throw new InvalidOperationException("Delivery output cannot be rolled back because it is neither staged nor committed.");
}
}
protected override void OnStage(Delivery delivery, int pass)
{
// get this from last 'Processed' history entry
string measuresFieldNamesSQL = delivery.Parameters[Consts.DeliveryOutputParameters.MeasureFieldsSql].ToString();
string measuresNamesSQL = delivery.Parameters[Consts.DeliveryOutputParameters.MeasureNamesSql].ToString();
string tablePerfix = delivery.Parameters[Consts.DeliveryOutputParameters.TablePerfix].ToString();
string deliveryId = delivery.DeliveryID.ToString("N");
// ...........................
// COMMIT data to OLTP
_stageCommand = _stageCommand ?? DataManager.CreateCommand(this.Options.SqlStageCommand, CommandType.StoredProcedure);
_stageCommand.Connection = _sqlConnection;
_stageCommand.Transaction = _stageTransaction;
_stageCommand.Parameters["@DeliveryFileName"].Size = 4000;
_stageCommand.Parameters["@DeliveryFileName"].Value = tablePerfix;
_stageCommand.Parameters["@CommitTableName"].Size = 4000;
_stageCommand.Parameters["@CommitTableName"].Value = delivery.Parameters["CommitTableName"];
_stageCommand.Parameters["@MeasuresNamesSQL"].Size = 4000;
_stageCommand.Parameters["@MeasuresNamesSQL"].Value = measuresNamesSQL;
_stageCommand.Parameters["@MeasuresFieldNamesSQL"].Size = 4000;
_stageCommand.Parameters["@MeasuresFieldNamesSQL"].Value = measuresFieldNamesSQL;
_stageCommand.Parameters["@OutputIDsPerSignature"].Size = 8000;
_stageCommand.Parameters["@OutputIDsPerSignature"].Direction = ParameterDirection.Output;
_stageCommand.Parameters["@DeliveryID"].Size = 4000;
_stageCommand.Parameters["@DeliveryID"].Value = deliveryId;
try
{
_stageCommand.ExecuteNonQuery();
// _commitTransaction.Commit();
string outPutsIDsPerSignature = _stageCommand.Parameters["@OutputIDsPerSignature"].Value.ToString();
string[] existsOutPuts;
if ((!string.IsNullOrEmpty(outPutsIDsPerSignature) && outPutsIDsPerSignature != "0"))
{
_stageTransaction.Rollback();
existsOutPuts = outPutsIDsPerSignature.Split(',');
List <DeliveryOutput> outputs = new List <DeliveryOutput>();
foreach (string existOutput in existsOutPuts)
{
DeliveryOutput o = DeliveryOutput.Get(Guid.Parse(existOutput));
o.Parameters[Consts.DeliveryOutputParameters.CommitTableName] = delivery.Parameters["CommitTableName"];
outputs.Add(o);
}
throw new DeliveryConflictException(string.Format("DeliveryOutputs with the same signature are already committed in the database\n Deliveries:\n {0}:", outPutsIDsPerSignature))
{
ConflictingOutputs = outputs.ToArray()
};
}
else
{
//already updated by sp, this is so we don't override it
foreach (var output in delivery.Outputs)
{
output.Status = DeliveryOutputStatus.Staged;
}
}
}
finally
{
this.State = DeliveryImportManagerState.Idle;
}
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
DeliveryOutput currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
#region Campaigns
List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.id),
};
//string campaignStatus = campaignsReader.Current.campaign_status;
//switch (campaignStatus)
//{
// case "ACTIVE":
// camp.Status = ObjectStatus.Active;
// break;
// case "PAUSED":
// camp.Status = ObjectStatus.Paused;
// break;
// case "DELETED":
// camp.Status = ObjectStatus.Deleted;
// break;
//}
campaignsData.Add(camp.OriginalID, camp);
}
}
}
#endregion
this.ReportProgress(0.1);
#region adGroups And Targeting
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
ad.Segments = new Dictionary <Segment, SegmentObject>();
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)];
ad.Name = adGroupsReader.Current.name;
ad.Channel = new Channel()
{
ID = 6
};
ad.Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
};
if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true")
{
string[] delimiter = new string[1];
delimiter[0] = string.Empty;
if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter"))
{
Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning);
}
else
{
delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"];
}
//Note for Ronen - this section if relevant for AD SETS
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0],
OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID) :
(ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID)
};
}
else
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Value = ad.Name,
OriginalID = ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID
};
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
}
}
#endregion
#region AdGroupStats start new import session
//GetAdGroupStats
#region for validation
foreach (var measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(measure.Key))
{
currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
#endregion
if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
{
List <string> convStatsFiles = filesByType[Consts.FileTypes.ConversionsStats];
foreach (var convStat in convStatsFiles)
{
DeliveryFile conversionStatsFile = this.Delivery.Files[convStat];
//Getting Next conversion file URL
var conversionStatsReader = new JsonDynamicReader(conversionStatsFile.OpenContents(), "$.data[*].*");
using (conversionStatsReader)
{
//Get Stats from conversion file
}
}
}
if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats))
{
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
Ad tempAd;
if (adGroupStatsReader.Current.adgroup_id != null)
{
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
//adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
//adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
// Common and Facebook specific meausures
/* Sets totals for validations */
if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
{
currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
{
currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
{
currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
/* Sets measures values */
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
//adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
this.ImportManager.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
}
this.ReportProgress(0.4);
}
#endregion
this.ReportProgress(0.6);
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
// Ronen , please see Naama's file of Ad types
ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
ad.Creatives = new List <Creative>();
switch ((string)adGroupCreativesReader.Current.type)
{
//case "8": // deprecated
//case "9": // deprecated
//case "10": // deprecated
//case "16": // deprecated
//case "17": // deprecated
//case "19": // deprecated
case "25":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "Sponsored Story",
};
ad.DestinationUrl = "Sponsored Story";
ad.Creatives.Add(sponserStory);
break;
}
case "27":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "Page Ads for a Page post"
};
ad.DestinationUrl = "Page Ads for a Page post";
ad.Creatives.Add(sponserStory);
break;
}
case "1":
case "2":
case "3":
case "4":
case "12":
{
ImageCreative ic = new ImageCreative()
{
ImageUrl = adGroupCreativesReader.Current.image_url,
OriginalID = adGroupCreativesReader.Current.id
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(ic.ImageUrl))
{
ad.Creatives.Add(ic);
}
TextCreative bc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Body,
Text = adGroupCreativesReader.Current.body
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(bc);
}
//bug creative type =9 story like
TextCreative tc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = adGroupCreativesReader.Current.title
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(tc);
}
break;
}
default:
{
TextCreative unknown = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "UnKnown creative"
};
ad.DestinationUrl = "UnKnown creative";
ad.Creatives.Add(unknown);
break;
}
}
this.ImportManager.ImportAd(ad);
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
}
#endregion
}
}
}
currentOutput.Status = DeliveryOutputStatus.Imported;
this.ImportManager.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override IEnumerable <ValidationResult> Validate()
{
Channel channel = new Channel();
progress += 0.1;
this.ReportProgress(progress);
#region Getting Service option params
//Getting Accounts list
string[] accounts;
if (this.Instance.AccountID == -1)
{
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["AccountsList"]))
{
throw new Exception("Missing Configuration option AccountsList");
}
accounts = this.Instance.Configuration.Options["AccountsList"].Split(',');
}
else
{
List <string> account = new List <string>()
{
this.Instance.AccountID.ToString()
};
accounts = account.ToArray();
}
//Getting Table
string comparisonTable;
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["SourceTable"]))
{
throw new Exception("Missing Configuration option SourceTable");
}
else
{
comparisonTable = this.Instance.Configuration.Options["SourceTable"];
}
//Getting Channel List
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["ChannelList"]))
{
throw new Exception("Missing Configuration option ChannelList");
}
string[] channels = this.Instance.Configuration.Options["ChannelList"].Split(',');
//Getting TimePeriod
DateTime fromDate, toDate;
if ((String.IsNullOrEmpty(this.Instance.Configuration.Options["fromDate"])) && (String.IsNullOrEmpty(this.Instance.Configuration.Options["toDate"])))
{
fromDate = this.TimePeriod.Start.ToDateTime();
toDate = this.TimePeriod.End.ToDateTime();
}
else
{
fromDate = Convert.ToDateTime(this.Instance.Configuration.Options["fromDate"]);
toDate = Convert.ToDateTime(this.Instance.Configuration.Options["toDate"]);
}
//Get Sub Channel (for cases such as Google Search and GDN)
//string subChannel = String.Empty;
//if (!String.IsNullOrEmpty(this.Instance.Configuration.Options["SubChannel"]))
//{
// subChannel = this.Instance.Configuration.Options["SubChannel"].ToString();
//}
#endregion
if (this.Delivery == null || this.Delivery.DeliveryID.Equals(Guid.Empty))
{
#region Creating Delivery Search List
List <DeliveryOutput> deliveryOutputSearchList = new List <DeliveryOutput>();
while (fromDate <= toDate)
{
// {start: {base : '2009-01-01', h:0}, end: {base: '2009-01-01', h:'*'}}
var subRange = new DateTimeRange()
{
Start = new DateTimeSpecification()
{
BaseDateTime = fromDate,
Hour = new DateTimeTransformation()
{
Type = DateTimeTransformationType.Exact, Value = 0
},
},
End = new DateTimeSpecification()
{
BaseDateTime = fromDate,
Hour = new DateTimeTransformation()
{
Type = DateTimeTransformationType.Max
},
}
};
foreach (var Channel in channels)
{
foreach (string account in accounts)
{
DeliveryOutput deliveryOutput = new DeliveryOutput();
deliveryOutput.Account = new Account()
{
ID = Convert.ToInt32(account)
};
deliveryOutput.Channel = new Channel()
{
ID = Convert.ToInt32(Channel)
};
deliveryOutput.TimePeriodStart = subRange.Start.ToDateTime();
deliveryOutput.TimePeriodEnd = subRange.End.ToDateTime();
deliveryOutputSearchList.Add(deliveryOutput);
progress += 0.3 * ((1 - progress) / (channels.LongLength + accounts.LongLength));
this.ReportProgress(progress);
}
}
fromDate = fromDate.AddDays(1);
}
#endregion
foreach (DeliveryOutput deliveryOutput in deliveryOutputSearchList)
{
//Getting criterion matched deliveries
DeliveryOutput[] outputToCheck = DeliveryOutput.GetByTimePeriod(deliveryOutput.TimePeriodStart, deliveryOutput.TimePeriodEnd, deliveryOutput.Channel, deliveryOutput.Account);
bool foundCommited = false;
progress += 0.3 * (1 - progress);
this.ReportProgress(progress);
//Get Relevant Outputs by source table
var outputToCheck_withSubChannel =
from o in outputToCheck
where o.Parameters.ContainsKey("CommitTableName") && o.Parameters["CommitTableName"].Equals(comparisonTable.Split('.')[1])
select o;
outputToCheck = outputToCheck_withSubChannel.ToArray <DeliveryOutput>();
foreach (DeliveryOutput output in outputToCheck)
{
// if (output.Parameters.ContainsKey("SubChannel") && output.Parameters["SubChannel"].Equals(subChannel))
if ((output.Status == DeliveryOutputStatus.Committed || output.Status == DeliveryOutputStatus.Staged) && output.Checksum != null && output.Checksum.Count > 0)
{
//Check Delivery data vs OLTP
foundCommited = true;
yield return(DeliveryDbCompare(output, output.Checksum, "OltpDB", comparisonTable));
}
}
//could not find deliveries by user criterions
if (outputToCheck.Length == 0)
{
yield return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Cannot find outputs in DB",
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
else if (!foundCommited)
{
yield return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Cannot find Commited output in DB",
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
} // End of foreach
}
else
{
//Getting current Delivery totals
foreach (DeliveryOutput output in this.Delivery.Outputs)
{
if ((output.Status == DeliveryOutputStatus.Committed || output.Status == DeliveryOutputStatus.Staged) && output.Checksum != null && output.Checksum.Count > 0)
{
yield return(DeliveryDbCompare(output, output.Checksum, "OltpDB", comparisonTable));
}
}
}
}
protected override ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
_currentOutput = Delivery.Outputs.First();
_currentOutput.Checksum = new Dictionary <string, double>();
var columns = new Dictionary <string, int>();
foreach (var reportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.columnHeaders[*].*");
using (reportReader)
{
var colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
}
// get sample unit to create metrics table
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
reportReader.Read();
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes))
{
ImportManager.BeginImport(Delivery, CurrentMetricsUnit);
//Get values
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
ProcessMetrics();
}
}
// finish import (insert edge objects)
ImportManager.EndImport();
}
}
}
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, AdGroup> adGroupsData = new Dictionary <string, AdGroup>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
var adStatIds = new Dictionary <string, string>();
var insertedAds = new Dictionary <string, string>();
var storyIds = new Dictionary <string, Dictionary <string, string> >();
var photoIds = new Dictionary <string, Dictionary <string, string> >();
DeliveryOutput currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
#region AdSets
if (filesByType.ContainsKey(Consts.FileTypes.CampaignGroups))
{
List <string> campaignsFiles = filesByType[Consts.FileTypes.CampaignGroups];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.id),
};
campaignsData.Add(camp.OriginalID, camp);
}
}
}
}
#endregion
#region AdGroups
if (filesByType.ContainsKey(Consts.FileTypes.AdSets))
{
List <string> adSetList = filesByType[Consts.FileTypes.AdSets];
foreach (var adSet in adSetList)
{
DeliveryFile adSetDF = this.Delivery.Files[adSet];
var adSetReader = new JsonDynamicReader(adSetDF.OpenContents(), "$.data[*].*");
using (adSetReader)
{
while (adSetReader.Read())
{
var adGroupObj = new AdGroup()
{
Value = adSetReader.Current.name,
OriginalID = Convert.ToString(adSetReader.Current.id),
};
if (campaignsData.ContainsKey(adSetReader.Current.campaign_group_id))
{
adGroupObj.Campaign = campaignsData[adSetReader.Current.campaign_group_id];
}
adGroupsData.Add(adGroupObj.OriginalID, adGroupObj);
}
}
}
}
#endregion
#region adGroups And Targeting
//******************************************************************************************************************************************************
if (filesByType.ContainsKey(Consts.FileTypes.AdGroups))
{
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
#region foreach
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
var campaignId = Convert.ToString(adGroupsReader.Current.campaign_id);
if (adGroupsData.ContainsKey(campaignId) && ((AdGroup)adGroupsData[campaignId]).Campaign != null)
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
ad.Segments = new Dictionary <Segment, SegmentObject>();
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] =
((AdGroup)adGroupsData[Convert.ToString(adGroupsReader.Current.campaign_id)]).Campaign;
ad.Name = adGroupsReader.Current.name;
ad.Channel = new Channel()
{
ID = 6
};
ad.Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
};
if (adGroupsData.ContainsKey(adGroupsReader.Current.campaign_id))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] =
adGroupsData[adGroupsReader.Current.campaign_id];
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min),
ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
}
}
#endregion
}
}
//******************************************************************************************************************************************************
#endregion adGroups And Targeting
//GetAdGroupStats
#region for validation
foreach (var measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(measure.Key))
{
currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
//**************************************************************************
#endregion
Dictionary <string, List <Dictionary <string, object> > > conversion_data = new Dictionary <string, List <Dictionary <string, object> > >();
if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
{
#region Conversions
List <string> conversionFiles = filesByType[Consts.FileTypes.ConversionsStats];
foreach (string conversionFile in conversionFiles)
{
DeliveryFile creativeFile = Delivery.Files[conversionFile];
var conversionCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (conversionCreativesReader)
{
this.Mappings.OnFieldRequired = field => conversionCreativesReader.Current[field];
while (conversionCreativesReader.Read())
{
if (((Edge.Data.Pipeline.DynamicDictionaryObject)(conversionCreativesReader.Current)).Values == null)
{
break; // in case of empty data (e.g. data [] )
}
if (!conversion_data.ContainsKey(conversionCreativesReader.Current.adgroup_id))
{
Dictionary <string, string> adConversionData = new Dictionary <string, string>();
//string TotalConversionsManyPerClick, Leads, Signups, Purchases;
List <object> actions = conversionCreativesReader.Current.actions;
if (actions.Count > 0)
{
List <Dictionary <string, object> > action_list = actions.Select(s => (Dictionary <string, object>)s).ToList();
List <Dictionary <string, object> > action_items = action_list.Where(dict => dict.ContainsKey("action_type") && dict.ContainsKey("value") &&
(
dict["action_type"].ToString().Equals("offsite_conversion") ||
dict["action_type"].ToString().Equals("offsite_conversion.lead") ||
dict["action_type"].ToString().Equals("offsite_conversion.registration") ||
dict["action_type"].ToString().Equals("offsite_conversion.add_to_cart") ||
dict["action_type"].ToString().Equals("offsite_conversion.checkout") ||
dict["action_type"].ToString().Equals("offsite_conversion.key_page_view") ||
dict["action_type"].ToString().Equals("offsite_conversion.other")
)
).ToList();
if (action_items.Count > 0)
{
conversion_data.Add(conversionCreativesReader.Current.adgroup_id, action_items);
}
}
}
} //End of While read
} // End of Using reader
} //End Foreach conversion file
#endregion
} //End if contains conversion file
#region AdGroupStats start new import session
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
#region Create Metrics
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
Ad tempAd;
try
{
var x = adGroupStatsReader.Current.adgroup_id;
}
catch (Exception)
{
continue;
}
if (adGroupStatsReader.Current.adgroup_id != null)
{
adStatIds[adGroupStatsReader.Current.adgroup_id] = adGroupStatsReader.Current.adgroup_id;
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
//adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
//adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
// Common and Facebook specific meausures
/* Sets totals for validations */
if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
{
currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
{
currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
{
currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
/* Sets measures values */
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
//adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
//Setting conversions from conversion files data
List <Dictionary <string, object> > adgroup_conversion_data;
if (conversion_data.TryGetValue(adGroupStatsReader.Current.adgroup_id, out adgroup_conversion_data))
{
var TotalConversionsManyPerClick = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion") select element["value"];
var Leads = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.lead") select element["value"];
var Signups = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.registration") select element["value"];
//TO DO : Get field from Configuration
var Purchases = from element in adgroup_conversion_data where element["action_type"].Equals(this.Delivery.Parameters["API_Purchases_Field_Name"]) select element["value"];
//add values to metrics unit
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.TotalConversionsManyPerClick], Convert.ToDouble(TotalConversionsManyPerClick.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Leads], Convert.ToDouble(Leads.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Signups], Convert.ToDouble(Signups.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Purchases], Convert.ToDouble(Purchases.DefaultIfEmpty("0").FirstOrDefault()));
}
this.ImportManager.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
#endregion
}
}
}
#endregion AdGroupStats start new import session
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
//this.Mappings.OnFieldRequired = field => if((field == "object_url" && adGroupCreativesReader.Current[field] != null) || field != "object_url")adGroupCreativesReader.Current[field];
this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
if (!adStatIds.ContainsKey(ad.OriginalID))
{
continue;
}
ad.Creatives = new List <Creative>();
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_type))
{
string objectType = adGroupCreativesReader.Current.object_type;
switch (objectType.ToUpper())
{
case "SHARE":
{
#region Ads Type SHARE
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_story_id))
{
Dictionary <string, string> shareCreativeData;
string object_story_id = adGroupCreativesReader.Current.object_story_id;
if (storyIds.ContainsKey(object_story_id))
{
shareCreativeData = storyIds[object_story_id];
}
else
{
var accessToken = this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken];
shareCreativeData = GetShareCreativeData(object_story_id, accessToken);
}
ad.DestinationUrl = shareCreativeData["link"];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
var trackersReadCommands = from n in this.Mappings.Objects[typeof(Ad)].ReadCommands
where n.Field.Equals("link_url")
select n;
foreach (var command in trackersReadCommands)
{
string trackerValue = ApplyRegex(ad.DestinationUrl, command.RegexPattern);
if (!String.IsNullOrEmpty(trackerValue))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value = trackerValue;
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].OriginalID = trackerValue;
}
}
}
ad.Creatives.Add(GetTextCreative(shareCreativeData["text"], adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(shareCreativeData["description"], adGroupCreativesReader));
ad.Creatives.Add(GetImageCreative(shareCreativeData["picture"], adGroupCreativesReader));
}
#endregion
break;
}
case "DOMAIN":
{
#region Ads Type DOMAIN
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_url))
{
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
}
}
else if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.link_url))
{
ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
}
else
{
ad.DestinationUrl = "UnKnown Url";
}
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
if (adGroupCreativesReader.Current.image_url != null)
{
CreateImageCreatives(ad, adGroupCreativesReader);
}
ad.Creatives.Add(GetTextCreative(adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(adGroupCreativesReader));
#endregion
break;
}
default:
{
#region Ads Type not handeled
string adtype = objectType.ToUpper();
ad.DestinationUrl = adtype + " Ad Type";
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
TextCreative unhandeledTextAd_title = new TextCreative()
{
OriginalID = ad.OriginalID,
TextType = TextCreativeType.Title,
Text = adtype + " Ad Type"
};
ad.Creatives.Add(unhandeledTextAd_title);
#endregion
break;
}
}
if (!insertedAds.ContainsKey(ad.OriginalID))
{
insertedAds[ad.OriginalID] = ad.OriginalID;
this.ImportManager.ImportAd(ad);
}
}
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
}
#endregion
}
// End foreach creative
currentOutput.Status = DeliveryOutputStatus.Imported;
this.ImportManager.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
// TODO: add checks for delivery state
string[] requiredHeaders;
requiredHeaders = new string[1];
requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString();
_campaignsCache = new Dictionary <string, Campaign>();
_adCache = new Dictionary <long, Ad>();
DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport];
DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport];
FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip);
string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles();
var campaignReportReader = new CsvDynamicReader(
campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders
);
#region Reading campaigns file
while (campaignReportReader.Read())
{
string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()];
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file
{
break;
}
Campaign campaign = CreateCampaign(campaignReportReader.Current);
_campaignsCache.Add(campaign.Name, campaign);
}
this.ReportProgress(0.2);
#endregion
#region Read the ad report, and build a lookup table for later
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
// create the ad report reader
requiredHeaders = new string[1];
requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString();
string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var adReportReader = new CsvDynamicReader(
adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//read
using (adReportReader)
{
while (adReportReader.Read())
{
// create the ad
this.Mappings.OnFieldRequired = field => adReportReader.Current[field];
string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim())
{
break;
}
Ad ad = CreateAd(adReportReader.Current, ImportManager);
//TRACKER
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
//ADDING CAMPAIGN
ad.Segments[this.ImportManager.
SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]];
//CREATING Adgroup
AdGroup adGroup = new AdGroup()
{
Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()],
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()]
};
ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup);
//CRAETING AD TYPE
string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]);
ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey];
ImportManager.ImportAd(ad);
_adCache.Add(long.Parse(ad.OriginalID), ad);
}
}
this.ReportProgress(0.7);
#endregion
// ...............................................................
#region Read the keyword report, cross reference it with the ad data, and commit
// Read the keyword report, cross reference it with the ad data, and commit
// The name of the time period column is specified by the initializer, depending on the report
DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport];
string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string;
// // create the keyword report reader
requiredHeaders = new string[1];
requiredHeaders[0] = "Keyword";
string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var keywordReportReader = new CsvDynamicReader(
keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//Added by Shay for validation
foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(measure.Key, 0);
}
}
// read and save in transaction
using (keywordReportReader)
{
while (keywordReportReader.Read())
{
string GregorianDateColVal = keywordReportReader.Current.GregorianDate;
if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file
{
break;
}
// get the unit from the keyword report, and add the missing ad data
AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager);
ImportManager.ImportMetrics(unit);
// Validation information:
currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]];
currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]];
currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]];
}
ImportManager.EndImport();
ReportProgress(1);
}
#endregion
}
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
foreach (var ReportFile in Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString());
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
//checksum init
foreach (var m in ImportManager.Measures.Values)
{
if (m.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum[m.Name] = 0;
}
}
using (ReportReader)
{
dynamic readerHelper;
while (ReportReader.Read())
{
if (isAttribute)
{
readerHelper = ReportReader.Current.Attributes;
}
else
{
readerHelper = ReportReader.Current;
}
this.Mappings.OnFieldRequired = field => readerHelper[field];
GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(genericMetricsUnit);
foreach (var m in genericMetricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
genericMetricsUnit.Output = currentOutput;
ImportManager.ImportMetrics(genericMetricsUnit);
}
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable)
{
Dictionary <string, double> oltpTotals = new Dictionary <string, double>();
Dictionary <string, double> dwhTotals = new Dictionary <string, double>();
string dayCode = Convert.ToDateTime(Params["Date"]).ToString("yyyyMMdd");
#region Get Totals from Oltp
using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, "OltpDB")))
{
sqlCon.Open();
SqlCommand sqlCommand = new SqlCommand(
"SELECT SUM(cost),sum(imps),sum(clicks) from " + OltpTable +
" where account_id = @Account_ID and Day_Code = @Daycode and Channel_ID = @Channel_ID"
);
SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int);
SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int);
SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int);
accountIdParam.Value = Params["AccountID"];
daycodeParam.Value = dayCode;
channelIdParam.Value = Params["ChannelID"];
sqlCommand.Parameters.Add(accountIdParam);
sqlCommand.Parameters.Add(daycodeParam);
sqlCommand.Parameters.Add(channelIdParam);
sqlCommand.Connection = sqlCon;
using (var _reader = sqlCommand.ExecuteReader())
{
progress += 0.5 * (1 - progress);
this.ReportProgress(progress);
if (!_reader.IsClosed)
{
while (_reader.Read())
{
if (!_reader[0].Equals(DBNull.Value))
{
oltpTotals.Add("Cost", Convert.ToDouble(_reader[0]));
oltpTotals.Add("Imps", Convert.ToDouble(_reader[1]));
oltpTotals.Add("Clicks", Convert.ToDouble(_reader[2]));
}
}
}
}
}
#endregion
#region Get Totals from DWH
using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, "DwhDB")))
{
sqlCon.Open();
SqlCommand sqlCommand = new SqlCommand(
"SELECT SUM(cost),sum(Impressions),sum(clicks) from " + DwhTabel +
" where account_id = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID"
);
SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int);
SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int);
SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int);
accountIdParam.Value = Params["AccountID"];
daycodeParam.Value = dayCode;
channelIdParam.Value = Params["ChannelID"];
sqlCommand.Parameters.Add(accountIdParam);
sqlCommand.Parameters.Add(daycodeParam);
sqlCommand.Parameters.Add(channelIdParam);
sqlCommand.Connection = sqlCon;
using (var _reader = sqlCommand.ExecuteReader())
{
progress += 0.5 * (1 - progress);
this.ReportProgress(progress);
if (!_reader.IsClosed)
{
while (_reader.Read())
{
if (!_reader[0].Equals(DBNull.Value))
{
dwhTotals.Add("Cost", Convert.ToDouble(_reader[0]));
dwhTotals.Add("Imps", Convert.ToDouble(_reader[1]));
dwhTotals.Add("Clicks", Convert.ToDouble(_reader[2]));
}
}
}
}
}
#endregion
#region Comparing totals results
return(IsEqual(Params, oltpTotals, dwhTotals, "Oltp", "Dwh"));
#endregion
}
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable)
{
string dayCode = deliveryOutput.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode
Dictionary <string, Diffrence> diffs = new Dictionary <string, Diffrence>();
string checksumThreshold = Instance.Configuration.Options[Consts.ConfigurationOptions.ChecksumTheshold];
double allowedChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold);
ValidationResult result;
using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName)))
{
sqlCon.Open();
//Get measures need to be check
StringBuilder sqlBuilder = new StringBuilder();;
Dictionary <string, Measure> measures = Measure.GetMeasures(deliveryOutput.Account, deliveryOutput.Channel, sqlCon, MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, OptionsOperator.Not);
foreach (var measure in measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
sqlBuilder.AppendFormat("{1}SUM([{0}]) as [{0}]", measure.Value.OltpName, sqlBuilder.Length > 0 ? ", " : null);
}
}
sqlBuilder.Insert(0, "SELECT \n");
sqlBuilder.AppendFormat(" FROM {0}\n ", comparisonTable);
sqlBuilder.Append("WHERE Account_ID=@accountID:int \nAND Day_Code=@daycode:int \nAND Channel_ID=@Channel_ID:int");
///*******Work around*******************////////
if (deliveryOutput.Account.OriginalID != null)
{
sqlBuilder.Append(" \nand Account_ID_SRC=@OriginalID:nvarchar");
}
SqlCommand sqlCommand = DataManager.CreateCommand(sqlBuilder.ToString());
sqlCommand.Parameters["@accountID"].Value = deliveryOutput.Account.ID;
sqlCommand.Parameters["@daycode"].Value = dayCode;;
sqlCommand.Parameters["@Channel_ID"].Value = deliveryOutput.Channel.ID;
///*******Work around*******************////////
if (deliveryOutput.Account.OriginalID != null)
{
sqlCommand.Parameters["@OriginalID"].Value = deliveryOutput.Account.OriginalID;
}
sqlCommand.Connection = sqlCon;
using (var _reader = sqlCommand.ExecuteReader())
{
if (!_reader.IsClosed)
{
if (_reader.Read())
{
foreach (var measure in measures)
{
Diffrence difference = new Diffrence();
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!_reader[measure.Value.OltpName].Equals(DBNull.Value))
{
difference.Source = totals[measure.Value.Name];
difference.Target = Convert.ToDouble(_reader[measure.Value.OltpName]);
difference.diff = Math.Abs((Convert.ToDouble(_reader[measure.Value.OltpName]) - totals[measure.Value.Name]));
diffs[measure.Value.Name] = difference;
}
#region Scenario : data exists in delivery and not in DB
else if (totals[measure.Value.Name] > 0)
{
return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = string.Format("Data exists in delivery but not in DB for Account {0} output {1}", deliveryOutput.Account.ID, deliveryOutput.OutputID.ToString("N")),
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
else //diff is 0
{
difference.Source = 0;
difference.Target = 0;
difference.diff = 0;
diffs[measure.Value.Name] = difference;
}
#endregion
}
}
StringBuilder errors = new StringBuilder();
foreach (KeyValuePair <string, Diffrence> diff in diffs)
{
if (diff.Value.diff / diff.Value.Source > allowedChecksumThreshold)
{
errors.AppendFormat("validation Error - differences has been found for Account-{0}-channel{1}-output{2}-measure{3} source{4} target {5}\n", deliveryOutput.Account.ID, deliveryOutput.Channel.ID, deliveryOutput.OutputID.ToString("N"), diff.Key, diff.Value.Source, diff.Value.Target); //TODO: USE
}
}
if (errors.Length > 0)
{
return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = errors.ToString(),
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
else
{
return(new ValidationResult()
{
ResultType = ValidationResultType.Information,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "validation Success - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
}
}
// If reader is closed
else
{
result = new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Cannot Read Data from DB connection closed - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
};
}
//Could not find check scenario
result = new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Could not find check scenario - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
};
}
return(result);
}
}
protected override ValidationResult DeliveryDbCompare(DeliveryOutput output, Dictionary <string, double> deliveryTotals, string DbConnectionStringName, string comparisonTable)
{
using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName)))
{
sqlCon.Open();
Dictionary <string, Measure> measurs = Measure.GetMeasures(output.Account, output.Channel, sqlCon, MeasureOptions.IsBackOffice, OptionsOperator.And);
string dayCode = output.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode
Dictionary <string, double> diff = new Dictionary <string, double>();
Dictionary <string, Measure> validationRequiredMeasure = new Dictionary <string, Measure>();
#region Creating Command txt
/*==========================================================*/
StringBuilder command = new StringBuilder();
command.Append("Select ");
foreach (var measure in measurs)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
validationRequiredMeasure.Add(measure.Value.SourceName, measure.Value);
command.Append(string.Format("SUM({0}) as {1}", measure.Value.OltpName, measure.Value.SourceName));
command.Append(",");
}
}
command.Remove(command.Length - 1, 1); //remove last comma character
command.Append(" from ");
command.Append(comparisonTable);
command.Append(" where Account_ID = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID");
/*===========================================================*/
#endregion
#region SQL Command
/*==============================================================================*/
SqlCommand sqlCommand = DataManager.CreateCommand(command.ToString());
sqlCommand.Connection = sqlCon;
SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int);
SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int);
SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int);
accountIdParam.Value = output.Account.ID;
daycodeParam.Value = dayCode;
channelIdParam.Value = output.Channel.ID;
sqlCommand.Parameters.Add(accountIdParam);
sqlCommand.Parameters.Add(daycodeParam);
sqlCommand.Parameters.Add(channelIdParam);
sqlCommand.Connection = sqlCon;
/*==============================================================================*/
#endregion
using (var _reader = sqlCommand.ExecuteReader())
{
progress += 0.5 / 1 - progress;
this.ReportProgress(progress);
if (!_reader.IsClosed)
{
while (_reader.Read())
{
if (!_reader[0].Equals(DBNull.Value))
{
foreach (var measureItem in validationRequiredMeasure)
{
diff.Add(measureItem.Value.SourceName,
Math.Abs(Convert.ToDouble(_reader[measureItem.Value.SourceName]) - deliveryTotals[measureItem.Value.SourceName]));
}
}
#region Scenario : data exists in delivery and not in DB
else
{
double sum = 0;
foreach (var measure in validationRequiredMeasure)
{
sum += deliveryTotals[measure.Value.SourceName];
}
if (sum > 0)
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = output.Account.ID,
TargetPeriodStart = output.TimePeriodStart,
TargetPeriodEnd = output.TimePeriodEnd,
Message = "Data exists in delivery but not in DB for Account ID: " + output.Account.ID,
ChannelID = output.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
}
;
}
#endregion
foreach (var item in diff)
{
#region Scenario: Found differences
if (item.Value > ALLOWED_DIFF)
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = output.Account.ID,
DeliveryID = output.DeliveryID,
TargetPeriodStart = output.TimePeriodStart,
TargetPeriodEnd = output.TimePeriodEnd,
Message = "validation Error - differences has been found - Account ID: " + output.Account.ID,
ChannelID = output.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
}
;
#endregion
#region Scenario: Differences were not found
else
{
return new ValidationResult()
{
ResultType = ValidationResultType.Information,
AccountID = output.Account.ID,
DeliveryID = output.DeliveryID,
TargetPeriodStart = output.TimePeriodStart,
TargetPeriodEnd = output.TimePeriodEnd,
Message = "validation Success - Account ID: " + output.Account.ID,
ChannelID = output.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
};
#endregion
}
}
}
// If reader is closed
else
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = output.Account.ID,
DeliveryID = output.DeliveryID,
TargetPeriodStart = output.TimePeriodStart,
TargetPeriodEnd = output.TimePeriodEnd,
Message = "Cannot Read Data from DB connection closed - Account Original ID: " + output.Account.OriginalID,
ChannelID = output.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
};
}
//Could not find check scenario
return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = output.Account.ID,
DeliveryID = output.DeliveryID,
TargetPeriodStart = output.TimePeriodStart,
TargetPeriodEnd = output.TimePeriodEnd,
Message = "Could not find check scenario - Account Original ID: " + output.Account.OriginalID,
ChannelID = output.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
}
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString());
bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString());
bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString());
//double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1;
//Status Members
Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
string[] requiredHeaders = new string[1];
requiredHeaders[0] = Const.AdPreRequiredHeader;
#region Getting Keywords Data
Dictionary <string, double> _totals = new Dictionary <string, double>();
DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]];
requiredHeaders[0] = Const.AdPreRequiredHeader;
var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
_keywordsReader.MatchExactColumns = false;
Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>();
this.ImportManager.BeginImport(this.Delivery);
using (_keywordsReader)
{
while (_keywordsReader.Read())
{
this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field];
if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF)
{
break;
}
KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]),
AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName])
};
KeywordTarget keyword = new KeywordTarget()
{
OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName],
Keyword = _keywordsReader.Current[Const.KeywordFieldName],
//Status = kwd_Status_Data[keywordPrimaryKey.ToString()]
};
keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]);
string matchType = _keywordsReader.Current[Const.MatchTypeFieldName];
keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true);
//Setting Tracker for Keyword
if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName])))
{
keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]);
//setting kwd tracker
//if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0)
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget)))
{
this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword);
}
}
}
_keywordsData.Add(keywordPrimaryKey.ToString(), keyword);
}
}
#endregion
Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>();
#region Getting Placements Data
string[] _placementsFileRequiredHeaders = new string[1];
_placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID;
DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]];
var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders);
using (_PlacementsReader)
{
while (_PlacementsReader.Read())
{
if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF)
{
break;
}
//Read data only if managed GDN, otherwise it is an automatic GDN so skip
if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--"))
{
KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]),
AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName])
};
PlacementTarget placement = new PlacementTarget()
{
OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID],
Placement = _PlacementsReader.Current[Const.PlacementFieldName],
PlacementType = PlacementType.Managed,
// Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()]
};
//Setting Tracker for placment
if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName])))
{
placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]);
}
_placementsData.Add(placementPrimaryKey.ToString(), placement);
}
}
}
#endregion
#region Getting Conversions Data
//Get Ads Conversion ( for ex. signup , purchase )
DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"];
var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >();
using (_conversionsReader)
{
while (_conversionsReader.Read())
{
if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report
{
break;
}
string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
//ADD conversionKey to importedAdsWithConv
//than add conversion field to importedAdsWithConv : <conversion name , conversion value>
Dictionary <string, long> conversion = new Dictionary <string, long>();
conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
importedAdsWithConv.Add(conversionKey, conversion);
}
else // if Key exists
{
// if current add already has current conversion type than add value to the current type
if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])))
{
conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
}
// else create new conversion type and add the value
else
{
conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]);
}
}
}
}
#endregion
#region Getting Ads Data
DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]];
var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>();
//session.Begin(false);
//this.ImportManager.BeginImport(this.Delivery);
DeliveryOutput currentOutput = Delivery.Outputs.First();
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_adsReader)
{
this.Mappings.OnFieldRequired = field => _adsReader.Current[field];
while (_adsReader.Read())
{
string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper();
// Adding totals line for validation (checksum)
if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF)
{
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] =
ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000;
break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break;
}
}
break;
}
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
Ad ad;
#region Try Get SearchKWD
/***********************************************************************************/
//Creating kwd primary key
KeywordPrimaryKey kwdKey = new KeywordPrimaryKey()
{
AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]),
KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]),
CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])
};
//Check if keyword file contains this kwdkey and not a GDN Keyword
String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(',');
bool IsSearchKwd = false;
KeywordTarget kwd = null;
PlacementTarget placement = null;
if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString()))
{
kwd = new KeywordTarget();
try
{
kwd = _keywordsData[kwdKey.ToString()];
}
catch (Exception)
{
//Creating KWD with OriginalID , since the KWD doesnt exists in KWD report.
kwd = new KeywordTarget {
OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName])
};
}
IsSearchKwd = true;
}
else
{
placement = new PlacementTarget();
try
{
placement = _placementsData[kwdKey.ToString()];
}
catch (Exception)
{
placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]);
placement.PlacementType = PlacementType.Automatic;
placement.Placement = Const.AutoDisplayNetworkName;
}
}
/***********************************************************************************/
#endregion
string adId = _adsReader.Current[Const.AdIDFieldName];
if (!importedAds.ContainsKey(adId))
{
ad = new Ad();
ad.OriginalID = adId;
ad.Channel = new Channel()
{
ID = 1
};
ad.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
// ad.Status = ad_Status_Data[Convert.ToInt64(adId)];
#region Ad Type
/****************************************************************/
string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]);
string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]);
if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue))
{
continue;
}
string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString();
//EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);
//is mobile ad ?
if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue))
{
string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]));
//Check if this mobile value exists on dictionary
if (GoogleAdTypeDic.ContainsKey(mobileValue))
{
adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString();
}
else
{
adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString();
}
}
ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);;
/****************************************************************/
#endregion
//Creative
ad.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName]
});
#region Ad Tracker segment
/******************************************************/
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName]))
{
ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
}
//if Ad doesnt contains tracker than check for kwd tracker
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker]))
{
if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null)
{
SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]];
//if value contains ADID than replace ADID with AD original id
tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID);
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker;
}
}
}
/******************************************************/
#endregion
#region Campaign
/****************************************************************/
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _adsReader.Current[Const.CampaignIdFieldName],
Name = _adsReader.Current[Const.CampaignFieldName],
//Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])]
};
/****************************************************************/
#endregion
#region Image
/****************************************************************/
//Image Type > Create Image
if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad"))
{
string adNameField = _adsReader.Current[Const.AdFieldName];
string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60
ad.Name = imageParams[1].Trim();
ad.Creatives.Add(new ImageCreative()
{
ImageUrl = imageParams[1].Trim(),
ImageSize = imageParams[2].Trim()
});
}
/****************************************************************/
#endregion
#region Text od Display
/****************************************************************/
else //Text ad or Display ad
{
ad.Name = _adsReader.Current[Const.AdFieldName];
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = _adsReader.Current.Ad,
});
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = _adsReader.Current["Description line 1"],
Text2 = _adsReader.Current["Description line 2"]
});
}
/****************************************************************/
#endregion
#region Adgroup
/****************************************************************/
//Insert Adgroup
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _adsReader.Current[Const.AdGroupFieldName],
OriginalID = _adsReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
/****************************************************************/
#endregion
#region Network
/****************************************************************/
//Insert Network Type Display Network / Search Network
//string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]);
//if (networkType.Equals(Const.GoogleSearchNetwork))
// networkType = Const.SystemSearchNetwork;
//else if (networkType.Equals(Const.GoogleDisplayNetwork))
// networkType = Const.SystemDisplayNetwork;
//ad.ExtraFields[NetworkType] = networkType;
/****************************************************************/
#endregion
importedAds.Add(adId, ad);
this.ImportManager.ImportAd(ad);
}
else
{
ad = importedAds[adId];
}
adMetricsUnit.Ad = ad;
//INSERTING SEARCH KEYWORD INTO METRICS
if (IsSearchKwd & kwd != null)
{
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(kwd);
}
//INSERTING GDN KEYWORD INTO METRICS
else if (placement != null)
{
//INSERTING KEYWORD INTO METRICS
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(placement);
}
//INSERTING METRICS DATA
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks));
//Currencies Conversion Support
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost],
ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
if (ConvertToUSD)
{
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue));
}
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName]));
//Inserting conversion values
string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
foreach (var pair in conversionDic)
{
if (GoogleMeasuresDic.ContainsKey(pair.Key))
{
adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value;
}
}
}
adMetricsUnit.Currency = new Currency
{
Code = Convert.ToString(_adsReader.Current.Currency)
};
this.ImportManager.ImportMetrics(adMetricsUnit);
}
#endregion
}
#region Getting Sitelinks Data without metrics
if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString()))
{
string[] sitelinksRequiredHeaders = new string[1];
sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID;
DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]];
var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders);
AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit();
siteLinkMetricsUnit.Output = currentOutput;
Ad sitelinkAd;
using (_sitelinkReader)
{
this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field];
//to do : get site link tracker
string[] sitelinkAttr;
while (_sitelinkReader.Read())
{
if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF))
{
break;
}
string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]);
sitelinkAd = new Ad();
sitelinkAd.OriginalID = sitelinkId;
sitelinkAd.Channel = new Channel()
{
ID = 1
};
sitelinkAd.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
//Creative
sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';');
bool legacy = sitelinkAttr.Count() != 4 ? true : false;
string destUrl = string.Empty;
bool destUrlParsingError = false;
//Checking desturl errors ( we would like to insert only sitelinks that contains desturl ).
try
{
destUrl = sitelinkAttr[1];
}
catch (Exception e)
{
Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e);
destUrlParsingError = true;
}
if (!destUrlParsingError)
{
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(sitelinkAttr[1]))
{
sitelinkAd.DestinationUrl = sitelinkAttr[1];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd);
}
}
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName],
Name = _sitelinkReader.Current[Const.CampaignFieldName],
};
//Insert Adgroup
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _sitelinkReader.Current[Const.AdGroupFieldName],
OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
sitelinkAd.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1]
});
if (!legacy) // only in case it doesnt contains legacy siteink
{
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = sitelinkAttr[2],
Text2 = sitelinkAttr[3]
});
}
sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0];
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = "[Sitelink] " + sitelinkAttr[0]
});
//Ad Type
//Note: changed to "sitelink" following Amir request
sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink);
siteLinkMetricsUnit.Ad = sitelinkAd;
//Setting Default Currency as USD following Amir's request from March 2014
siteLinkMetricsUnit.Currency = new Currency
{
Code = "USD"
};
//INSERTING METRICS DATA
siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0);
ImportManager.ImportMetrics(siteLinkMetricsUnit);
ImportManager.ImportAd(sitelinkAd);
}
}
}
}// end if
#endregion
currentOutput.Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader };
DeliveryOutput currentOutput = Delivery.Outputs.First();
//Open Auto Plac file
DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]];
var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
Dictionary <string, double> _totals = new Dictionary <string, double>();
this.ImportManager.BeginImport(this.Delivery);
//Intializing totals for validation
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_autoPlacReader)
{
this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field];
while (_autoPlacReader.Read())
{
#region Setting Totals
/*==================================================================================================================*/
// If end of file
if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF)
{
//Setting totals for validation from totals line in adowrds file
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break;
}
}
break;
}
/*==================================================================================================================*/
#endregion
GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit();
autoPlacMetricsUnit.Output = currentOutput;
autoPlacMetricsUnit.Channel = new Channel()
{
ID = 1
};
autoPlacMetricsUnit.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"]
};
autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>();
//ADDING CAMPAIGN
Campaign campaign = new Campaign()
{
OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName],
Name = _autoPlacReader.Current[Const.CampaignFieldName],
Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign);
//ADDING ADGROUP
AdGroup adgroup = new AdGroup()
{
Campaign = campaign,
Value = _autoPlacReader.Current[Const.AdGroupFieldName],
OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup);
//INSERTING METRICS DATA
autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000);
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName]));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName]));
//CREATING PLACEMENT
autoPlacMetricsUnit.TargetDimensions = new List <Target>();
PlacementTarget placement = new PlacementTarget()
{
Placement = _autoPlacReader.Current[Const.DomainFieldName],
PlacementType = PlacementType.Automatic
// Add status !!!
};
autoPlacMetricsUnit.TargetDimensions.Add(placement);
//SETTING TIME PERIOD
autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
this.ImportManager.ImportMetrics(autoPlacMetricsUnit);
}
}
this.Delivery.Outputs.First().Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
foreach (var ReportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*");
using (reportReader)
{
int colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
///sssss
}
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
// Checksums
_isChecksum = true;
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current[field];
if (reportReader.Read())
{
GenericMetricsUnit checksumUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(checksumUnit);
foreach (var m in checksumUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
}
}
}
_isChecksum = false;
//Get Valuees
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
GenericMetricsUnit tempUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(tempUnit);
SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit existingUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out existingUnit))
{
tempUnit.Output = currentOutput;
data.Add(tracker.Value, tempUnit);
}
else
{
// if tracker already exists, merge with existing values
foreach (var m in tempUnit.MeasureValues)
{
if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
existingUnit.MeasureValues[m.Key] += m.Value;
}
}
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable)
{
double costDif = 0;
double impsDif = 0;
double clicksDif = 0;
string dayCode = deliveryOutput.TimePeriodStart.ToString("yyyyMMdd"); // Delivery Per Day = > TargetPeriod.Start = daycode
using (SqlConnection sqlCon = new SqlConnection(AppSettings.GetConnectionString(this, DbConnectionStringName)))
{
// This option is relevant only for account with one commited delivery.
// for example it is not relevant for Easy forex.
sqlCon.Open();
SqlCommand sqlCommand = DataManager.CreateCommand(
"SELECT SUM(cost),sum(imps),sum(clicks) from " + comparisonTable +
" where Account_ID = @Account_ID and Day_ID = @Daycode and Channel_ID = @Channel_ID"
);
SqlParameter accountIdParam = new SqlParameter("@Account_ID", System.Data.SqlDbType.Int);
SqlParameter daycodeParam = new SqlParameter("@Daycode", System.Data.SqlDbType.Int);
SqlParameter channelIdParam = new SqlParameter("@Channel_ID", System.Data.SqlDbType.Int);
accountIdParam.Value = deliveryOutput.Account.ID;
daycodeParam.Value = dayCode;
channelIdParam.Value = deliveryOutput.Channel.ID;
sqlCommand.Parameters.Add(accountIdParam);
sqlCommand.Parameters.Add(daycodeParam);
sqlCommand.Parameters.Add(channelIdParam);
sqlCommand.Connection = sqlCon;
using (var _reader = sqlCommand.ExecuteReader())
{
progress += 0.5 / 1 - progress;
this.ReportProgress(progress);
if (!_reader.IsClosed)
{
while (_reader.Read())
{
if (!_reader[0].Equals(DBNull.Value))
{
costDif = Math.Abs(Convert.ToUInt64(_reader[0]) - totals["Cost"]);
clicksDif = Math.Abs(Convert.ToUInt64(_reader[2]) - totals["Clicks"]);
impsDif = Math.Abs(Convert.ToUInt64(_reader[1]) - totals["Impressions"]);
}
#region Scenario : data exists in delivery and not in DB
else if (totals["Cost"] > 0 || totals["Clicks"] > 0 || totals["Impressions"] > 0)
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Data exists in delivery but not in DB for Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
}
;
#endregion
#region Scenario: Found differences
if ((costDif != 0 && (costDif / totals["Cost"] > ALLOWED_DIFF)) ||
(clicksDif != 0 && (clicksDif / totals["Clicks"] > ALLOWED_DIFF)) ||
(impsDif != 0 && (impsDif / totals["Impressions"] > ALLOWED_DIFF)))
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.Delivery.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "validation Error - differences has been found - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
}
;
#endregion
#region Scenario: Differences were not found
else
{
return new ValidationResult()
{
ResultType = ValidationResultType.Information,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.Delivery.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "validation Success - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
};
#endregion
}
}
// If reader is closed
else
{
return new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.Delivery.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Cannot Read Data from DB connection closed - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
}
};
}
//Could not find check scenario
return(new ValidationResult()
{
ResultType = ValidationResultType.Error,
AccountID = deliveryOutput.Account.ID,
DeliveryID = deliveryOutput.Delivery.DeliveryID,
TargetPeriodStart = deliveryOutput.TimePeriodStart,
TargetPeriodEnd = deliveryOutput.TimePeriodEnd,
Message = "Could not find check scenario - Account Original ID: " + deliveryOutput.Account.OriginalID,
ChannelID = deliveryOutput.Channel.ID,
CheckType = this.Instance.Configuration.Name
});
}
}
}
}
abstract protected ValidationResult DeliveryDbCompare(DeliveryOutput deliveryOutput, Dictionary <string, double> totals, string DbConnectionStringName, string comparisonTable);
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
foreach (var ReportFile in Delivery.Files)
{
//check number of recordes
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize");
int numOfRecordes = 0;
if (reportReader.Read())
{
numOfRecordes = int.Parse(reportReader.Current.totalSize);
}
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
if (reportReader.Read())
{
Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning);
}
if (numOfRecordes > 0)
{
//Get Values
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field =>
{
object value = new object();
string[] nestedFields;
try
{
if (field.Contains('.'))
{
nestedFields = field.Split('.');
value = reportReader.Current[nestedFields[0]];
foreach (var item in nestedFields)
{
if (item.Equals(nestedFields[0]))
{
continue;
}
value = ((Dictionary <string, object>)value)[item];
}
value = reportReader.Current[nestedFields[0]][nestedFields[1]];
Console.Write(value.ToString());
return(value);
}
else
{
Console.Write(value.ToString());
return(reportReader.Current[field]);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex);
}
};
while (reportReader.Read())
{
GenericMetricsUnit metricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(metricsUnit);
if (metricsUnit.MeasureValues != null)
{
SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit importedUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out importedUnit))
{
metricsUnit.Output = currentOutput;
data.Add(tracker.Value, metricsUnit);
}
else //Tracker already exists
{
// Merge captured measure with existing measures
foreach (var capturedMeasure in metricsUnit.MeasureValues)
{
if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
//Measure already exists per tracker than aggregate:
if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key))
{
importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value;
}
else
{
//Captured Measure doest exists with this tracker:
importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value);
}
}
}
#region Validation
// For validations
foreach (var m in metricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
#endregion
}
}
}
}
else
{
Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information);
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}