protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
foreach (var ReportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*");
using (reportReader)
{
int colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
///sssss
}
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
// Checksums
_isChecksum = true;
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current[field];
if (reportReader.Read())
{
GenericMetricsUnit checksumUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(checksumUnit);
foreach (var m in checksumUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
}
}
}
_isChecksum = false;
//Get Valuees
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
GenericMetricsUnit tempUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(tempUnit);
SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit existingUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out existingUnit))
{
tempUnit.Output = currentOutput;
data.Add(tracker.Value, tempUnit);
}
else
{
// if tracker already exists, merge with existing values
foreach (var m in tempUnit.MeasureValues)
{
if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
existingUnit.MeasureValues[m.Key] += m.Value;
}
}
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string baseAddress;
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
bool multibo = false;
if (this.Instance.Configuration.Options.ContainsKey("URL.SignatureAppend"))
{
multibo = Convert.ToBoolean(this.Instance.Configuration.Options["URL.SignatureAppend"]);
}
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}{2}]", this.Instance.AccountID, this.TimePeriod.ToAbsolute(), multibo?"-" + this.Instance.Configuration.Options["Bo.ServiceAdress"]:string.Empty)),
Account = Delivery.Account,
Channel = Delivery.Channel,
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
});
// Create an import manager that will handle rollback, if necessary
var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("base url must be configured!");
}
baseAddress = this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress];
int utcOffset = 0;
if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset))
{
utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
}
Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset);
int timeZone = 0;
if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone))
{
timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]);
}
Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone);
this.ReportProgress(0.2);
#endregion
#region DeliveryFile
Dictionary <string, string> UrlParams = new Dictionary <string, string>();
DeliveryFile boFile = new DeliveryFile();
DateTime start = TimePeriod.Start.ToDateTime();
DateTime end = TimePeriod.End.ToDateTime();
boFile.Parameters[BoConfigurationOptions.BO_XPath_Trackers] = Instance.Configuration.Options[BoConfigurationOptions.BO_XPath_Trackers];
boFile.Name = BoConfigurationOptions.BoFileName;
if (utcOffset != 0)
{
start = start.AddHours(utcOffset);
end = end.AddHours(utcOffset);
}
if (timeZone != 0)
{
start = start.AddHours(-timeZone);
end = end.AddHours(-timeZone);
}
boFile.SourceUrl = string.Format(baseAddress, start, end);
boFile.Parameters.Add(BoConfigurationOptions.IsAttribute, Instance.Configuration.Options[BoConfigurationOptions.IsAttribute]);
boFile.Parameters.Add(BoConfigurationOptions.TrackerFieldName, Instance.Configuration.Options[BoConfigurationOptions.TrackerFieldName]);
this.Delivery.Files.Add(boFile);
this.Delivery.Save();
#endregion
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]",
this.Instance.AccountID,
this.TimePeriod.ToAbsolute())),
Account = Delivery.Account,
Channel = Delivery.Channel,
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
});
// Create an import manager that will handle rollback, if necessary
var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
this.ReportProgress(0.2);
#endregion
//Create File
DeliveryFile _file = new DeliveryFile()
{
Name = "EasyForexBackOffice",
};
_file.SourceUrl = Instance.Configuration.Options["SourceUrl"]; // "https://classic.easy-forex.com/BackOffice/API/Marketing.asmx";
//_file.Parameters.Add("SOAPAction", "http://www.easy-forex.com/GetGatewayStatistics");
_file.Parameters.Add("Content-Type", "text/xml; charset=utf-8");
_file.Parameters.Add("SOAPAction", Instance.Configuration.Options["SOAPAction"]);
Delivery.Parameters.Add("User", Instance.Configuration.Options["User"]);
Delivery.Parameters.Add("Pass", Core.Utilities.Encryptor.Dec(Instance.Configuration.Options["Pass"].ToString()));
Delivery.Parameters.Add("SoapMethod", Instance.Configuration.Options["SoapMethod"]);
Delivery.Parameters.Add("StartGid", Instance.Configuration.Options["StartGid"]);
Delivery.Parameters.Add("EndGid", Instance.Configuration.Options["EndGid"]);
//Creating Soap Body
string strSoapEnvelope = GetSoapEnvelope(
Delivery.Parameters["User"].ToString(),
Delivery.Parameters["Pass"].ToString(),
Delivery.Parameters["SoapMethod"].ToString(),
Delivery.Parameters["StartGid"].ToString(),
Delivery.Parameters["EndGid"].ToString(),
this.Delivery.TimePeriodStart.ToString("yyyy-MM-ddTHH:mm:ss"),
this.Delivery.TimePeriodEnd.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")
);
#region Soap 1.2
//strSoapEnvelope += "<?xml version=\"1.0\" encoding=\"utf-8\"?>";
//strSoapEnvelope += "<soap12:Envelope xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:soap12=\"http://www.w3.org/2003/05/soap-envelope\">";
//strSoapEnvelope += " <soap12:Header>";
//strSoapEnvelope += " <AuthHeader xmlns=\"http://www.easy-forex.com\">";
//strSoapEnvelope += " <Username>Seperia</Username>";
//strSoapEnvelope += " <Password>Seperia909</Password>";
//strSoapEnvelope += " </AuthHeader>";
//strSoapEnvelope += " </soap12:Header>";
//strSoapEnvelope += " <soap12:Body>";
//strSoapEnvelope += "<GetGatewayStatistics xmlns=\"http://www.easy-forex.com\">";
//strSoapEnvelope += " <startGid>1</startGid>";
//strSoapEnvelope += " <finishGid>1000000</finishGid>";
//strSoapEnvelope += string.Format(" <fromDateTime>{0}</fromDateTime>", _requiredDay);
//strSoapEnvelope += string.Format(" <toDateTime>{0}</toDateTime>", _requiredDay.AddDays(1).AddTicks(-1));
//strSoapEnvelope += " </GetGatewayStatistics>";
//strSoapEnvelope += "</soap12:Body>";
//strSoapEnvelope += "</soap12:Envelope>";
#endregion
_file.Parameters.Add("Body", strSoapEnvelope);
_file.Parameters.Add("Bo.IsAttribute", Instance.Configuration.Options["Bo.IsAttribute"]);
_file.Parameters.Add("Bo.Xpath", Instance.Configuration.Options["Bo.Xpath"]);
_file.Parameters.Add("Bo.TrackerIDField", Instance.Configuration.Options["Bo.TrackerIDField"]);
this.Delivery.Files.Add(_file);
// Save with success
this.Delivery.Save();
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader };
DeliveryOutput currentOutput = Delivery.Outputs.First();
//Open Auto Plac file
DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]];
var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
Dictionary <string, double> _totals = new Dictionary <string, double>();
this.ImportManager.BeginImport(this.Delivery);
//Intializing totals for validation
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_autoPlacReader)
{
this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field];
while (_autoPlacReader.Read())
{
#region Setting Totals
/*==================================================================================================================*/
// If end of file
if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF)
{
//Setting totals for validation from totals line in adowrds file
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break;
}
}
break;
}
/*==================================================================================================================*/
#endregion
GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit();
autoPlacMetricsUnit.Output = currentOutput;
autoPlacMetricsUnit.Channel = new Channel()
{
ID = 1
};
autoPlacMetricsUnit.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"]
};
autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>();
//ADDING CAMPAIGN
Campaign campaign = new Campaign()
{
OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName],
Name = _autoPlacReader.Current[Const.CampaignFieldName],
Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign);
//ADDING ADGROUP
AdGroup adgroup = new AdGroup()
{
Campaign = campaign,
Value = _autoPlacReader.Current[Const.AdGroupFieldName],
OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName]
};
autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup);
//INSERTING METRICS DATA
autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000);
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName]));
autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName]));
//CREATING PLACEMENT
autoPlacMetricsUnit.TargetDimensions = new List <Target>();
PlacementTarget placement = new PlacementTarget()
{
Placement = _autoPlacReader.Current[Const.DomainFieldName],
PlacementType = PlacementType.Automatic
// Add status !!!
};
autoPlacMetricsUnit.TargetDimensions.Add(placement);
//SETTING TIME PERIOD
autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
this.ImportManager.ImportMetrics(autoPlacMetricsUnit);
}
}
this.Delivery.Outputs.First().Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
foreach (var ReportFile in Delivery.Files)
{
bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString());
var ReportReader = new XmlDynamicReader
(ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString());
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
//checksum init
foreach (var m in ImportManager.Measures.Values)
{
if (m.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum[m.Name] = 0;
}
}
using (ReportReader)
{
dynamic readerHelper;
while (ReportReader.Read())
{
if (isAttribute)
{
readerHelper = ReportReader.Current.Attributes;
}
else
{
readerHelper = ReportReader.Current;
}
this.Mappings.OnFieldRequired = field => readerHelper[field];
GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(genericMetricsUnit);
foreach (var m in genericMetricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
genericMetricsUnit.Output = currentOutput;
ImportManager.ImportMetrics(genericMetricsUnit);
}
}
ImportManager.EndImport();
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (!this.Instance.Configuration.Options.ContainsKey("AuthenticationUrl"))
{
throw new Exception("AuthenticationUrl must be configured in configuration file");
}
this.Delivery.Parameters["AuthenticationUrl"] = this.Instance.Configuration.Options["AuthenticationUrl"]; //https://test.salesforce.com/services/oauth2/token
if (!this.Instance.Configuration.Options.ContainsKey("ConsumerKey"))
{
throw new Exception("ClientID must be configured in configuration file");
}
this.Delivery.Parameters["SalesForceClientID"] = this.Instance.Configuration.Options["ConsumerKey"];//3MVG9GiqKapCZBwG.OqpT.DCgHmIXOlszzCpZPxbRyvzPDNlshB5LD0x94rQO5SzGOAZrWPNIPm_aGR7nBeXe
if (!this.Instance.Configuration.Options.ContainsKey("ConsentCode"))
{
throw new Exception("ConsentCode must be configured in configuration file");
}
this.Delivery.Parameters["ConsentCode"] = this.Instance.Configuration.Options["ConsentCode"]; //(accesstoken
if (!this.Instance.Configuration.Options.ContainsKey("ConsumerSecret"))
{
throw new Exception("ClientSecret must be configured in configuration file");
}
this.Delivery.Parameters["ClientSecret"] = this.Instance.Configuration.Options["ConsumerSecret"];//321506373515061074
if (!this.Instance.Configuration.Options.ContainsKey("Redirect_URI"))
{
throw new Exception("Redirect_URI must be configured in configuration file");
}
this.Delivery.Parameters["Redirect_URI"] = this.Instance.Configuration.Options["Redirect_URI"]; //http://localhost:8080/RestTest/oauth/_callback
// This is for finding conflicting services
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.TimePeriod.ToAbsolute(), this.Delivery.Parameters["SalesForceClientID"])),
Account = Delivery.Account,
Channel = Delivery.Channel,
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
});
// Create an import manager that will handle rollback, if necessary
var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
int utcOffset = 0;
if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset))
{
utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
}
Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset);
int timeZone = 0;
if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone))
{
timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]);
}
Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone);
this.ReportProgress(0.2);
//if (!Instance.Configuration.Options.ContainsKey("ServiceAddress"))
// throw new Exception("ServiceAddress must be configured in configuration file");
//serviceAddress = Instance.Configuration.Options["ServiceAddress"];
#endregion
#region DeliveryFile
string query = Instance.Configuration.Options["Query"];
//@SMBLeads:SELECT CreatedDate,Edge_BI_Tracker__c ,Company_Type__c FROM Lead WHERE CreatedDate > 2013-06-28T00:00:00.00Z AND Edge_BI_Tracker__c != null
string[] queries = query.Split('@');
foreach (string qwry in queries)
{
DeliveryFile file = new DeliveryFile();
string[] q = qwry.Split(':');
file.Name = q[0] + "-0";
DateTime start = TimePeriod.Start.ToDateTime();
DateTime end = TimePeriod.End.ToDateTime();
if (utcOffset != 0)
{
start = start.AddHours(utcOffset);
end = end.AddHours(utcOffset);
}
if (timeZone != 0)
{
start = start.AddHours(-timeZone);
end = end.AddHours(-timeZone);
}
file.Parameters["Query"] = q[1];
this.Delivery.Files.Add(file);
}
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
#endregion
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
#region Init General
int backOfficeChannel = -888;
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
#region two profiles
//serviceAddress = Instance.Configuration.Options["ServiceAddress"];
//Match m=Regex.Match(serviceAddress,@"\bids=(?<profileID>[^&]+)",RegexOptions.IgnoreCase);
//Group g=m.Groups["profileID"];
//string profileID=g.Value;
//**************in case amir will want to run two profiles
//this.Delivery.Outputs.Add(new DeliveryOutput()
//{
// Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]-[{2}]",
// this.Instance.AccountID,
// this.TimePeriod.ToAbsolute(),
// profileID)),
// Account = Delivery.Account,
// Channel = Delivery.Channel,
// TimePeriodStart=Delivery.TimePeriodStart,
// TimePeriodEnd=Delivery.TimePeriodEnd
//});
#endregion
// This is for finding conflicting services
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]",
this.Instance.AccountID,
this.TimePeriod.ToAbsolute())),
Account = Delivery.Account,
Channel = Delivery.Channel,
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
});
// Create an import manager that will handle rollback, if necessary
var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (!this.Instance.Configuration.Options.ContainsKey("AuthenticationUrl"))
{
throw new Exception("AuthenticationUrl must be configured in configuration file");
}
this.Delivery.Parameters["AuthenticationUrl"] = this.Instance.Configuration.Options["AuthenticationUrl"];
if (!this.Instance.Configuration.Options.ContainsKey("ClientID"))
{
throw new Exception("ClientID must be configured in configuration file");
}
this.Delivery.Parameters["ClientID"] = this.Instance.Configuration.Options["ClientID"];
if (!this.Instance.Configuration.Options.ContainsKey("ConsentCode"))
{
throw new Exception("ConsentCode must be configured in configuration file");
}
this.Delivery.Parameters["ConsentCode"] = this.Instance.Configuration.Options["ConsentCode"];
if (!this.Instance.Configuration.Options.ContainsKey("ClientSecret"))
{
throw new Exception("ClientSecret must be configured in configuration file");
}
this.Delivery.Parameters["ClientSecret"] = this.Instance.Configuration.Options["ClientSecret"];
if (!this.Instance.Configuration.Options.ContainsKey("Redirect_URI"))
{
throw new Exception("Redirect_URI must be configured in configuration file");
}
this.Delivery.Parameters["Redirect_URI"] = this.Instance.Configuration.Options["Redirect_URI"];
int utcOffset = 0;
if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset))
{
utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
}
Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset);
int timeZone = 0;
if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone))
{
timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]);
}
Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone);
this.ReportProgress(0.2);
if (!Instance.Configuration.Options.ContainsKey("ServiceAddress"))
{
throw new Exception("ServiceAddress must be configured in configuration file");
}
serviceAddress = Instance.Configuration.Options["ServiceAddress"];
#endregion
#region DeliveryFile
DeliveryFile file = new DeliveryFile();
file.Name = "GoalsByCountryAndPagePath.GZIP";
DateTime start = TimePeriod.Start.ToDateTime();
DateTime end = TimePeriod.End.ToDateTime();
if (utcOffset != 0)
{
start = start.AddHours(utcOffset);
end = end.AddHours(utcOffset);
}
if (timeZone != 0)
{
start = start.AddHours(-timeZone);
end = end.AddHours(-timeZone);
}
file.SourceUrl = string.Format(serviceAddress, this.Delivery.TimePeriodDefinition.Start.ToDateTime(), this.Delivery.TimePeriodDefinition.End.ToDateTime());
this.Delivery.Files.Add(file);
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
#endregion
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
currentOutput = this.Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
Dictionary <string, int> columns = new Dictionary <string, int>();
using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Or,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>();
foreach (var ReportFile in Delivery.Files)
{
//check number of recordes
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize");
int numOfRecordes = 0;
if (reportReader.Read())
{
numOfRecordes = int.Parse(reportReader.Current.totalSize);
}
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
if (reportReader.Read())
{
Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning);
}
if (numOfRecordes > 0)
{
//Get Values
reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*");
using (reportReader)
{
this.Mappings.OnFieldRequired = field =>
{
object value = new object();
string[] nestedFields;
try
{
if (field.Contains('.'))
{
nestedFields = field.Split('.');
value = reportReader.Current[nestedFields[0]];
foreach (var item in nestedFields)
{
if (item.Equals(nestedFields[0]))
{
continue;
}
value = ((Dictionary <string, object>)value)[item];
}
value = reportReader.Current[nestedFields[0]][nestedFields[1]];
Console.Write(value.ToString());
return(value);
}
else
{
Console.Write(value.ToString());
return(reportReader.Current[field]);
}
}
catch (Exception ex)
{
throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex);
}
};
while (reportReader.Read())
{
GenericMetricsUnit metricsUnit = new GenericMetricsUnit();
metricsUnitMapping.Apply(metricsUnit);
if (metricsUnit.MeasureValues != null)
{
SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]];
GenericMetricsUnit importedUnit = null;
// check if we already found a metrics unit with the same tracker
if (!data.TryGetValue(tracker.Value, out importedUnit))
{
metricsUnit.Output = currentOutput;
data.Add(tracker.Value, metricsUnit);
}
else //Tracker already exists
{
// Merge captured measure with existing measures
foreach (var capturedMeasure in metricsUnit.MeasureValues)
{
if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice))
{
continue;
}
//Measure already exists per tracker than aggregate:
if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key))
{
importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value;
}
else
{
//Captured Measure doest exists with this tracker:
importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value);
}
}
}
#region Validation
// For validations
foreach (var m in metricsUnit.MeasureValues)
{
if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(m.Key.Name))
{
currentOutput.Checksum.Add(m.Key.Name, m.Value);
}
else
{
currentOutput.Checksum[m.Key.Name] += m.Value;
}
}
}
#endregion
}
}
}
}
else
{
Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information);
}
}
// Import all unique units per tracker
foreach (GenericMetricsUnit metricsUnit in data.Values)
{
this.ImportManager.ImportMetrics(metricsUnit);
}
ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}