protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); foreach (var ReportFile in Delivery.Files) { //Get Columns var reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.columnHeaders[*].*"); using (reportReader) { int colIndex = 0; while (reportReader.Read()) { columns.Add(reportReader.Current.name, colIndex); colIndex++; } ///sssss } using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); // Checksums _isChecksum = true; reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.totalsForAllResults.*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current[field]; if (reportReader.Read()) { GenericMetricsUnit checksumUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(checksumUnit); foreach (var m in checksumUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } } } } _isChecksum = false; //Get Valuees reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.Gzip), "$.rows[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]]; while (reportReader.Read()) { GenericMetricsUnit tempUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(tempUnit); SegmentObject tracker = tempUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit existingUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out existingUnit)) { tempUnit.Output = currentOutput; data.Add(tracker.Value, tempUnit); } else { // if tracker already exists, merge with existing values foreach (var m in tempUnit.MeasureValues) { if (!m.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } existingUnit.MeasureValues[m.Key] += m.Value; } } } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { string baseAddress; #region Init General // ............................... // SETUP this.Delivery = this.NewDelivery(); this.Delivery.Account = new Data.Objects.Account() { ID = this.Instance.AccountID, }; this.Delivery.Channel = new Data.Objects.Channel() { ID = -1 }; this.Delivery.TimePeriodDefinition = this.TimePeriod; this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory]; if (string.IsNullOrEmpty(this.Delivery.FileDirectory)) { throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)"); } bool multibo = false; if (this.Instance.Configuration.Options.ContainsKey("URL.SignatureAppend")) { multibo = Convert.ToBoolean(this.Instance.Configuration.Options["URL.SignatureAppend"]); } this.Delivery.Outputs.Add(new DeliveryOutput() { Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}{2}]", this.Instance.AccountID, this.TimePeriod.ToAbsolute(), multibo?"-" + this.Instance.Configuration.Options["Bo.ServiceAdress"]:string.Empty)), Account = Delivery.Account, Channel = Delivery.Channel, TimePeriodStart = Delivery.TimePeriodStart, TimePeriodEnd = Delivery.TimePeriodEnd }); // Create an import manager that will handle rollback, if necessary var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions() { SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand] }); // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur) this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort); if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress])) { throw new Exception("base url must be configured!"); } baseAddress = this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress]; int utcOffset = 0; if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset)) { utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]); } Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset); int timeZone = 0; if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone)) { timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]); } Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone); this.ReportProgress(0.2); #endregion #region DeliveryFile Dictionary <string, string> UrlParams = new Dictionary <string, string>(); DeliveryFile boFile = new DeliveryFile(); DateTime start = TimePeriod.Start.ToDateTime(); DateTime end = TimePeriod.End.ToDateTime(); boFile.Parameters[BoConfigurationOptions.BO_XPath_Trackers] = Instance.Configuration.Options[BoConfigurationOptions.BO_XPath_Trackers]; boFile.Name = BoConfigurationOptions.BoFileName; if (utcOffset != 0) { start = start.AddHours(utcOffset); end = end.AddHours(utcOffset); } if (timeZone != 0) { start = start.AddHours(-timeZone); end = end.AddHours(-timeZone); } boFile.SourceUrl = string.Format(baseAddress, start, end); boFile.Parameters.Add(BoConfigurationOptions.IsAttribute, Instance.Configuration.Options[BoConfigurationOptions.IsAttribute]); boFile.Parameters.Add(BoConfigurationOptions.TrackerFieldName, Instance.Configuration.Options[BoConfigurationOptions.TrackerFieldName]); this.Delivery.Files.Add(boFile); this.Delivery.Save(); #endregion return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { #region Init General // ............................... // SETUP this.Delivery = this.NewDelivery(); this.Delivery.Account = new Data.Objects.Account() { ID = this.Instance.AccountID, }; this.Delivery.Channel = new Data.Objects.Channel() { ID = -1 }; this.Delivery.TimePeriodDefinition = this.TimePeriod; this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory]; if (string.IsNullOrEmpty(this.Delivery.FileDirectory)) { throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)"); } this.Delivery.Outputs.Add(new DeliveryOutput() { Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]", this.Instance.AccountID, this.TimePeriod.ToAbsolute())), Account = Delivery.Account, Channel = Delivery.Channel, TimePeriodStart = Delivery.TimePeriodStart, TimePeriodEnd = Delivery.TimePeriodEnd }); // Create an import manager that will handle rollback, if necessary var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions() { SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand] }); this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort); this.ReportProgress(0.2); #endregion //Create File DeliveryFile _file = new DeliveryFile() { Name = "EasyForexBackOffice", }; _file.SourceUrl = Instance.Configuration.Options["SourceUrl"]; // "https://classic.easy-forex.com/BackOffice/API/Marketing.asmx"; //_file.Parameters.Add("SOAPAction", "http://www.easy-forex.com/GetGatewayStatistics"); _file.Parameters.Add("Content-Type", "text/xml; charset=utf-8"); _file.Parameters.Add("SOAPAction", Instance.Configuration.Options["SOAPAction"]); Delivery.Parameters.Add("User", Instance.Configuration.Options["User"]); Delivery.Parameters.Add("Pass", Core.Utilities.Encryptor.Dec(Instance.Configuration.Options["Pass"].ToString())); Delivery.Parameters.Add("SoapMethod", Instance.Configuration.Options["SoapMethod"]); Delivery.Parameters.Add("StartGid", Instance.Configuration.Options["StartGid"]); Delivery.Parameters.Add("EndGid", Instance.Configuration.Options["EndGid"]); //Creating Soap Body string strSoapEnvelope = GetSoapEnvelope( Delivery.Parameters["User"].ToString(), Delivery.Parameters["Pass"].ToString(), Delivery.Parameters["SoapMethod"].ToString(), Delivery.Parameters["StartGid"].ToString(), Delivery.Parameters["EndGid"].ToString(), this.Delivery.TimePeriodStart.ToString("yyyy-MM-ddTHH:mm:ss"), this.Delivery.TimePeriodEnd.ToString("yyyy-MM-ddTHH:mm:ss.fffffff") ); #region Soap 1.2 //strSoapEnvelope += "<?xml version=\"1.0\" encoding=\"utf-8\"?>"; //strSoapEnvelope += "<soap12:Envelope xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:soap12=\"http://www.w3.org/2003/05/soap-envelope\">"; //strSoapEnvelope += " <soap12:Header>"; //strSoapEnvelope += " <AuthHeader xmlns=\"http://www.easy-forex.com\">"; //strSoapEnvelope += " <Username>Seperia</Username>"; //strSoapEnvelope += " <Password>Seperia909</Password>"; //strSoapEnvelope += " </AuthHeader>"; //strSoapEnvelope += " </soap12:Header>"; //strSoapEnvelope += " <soap12:Body>"; //strSoapEnvelope += "<GetGatewayStatistics xmlns=\"http://www.easy-forex.com\">"; //strSoapEnvelope += " <startGid>1</startGid>"; //strSoapEnvelope += " <finishGid>1000000</finishGid>"; //strSoapEnvelope += string.Format(" <fromDateTime>{0}</fromDateTime>", _requiredDay); //strSoapEnvelope += string.Format(" <toDateTime>{0}</toDateTime>", _requiredDay.AddDays(1).AddTicks(-1)); //strSoapEnvelope += " </GetGatewayStatistics>"; //strSoapEnvelope += "</soap12:Body>"; //strSoapEnvelope += "</soap12:Envelope>"; #endregion _file.Parameters.Add("Body", strSoapEnvelope); _file.Parameters.Add("Bo.IsAttribute", Instance.Configuration.Options["Bo.IsAttribute"]); _file.Parameters.Add("Bo.Xpath", Instance.Configuration.Options["Bo.Xpath"]); _file.Parameters.Add("Bo.TrackerIDField", Instance.Configuration.Options["Bo.TrackerIDField"]); this.Delivery.Files.Add(_file); // Save with success this.Delivery.Save(); return(ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader }; DeliveryOutput currentOutput = Delivery.Outputs.First(); //Open Auto Plac file DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]]; var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { Dictionary <string, double> _totals = new Dictionary <string, double>(); this.ImportManager.BeginImport(this.Delivery); //Intializing totals for validation foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_autoPlacReader) { this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field]; while (_autoPlacReader.Read()) { #region Setting Totals /*==================================================================================================================*/ // If end of file if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF) { //Setting totals for validation from totals line in adowrds file foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break; } } break; } /*==================================================================================================================*/ #endregion GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit(); autoPlacMetricsUnit.Output = currentOutput; autoPlacMetricsUnit.Channel = new Channel() { ID = 1 }; autoPlacMetricsUnit.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"] }; autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>(); //ADDING CAMPAIGN Campaign campaign = new Campaign() { OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName], Name = _autoPlacReader.Current[Const.CampaignFieldName], Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign); //ADDING ADGROUP AdGroup adgroup = new AdGroup() { Campaign = campaign, Value = _autoPlacReader.Current[Const.AdGroupFieldName], OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup); //INSERTING METRICS DATA autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName])); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName])); //CREATING PLACEMENT autoPlacMetricsUnit.TargetDimensions = new List <Target>(); PlacementTarget placement = new PlacementTarget() { Placement = _autoPlacReader.Current[Const.DomainFieldName], PlacementType = PlacementType.Automatic // Add status !!! }; autoPlacMetricsUnit.TargetDimensions.Add(placement); //SETTING TIME PERIOD autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); this.ImportManager.ImportMetrics(autoPlacMetricsUnit); } } this.Delivery.Outputs.First().Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); foreach (var ReportFile in Delivery.Files) { bool isAttribute = Boolean.Parse(ReportFile.Parameters["Bo.IsAttribute"].ToString()); var ReportReader = new XmlDynamicReader (ReportFile.OpenContents(), ReportFile.Parameters["Bo.Xpath"].ToString()); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); //checksum init foreach (var m in ImportManager.Measures.Values) { if (m.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum[m.Name] = 0; } } using (ReportReader) { dynamic readerHelper; while (ReportReader.Read()) { if (isAttribute) { readerHelper = ReportReader.Current.Attributes; } else { readerHelper = ReportReader.Current; } this.Mappings.OnFieldRequired = field => readerHelper[field]; GenericMetricsUnit genericMetricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(genericMetricsUnit); foreach (var m in genericMetricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } genericMetricsUnit.Output = currentOutput; ImportManager.ImportMetrics(genericMetricsUnit); } } ImportManager.EndImport(); } } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { #region Init General // ............................... // SETUP this.Delivery = this.NewDelivery(); this.Delivery.TimePeriodDefinition = this.TimePeriod; this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory]; this.Delivery.Account = new Data.Objects.Account() { ID = this.Instance.AccountID, }; this.Delivery.Channel = new Data.Objects.Channel() { ID = -1 }; if (string.IsNullOrEmpty(this.Delivery.FileDirectory)) { throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)"); } if (!this.Instance.Configuration.Options.ContainsKey("AuthenticationUrl")) { throw new Exception("AuthenticationUrl must be configured in configuration file"); } this.Delivery.Parameters["AuthenticationUrl"] = this.Instance.Configuration.Options["AuthenticationUrl"]; //https://test.salesforce.com/services/oauth2/token if (!this.Instance.Configuration.Options.ContainsKey("ConsumerKey")) { throw new Exception("ClientID must be configured in configuration file"); } this.Delivery.Parameters["SalesForceClientID"] = this.Instance.Configuration.Options["ConsumerKey"];//3MVG9GiqKapCZBwG.OqpT.DCgHmIXOlszzCpZPxbRyvzPDNlshB5LD0x94rQO5SzGOAZrWPNIPm_aGR7nBeXe if (!this.Instance.Configuration.Options.ContainsKey("ConsentCode")) { throw new Exception("ConsentCode must be configured in configuration file"); } this.Delivery.Parameters["ConsentCode"] = this.Instance.Configuration.Options["ConsentCode"]; //(accesstoken if (!this.Instance.Configuration.Options.ContainsKey("ConsumerSecret")) { throw new Exception("ClientSecret must be configured in configuration file"); } this.Delivery.Parameters["ClientSecret"] = this.Instance.Configuration.Options["ConsumerSecret"];//321506373515061074 if (!this.Instance.Configuration.Options.ContainsKey("Redirect_URI")) { throw new Exception("Redirect_URI must be configured in configuration file"); } this.Delivery.Parameters["Redirect_URI"] = this.Instance.Configuration.Options["Redirect_URI"]; //http://localhost:8080/RestTest/oauth/_callback // This is for finding conflicting services this.Delivery.Outputs.Add(new DeliveryOutput() { Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]-[{2}]", this.Instance.AccountID, this.TimePeriod.ToAbsolute(), this.Delivery.Parameters["SalesForceClientID"])), Account = Delivery.Account, Channel = Delivery.Channel, TimePeriodStart = Delivery.TimePeriodStart, TimePeriodEnd = Delivery.TimePeriodEnd }); // Create an import manager that will handle rollback, if necessary var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions() { SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand] }); // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur) this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort); // ............................... // Now that we have a new delivery, start adding values int utcOffset = 0; if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset)) { utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]); } Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset); int timeZone = 0; if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone)) { timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]); } Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone); this.ReportProgress(0.2); //if (!Instance.Configuration.Options.ContainsKey("ServiceAddress")) // throw new Exception("ServiceAddress must be configured in configuration file"); //serviceAddress = Instance.Configuration.Options["ServiceAddress"]; #endregion #region DeliveryFile string query = Instance.Configuration.Options["Query"]; //@SMBLeads:SELECT CreatedDate,Edge_BI_Tracker__c ,Company_Type__c FROM Lead WHERE CreatedDate > 2013-06-28T00:00:00.00Z AND Edge_BI_Tracker__c != null string[] queries = query.Split('@'); foreach (string qwry in queries) { DeliveryFile file = new DeliveryFile(); string[] q = qwry.Split(':'); file.Name = q[0] + "-0"; DateTime start = TimePeriod.Start.ToDateTime(); DateTime end = TimePeriod.End.ToDateTime(); if (utcOffset != 0) { start = start.AddHours(utcOffset); end = end.AddHours(utcOffset); } if (timeZone != 0) { start = start.AddHours(-timeZone); end = end.AddHours(-timeZone); } file.Parameters["Query"] = q[1]; this.Delivery.Files.Add(file); } this.Delivery.Save(); return(Core.Services.ServiceOutcome.Success); #endregion }
protected override Core.Services.ServiceOutcome DoPipelineWork() { #region Init General int backOfficeChannel = -888; // ............................... // SETUP this.Delivery = this.NewDelivery(); this.Delivery.Account = new Data.Objects.Account() { ID = this.Instance.AccountID, }; this.Delivery.Channel = new Data.Objects.Channel() { ID = -1 }; this.Delivery.TimePeriodDefinition = this.TimePeriod; this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory]; #region two profiles //serviceAddress = Instance.Configuration.Options["ServiceAddress"]; //Match m=Regex.Match(serviceAddress,@"\bids=(?<profileID>[^&]+)",RegexOptions.IgnoreCase); //Group g=m.Groups["profileID"]; //string profileID=g.Value; //**************in case amir will want to run two profiles //this.Delivery.Outputs.Add(new DeliveryOutput() //{ // Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]-[{2}]", // this.Instance.AccountID, // this.TimePeriod.ToAbsolute(), // profileID)), // Account = Delivery.Account, // Channel = Delivery.Channel, // TimePeriodStart=Delivery.TimePeriodStart, // TimePeriodEnd=Delivery.TimePeriodEnd //}); #endregion // This is for finding conflicting services this.Delivery.Outputs.Add(new DeliveryOutput() { Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]", this.Instance.AccountID, this.TimePeriod.ToAbsolute())), Account = Delivery.Account, Channel = Delivery.Channel, TimePeriodStart = Delivery.TimePeriodStart, TimePeriodEnd = Delivery.TimePeriodEnd }); // Create an import manager that will handle rollback, if necessary var importManager = new GenericMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.MetricsImportManagerOptions() { SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand] }); // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur) this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort); // ............................... // Now that we have a new delivery, start adding values if (string.IsNullOrEmpty(this.Delivery.FileDirectory)) { throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)"); } if (!this.Instance.Configuration.Options.ContainsKey("AuthenticationUrl")) { throw new Exception("AuthenticationUrl must be configured in configuration file"); } this.Delivery.Parameters["AuthenticationUrl"] = this.Instance.Configuration.Options["AuthenticationUrl"]; if (!this.Instance.Configuration.Options.ContainsKey("ClientID")) { throw new Exception("ClientID must be configured in configuration file"); } this.Delivery.Parameters["ClientID"] = this.Instance.Configuration.Options["ClientID"]; if (!this.Instance.Configuration.Options.ContainsKey("ConsentCode")) { throw new Exception("ConsentCode must be configured in configuration file"); } this.Delivery.Parameters["ConsentCode"] = this.Instance.Configuration.Options["ConsentCode"]; if (!this.Instance.Configuration.Options.ContainsKey("ClientSecret")) { throw new Exception("ClientSecret must be configured in configuration file"); } this.Delivery.Parameters["ClientSecret"] = this.Instance.Configuration.Options["ClientSecret"]; if (!this.Instance.Configuration.Options.ContainsKey("Redirect_URI")) { throw new Exception("Redirect_URI must be configured in configuration file"); } this.Delivery.Parameters["Redirect_URI"] = this.Instance.Configuration.Options["Redirect_URI"]; int utcOffset = 0; if (this.Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.UtcOffset)) { utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]); } Delivery.Parameters.Add(BoConfigurationOptions.UtcOffset, utcOffset); int timeZone = 0; if (Instance.Configuration.Options.ContainsKey(BoConfigurationOptions.TimeZone)) { timeZone = int.Parse(Instance.Configuration.Options[BoConfigurationOptions.TimeZone]); } Delivery.Parameters.Add(BoConfigurationOptions.TimeZone, timeZone); this.ReportProgress(0.2); if (!Instance.Configuration.Options.ContainsKey("ServiceAddress")) { throw new Exception("ServiceAddress must be configured in configuration file"); } serviceAddress = Instance.Configuration.Options["ServiceAddress"]; #endregion #region DeliveryFile DeliveryFile file = new DeliveryFile(); file.Name = "GoalsByCountryAndPagePath.GZIP"; DateTime start = TimePeriod.Start.ToDateTime(); DateTime end = TimePeriod.End.ToDateTime(); if (utcOffset != 0) { start = start.AddHours(utcOffset); end = end.AddHours(utcOffset); } if (timeZone != 0) { start = start.AddHours(-timeZone); end = end.AddHours(-timeZone); } file.SourceUrl = string.Format(serviceAddress, this.Delivery.TimePeriodDefinition.Start.ToDateTime(), this.Delivery.TimePeriodDefinition.End.ToDateTime()); this.Delivery.Files.Add(file); this.Delivery.Save(); return(Core.Services.ServiceOutcome.Success); #endregion }
protected override Core.Services.ServiceOutcome DoPipelineWork() { MappingContainer metricsUnitMapping; if (!this.Mappings.Objects.TryGetValue(typeof(GenericMetricsUnit), out metricsUnitMapping)) { throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit."); } currentOutput = this.Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); Dictionary <string, int> columns = new Dictionary <string, int>(); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Or, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { this.ImportManager.BeginImport(this.Delivery); Dictionary <string, GenericMetricsUnit> data = new Dictionary <string, GenericMetricsUnit>(); foreach (var ReportFile in Delivery.Files) { //check number of recordes JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.totalSize"); int numOfRecordes = 0; if (reportReader.Read()) { numOfRecordes = int.Parse(reportReader.Current.totalSize); } reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl"); if (reportReader.Read()) { Log.Write(string.Format("Salesforce Attention - account {0} contains more than 1 file per query {1}", this.Delivery.Account.ID, ReportFile.Parameters["Query"].ToString()), LogMessageType.Warning); } if (numOfRecordes > 0) { //Get Values reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.records[*].*"); using (reportReader) { this.Mappings.OnFieldRequired = field => { object value = new object(); string[] nestedFields; try { if (field.Contains('.')) { nestedFields = field.Split('.'); value = reportReader.Current[nestedFields[0]]; foreach (var item in nestedFields) { if (item.Equals(nestedFields[0])) { continue; } value = ((Dictionary <string, object>)value)[item]; } value = reportReader.Current[nestedFields[0]][nestedFields[1]]; Console.Write(value.ToString()); return(value); } else { Console.Write(value.ToString()); return(reportReader.Current[field]); } } catch (Exception ex) { throw new Exception(string.Format("Error while trying to map field {0} from mapper", field), ex); } }; while (reportReader.Read()) { GenericMetricsUnit metricsUnit = new GenericMetricsUnit(); metricsUnitMapping.Apply(metricsUnit); if (metricsUnit.MeasureValues != null) { SegmentObject tracker = metricsUnit.SegmentDimensions[ImportManager.SegmentTypes[Segment.Common.Tracker]]; GenericMetricsUnit importedUnit = null; // check if we already found a metrics unit with the same tracker if (!data.TryGetValue(tracker.Value, out importedUnit)) { metricsUnit.Output = currentOutput; data.Add(tracker.Value, metricsUnit); } else //Tracker already exists { // Merge captured measure with existing measures foreach (var capturedMeasure in metricsUnit.MeasureValues) { if (!capturedMeasure.Key.Options.HasFlag(MeasureOptions.IsBackOffice)) { continue; } //Measure already exists per tracker than aggregate: if (importedUnit.MeasureValues.ContainsKey(capturedMeasure.Key)) { importedUnit.MeasureValues[capturedMeasure.Key] += capturedMeasure.Value; } else { //Captured Measure doest exists with this tracker: importedUnit.MeasureValues.Add(capturedMeasure.Key, capturedMeasure.Value); } } } #region Validation // For validations foreach (var m in metricsUnit.MeasureValues) { if (m.Key.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!currentOutput.Checksum.ContainsKey(m.Key.Name)) { currentOutput.Checksum.Add(m.Key.Name, m.Value); } else { currentOutput.Checksum[m.Key.Name] += m.Value; } } } #endregion } } } } else { Log.Write("No Records Found in File " + ReportFile.Name, LogMessageType.Information); } } // Import all unique units per tracker foreach (GenericMetricsUnit metricsUnit in data.Values) { this.ImportManager.ImportMetrics(metricsUnit); } ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }