Exemplo n.º 1
0
        protected override MetricsUnit GetSampleMetrics()
        {
            var headers = new[] { AdWordsConst.AdPreRequiredHeader };

            // load sample keywords
            var file = new DeliveryFile {
                Location = Configuration.Parameters.Get <string>("KeywordSampleFile")
            };

            LoadKeywords(file, headers, FileCompression.None);

            file = new DeliveryFile {
                Location = Configuration.Parameters.Get <string>("PlacementSampleFile")
            };
            LoadPlacements(file, new[] { AdWordsConst.AutoPlacRequiredHeader }, FileCompression.None);

            // load ad
            using (_adsReader = new CsvDynamicReader(Configuration.Parameters.Get <string>("AdSampleFile"), headers))
            {
                Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName];
                if (_adsReader.Read())
                {
                    CurrentMetricsUnit = new MetricsUnit {
                        GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
                    };
                    MetricsMappings.Apply(CurrentMetricsUnit);
                    return(CurrentMetricsUnit);
                }
            }
            throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.Parameters.Get <string>("AdSampleFile")));
        }
		private void LoadKeywords(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip)
		{
			if (file == null)
				throw new ArgumentException("Keywords delivery file does not exist");

			_keywordsCache.Clear();
			using (var keywordsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers))
			{
				Mappings.OnFieldRequired = fieldName => keywordsReader.Current[fieldName];
				keywordsReader.MatchExactColumns = false;
				while (keywordsReader.Read())
				{
					if (keywordsReader.Current[AdWordsConst.KeywordIdFieldName] == AdWordsConst.EOF)
						break;
					var keywordPrimaryKey = new KeywordPrimaryKey
					{
						KeywordId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.KeywordIdFieldName]),
						AdgroupId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.AdGroupIdFieldName]),
						CampaignId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.CampaignIdFieldName])
					};

					var keyword = new KeywordTarget();
					KeywordMappings.Apply(keyword);

					_keywordsCache.Add(keywordPrimaryKey.ToString(), keyword);
				}
			}
		}
Exemplo n.º 3
0
        private void CreateNextFiles(DeliveryFile file, int counter, List <DeliveryFile> list)
        {
            string next = GetNextUrl(file);

            if (!string.IsNullOrEmpty(next))
            {
                var url        = FixUrlChars(next);
                var name       = file.Name.Substring(0, file.Name.IndexOf('-'));
                var nameFormat = "{0}-{1}.json";
                var fileName   = string.Format(nameFormat, name, counter);

                var fileType = file.Parameters[Consts.DeliveryFileParameters.FileType].ToString();
                var f        = CreateDeliveryFile(fileName, url, fileType);


                if (!filesByType.ContainsKey((Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType]))
                {
                    filesByType.Add((Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType],
                                    new List <string>());
                }
                filesByType[(Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType]].Add(f.Name);

                list.Add(f);
            }
        }
Exemplo n.º 4
0
        private void LoadCreatives(DeliveryFile file)
        {
            using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
            {
                Mappings.OnFieldRequired = fieldName =>
                {
                    try
                    {
                        return(reader.Current[fieldName]);
                    }
                    catch (Exception ex)
                    {
                        Log(String.Format("Failed to read field '{0}' for creative id = {1} in file {2}, ex: {3}", fieldName, reader.Current.creative_id, file.Name, ex.Message), LogMessageType.Error);
                        return(null);
                    }
                };
                while (reader.Read())
                {
                    var creative = new CompositeCreative();
                    CreativeMappings.Apply(creative);

                    _creativeCache.Add(reader.Current.creative_id, creative);
                }
            }
        }
Exemplo n.º 5
0
        private void LoadPlacements(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip)
        {
            if (file == null)
            {
                throw new ArgumentException("Placement delivery file does not exist");
            }

            _placementsCache.Clear();
            using (var placementsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers))
            {
                Mappings.OnFieldRequired           = fieldName => placementsReader.Current[fieldName];
                placementsReader.MatchExactColumns = false;
                while (placementsReader.Read())
                {
                    if (placementsReader.Current[AdWordsConst.PlacementIdFieldName] == AdWordsConst.EOF)
                    {
                        break;
                    }
                    var placementPrimaryKey = new KeywordPrimaryKey
                    {
                        KeywordId  = Convert.ToInt64(placementsReader.Current[AdWordsConst.PlacementIdFieldName]),
                        AdgroupId  = Convert.ToInt64(placementsReader.Current[AdWordsConst.AdGroupIdFieldName]),
                        CampaignId = Convert.ToInt64(placementsReader.Current[AdWordsConst.CampaignIdFieldName])
                    };

                    var placement = new PlacementTarget();
                    PlacementMappings.Apply(placement);

                    _placementsCache.Add(placementPrimaryKey.ToString(), placement);
                }
            }
        }
Exemplo n.º 6
0
        protected override Core.Services.ServiceOutcome DoPipelineWork()
        {
            if (this.Delivery == null)
            {
                int accountID = this.Configuration.Parameters.Get <int>("AccountID", emptyIsError: false, defaultValue: -1);
                int channelID = this.Configuration.Parameters.Get <int>("ChannelID", emptyIsError: false, defaultValue: -1);

                this.Delivery = NewDelivery();
                this.Delivery.TimePeriodDefinition = this.Configuration.TimePeriod.Value;
                this.Delivery.Account = accountID != -1 ? new Account()
                {
                    ID = accountID
                } : null;
                this.Delivery.FileDirectory = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.FileDirectory);

                if (channelID != -1)
                {
                    this.Delivery.Channel = new Channel()
                    {
                        ID = channelID
                    }
                }
                ;
            }

            DeliveryFile deliveryFile = new DeliveryFile()
            {
                Name      = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.DeliveryFileName, false) ?? "File",
                SourceUrl = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.SourceUrl, false)
            };

            this.Delivery.Files.Add(deliveryFile);


            WebRequest request = FileWebRequest.Create(this.Configuration.Parameters.Get <string>("SourceUrl"));

            if (request is FtpWebRequest)
            {
                // FTP Configuration
                this.Delivery.Parameters["UsePassive"] = this.Configuration.Parameters.Get <bool>("UsePassive", emptyIsError: false, defaultValue: true);
                this.Delivery.Parameters["UseBinary"]  = this.Configuration.Parameters.Get <bool>("UseBinary");
                this.Delivery.Parameters["SourceUrl"]  = this.Configuration.Parameters.Get <string>("SourceUrl");
                this.Delivery.Parameters["UserID"]     = this.Configuration.Parameters.Get <string>("UserID");
                this.Delivery.Parameters["Password"]   = Core.Utilities.Encryptor.Dec(this.Configuration.Parameters.Get <string>("Password"));
                this.Delivery.Parameters["DirectoryWatcherLocation"] = this.Configuration.Parameters.Get <string>("DirectoryWatcherLocation", false);


                deliveryFile.FileSignature = string.Format("{0}-{1}-{2}", this.Configuration.Parameters.Get <object>("DeliveryFileName"),
                                                           this.Configuration.Parameters.Get <object>("FileModifyDate"),
                                                           this.Configuration.Parameters.Get <object>("FileSize"));

                deliveryFile.Parameters.Add("Size", this.Configuration.Parameters.Get <string>("Size", emptyIsError: false));
            }

            this.Delivery.Save();

            return(Core.Services.ServiceOutcome.Success);
        }
    }
Exemplo n.º 7
0
        IEnumerable <string> IRetriever.RetrieveData(DeliveryFile file, AdWordsUser user)
        {
            var offset = 0;
            var sb     = new StringBuilder();
            // 1st row is a header (field list)
            var outputLines = new List <string> {
                file.Parameters["ReportFields"].ToString()
            };

            var service = user.GetService(AdWordsService.v201302.CampaignCriterionService) as CampaignCriterionService;

            if (service == null)
            {
                throw new Exception("Failed to create CampaignCriterionService");
            }

            var selector = new Selector
            {
                fields = file.Parameters["ReportFields"].ToString().Split(','),
                paging = new Paging {
                    numberResults = PAGE_SIZE
                },
            };

            if (!String.IsNullOrEmpty(file.Parameters["ReportFilter"].ToString()))
            {
                selector.predicates = new[] { new Predicate {
                                                  field = "CriteriaType", @operator = PredicateOperator.IN, values = file.Parameters["ReportFilter"].ToString().Split('|')
                                              } }
            }
            ;

            //var query = String.Format("SELECT {0} WHERE CriteriaType IN ('LOCATION', 'LANGUAGE') LIMIT {{0}}, {1} ", file.Parameters["ReportFields"], PAGE_SIZE);
            CampaignCriterionPage page;

            do
            {
                // get next page
                selector.paging.startIndex = offset;
                page = service.get(selector);

                if (page != null && page.entries != null)
                {
                    // for each criterion create row with all fields
                    foreach (var campaignCriterion in page.entries)
                    {
                        sb.Clear();
                        foreach (var field in selector.fields)
                        {
                            sb.AppendFormat("{0}{1}", sb.Length > 0 ? "," : "", GetFieldValue(campaignCriterion, field));
                        }
                        outputLines.Add(sb.ToString());
                    }
                }
                offset += PAGE_SIZE;
            } while (page != null && offset < page.totalNumEntries);

            return(outputLines);
        }
        private void DownloadFile(DeliveryFile file, AdWordsUser user, string postData)
        {
            WebRequest            request   = CreateAdwordsReportRequest(file, user, postData);
            FileDownloadOperation operation = file.Download(request);

            operation.RequestBody = postData;

            _batchDownloadOperation.Add(operation);
        }
Exemplo n.º 9
0
        private DeliveryFile CreateDeliveryFile(string name, string url, string type)
        {
            var f = new DeliveryFile();

            f.Name = name;
            f.Parameters.Add(Consts.DeliveryFileParameters.Url, string.Format("{0}", url));
            f.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, (long)Consts.FileSubType.New);
            f.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), type));
            return(f);
        }
Exemplo n.º 10
0
        protected override ServiceOutcome DoPipelineWork()
        {
            this.Delivery = this.NewDelivery(); // setup delivery
            this.Delivery.FileDirectory = Instance.Configuration.Options[Edge.Data.Pipeline.Services.Const.DeliveryServiceConfigurationOptions.FileDirectory];
            this.Delivery.Account       = new Account()
            {
                ID = 0
            };

            string fileName = string.Empty;

            //FileName
            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["FileName"]))
            {
                throw new Exception("Missing Configuration Param , FileName");
            }
            else
            {
                fileName = this.Instance.Configuration.Options["FileName"];
            }

            DeliveryFile _file = new DeliveryFile()
            {
                Name = fileName
            };

            _file.SourceUrl = Instance.Configuration.Options["SourceUrl"];

            _file.Parameters.Add("XML.IsAttribute", Instance.Configuration.Options["XML.IsAttribute"]);
            _file.Parameters.Add("XML.Path", Instance.Configuration.Options["XML.Path"]);

            this.Delivery.Files.Add(_file);


            //Set Output


            this.Delivery.Outputs.Add(new DeliveryOutput()
            {
                Signature = Delivery.CreateSignature(String.Format("[{0}]-[{1}]",
                                                                   this.TimePeriod.ToAbsolute(),
                                                                   Instance.Configuration.Options["SourceUrl"]
                                                                   )),
                Account = new Data.Objects.Account()
                {
                    ID = 0
                },
                TimePeriodStart = Delivery.TimePeriodStart,
                TimePeriodEnd   = Delivery.TimePeriodEnd
            });

            this.Delivery.Save();
            return(Core.Services.ServiceOutcome.Success);
        }
Exemplo n.º 11
0
        private FileInfo AddDataFilesToBatch(BatchDownloadOperation batch, DeliveryFile file)
        {
            if (file.Parameters[Consts.DeliveryFileParameters.FileSubType].Equals((long)Consts.FileSubType.Data))
            {
                FileDownloadOperation fileDownloadOperation = file.Download(CreateRequest(file.Parameters[Consts.DeliveryFileParameters.Url].ToString()));
                batch.Add(fileDownloadOperation);
                return(fileDownloadOperation.FileInfo);
            }

            return(null);
        }
        private WebRequest CreateAdwordsReportRequest(DeliveryFile file, AdWordsUser user, string postBody)
        {
            WebRequest request = HttpWebRequest.Create(file.SourceUrl);

            request.Timeout = 100000;

            if (!string.IsNullOrEmpty(postBody))
            {
                request.Method = "POST";
            }
            if (!string.IsNullOrEmpty((user.Config as AdWordsAppConfig).ClientCustomerId))
            {
                request.Headers.Add("clientCustomerId: " + (user.Config as AdWordsAppConfig).ClientCustomerId);
            }

            request.ContentType = "application/x-www-form-urlencoded";

            if ((user.Config as AdWordsAppConfig).EnableGzipCompression)
            {
                (request as HttpWebRequest).AutomaticDecompression = DecompressionMethods.GZip
                                                                     | DecompressionMethods.Deflate;
            }
            else
            {
                (request as HttpWebRequest).AutomaticDecompression = DecompressionMethods.None;
            }

            if (user.OAuthProvider != null)
            {
                request.Headers["Authorization"] = user.OAuthProvider.GetAuthHeader();
            }
            else
            {
                throw new Exception("AdWordsErrorMessages.OAuthProviderCannotBeNull");
            }

            //if ((user.Config as AdWordsAppConfig).AuthorizationMethod == AdWordsAuthorizationMethod.ClientLogin)
            //{
            //    string authToken = AdwordsUtill.GetAuthToken(user);
            //    (user.Config as AdWordsAppConfig).AuthToken = authToken;
            //    request.Headers["Authorization"] = "GoogleLogin auth=" + authToken;
            //}

            // request.Headers.Add("returnMoneyInMicros: true");
            request.Headers.Add("developerToken: " + (user.Config as AdWordsAppConfig).DeveloperToken);

            //Try to unmark the following comment in case of api error
            //The client library will use only apiMode = true.
            request.Headers.Add("apiMode", "true");

            return(request);
        }
        protected void LoadConfiguration()
        {
            var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold);

            _importManagerOptions = new MetricsDeliveryManagerOptions
            {
                SqlTransformCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlTransformCommand),
                SqlStageCommand     = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlStageCommand),
                SqlRollbackCommand  = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlRollbackCommand),
                ChecksumThreshold   = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold)
            };

            _deliveryFile = Delivery.Files[Configuration.DeliveryFileName];
            if (_deliveryFile == null)
            {
                throw new Exception(String.Format("Could not find delivery file '{0}' in the delivery.", Configuration.DeliveryFileName));
            }

            if (!Enum.TryParse(Configuration.Compression, out _compression))
            {
                throw new ConfigurationErrorsException(String.Format("Invalid compression type '{0}'.", Configuration.Compression));
            }

            if (Configuration.Parameters.ContainsKey("EOF"))
            {
                _eofIndication = Configuration.Parameters.Get <string>("EOF");
            }
            if (Configuration.Parameters.ContainsKey("EOF_FieldName"))
            {
                _eofFieldName = Configuration.Parameters.Get <string>("EOF_FieldName");
            }

            // Create format processor from configuration
            var readerAdapterType = Type.GetType(Configuration.ReaderAdapterType, true);

            ReaderAdapter = (ReaderAdapter)Activator.CreateInstance(readerAdapterType);

            Mappings.OnFieldRequired  = ReaderAdapter.GetField;
            Mappings.OnMappingApplied = SetEdgeType;

            if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
            {
                throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.", "Object");
            }

            if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings))
            {
                throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object");
            }
        }
Exemplo n.º 14
0
        private void LoadCampaigns(DeliveryFile file)
        {
            using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
            {
                Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
                while (reader.Read())
                {
                    var campaign = new Campaign();
                    CampaignMappings.Apply(campaign);

                    _campaignCache.Add(campaign.OriginalID, campaign);
                }
            }
        }
Exemplo n.º 15
0
        private void DownloadFile(DeliveryFile file)
        {
            FtpWebRequest request = (FtpWebRequest)FtpWebRequest.Create(file.SourceUrl);

            request.UseBinary   = true;
            request.Credentials = new NetworkCredential
                                  (
                this.Delivery.Parameters["UserID"].ToString(),
                this.Delivery.Parameters["Password"].ToString()
                                  );
            request.Method     = WebRequestMethods.Ftp.DownloadFile;
            request.UsePassive = true;

            _batchDownloadOperation.Add(file.Download(request, Convert.ToInt64(file.Parameters["Size"])));
        }
        private WebRequest CreateAdwordsReportRequest(DeliveryFile file, AdWordsUser user, string postBody)
        {
            var config  = user.Config as AdWordsAppConfig;
            var request = WebRequest.Create(file.SourceUrl) as HttpWebRequest;

            if (config == null || request == null)
            {
                return(null);
            }

            request.Timeout = 100000;
            if (!string.IsNullOrEmpty(postBody))
            {
                request.Method = "POST";
            }
            if (!string.IsNullOrEmpty(config.ClientCustomerId))
            {
                request.Headers.Add("clientCustomerId: " + (user.Config as AdWordsAppConfig).ClientCustomerId);
            }

            request.ContentType = "application/x-www-form-urlencoded";

            if (config.EnableGzipCompression)
            {
                request.AutomaticDecompression = DecompressionMethods.GZip
                                                 | DecompressionMethods.Deflate;
            }
            else
            {
                request.AutomaticDecompression = DecompressionMethods.None;
            }

            if (config.AuthorizationMethod == AdWordsAuthorizationMethod.ClientLogin)
            {
                string authToken = AdwordsUtill.GetAuthToken(user);
                (user.Config as AdWordsAppConfig).AuthToken = authToken;
                request.Headers["Authorization"]            = "GoogleLogin auth=" + authToken;
            }

            request.Headers.Add("returnMoneyInMicros: true");
            request.Headers.Add("developerToken: " + (user.Config as AdWordsAppConfig).DeveloperToken);

            //Try to unmark the following comment in case of api error
            //The client library will use only apiMode = true.
            //request.Headers.Add("apiMode", "true");

            return(request);
        }
Exemplo n.º 17
0
        private string GetNextUrl(DeliveryFile file)
        {
            string next = string.Empty;

            using (StreamReader reader = new StreamReader(file.OpenContents()))
            {
                string  json          = reader.ReadToEnd();
                dynamic dynamicObject = JsonConvert.DeserializeObject(json);

                if (dynamicObject.paging != null)
                {
                    next = dynamicObject.paging.next;
                }
            }

            return(next);
        }
Exemplo n.º 18
0
        private List <DeliveryFile> FetchNext(List <DeliveryFile> fetchFrom, int offset)
        {
            BatchDownloadOperation nextBatch        = new BatchDownloadOperation();
            List <DeliveryFile>    nextRecordsFiles = new List <DeliveryFile>();

            foreach (DeliveryFile ReportFile in fetchFrom)
            {
                //setting cuurent file has batched and batching next file
                ReportFile.Parameters.Add("Batch", true);


                string fileName = ReportFile.Name + "-" + offset;

                JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
                string            nextRecordPath;
                if (reportReader.Read())
                {
                    nextRecordPath = reportReader.Current.nextRecordsUrl;
                    DeliveryFile nextRecordFile = new DeliveryFile();
                    nextRecordFile.SourceUrl = ((Token)(ReportFile.Parameters["Token"])).instance_url + nextRecordPath;

                    HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(nextRecordFile.SourceUrl);
                    request.Headers.Add("Authorization: OAuth " + ((Token)(ReportFile.Parameters["Token"])).access_token);

                    //check if response contains more than one file

                    FileDownloadOperation fileDownloadOperation = nextRecordFile.Download(request);
                    nextBatch.Add(fileDownloadOperation);

                    nextRecordsFiles.Add(nextRecordFile);
                }
            }
            if (nextRecordsFiles.Count > 0)
            {
                nextBatch.Start();
                nextBatch.Wait();
                nextBatch.EnsureSuccess();

                foreach (DeliveryFile file in FetchNext(nextRecordsFiles, offset))
                {
                    this.Delivery.Files.Add(file);
                }
            }

            return(nextRecordsFiles);
        }
Exemplo n.º 19
0
        private void LoadConversions(DeliveryFile file, string[] headers)
        {
            if (file == null)
            {
                throw new ArgumentException("Ad conversions delivery file does not exist");
            }

            _importedAdsWithConv.Clear();
            using (var conversionsReader = new CsvDynamicReader(file.OpenContents(compression: FileCompression.Gzip), headers))
            {
                while (conversionsReader.Read())
                {
                    if (conversionsReader.Current[AdWordsConst.AdIDFieldName] == AdWordsConst.EOF)
                    {
                        break;                                                                                                // if end of report
                    }
                    string conversionKey = String.Format("{0}#{1}#{2}", conversionsReader.Current[AdWordsConst.AdIDFieldName],
                                                         conversionsReader.Current[AdWordsConst.KeywordIdFieldName],
                                                         conversionsReader.Current[AdWordsConst.DateFieldName]);
                    var trackingPurpose = Convert.ToString(conversionsReader.Current[AdWordsConst.ConversionTrackingPurposeFieldName]);
                    var manyPerClick    = Convert.ToInt64(conversionsReader.Current[AdWordsConst.ConversionManyPerClickFieldName]);

                    if (!_importedAdsWithConv.ContainsKey(conversionKey))
                    {
                        // add new conversion key with new dictionary of tracking purpose and clicks
                        _importedAdsWithConv.Add(conversionKey, new Dictionary <string, long> {
                            { trackingPurpose, manyPerClick }
                        });
                    }
                    else                     // if conversion key exists
                    {
                        if (_importedAdsWithConv[conversionKey].ContainsKey(trackingPurpose))
                        {
                            // if purpose exists --> sum to existing value
                            _importedAdsWithConv[conversionKey][trackingPurpose] += manyPerClick;
                        }
                        else
                        {
                            // create new entry for new tracking purpose
                            _importedAdsWithConv[conversionKey].Add(trackingPurpose, manyPerClick);
                        }
                    }
                }
            }
        }
        private bool ValidateReport(DeliveryFile file, AdWordsUser user, string postData, out string ErrorMsg)
        {
            ErrorMsg = string.Empty;
            WebRequest request = CreateAdwordsReportRequest(file, user, postData);

            request.Proxy = null;
            request.Headers.Add("validateOnly: true"); // Add Validate

            using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
            {
                writer.Write(postData);
            }

            //Try to ger response from server
            WebResponse response = null;

            try
            {
                response = request.GetResponse();
            }
            catch (WebException we)
            {
                response = we.Response;
                byte[] preview       = ConvertStreamToByteArray(response.GetResponseStream(), MAX_ERROR_LENGTH);
                string previewString = ConvertPreviewBytesToString(preview);

                if (!string.IsNullOrEmpty(previewString))
                {
                    //BUG fix due to adwords new version changes
                    //if (Regex.IsMatch(previewString, REPORT_ERROR_REGEX))
                    if ((previewString.Contains(typeof(GA201406.AuthorizationError).Name)) || (previewString.Contains(typeof(GA201406.AuthenticationError).Name)))
                    {
                        ErrorMsg = previewString;
                        return(false);
                    }
                    else
                    {
                        throw new Exception(previewString);
                    }
                }
            }

            return(true);
        }
Exemplo n.º 21
0
        private void LoadAds(DeliveryFile file)
        {
            using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
            {
                Mappings.OnFieldRequired = fieldName =>
                {
                    var parts = fieldName.Split('@');
                    if (parts.Length > 1)
                    {
                        // value from the dictionary by key
                        var fieldValue = reader.Current[parts[1]].ContainsKey(parts[0]) ?
                                         reader.Current[parts[1]][parts[0]] is List <object>?
                                         String.Join(",", reader.Current[parts[1]][parts[0]]) :
                                         reader.Current[parts[1]][parts[0]] : null;

                        return(fieldValue);
                    }

                    // 1st item in the list
                    if (reader.Current[fieldName] is IList <object> )
                    {
                        return((reader.Current[fieldName] as IList <object>)[0]);
                    }

                    // just a string value
                    return(reader.Current[fieldName]);
                };

                while (reader.Read())
                {
                    var ad = new Ad();
                    AdMappings.Apply(ad);
                    _adCache.Add(ad.OriginalID, ad);
                }
            }
        }
Exemplo n.º 22
0
        private void DownloadFile(DeliveryFile file)
        {
            WebRequest request = FileWebRequest.Create(file.SourceUrl);

            /* FTP */
            if (request.GetType().Equals(typeof(FtpWebRequest)))
            {
                FtpWebRequest ftpRequest = (FtpWebRequest)request;
                ftpRequest.UseBinary   = true;
                ftpRequest.Credentials = new NetworkCredential
                                         (
                    this.Delivery.Parameters["UserID"].ToString(),
                    this.Delivery.Parameters["Password"].ToString()
                                         );
                ftpRequest.Method     = WebRequestMethods.Ftp.DownloadFile;
                ftpRequest.UsePassive = true;
                _batch.Add(file.Download(request, Convert.ToInt64(file.Parameters["Size"])));
            }
            /*OTHER*/
            else
            {
                _batch.Add(file.Download(request));
            }
        }
Exemplo n.º 23
0
        private void CreateRequests()
        {
            List <ManualResetEvent> manualEvents = new List <ManualResetEvent>();

            _waitHandle = new AutoResetEvent(false);

            #region ADREPORT
            DeliveryFile     adReportFile = this.Delivery.Files[Const.Files.AdReport];
            WS.ReportRequest adReportRequest;
            adReportRequest = _adCenterApi.NewAdPerformanceReportRequest
                              (
                WS.AdPerformanceReportColumn.AccountName,   //required field
                WS.AdPerformanceReportColumn.AccountNumber, // not necessary for this service version
                WS.AdPerformanceReportColumn.CampaignName,  //required field
                WS.AdPerformanceReportColumn.TimePeriod,    //required field if aggregation time!=Summary
                //WS.AdPerformanceReportColumn.CampaignId, // why did MS leave this out? stupid f***s
                WS.AdPerformanceReportColumn.AdGroupName,
                WS.AdPerformanceReportColumn.AdGroupId,
                WS.AdPerformanceReportColumn.AdId,
                WS.AdPerformanceReportColumn.AdTitle,
                WS.AdPerformanceReportColumn.AdDescription,
                WS.AdPerformanceReportColumn.DestinationUrl,
                WS.AdPerformanceReportColumn.Clicks,
                WS.AdPerformanceReportColumn.AdType
                              );

            if (string.IsNullOrEmpty(adReportFile.SourceUrl))
            {
                ManualResetEvent asyncWait = new ManualResetEvent(false);

                manualEvents.Add(asyncWait);
                Action getAdReportUrl = () =>
                {
                    string innerFileName;
                    adReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(adReportRequest, out innerFileName);
                    adReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
                };

                getAdReportUrl.BeginInvoke(result =>
                {
                    asyncWait.Set();
                },
                                           null);
            }
            #endregion

            #region CAMPAIGNREPORT
            DeliveryFile     campaignReportFile = this.Delivery.Files[Const.Files.CampaignReport];
            WS.ReportRequest campaignReportRequest;
            campaignReportRequest = _adCenterApi.NewCampaignPerformanceReportRequest
                                    (
                WS.CampaignPerformanceReportColumn.AccountName,
                WS.CampaignPerformanceReportColumn.CampaignName,
                WS.CampaignPerformanceReportColumn.TimePeriod,
                WS.CampaignPerformanceReportColumn.CampaignId,
                WS.CampaignPerformanceReportColumn.Status,
                WS.CampaignPerformanceReportColumn.Clicks

                                    );

            if (string.IsNullOrEmpty(campaignReportFile.SourceUrl))
            {
                ManualResetEvent asyncWait = new ManualResetEvent(false);
                manualEvents.Add(asyncWait);
                Action getCampaignReportUrl = () =>
                {
                    string innerFileName;
                    campaignReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(campaignReportRequest, out innerFileName);
                    campaignReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
                };

                getCampaignReportUrl.BeginInvoke(result =>
                {
                    asyncWait.Set();
                },
                                                 null);
            }
            #endregion


            #region KEYWORDREPORT
            DeliveryFile     keywordReportFile = this.Delivery.Files[Const.Files.KeywordReport];
            WS.ReportRequest keywordReportRequest;
            keywordReportRequest = _adCenterApi.NewKeywordPerformanceReportRequest(
                WS.KeywordPerformanceReportColumn.TimePeriod,     // special column
                WS.KeywordPerformanceReportColumn.AdId,
                WS.KeywordPerformanceReportColumn.AdGroupId,
                WS.KeywordPerformanceReportColumn.CampaignId,
                WS.KeywordPerformanceReportColumn.Keyword,
                WS.KeywordPerformanceReportColumn.KeywordId,
                WS.KeywordPerformanceReportColumn.DestinationUrl,
                WS.KeywordPerformanceReportColumn.DeliveredMatchType,
                WS.KeywordPerformanceReportColumn.CurrencyCode,
                WS.KeywordPerformanceReportColumn.Impressions,
                WS.KeywordPerformanceReportColumn.Clicks,
                WS.KeywordPerformanceReportColumn.Spend,
                WS.KeywordPerformanceReportColumn.AveragePosition,
                WS.KeywordPerformanceReportColumn.Conversions,
                WS.KeywordPerformanceReportColumn.QualityScore

                );

            if (string.IsNullOrEmpty(keywordReportFile.SourceUrl))
            {
                ManualResetEvent asyncWait = new ManualResetEvent(false);
                manualEvents.Add(asyncWait);
                Action getKeywordReportUrl = () =>
                {
                    string innerFileName;
                    keywordReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(keywordReportRequest, out innerFileName);
                    keywordReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
                };

                getKeywordReportUrl.BeginInvoke(result =>
                {
                    asyncWait.Set();
                },
                                                null);
            }
            #endregion

            if (manualEvents.Count > 1)
            {
                WaitHandle.WaitAll(manualEvents.ToArray());
            }
        }
        protected override ServiceOutcome DoPipelineWork()
        {
            #region Check config parameters
            if (!Configuration.Parameters.ContainsKey("FilterDeleted"))
            {
                throw new Exception("Missing Configuration Param: FilterDeleted");
            }

            if (!Configuration.Parameters.ContainsKey("KeywordContentId"))
            {
                throw new Exception("Missing Configuration Param: KeywordContentId");
            }

            if (!Configuration.Parameters.ContainsKey("DeveloperToken"))
            {
                throw new Exception("Missing Configuration Param: DeveloperToken");
            }

            if (!Configuration.Parameters.ContainsKey("Adwords.MccEmail"))
            {
                throw new Exception("Missing Configuration Param: Adwords.MccEmail");
            }

            if (!Configuration.Parameters.ContainsKey("Adwords.ClientID"))
            {
                throw new Exception("Missing Configuration Param: Adwords.ClientID");
            }

            if (!Configuration.Parameters.ContainsKey("SubChannelName"))
            {
                throw new Exception("Missing Configuration Param: SubChannelName");
            }

            if (!Configuration.TimePeriod.HasValue)
            {
                throw new Exception("No time period is set for Service");
            }

            if (!Configuration.Parameters.ContainsKey("Adwords.ReportConfig"))
            {
                throw new Exception("Missing Configuration Param: Adwords.ReportConfig");
            }
            #endregion

            var accountId = Configuration.Parameters.Get("AccountID", false, -1);
            var channelId = Configuration.Parameters.Get("ChannelID", false, -1);

            // create Delivery
            if (Delivery == null)
            {
                Delivery = NewDelivery();
                Delivery.TimePeriodDefinition = Configuration.TimePeriod.Value;
                Delivery.Account = accountId != -1 ? new Account {
                    ID = accountId
                } : null;
                Delivery.FileDirectory = Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.FileDirectory);
                if (channelId != -1)
                {
                    Delivery.Channel = new Channel {
                        ID = channelId
                    }
                }
                ;
            }
            Progress = 0.3;

            #region Must Have Params
            Delivery.Parameters["FilterDeleted"] = Configuration.Parameters["FilterDeleted"];
            //Delivery.Parameters["IncludeStatus"] = Configuration.Parameters.Get<string>("IncludeStatus");

            //Get MCC Paramerters
            Delivery.Parameters["DeveloperToken"]   = Configuration.Parameters.Get <string>("DeveloperToken");
            Delivery.Parameters["MccEmail"]         = Configuration.Parameters.Get <string>("Adwords.MccEmail");
            Delivery.Parameters["KeywordContentId"] = Configuration.Parameters.Get <string>("KeywordContentId");

            //Get Account Client Id's
            var adwordsClientIds = Configuration.Parameters.Get <string>("Adwords.ClientID").Split('|');
            Delivery.Parameters["AdwordsClientIDs"] = adwordsClientIds;

            // Get Reports configuration
            var reportConfig = AdwordsReportConfig.Deserialize(Configuration.Parameters.Get <string>("Adwords.ReportConfig"));

            #endregion

            #region Nice to have params

            //Check for includeZeroImpression
            if (Configuration.Parameters.ContainsKey("includeZeroImpression"))
            {
                Delivery.Parameters["includeZeroImpression"] = Configuration.Parameters.Get <string>("includeZeroImpression");
            }
            else
            {
                Delivery.Parameters["includeZeroImpression"] = false;
            }

            //Check for includeConversionTypes
            if (Configuration.Parameters.ContainsKey("includeConversionTypes"))
            {
                Delivery.Parameters["includeConversionTypes"] = Configuration.Parameters.Get <string>("includeConversionTypes");
            }
            else
            {
                Delivery.Parameters["includeConversionTypes"] = false;                 // default
            }
            //Check for includeDisplaytData
            if (Configuration.Parameters.ContainsKey("includeDisplaytData"))
            {
                Delivery.Parameters["includeDisplaytData"] = Configuration.Parameters.Get <string>("includeDisplaytData");
            }
            else
            {
                Delivery.Parameters["includeDisplaytData"] = false;                 // default
            }
            #endregion

            Progress = 0.5;
            // create Delivery files for each Client ID and Report type
            foreach (var clientId in adwordsClientIds)
            {
                foreach (var report in reportConfig.Reports.Where(x => x.Enable))
                {
                    var file = new DeliveryFile {
                        Name = report.Name
                    };
                    file.Parameters.Add("ReportType", report.Type);
                    file.Parameters.Add("ReportFields", report.GetFieldList());
                    file.Parameters.Add("AdwordsClientID", clientId);
                    Delivery.Files.Add(file);
                }
            }

            Progress = 0.8;
            Delivery.Save();
            return(ServiceOutcome.Success);
        }

        #endregion
    }
Exemplo n.º 25
0
        protected override Core.Services.ServiceOutcome DoPipelineWork()
        {
            Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
            StringBuilder warningsStr = new StringBuilder();
            Dictionary <string, Campaign>   campaignsData  = new Dictionary <string, Campaign>();
            Dictionary <string, AdGroup>    adGroupsData   = new Dictionary <string, AdGroup>();
            Dictionary <string, Ad>         ads            = new Dictionary <string, Ad>();
            Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
            var            adStatIds     = new Dictionary <string, string>();
            var            insertedAds   = new Dictionary <string, string>();
            var            storyIds      = new Dictionary <string, Dictionary <string, string> >();
            var            photoIds      = new Dictionary <string, Dictionary <string, string> >();
            DeliveryOutput currentOutput = Delivery.Outputs.First();

            currentOutput.Checksum = new Dictionary <string, double>();
            using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
            {
                MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
                MeasureOptionsOperator = OptionsOperator.Not,
                SegmentOptions = Data.Objects.SegmentOptions.All,
                SegmentOptionsOperator = OptionsOperator.And
            }))
            {
                this.ImportManager.BeginImport(this.Delivery);
                #region AdSets
                if (filesByType.ContainsKey(Consts.FileTypes.CampaignGroups))
                {
                    List <string> campaignsFiles = filesByType[Consts.FileTypes.CampaignGroups];
                    foreach (var campaignFile in campaignsFiles)
                    {
                        DeliveryFile campaigns       = this.Delivery.Files[campaignFile];
                        var          campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
                        using (campaignsReader)
                        {
                            while (campaignsReader.Read())
                            {
                                Campaign camp = new Campaign()
                                {
                                    Name       = campaignsReader.Current.name,
                                    OriginalID = Convert.ToString(campaignsReader.Current.id),
                                };

                                campaignsData.Add(camp.OriginalID, camp);
                            }
                        }
                    }
                }
                #endregion

                #region AdGroups
                if (filesByType.ContainsKey(Consts.FileTypes.AdSets))
                {
                    List <string> adSetList = filesByType[Consts.FileTypes.AdSets];
                    foreach (var adSet in adSetList)
                    {
                        DeliveryFile adSetDF     = this.Delivery.Files[adSet];
                        var          adSetReader = new JsonDynamicReader(adSetDF.OpenContents(), "$.data[*].*");
                        using (adSetReader)
                        {
                            while (adSetReader.Read())
                            {
                                var adGroupObj = new AdGroup()
                                {
                                    Value      = adSetReader.Current.name,
                                    OriginalID = Convert.ToString(adSetReader.Current.id),
                                };

                                if (campaignsData.ContainsKey(adSetReader.Current.campaign_group_id))
                                {
                                    adGroupObj.Campaign = campaignsData[adSetReader.Current.campaign_group_id];
                                }

                                adGroupsData.Add(adGroupObj.OriginalID, adGroupObj);
                            }
                        }
                    }
                }
                #endregion


                #region adGroups And Targeting
                //******************************************************************************************************************************************************
                if (filesByType.ContainsKey(Consts.FileTypes.AdGroups))
                {
                    List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
                    foreach (var adGroup in adGroupsFiles)
                    {
                        #region foreach
                        DeliveryFile adGroups = this.Delivery.Files[adGroup];

                        var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");

                        using (adGroupsReader)
                        {
                            while (adGroupsReader.Read())
                            {
                                var campaignId = Convert.ToString(adGroupsReader.Current.campaign_id);
                                if (adGroupsData.ContainsKey(campaignId) && ((AdGroup)adGroupsData[campaignId]).Campaign != null)
                                {
                                    Ad ad = new Ad();
                                    ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
                                    ad.Segments   = new Dictionary <Segment, SegmentObject>();
                                    ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] =
                                        ((AdGroup)adGroupsData[Convert.ToString(adGroupsReader.Current.campaign_id)]).Campaign;

                                    ad.Name = adGroupsReader.Current.name;

                                    ad.Channel = new Channel()
                                    {
                                        ID = 6
                                    };

                                    ad.Account = new Account()
                                    {
                                        ID         = this.Delivery.Account.ID,
                                        OriginalID = this.Delivery.Account.OriginalID.ToString()
                                    };


                                    if (adGroupsData.ContainsKey(adGroupsReader.Current.campaign_id))
                                    {
                                        ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] =
                                            adGroupsData[adGroupsReader.Current.campaign_id];
                                    }

                                    // adgroup targeting
                                    string age_min = string.Empty;
                                    if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
                                    {
                                        age_min = adGroupsReader.Current.targeting["age_min"];
                                    }

                                    if (!string.IsNullOrEmpty(age_min))
                                    {
                                        AgeTarget ageTarget = new AgeTarget()
                                        {
                                            FromAge = int.Parse(age_min),
                                            ToAge   = int.Parse(adGroupsReader.Current.targeting["age_max"])
                                        };
                                        ad.Targets.Add(ageTarget);
                                    }
                                    List <object> genders = null;
                                    if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
                                    {
                                        genders = adGroupsReader.Current.targeting["genders"];
                                    }

                                    if (genders != null)
                                    {
                                        foreach (object gender in genders)
                                        {
                                            GenderTarget genderTarget = new GenderTarget();
                                            if (gender.ToString() == "1")
                                            {
                                                genderTarget.Gender = Gender.Male;
                                            }
                                            else if (gender.ToString() == "2")
                                            {
                                                genderTarget.Gender = Gender.Female;
                                            }
                                            else
                                            {
                                                genderTarget.Gender = Gender.Unspecified;
                                            }

                                            genderTarget.OriginalID = gender.ToString();
                                            ad.Targets.Add(genderTarget);
                                        }
                                    }
                                    if (adGroupsReader.Current.creative_ids != null)
                                    {
                                        foreach (string creative in adGroupsReader.Current.creative_ids)
                                        {
                                            if (!adsBycreatives.ContainsKey(creative))
                                            {
                                                adsBycreatives.Add(creative, new List <Ad>());
                                            }
                                            adsBycreatives[creative].Add(ad);
                                        }
                                    }
                                    ads.Add(ad.OriginalID, ad);
                                }
                            }
                        }
                        #endregion
                    }
                }

                //******************************************************************************************************************************************************
                #endregion adGroups And Targeting



                //GetAdGroupStats

                #region for validation

                foreach (var measure in this.ImportManager.Measures)
                {
                    if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
                    {
                        if (!currentOutput.Checksum.ContainsKey(measure.Key))
                        {
                            currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
                        }
                    }
                }
                //**************************************************************************
                #endregion


                Dictionary <string, List <Dictionary <string, object> > > conversion_data = new Dictionary <string, List <Dictionary <string, object> > >();
                if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
                {
                    #region Conversions
                    List <string> conversionFiles = filesByType[Consts.FileTypes.ConversionsStats];



                    foreach (string conversionFile in conversionFiles)
                    {
                        DeliveryFile creativeFile = Delivery.Files[conversionFile];
                        var          conversionCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");



                        using (conversionCreativesReader)
                        {
                            this.Mappings.OnFieldRequired = field => conversionCreativesReader.Current[field];
                            while (conversionCreativesReader.Read())
                            {
                                if (((Edge.Data.Pipeline.DynamicDictionaryObject)(conversionCreativesReader.Current)).Values == null)
                                {
                                    break;                                                                                                   // in case of empty data (e.g. data [] )
                                }
                                if (!conversion_data.ContainsKey(conversionCreativesReader.Current.adgroup_id))
                                {
                                    Dictionary <string, string> adConversionData = new Dictionary <string, string>();

                                    //string TotalConversionsManyPerClick, Leads, Signups, Purchases;


                                    List <object> actions = conversionCreativesReader.Current.actions;
                                    if (actions.Count > 0)
                                    {
                                        List <Dictionary <string, object> > action_list = actions.Select(s => (Dictionary <string, object>)s).ToList();

                                        List <Dictionary <string, object> > action_items = action_list.Where(dict => dict.ContainsKey("action_type") && dict.ContainsKey("value") &&
                                                                                                             (
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.lead") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.registration") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.add_to_cart") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.checkout") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.key_page_view") ||
                                                                                                                 dict["action_type"].ToString().Equals("offsite_conversion.other")
                                                                                                             )
                                                                                                             ).ToList();

                                        if (action_items.Count > 0)
                                        {
                                            conversion_data.Add(conversionCreativesReader.Current.adgroup_id, action_items);
                                        }
                                    }
                                }
                            } //End of While read
                        }     // End of Using reader
                    }         //End Foreach conversion file
                    #endregion
                }             //End if contains conversion file


                #region AdGroupStats start new import session
                List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
                foreach (var adGroupStat in adGroupStatsFiles)
                {
                    DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];

                    var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");

                    using (adGroupStatsReader)
                    {
                        while (adGroupStatsReader.Read())
                        {
                            #region Create Metrics
                            AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
                            adMetricsUnit.Output        = currentOutput;
                            adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
                            Ad tempAd;
                            try
                            {
                                var x = adGroupStatsReader.Current.adgroup_id;
                            }
                            catch (Exception)
                            {
                                continue;
                            }

                            if (adGroupStatsReader.Current.adgroup_id != null)
                            {
                                adStatIds[adGroupStatsReader.Current.adgroup_id] = adGroupStatsReader.Current.adgroup_id;

                                if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
                                {
                                    adMetricsUnit.Ad = tempAd;

                                    //adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
                                    //adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();

                                    // Common and Facebook specific meausures

                                    /* Sets totals for validations */
                                    if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
                                    {
                                        currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
                                    }
                                    if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
                                    {
                                        currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
                                    }
                                    if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
                                    {
                                        currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
                                    }

                                    /* Sets measures values */

                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
                                    adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
                                    //adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));



                                    //Setting conversions from conversion files data
                                    List <Dictionary <string, object> > adgroup_conversion_data;

                                    if (conversion_data.TryGetValue(adGroupStatsReader.Current.adgroup_id, out adgroup_conversion_data))
                                    {
                                        var TotalConversionsManyPerClick = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion") select element["value"];
                                        var Leads   = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.lead") select element["value"];
                                        var Signups = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.registration") select element["value"];

                                        //TO DO : Get field from Configuration
                                        var Purchases = from element in adgroup_conversion_data where element["action_type"].Equals(this.Delivery.Parameters["API_Purchases_Field_Name"]) select element["value"];

                                        //add values to metrics unit



                                        adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.TotalConversionsManyPerClick], Convert.ToDouble(TotalConversionsManyPerClick.DefaultIfEmpty("0").FirstOrDefault()));
                                        adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Leads], Convert.ToDouble(Leads.DefaultIfEmpty("0").FirstOrDefault()));
                                        adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Signups], Convert.ToDouble(Signups.DefaultIfEmpty("0").FirstOrDefault()));
                                        adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Purchases], Convert.ToDouble(Purchases.DefaultIfEmpty("0").FirstOrDefault()));
                                    }



                                    this.ImportManager.ImportMetrics(adMetricsUnit);
                                }
                                else
                                {
                                    warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
                                }
                            }
                            else
                            {
                                warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
                            }
                            #endregion
                        }
                    }
                }

                #endregion AdGroupStats start new import session



                #region Creatives
                List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];

                Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
                foreach (string creative in creativeFiles)
                {
                    DeliveryFile creativeFile           = Delivery.Files[creative];
                    var          adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");



                    using (adGroupCreativesReader)
                    {
                        //this.Mappings.OnFieldRequired = field => if((field == "object_url" && adGroupCreativesReader.Current[field] != null) || field != "object_url")adGroupCreativesReader.Current[field];
                        this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
                        while (adGroupCreativesReader.Read())
                        {
                            List <Ad> adsByCreativeID = null;
                            if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
                            {
                                if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
                                {
                                    usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
                                    adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
                                }
                            }
                            if (adsByCreativeID != null)
                            {
                                foreach (Ad ad in adsByCreativeID)
                                {
                                    if (!adStatIds.ContainsKey(ad.OriginalID))
                                    {
                                        continue;
                                    }

                                    ad.Creatives = new List <Creative>();

                                    if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_type))
                                    {
                                        string objectType = adGroupCreativesReader.Current.object_type;

                                        switch (objectType.ToUpper())
                                        {
                                        case "SHARE":
                                        {
                                            #region Ads Type SHARE
                                            if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_story_id))
                                            {
                                                Dictionary <string, string> shareCreativeData;
                                                string object_story_id = adGroupCreativesReader.Current.object_story_id;

                                                if (storyIds.ContainsKey(object_story_id))
                                                {
                                                    shareCreativeData = storyIds[object_story_id];
                                                }
                                                else
                                                {
                                                    var accessToken = this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken];
                                                    shareCreativeData = GetShareCreativeData(object_story_id, accessToken);
                                                }

                                                ad.DestinationUrl = shareCreativeData["link"];

                                                if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
                                                {
                                                    this.Mappings.Objects[typeof(Ad)].Apply(ad);

                                                    var trackersReadCommands = from n in this.Mappings.Objects[typeof(Ad)].ReadCommands
                                                                               where n.Field.Equals("link_url")
                                                                               select n;

                                                    foreach (var command in trackersReadCommands)
                                                    {
                                                        string trackerValue = ApplyRegex(ad.DestinationUrl, command.RegexPattern);

                                                        if (!String.IsNullOrEmpty(trackerValue))
                                                        {
                                                            ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value      = trackerValue;
                                                            ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].OriginalID = trackerValue;
                                                        }
                                                    }
                                                }
                                                ad.Creatives.Add(GetTextCreative(shareCreativeData["text"], adGroupCreativesReader));
                                                ad.Creatives.Add(GetBodyCreative(shareCreativeData["description"], adGroupCreativesReader));
                                                ad.Creatives.Add(GetImageCreative(shareCreativeData["picture"], adGroupCreativesReader));
                                            }
                                            #endregion
                                            break;
                                        }

                                        case "DOMAIN":
                                        {
                                            #region Ads Type DOMAIN
                                            if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_url))
                                            {
                                                if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
                                                {
                                                    ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
                                                }
                                            }

                                            else if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.link_url))
                                            {
                                                ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
                                            }
                                            else
                                            {
                                                ad.DestinationUrl = "UnKnown Url";
                                            }

                                            /*Get Data from Mapping E.g Tracker*/
                                            if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
                                            {
                                                this.Mappings.Objects[typeof(Ad)].Apply(ad);
                                            }


                                            if (adGroupCreativesReader.Current.image_url != null)
                                            {
                                                CreateImageCreatives(ad, adGroupCreativesReader);
                                            }



                                            ad.Creatives.Add(GetTextCreative(adGroupCreativesReader));
                                            ad.Creatives.Add(GetBodyCreative(adGroupCreativesReader));

                                            #endregion
                                            break;
                                        }

                                        default:
                                        {
                                            #region Ads Type  not handeled

                                            string adtype = objectType.ToUpper();

                                            ad.DestinationUrl = adtype + " Ad Type";

                                            /*Get Data from Mapping E.g Tracker*/
                                            if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
                                            {
                                                this.Mappings.Objects[typeof(Ad)].Apply(ad);
                                            }

                                            TextCreative unhandeledTextAd_title = new TextCreative()
                                            {
                                                OriginalID = ad.OriginalID,
                                                TextType   = TextCreativeType.Title,
                                                Text       = adtype + " Ad Type"
                                            };

                                            ad.Creatives.Add(unhandeledTextAd_title);


                                            #endregion
                                            break;
                                        }
                                        }



                                        if (!insertedAds.ContainsKey(ad.OriginalID))
                                        {
                                            insertedAds[ad.OriginalID] = ad.OriginalID;
                                            this.ImportManager.ImportAd(ad);
                                        }
                                    }
                                }
                            }

                            //TODO: REPORT PROGRESS 2	 ReportProgress(PROGRESS)
                        }
                    }
                    #endregion
                }


                // End foreach creative



                currentOutput.Status = DeliveryOutputStatus.Imported;
                this.ImportManager.EndImport();
                if (!string.IsNullOrEmpty(warningsStr.ToString()))
                {
                    Log.Write(warningsStr.ToString(), LogMessageType.Warning);
                }
            }
            return(Core.Services.ServiceOutcome.Success);
        }
Exemplo n.º 26
0
        protected override ServiceOutcome DoPipelineWork()
        {
            #region Init General
            // ...............................
            // SETUP
            this.Delivery = this.NewDelivery();

            // This is for finding conflicting services
            this.Delivery.Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
                                                                             this.Instance.AccountID,
                                                                             this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
                                                                             this.TargetPeriod.ToAbsolute()));

            // Create an import manager that will handle rollback, if necessary
            AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
            {
                SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
            });

            // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
            this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);

            // ...............................

            // Now that we have a new delivery, start adding values
            this.Delivery.Account = new Data.Objects.Account()
            {
                ID         = this.Instance.AccountID,
                OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
            };
            this.Delivery.TargetPeriod = this.TargetPeriod;
            this.Delivery.Channel      = new Data.Objects.Channel()
            {
                ID = 6
            };

            this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];

            if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
            {
                throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
            }
            // Copy some options as delivery parameters
            var configOptionsToCopyToDelivery = new string[] {
                FacebookConfigurationOptions.Account_ID,
                FacebookConfigurationOptions.Account_Name,
                FacebookConfigurationOptions.Auth_ApiKey,
                FacebookConfigurationOptions.Auth_AppSecret,
                FacebookConfigurationOptions.Auth_SessionKey,
                FacebookConfigurationOptions.Auth_SessionSecret,
                FacebookConfigurationOptions.Auth_RedirectUri,
                FacebookConfigurationOptions.Auth_AuthenticationUrl
            };
            foreach (string option in configOptionsToCopyToDelivery)
            {
                this.Delivery.Parameters[option] = this.Instance.Configuration.Options[option];
            }
            if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
            {
                throw new Exception("facebook base url must be configured!");
            }
            _baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
            this.ReportProgress(0.2);
            #endregion

            #region Init Delivery Files

            Dictionary <string, string> methodParams = new Dictionary <string, string>();
            string       methodUrl;
            DeliveryFile deliveryFile = new DeliveryFile();
            #region adgroupstats

            deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
            methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TargetPeriod.Start.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TargetPeriod.End.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
            this.Delivery.Files.Add(deliveryFile);
            #endregion

            this.ReportProgress(0.4);
            #region adgroup

            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
            methodUrl         = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);

            this.Delivery.Files.Add(deliveryFile);
            #endregion

            this.ReportProgress(0.6);

            #region Campaigns


            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaigns);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
            this.Delivery.Files.Add(deliveryFile);

            #endregion

            #region Creatives
            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
            this.Delivery.Files.Add(deliveryFile);
            #endregion

            //#region AdGroupTargeting
            //deliveryFile = new DeliveryFile();
            //deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
            //methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
            //deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            //deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
            //this.Delivery.Files.Add(deliveryFile);
            //#endregion



            #endregion

            this.ReportProgress(0.9);
            this.Delivery.Save();

            this.ReportProgress(1);

            return(ServiceOutcome.Success);
        }
        protected override ServiceOutcome DoPipelineWork()
        {
            this.Delivery = this.NewDelivery();             // setup delivery
            this.Delivery.FileDirectory = Instance.Configuration.Options[Edge.Data.Pipeline.Services.Const.DeliveryServiceConfigurationOptions.FileDirectory];
            this.Delivery.Account       = new Account()
            {
                ID = 0
            };



            string fileName = string.Empty;

            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["YahhoApiURL"]))
            {
                this.Delivery.Parameters.Add("YahhoApiURL", this.Instance.Configuration.Options["YahhoApiURL"]);
            }

            //FileName
            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["FileName"]))
            {
                throw new Exception("Missing Configuration Param , FileName");
            }
            else
            {
                fileName = this.Instance.Configuration.Options["FileName"];
            }

            //CrossRateSymbols
            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["CrossRateSymbols"]))
            {
                throw new Exception("Missing Configuration Param , CrossRateSymbols");
            }
            else
            {
                this.Delivery.Parameters.Add("CrossRateSymbols", this.Instance.Configuration.Options["CrossRateSymbols"]);
            }

            //UserName
            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["UserName"]))
            {
                throw new Exception("Missing Configuration Param , UserName");
            }
            else
            {
                this.Delivery.Parameters.Add("UserName", this.Instance.Configuration.Options["UserName"]);
            }

            //UserPassword
            if (String.IsNullOrEmpty(this.Instance.Configuration.Options["UserPassword"]))
            {
                throw new Exception("Missing Configuration Param , UserPassword");
            }
            else
            {
                this.Delivery.Parameters.Add("UserPassword", this.Instance.Configuration.Options["UserPassword"]);
            }


            DeliveryFile _file = new DeliveryFile()
            {
                Name = fileName
            };

            _file.SourceUrl = Instance.Configuration.Options["SourceUrl"];            // "http://www.xignite.com/xCurrencies.asmx";
            _file.Parameters.Add("Content-Type", "text/xml; charset=utf-8");
            _file.Parameters.Add("SOAPAction", Instance.Configuration.Options["SOAPAction"]);
            _file.Parameters.Add("SoapMethod", Instance.Configuration.Options["SoapMethod"]);
            _file.Parameters.Add("Body", GetSoapEnvelope(
                                     this.Instance.Configuration.Options["UserName"].ToString(),
                                     this.Instance.Configuration.Options["UserPassword"].ToString(),
                                     this.Instance.Configuration.Options["SoapMethod"].ToString(),
                                     this.Instance.Configuration.Options["CrossRateSymbols"].ToString(),
                                     string.Empty,
                                     this.Delivery.TimePeriodStart.ToString("MM/dd/yyyy")
                                     ));

            _file.Parameters.Add("XML.IsAttribute", Instance.Configuration.Options["XML.IsAttribute"]);
            _file.Parameters.Add("XML.Path", Instance.Configuration.Options["XML.Path"]);

            this.Delivery.Files.Add(_file);


            //Set Output

            foreach (string crossRate in this.Delivery.Parameters["CrossRateSymbols"].ToString().Split(','))
            {
                this.Delivery.Outputs.Add(new DeliveryOutput()
                {
                    Signature = Delivery.CreateSignature(String.Format("[{0}]-[{1}]",
                                                                       this.TimePeriod.ToAbsolute(),
                                                                       crossRate
                                                                       )),
                    Account = new Data.Objects.Account()
                    {
                        ID = 0
                    },
                    TimePeriodStart = Delivery.TimePeriodStart,
                    TimePeriodEnd   = Delivery.TimePeriodEnd
                }
                                          );
            }

            // Create an import manager that will handle rollback, if necessary
            //CurrencyImportManager importManager = new CurrencyImportManager(this.Instance.InstanceID,null);
            //TO DO: Add rollback


            // will use ConflictBehavior configuration option to abort or rollback if any conflicts occur
            //this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);


            this.Delivery.Save();
            return(Core.Services.ServiceOutcome.Success);
        }
        protected override ServiceOutcome DoPipelineWork()
        {
            #region Init General
            // ...............................
            // SETUP
            this.Delivery = NewDelivery();
            // This is for finding conflicting services
            this.Delivery.Account = new Data.Objects.Account()
            {
                ID         = this.Instance.AccountID,
                OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
            };
            this.Delivery.TimePeriodDefinition = this.TimePeriod;
            this.Delivery.Channel = new Data.Objects.Channel()
            {
                ID = 6
            };

            this.Delivery.Outputs.Add(new DeliveryOutput()
            {
                Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
                                                                   this.Instance.AccountID,
                                                                   this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
                                                                   this.Delivery.TimePeriodDefinition.ToAbsolute())),
                TimePeriodStart = Delivery.TimePeriodStart,
                TimePeriodEnd   = Delivery.TimePeriodEnd,
                Account         = this.Delivery.Account,
                Channel         = this.Delivery.Channel
            });

            // Now that we have a new delivery, start adding values
            this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];

            // Create an import manager that will handle rollback, if necessary
            AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
            {
                SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
            });

            // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
            this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);

            if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
            {
                throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
            }

            if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
            {
                throw new Exception("facebook base url must be configured!");
            }
            _baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
            //this.ReportProgress(0.2);
            #endregion

            #region Init Delivery Files

            Dictionary <string, string> methodParams = new Dictionary <string, string>();
            string       methodUrl;
            DeliveryFile deliveryFile = new DeliveryFile();

            #region adgroupstats

            deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
            methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
            this.Delivery.Files.Add(deliveryFile);
            #endregion

            //this.ReportProgress(0.4);


            #region Conversions
            //======================================================================================
            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.ConversionsStats;
            methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetConversionStats);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.ConversionsStats.ToString()));
            this.Delivery.Files.Add(deliveryFile);
            //======================================================================================
            #endregion Conversions

            //this.ReportProgress(0.4);
            #region adgroup

            /*
             * Summary
             * An ad group contains the data necessary for an ad, such as bid type, bid info,
             * targeting data, creative elements, and campaign information. Each ad group is
             * associated with a campaign and all ad groups in a campaign have the same daily
             * or lifetime budget and schedule.
             * */

            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
            methodUrl         = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupFields))
            {
                methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupFields].ToString());
            }

            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);

            this.Delivery.Files.Add(deliveryFile);
            #endregion

            //this.ReportProgress(0.6);

            #region AdSet- Formally Campaigns

            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignFields))
            {
                methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignFields].ToString());
            }

            // methodParams.Add("redownload", "true");

            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsAdSets);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
            this.Delivery.Files.Add(deliveryFile);

            #endregion


            #region Campaigns - New Structure

            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.CampaignGroups;
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignGroupsFields))
            {
                methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignGroupsFields].ToString());
            }

            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsGroups);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.CampaignGroups);
            this.Delivery.Files.Add(deliveryFile);

            #endregion

            #region Creatives
            deliveryFile      = new DeliveryFile();
            deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
            methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
            if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
            {
                methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupCreativeFields].ToString());
            }

            methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
            deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
            this.Delivery.Files.Add(deliveryFile);
            #endregion

            //#region AdGroupTargeting
            //deliveryFile = new DeliveryFile();
            //deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
            //methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
            //deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
            //deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
            //this.Delivery.Files.Add(deliveryFile);

            #endregion


            //this.ReportProgress(0.9);
            this.Delivery.Save();

            this.ReportProgress(1);

            return(ServiceOutcome.Success);
        }
        protected override Core.Services.ServiceOutcome DoPipelineWork()
        {
            string baseAddress;

            #region Init General
            // ...............................
            // SETUP
            this.Delivery = this.NewDelivery();

            // This is for finding conflicting services
            this.Delivery.Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]",
                                                                             this.Instance.AccountID,
                                                                             this.TargetPeriod.ToAbsolute()));

            // Create an import manager that will handle rollback, if necessary
            AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
            {
                SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
            });

            // Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
            this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);

            // ...............................

            // Now that we have a new delivery, start adding values
            this.Delivery.Account = new Data.Objects.Account()
            {
                ID = this.Instance.AccountID,
            };
            this.Delivery.TargetPeriod = this.TargetPeriod;
            this.Delivery.Channel      = new Data.Objects.Channel()
            {
                ID = -1
            };

            this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];

            if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
            {
                throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
            }

            if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress]))
            {
                throw new Exception("base url must be configured!");
            }
            baseAddress = this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress];

            if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UserName]))
            {
                throw new Exception("base url must be configured!");
            }
            this.Delivery.Parameters["UserName"] = this.Instance.Configuration.Options[BoConfigurationOptions.UserName];

            if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.Password]))
            {
                throw new Exception("base url must be configured!");
            }
            this.Delivery.Parameters["Password"] = this.Instance.Configuration.Options[BoConfigurationOptions.Password];

            int utcOffset = 0;
            if (!string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]))
            {
                utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
            }


            this.ReportProgress(0.2);
            #endregion

            #region DeliveryFile
            Dictionary <string, string> UrlParams = new Dictionary <string, string>();
            DeliveryFile boFile = new DeliveryFile();
            boFile.Parameters[BoConfigurationOptions.BO_XPath_Trackers] = Instance.Configuration.Options[BoConfigurationOptions.BO_XPath_Trackers];
            boFile.Name = BoConfigurationOptions.BoFileName;
            UrlParams.Add("from", utcOffset == 0 ? Delivery.TargetPeriod.Start.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
            UrlParams.Add("to", utcOffset == 0 ? Delivery.TargetPeriod.End.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
            UrlParams.Add("vendor", this.Delivery.Parameters["UserName"].ToString());
            UrlParams.Add("Password", this.Delivery.Parameters["Password"].ToString());
            boFile.SourceUrl = CreateUrl(UrlParams, baseAddress.ToString());

            this.Delivery.Files.Add(boFile);
            this.Delivery.Save();


            #endregion
            return(Core.Services.ServiceOutcome.Success);
        }
Exemplo n.º 30
0
        protected override Core.Services.ServiceOutcome DoPipelineWork()
        {
            // Setup/defaults/configuration/etc.
            // ------------------------------------------

            string checksumThreshold    = Configuration.Parameters.Get <T>(Consts.ConfigurationOptions.ChecksumTheshold);
            var    importManagerOptions = new MetricsDeliveryManagerOptions()
            {
                SqlTransformCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlTransformCommand),
                SqlStageCommand     = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlStageCommand),
                SqlRollbackCommand  = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlRollbackCommand),
                ChecksumThreshold   = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold)
            };

            string fileName;

            if (!this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.DeliveryFileName, out fileName))
            {
                throw new ConfigurationException(String.Format("{0} is missing in the service configuration options.", Const.DeliveryServiceConfigurationOptions.DeliveryFileName));
            }

            DeliveryFile file = this.Delivery.Files[fileName];

            if (file == null)
            {
                throw new Exception(String.Format("Could not find delivery file '{0}' in the delivery.", fileName));
            }

            FileCompression compression;
            string          compressionOption;

            if (this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.Compression, out compressionOption))
            {
                if (!Enum.TryParse <FileCompression>(compressionOption, out compression))
                {
                    throw new ConfigurationException(String.Format("Invalid compression type '{0}'.", compressionOption));
                }
            }
            else
            {
                compression = FileCompression.None;
            }

            // Create format processor from configuration
            string adapterTypeName   = Configuration.GetOption(Consts.ConfigurationOptions.ReaderAdapterType);
            Type   readerAdapterType = Type.GetType(adapterTypeName, true);

            this.ReaderAdapter = (ReaderAdapter)Activator.CreateInstance(readerAdapterType);

            this.Mappings.OnFieldRequired = this.ReaderAdapter.GetField;

            LoadConfiguration();

            // ------------------------------------------

            using (var stream = file.OpenContents(compression: compression))
            {
                using (this.ReaderAdapter)
                {
                    this.ReaderAdapter.Init(stream, Configuration);

                    using (this.ImportManager = CreateImportManager(InstanceID, importManagerOptions))
                    {
                        this.ImportManager.BeginImport(this.Delivery);
                        bool readSuccess = false;
                        while (this.ReaderAdapter.Reader.Read())
                        {
                            readSuccess = true;
                            OnRead();
                        }

                        if (!readSuccess)
                        {
                            Edge.Core.Utilities.Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning);
                        }

                        this.ImportManager.EndImport();
                    }
                }
            }

            return(Core.Services.ServiceOutcome.Success);
        }