public static List <ExtractResult> Extract(ExtractRequest request) { if (NodeConfigurationSection.Standalone) { var result = RuiJiExtractor.Extract(request); return(result); } else { var proxyUrl = ProxyManager.Instance.Elect(NodeProxyTypeEnum.FEEDPROXY); if (string.IsNullOrEmpty(proxyUrl)) { throw new Exception("no available Extractor proxy servers"); } proxyUrl = IPHelper.FixLocalUrl(proxyUrl); var client = new RestClient("http://" + proxyUrl); var restRequest = new RestRequest("api/ep/extract"); restRequest.Method = Method.POST; restRequest.JsonSerializer = new NewtonJsonSerializer(); var json = JsonConvert.SerializeObject(request); restRequest.AddJsonBody(json); restRequest.Timeout = 15000; var restResponse = client.Execute(restRequest); var response = JsonConvert.DeserializeObject <List <ExtractResult> >(restResponse.Content); return(response); } }
private bool DoExtractTaskWithPod(Bot bot, Pod pod) { // Bring items to the randomly selected first fitting station foreach (var oStation in Instance.OutputStations.OrderBy(s => GetOrderValue(s, bot))) { // Search for requests matching the items of the pod List <ExtractRequest> fittingRequests = GetPossibleRequests(pod, oStation, PodSelectionExtractRequestFilteringMode.AssignedOnly); if (fittingRequests.Any()) { ExtractRequest oldestRequest = fittingRequests.OrderBy(o => o.Order.TimeStamp).First(); // Simply execute the next task with the pod EnqueueExtract( bot, // The bot itself oStation, // The random station pod, // Keep the pod new List <ExtractRequest> { oldestRequest }); // The first requests to serve // Finished search for next task return(true); } } // No fitting request return(false); }
public static void Run() { var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey); var apiInstance = new PagesApi(configuration); try { var fileInfo = new FileInfo { FilePath = "WordProcessing/sample-10-pages.docx" }; var options = new ExtractOptions { FileInfo = fileInfo, OutputPath = "Output/extract-pages-by-range.docx", StartPageNumber = 1, EndPageNumber = 10, RangeMode = PageOptions.RangeModeEnum.EvenPages }; var request = new ExtractRequest(options); var response = apiInstance.Extract(request); Console.WriteLine("Output file path: " + response.Path); } catch (Exception e) { Console.WriteLine("Exception while calling api: " + e.Message); } }
public void ExtractApiTest_Tag() { var options = new ExtractOptions { FileInfo = TestFiles.Docx.ToFileInfo(), SearchCriteria = new SearchCriteria { TagOptions = new TagOptions { ExactTag = new Tag { Name = "Created", Category = "Time" } } } }; var request = new ExtractRequest(options); var result = MetadataApi.Extract(request); Assert.IsNotNull(result); Assert.IsNotEmpty(result.Properties); Assert.IsTrue(result.Properties.Any(x => string.Equals(x.Name, "CreateTime"))); }
public void ExtractApiTest_PropertyNameExactPhrase() { var options = new ExtractOptions { FileInfo = TestFiles.Docx.ToFileInfo(), SearchCriteria = new SearchCriteria { NameOptions = new NameOptions { Value = "MimeType", MatchOptions = new MatchOptions { ExactPhrase = true } } } }; var request = new ExtractRequest(options); var result = MetadataApi.Extract(request); Assert.IsNotNull(result); Assert.IsNotEmpty(result.Properties); Assert.IsTrue(result.Properties.Any(x => x.Name.Equals("MimeType", StringComparison.OrdinalIgnoreCase))); }
/// <summary> /// Finds the best delivery task for the specified pod. /// Sets <code>bestDeliveryRequest</code> to null if none found, otherwise <code>bestDeliveryRequest</code> and <code>bestTimeForDeliveryRequest</code> are initialized. /// </summary> /// <param name="bot">The bot to consider.</param> /// <param name="pod">Pod to take.</param> /// <param name="podLocation">Current location of the pod.</param> /// <param name="bestExtractTask">The best extract task set by this method.</param> /// <param name="bestTimeForExtract">The time of the best extract set by this method.</param> void GetBestTaskForPod(Bot bot, Pod pod, Waypoint podLocation, out ExtractRequest bestExtractTask, out double bestTimeForExtract) { bestExtractTask = null; bestTimeForExtract = 0.0; bestExtractTask = null; if (pod == null || podLocation == null) { return; } bestTimeForExtract = double.PositiveInfinity; // Check all tasks foreach (var delivery in Instance.ResourceManager.AvailableAndAssignedExtractRequests) { // If it has the item if (pod.IsContained(delivery.Item)) { // See how long it would take to get to the output-station // Choose the worst of delivering or waiting Waypoint sw = delivery.Station.Waypoint; double time = Math.Max(Estimators.EstimateTravelTimeEuclid(bot, podLocation, sw), Estimators.EstimateOutputStationWaitTime(bot, sw)); // If it's the best, then use it if (time < bestTimeForExtract) { bestExtractTask = delivery; bestTimeForExtract = time; } } } }
public void ExtractApiTest_PropertyNameRegex() { var options = new ExtractOptions { FileInfo = TestFiles.Docx.ToFileInfo(), SearchCriteria = new SearchCriteria { NameOptions = new NameOptions { Value = "^dc:.*", MatchOptions = new MatchOptions { IsRegex = true } } } }; var request = new ExtractRequest(options); var result = MetadataApi.Extract(request); Assert.IsNotNull(result); Assert.IsNotEmpty(result.Properties); Assert.IsTrue(result.Properties.Any(x => x.Name.StartsWith("dc:", StringComparison.OrdinalIgnoreCase))); }
public static void Run() { var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey); var apiInstance = new PagesApi(configuration); try { var fileInfo = new FileInfo { FilePath = "WordProcessing/sample-10-pages.docx" }; var options = new ExtractOptions { FileInfo = fileInfo, OutputPath = "Output/extract-pages-by-numbers.docx", Pages = new List <int?> { 2, 4, 7 } }; var request = new ExtractRequest(options); var response = apiInstance.Extract(request); Console.WriteLine("Output file path: " + response.Path); } catch (Exception e) { Console.WriteLine("Exception while calling api: " + e.Message); } }
public static ExtractResult Extract(ExtractRequest request) { var proxyUrl = ProxyManager.Instance.Elect(ProxyTypeEnum.Extracter); if (string.IsNullOrEmpty(proxyUrl)) { throw new Exception("no available extracter proxy servers"); } var client = new RestClient("http://" + proxyUrl); var restRequest = new RestRequest("api/ep/extract"); restRequest.Method = Method.POST; restRequest.JsonSerializer = new NewtonJsonSerializer(); var json = JsonConvert.SerializeObject(request); restRequest.AddJsonBody(json); restRequest.Timeout = 15000; var restResponse = client.Execute(restRequest); var response = JsonConvert.DeserializeObject <ExtractResult>(restResponse.Content); return(response); }
// pagenation is ?page=2&per_page=100 /// <summary> /// the url is like this: https://github.com/andyliuliming/WALinuxAgent.git /// </summary> /// <param name="url"></param> /// <returns></returns> public List <User> Extract(ExtractRequest extractRequest, List <string> user_logins) { GithubFeed githubFee = this.Parse(extractRequest.URL); List <User> users = new List <User>(); RestApiCaller <User> userInfoCaller = new RestApiCaller <User>(ApiFormats.BaseUri); // 3. extract the commit info ("login") int leftUserLength = user_logins.Count - extractRequest.StartIndex; int numberToRetrieve = leftUserLength > extractRequest.Left ? extractRequest.Left : leftUserLength; for (int i = 0; i < numberToRetrieve; i++) { Console.WriteLine("getting the user's info: " + user_logins[extractRequest.StartIndex + i]); string urlParameters = string.Format(ApiFormats.UserApi, user_logins[extractRequest.StartIndex + i]); User user = null; Retry( () => { user = userInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null); }, () => { extractRequest.Left--; } ); users.Add(user); } extractRequest.StartIndex += numberToRetrieve; return(users); }
/// <summary> /// Retrieve document metadata. /// </summary> /// <param name="request">Request. <see cref="ExtractRequest" /></param> /// <returns><see cref="ExtractResult"/></returns> public ExtractResult Extract(ExtractRequest request) { // verify the required parameter 'options' is set if (request.options == null) { throw new ApiException(400, "Missing required parameter 'options' when calling Extract"); } // create path and map variables var resourcePath = this.configuration.GetServerUrl() + "/metadata"; resourcePath = Regex .Replace(resourcePath, "\\*", string.Empty) .Replace("&", "&") .Replace("/?", "?"); var postBody = SerializationHelper.Serialize(request.options); // http body (model) parameter var response = this.apiInvoker.InvokeApi( resourcePath, "POST", postBody, null, null); if (response != null) { return((ExtractResult)SerializationHelper.Deserialize(response, typeof(ExtractResult))); } return(null); }
public async Task <string> LoadDataFromGoogleScholar(ExtractRequest extractRequest) { BibTeXclient.BaseAddress = new Uri(ProgramAddresses.GoogleScholarWrapperAddress); return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), BibTeXclient)); //return File.ReadAllText("./archivoBibtex.json"); }
public async Task <string> LoadDataFromIEEEXplore(ExtractRequest extractRequest) { IEEEXclient.BaseAddress = new Uri(ProgramAddresses.IEEEXploreWrapperAddress); return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), IEEEXclient)); //return File.ReadAllText("../IEEEXmini.json"); }
public async Task <string> LoadDataFromDBLP(ExtractRequest extractRequest) { DBLPclient.BaseAddress = new Uri(ProgramAddresses.DBLPWrapperAddress); return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), DBLPclient)); //return File.ReadAllText("../DBLPreal.json"); }
/// <summary> /// Adds the request to pick the item to the order. This can be used by other components to see the particular requests' status. /// </summary> /// <param name="itemDescription">The SKU to add the request for.</param> /// <param name="request">The particular request to pick the item.</param> public void AddRequest(ItemDescription itemDescription, ExtractRequest request) { if (!_requests.ContainsKey(itemDescription)) { _requests[itemDescription] = new HashSet <ExtractRequest>(); } _requests[itemDescription].Add(request); }
/// <summary> /// Revokes a reservation of an item. /// </summary> /// <param name="item">The type of the item.</param> /// <param name="extractRequest">The request which should have been done with the item.</param> internal void UnregisterItem(ItemDescription item, ExtractRequest extractRequest) { _itemDescriptionCountAvailable[item]++; _extractRequestsRegistered.Remove(extractRequest); extractRequest.Unassign(this); // Notify instance Instance.NotifyPodItemUnreserved(this, item, extractRequest); }
public static void ExtractDataExtension(SoapClient soapClient, string DataExtensionCustomerKey, string FileName) { ExtractRequest er = new ExtractRequest(); er.ID = "bb94a04d-9632-4623-be47-daabc3f588a6"; // Always set an StartDate to the value specified ExtractParameter epOne = new ExtractParameter(); epOne.Name = "StartDate"; epOne.Value = "1/1/1900 1:00:00 AM"; // Always set an StartDate to the value specified ExtractParameter epTwo = new ExtractParameter(); epTwo.Name = "EndDate"; epTwo.Value = "1/1/1900 1:00:00 AM"; // Always set an _Async to 0 ExtractParameter epThree = new ExtractParameter(); epThree.Name = "_AsyncID"; epThree.Value = "0"; ExtractParameter epFour = new ExtractParameter(); epFour.Name = "OutputFileName"; epFour.Value = FileName; ExtractParameter epFive = new ExtractParameter(); epFive.Name = "DECustomerKey"; epFive.Value = DataExtensionCustomerKey; ExtractParameter epSix = new ExtractParameter(); epSix.Name = "HasColumnHeaders"; epSix.Value = "true"; er.Parameters = new ExtractParameter[] { epOne, epTwo, epThree, epFour, epFive, epSix }; string sRequestId; string sStatus; ExtractResult[] results; sStatus = soapClient.Extract(new ExtractRequest[] { er }, out sRequestId, out results); Console.WriteLine("Status: " + sStatus); Console.WriteLine("Request ID: " + sRequestId); foreach (ExtractResult eresult in results) { Console.WriteLine("StatusCode: " + eresult.StatusCode); Console.WriteLine("ErrorCode: " + eresult.ErrorCode); Console.WriteLine("StatusMessage: " + eresult.StatusMessage); } }
/// <summary> /// Adds another request to this task on-the-fly. /// </summary> /// <param name="request">The request that shall also be completed by this task.</param> public void AddRequest(ExtractRequest request) { Instance.ResourceManager.RemoveExtractRequest(request); if (!ReservedPod.IsContained(request.Item)) { throw new InvalidOperationException("Cannot add a request for an item that is not available!"); } ReservedPod.RegisterItem(request.Item, request); Requests.Add(request); request.StatInjected = true; }
public async Task <string> ExtractData(string request) { ExtractRequest extractRequest = GetExtractRequest(request); if (extractRequest == null) { return(null); } return(await ExtractDataFromIEEEXploreAPIAsync(extractRequest.InitialYear, extractRequest.FinalYear)); }
public string ExtractData(string request) { ExtractRequest extractRequest = GetExtractRequest(request); if (extractRequest == null) { return(null); } return(ExtractDataFromXmlFile(extractRequest.InitialYear, extractRequest.FinalYear)); }
public static void Run() { var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey); var apiInstance = new MetadataApi(configuration); try { var fileInfo = new FileInfo { FilePath = "documents/input.docx", StorageName = Common.MyStorage }; var options = new ExtractOptions { FileInfo = fileInfo, SearchCriteria = new SearchCriteria { TagOptions = new TagOptions { ExactTag = new Tag { Name = "Created", Category = "Time" } } } }; var request = new ExtractRequest(options); var response = apiInstance.Extract(request); foreach (var property in response.Properties) { Console.WriteLine($"Property: {property.Name}. Value: {property.Value}"); if (property.Tags == null) { continue; } foreach (var tag in property.Tags) { Console.WriteLine($"Property tag: {tag.Category} {tag.Name} "); } } } catch (Exception e) { Console.WriteLine("Exception while calling MetadataApi: " + e.Message); } }
public void ExtractApiTest_DocumentProtectedException() { var testFile = TestFiles.PasswordProtected; var options = new ExtractOptions { FileInfo = testFile.ToFileInfo(), }; var request = new ExtractRequest(options); var ex = Assert.Throws <ApiException>(() => { MetadataApi.Extract(request); }); Assert.AreEqual($"The specified file '{testFile.FullName}' is protected.", ex.Message); }
public void ExtractApiTest_FileNotFound() { var testFile = TestFiles.NotExist; var options = new ExtractOptions { FileInfo = testFile.ToFileInfo(), }; var request = new ExtractRequest(options); var ex = Assert.Throws <ApiException>(() => { MetadataApi.Extract(request); }); Assert.AreEqual($"Can't find file located at '{testFile.FullName}'.", ex.Message); }
public static void ExtractDataExtension(SoapClient soapClient, string DataExtensionCustomerKey, string FileName) { ExtractRequest er = new ExtractRequest(); er.ID = "bb94a04d-9632-4623-be47-daabc3f588a6"; // Always set an StartDate to the value specified ExtractParameter epOne = new ExtractParameter(); epOne.Name = "StartDate"; epOne.Value = "1/1/1900 1:00:00 AM"; // Always set an StartDate to the value specified ExtractParameter epTwo = new ExtractParameter(); epTwo.Name = "EndDate"; epTwo.Value = "1/1/1900 1:00:00 AM"; // Always set an _Async to 0 ExtractParameter epThree = new ExtractParameter(); epThree.Name = "_AsyncID"; epThree.Value = "0"; ExtractParameter epFour = new ExtractParameter(); epFour.Name = "OutputFileName"; epFour.Value = FileName; ExtractParameter epFive = new ExtractParameter(); epFive.Name = "DECustomerKey"; epFive.Value = DataExtensionCustomerKey; ExtractParameter epSix = new ExtractParameter(); epSix.Name = "HasColumnHeaders"; epSix.Value = "true"; er.Parameters = new ExtractParameter[] {epOne, epTwo, epThree, epFour, epFive, epSix }; string sRequestId; string sStatus; ExtractResult[] results; sStatus = soapClient.Extract(new ExtractRequest[] { er }, out sRequestId, out results); Console.WriteLine("Status: " + sStatus); Console.WriteLine("Request ID: " + sRequestId); foreach (ExtractResult eresult in results) { Console.WriteLine("StatusCode: " + eresult.StatusCode); Console.WriteLine("ErrorCode: " + eresult.ErrorCode); Console.WriteLine("StatusMessage: " + eresult.StatusMessage); } }
public void ExtractApiTest() { var options = new ExtractOptions { FileInfo = TestFiles.Docx.ToFileInfo() }; var request = new ExtractRequest(options); var result = MetadataApi.Extract(request); Assert.IsNotNull(result.MetadataTree); Assert.IsNotEmpty(result.MetadataTree.InnerPackages); Assert.IsTrue(result.MetadataTree.InnerPackages.Any(x => string.Equals(x.PackageName, "FileFormat"))); }
/// <summary> /// Requests the station to pick the given item for the given order. /// </summary> /// <param name="bot">The bot that requests the pick.</param> /// <param name="request">The request to handle.</param> public void RequestItemTake(Bot bot, ExtractRequest request) { if (bot.Pod != null && _assignedOrders.Contains(request.Order)) { // Add the request to the list of requests to handle _requestsBot.Enqueue(bot); _requestsExtract.Enqueue(request); } else { // Something went wrong, refuse to handle the request request.Abort(); bot.WaitUntil(Instance.Controller.CurrentTime + Instance.RefusedRequestPenaltyTime); } }
public async Task <ActionResult> Post([FromBody] ExtractRequest request) { try { var extractedMediaUrl = await _mediaExtractorService.Extract(request.MediaUrl); return(Ok(new ExtractResponse { MediaUrl = extractedMediaUrl })); } catch (Exception e) { return(BadRequest(new { error = e.Message })); } }
private async Task <LoadAnswer> ProcessLoadRequest(LoadRequest loadRequest) { LoadAnswer loadAnswer = new LoadAnswer(); ExtractRequest extractRequest = new ExtractRequest(loadRequest.InitialYear, loadRequest.FinalYear); if (loadRequest.LoadFromDBLP) { (int numberOfResults, List <string> errorList) = await LoadFromSource( "DBLP", loadRequest, extractRequest, requestsManager.LoadDataFromDBLP, dBLPExtractor.ExtractData); loadAnswer.DBLPNumberOfResults = numberOfResults; loadAnswer.DBLPErrors = errorList; } if (loadRequest.LoadFromIEEEXplore) { (int numberOfResults, List <string> errorList) = await LoadFromSource( "IEEEXplore", loadRequest, extractRequest, requestsManager.LoadDataFromIEEEXplore, iEEEXploreExtractor.ExtractData); loadAnswer.IEEEXploreNumberOfResults = numberOfResults; loadAnswer.IEEEXploreErrors = errorList; } if (loadRequest.LoadFromGoogleScholar) { (int numberOfResults, List <string> errorList) = await LoadFromSource( "Google Scholar", loadRequest, extractRequest, requestsManager.LoadDataFromGoogleScholar, bibTeXExtractor.ExtractData); loadAnswer.GoogleScholarNumberOfResults = numberOfResults; loadAnswer.GoogleScholarErrors = errorList; } return(loadAnswer); }
/// <summary> /// Reserves an item that is going to be picked at a station. /// </summary> /// <param name="item">The item that is going to be reserved for picking.</param> /// <param name="extractRequest">The request for which the item shall be reserved.</param> internal void RegisterItem(ItemDescription item, ExtractRequest extractRequest) { // Init, if not done yet if (_itemDescriptionCountContained == null) { InitPodContentInfo(); } if (_itemDescriptionCountAvailable[item] <= 0) { throw new InvalidOperationException("Cannot reserve an item for picking, if there is none left of the kind!"); } _itemDescriptionCountAvailable[item]--; _extractRequestsRegistered.Add(extractRequest); extractRequest.Assign(this); // Notify instance Instance.NotifyPodItemReserved(this, item, extractRequest); }
public HashSet <string> ExtractUserLogin(ExtractRequest extractRequest) { GithubFeed githubFee = this.Parse(extractRequest.URL); HashSet <string> user_logins = new HashSet <string>(); // 1. first get the branches https://api.github.com/repos/torvalds/linux // https://api.github.com/repos/apache/hadoop // 2. get the commits from the branches. https://api.github.com/repos/torvalds/linux/commits?page=3&per_page=100 RestApiCaller <List <CommitDetail> > commitInfoCaller = new RestApiCaller <List <CommitDetail> >(ApiFormats.BaseUri); string repoBaseUri = string.Format(ApiFormats.CommitRelativePathPattern, githubFee.owner, githubFee.repo); while (extractRequest.Left > 0) { Console.WriteLine("getting the page: " + extractRequest.StartPage); string urlParameters = repoBaseUri + "?page=" + extractRequest.StartPage + "&per_page=" + extractRequest.PerPage; List <CommitDetail> pagedDetails = null; Retry( () => { pagedDetails = commitInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null); }, () => { extractRequest.Left--; } ); if (pagedDetails == null || pagedDetails.Count == 0) { break; } else { for (int i = 0; i < pagedDetails.Count; i++) { if (pagedDetails[i].author != null) { user_logins.Add(pagedDetails[i].author.login); } } extractRequest.StartPage++; } } return(user_logins); }
public object TestRule([FromBody] RuleModel rule, bool debug = false) { var request = new Request(rule.Url); request.Method = rule.Method; request.RunJS = (rule.RunJS == Status.ON); if (request.RunJS) { request.WaitDom = request.WaitDom; } var response = Crawler.Request(request); if (response != null && response.Data != null) { var content = response.Data.ToString(); var block = RuiJiBlockParser.ParserBlock(rule.RuiJiExpression); var r = new ExtractRequest(); r.Content = content; r.Blocks = new List <ExtractFeatureBlock> { new ExtractFeatureBlock(block, rule.Feature) }; var results = Extractor.Extract(r); var result = results.OrderByDescending(m => m.Metas.Count).FirstOrDefault(); if (result != null && result.Paging != null && result.Paging.Count > 0 && result.Metas != null && result.Metas.ContainsKey("content")) { result = PagingExtractor.MergeContent(new Uri(rule.Url), result, block); } if (!debug) { CrawlTaskFunc.ClearContent(result); } return(result); } return(new { }); }
public static void Go() { var dict_fiidToAuctionID = new Dictionary<int, int>(); { string sql = "select * from FIBondAuction"; DataSet ds = Singleton<ConnMngr>.Instance.Execute(DBNames.SQLSERVER_SYMMETRYTS, sql); foreach (DataRow row in ds.Tables[0].Rows) if(!dict_fiidToAuctionID.ContainsKey(Convert.ToInt32(row["FIID"]))) dict_fiidToAuctionID.Add(Convert.ToInt32(row["FIID"]), Convert.ToInt32(row["FIBondAuctionID"])); } var dict_isinToMissingBond = new Dictionary<string, Bond>(); var rss_identifiers = new List<InstrumentIdentifier>(); foreach (var bond in Singleton<Bonds>.Instance.Where(x => x.SymmetryCode.StartsWith("US912828"))) { if (!dict_fiidToAuctionID.ContainsKey(bond.FIID)) { dict_isinToMissingBond.Add(bond.SymmetryCode, bond); rss_identifiers.Add(new InstrumentIdentifier { IdentifierType = "ISN", IdentifierValue = bond.SymmetryCode, Source="TWB"}); Logger.Debug(string.Format("{0}/{1} is missing auction information", bond.SymmetryCode, bond.DisplayName), typeof(UpdateBondAuctionData)); } } System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Ssl3; if (rss_identifiers.Count() == 0) return; // nothing to do try { var cred = ServiceHelpers.GetCredentials(); { using (var dsClient = new ExtractionServiceClient()) { var validatedInstruments = ServiceHelpers.ValidateInstruments(rss_identifiers.ToArray(), dsClient, ref cred); { var request = new ExtractRequest { CredentialsHeader = cred, request = new InstrumentExtractionRequestTermsAndConditions { Instruments = validatedInstruments, OutputFields = Enum.GetNames(typeof(FieldsToReturn)) .Select(x => typeof(FieldsToReturn).GetField(x.ToString())) .Select(x => x.GetCustomAttribute<RSSFieldAttribute>().FieldName).ToArray() } }; var response = dsClient.Extract(request); var pivoted = ExtractionUtility.PivotExtractionResult(response.ExtractResult.Columns, response.ExtractResult.RowCount, response.ExtractResult.Messages); StringBuilder b = new StringBuilder(); b.AppendLine(ServiceHelpers.ToCsvLine(pivoted.ColumnNames.Concat(new [] {"FIID"}))); foreach (var row in pivoted.Rows) { string isin = row[(int)FieldsToReturn.InstrumentID] as string; b.AppendLine(ServiceHelpers.ToCsvLine(row.Concat(new string[] { dict_isinToMissingBond[isin].FIID.ToString() }))); } var path = @"e:\temp\missingAuctions.csv"; FileInfo i = new FileInfo(path); StreamWriter writer = i.CreateText(); writer.Write(b.ToString()); writer.Close(); } } } } catch (Exception ex_) { Logger.Error("blast", typeof(UpdateCTBondData), ex_); } }
public static void Go() { var ctList = SI.Strategy.CTDs.Generator.Go(); var listOfMissing = new List<MonthYear>(); foreach (var v in ctList) { if (v.CT2 == null || v.CT3 == null || v.CT5 == null || v.CT7 == null || v.CT10 == null || v.CT30 == null) if (!listOfMissing.Contains(v.RefundingMonth)) { Logger.Debug(string.Format("Missing bonds for {0}", v.RefundingMonth), typeof(UpdateCTBondData)); listOfMissing.Add(v.RefundingMonth); } } System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Ssl3; try { var cred = ServiceHelpers.GetCredentials(); using (var dc = SymmetryDataSource.ReadOnly()) { using (var dsClient = new ExtractionServiceClient()) { var isins = new Dictionary<string, ValidatedInstrument>(); foreach (var v in listOfMissing) { var request = new SearchInstrumentsRequest { CredentialsHeader = cred, request = new InstrumentSearchRequestGovCorp { IdentifierType = "ISN", IdentifierSearchValue = "US9128*", PreferredIdentifierType = "ISN", IsGovernmentGroup = true, IssueDatePrimaryValue = new DateTime(v.Year, v.Month, 3), IssueDateSecondaryValue = new DateTime(v.Year, v.Month, 25), IssueDateComparisonOperator = "FROM", } }; var response = dsClient.SearchInstruments(request); Array.ForEach(response.SearchInstrumentsResult.Instruments.Select(x => x.IdentifierValue).Distinct().ToArray(), (x) => Logger.Debug(x, typeof(UpdateCTBondData))); foreach (var validatedInstrument in response.SearchInstrumentsResult.Instruments) { var isin = validatedInstrument.IdentifierValue; if (isins.ContainsKey(isin)) { if (validatedInstrument.Source.Equals("TWB")) isins[isin] = validatedInstrument; continue; } var fi = FIHelpers.GetFIBySymmetryCode(isin, dc, Symmetry.Core.ThrowBehavior.DontThrow); if (fi == null) isins.Add(isin, validatedInstrument); } } if (isins.Count == 0) return; { var request = new ExtractRequest { CredentialsHeader = cred, request = new InstrumentExtractionRequestTermsAndConditions { Instruments = isins.ToList().Select(x => x.Value).ToArray(), OutputFields = Enum.GetNames(typeof(FieldsToReturn)) .Select(x => typeof(FieldsToReturn).GetField(x.ToString())) .Select(x => x.GetCustomAttribute<RSSFieldAttribute>().FieldName).ToArray() } }; var response = dsClient.Extract(request); var pivoted = ExtractionUtility.PivotExtractionResult(response.ExtractResult.Columns, response.ExtractResult.RowCount, response.ExtractResult.Messages); StringBuilder b = new StringBuilder(); b.AppendLine(ServiceHelpers.ToCsvLine(pivoted.ColumnNames)); foreach (var row in pivoted.Rows) { b.AppendLine(ServiceHelpers.ToCsvLine(row)); } var path = @"e:\temp\missingBonds.csv"; FileInfo i = new FileInfo(path); StreamWriter writer = i.CreateText(); writer.Write(b.ToString()); writer.Close(); } } } } catch (Exception ex_) { Logger.Error("blast", typeof(UpdateCTBondData), ex_); } }