public async Task <ChangeSet[]> GetChangeSetsRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, int take = 30) { if (fromDate > toDate) { throw new ArgumentException("fromDate parameter must be before toDate."); } var table = await CreateOrGetTableAsync(); var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_* queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (int)type) ); queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, "01_" + ChangeSetEntity.ConvertDateTimeToRowKey(fromDate.Date)) // RowKey <= fromDate.Date (Higher RowKey is older) ); queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, "01_" + ChangeSetEntity.ConvertDateTimeToRowKey(toDate.Date)) // RowKey >= toDate.Date (Higher RowKey is older) ); var entities = await Utility.Measure(() => table.ExecuteQuerySegmentedAsync(new TableQuery <ChangeSetEntity>().Where(queryFilter).Take(take), null)).ConfigureAwait(false); return(entities.Select(x => x.ToChangeSet()).ToArray()); }
public async Task <string> GetByDateAsync(StatusDataType dataType, DateTime date) { var container = await GetContainerAsync(); var typeNameV2 = dataType == StatusDataType.InternetExplorer ? "Edge" : dataType.ToString(); return(await container.GetBlockBlobReference($"{typeNameV2}/{date.ToString("yyyyMMdd")}.json").DownloadTextAsync().ConfigureAwait(false)); }
public async Task InsertAsync(StatusDataType dataType, DateTime date, string content) { var container = await GetContainerAsync(); var typeNameV2 = dataType == StatusDataType.InternetExplorer ? "Edge" : dataType.ToString(); var data = Encoding.UTF8.GetBytes(content); await container.GetBlockBlobReference($"{typeNameV2}/{date.ToString("yyyyMMdd")}.json").UploadFromByteArrayAsync(data, 0, data.Length).ConfigureAwait(false); }
public ChangeSetEntity(StatusDataType dataType, DateTime date, String changeSetContent, DateTime from) { PartitionKey = "01_" + ((int)dataType).ToString(); RowKey = "01_" + (ConvertDateTimeToRowKey(date.Date)); // Date only DataType = (int)dataType; Date = date.Date; Content = LZ4Codec.Wrap(Encoding.UTF8.GetBytes(changeSetContent)); From = from; UpdatedAt = DateTime.UtcNow; }
private static async Task<ChangeSet[]> GetChangeSetsByBrowserAsync(IStatusDataRepository statusDataRepository, StatusDataType type) { var statusDataSet = (await statusDataRepository.GetPlatformStatusesRangeAsync(type, DateTime.UtcNow.AddMonths(-6), DateTime.UtcNow, take: 30)) .OrderByDescending(x => x.Date) .ToArray(); if (statusDataSet.Length < 2) { return new ChangeSet[0]; } return ChangeSet.GetChangeSetsFromPlatformStatuses(statusDataSet); }
private static async Task<ChangeSet[]> GetChangeSetsByBrowser(IStatusDataRepository statusDataRepository, StatusDataType type, DateTime date) { var statusDataSet = (await Utility.Measure(() => statusDataRepository.GetPlatformStatusesRangeAsync(type, date.AddMonths(-6), date, take: 2))) .OrderByDescending(x => x.Date) .Take(2) .ToArray(); if (statusDataSet.Length < 2) { return new ChangeSet[0]; } return ChangeSet.GetChangeSetsFromPlatformStatuses(statusDataSet); }
public async Task <ChangeSet> GetLatestChangeSetAsync(StatusDataType type) { var table = await CreateOrGetTableAsync(); var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_* queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (int)type) ); return((await table.ExecuteQuerySegmentedAsync(new TableQuery <ChangeSetEntity>().Where(queryFilter).Take(1), null)) .Select(x => x.ToChangeSet()) .FirstOrDefault()); }
public static IPlatformStatus[] Deserialize(StatusDataType dataType, string jsonValue) { switch (dataType) { case StatusDataType.Chromium: return(DeserializeForChromiumStatus(jsonValue)); case StatusDataType.InternetExplorer: return(DeserializeForIeStatus(jsonValue)); case StatusDataType.WebKitWebCore: return(DeserializeForWebKitStatus(jsonValue)); case StatusDataType.WebKitJavaScriptCore: return(DeserializeForWebKitStatus(jsonValue)); case StatusDataType.Mozilla: return(DeserializeForMozillaStatus(jsonValue)); default: throw new NotSupportedException(); } }
public async Task<PlatformStatuses[]> GetPlatformStatusesRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, Int32 take = 30) { if (fromDate > toDate) throw new ArgumentException("fromDate parameter must be before toDate."); var table = await CreateOrGetTable(); var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_* queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (Int32)type) ); queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, "01_" + ConvertDateTimeToRowKey(fromDate.Date)) // RowKey <= fromDate.Date (Higher RowKey is older) ); queryFilter = TableQuery.CombineFilters( queryFilter, TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, "01_" + ConvertDateTimeToRowKey(toDate.Date)) // RowKey >= toDate.Date (Higher RowKey is older) ); var entities = await Utility.Measure(() => table.ExecuteQuerySegmentedAsync(new TableQuery<StatusDataEntity>().Where(queryFilter).Take(take), null)); var tasks = entities.Select(x => { // return null if JSON data was broken. try { return CreatePlatformStatusesFromEntityAsync(x); } catch (JsonReaderException) { return null; } }) .Where(x => x != null) .ToArray(); return await Task.WhenAll(tasks); }
public async Task InsertOrReplaceAsync(StatusDataType dataType, DateTime date, string changeSetContent, DateTime from) { var table = await CreateOrGetTableAsync(); await table.ExecuteAsync(TableOperation.InsertOrReplace(new ChangeSetEntity(dataType, date, changeSetContent, from))).ConfigureAwait(false); }
private static async Task <ChangeSet[]> GetChangeSetsByBrowserAsync(IChangeSetRepository changeSetRepository, StatusDataType type) { return((await changeSetRepository.GetChangeSetsRangeAsync(type, DateTime.UtcNow.AddMonths(-6), DateTime.UtcNow, take: 30)) .OrderByDescending(x => x.Date) .ToArray()); }
public async Task UpdateRange([FromService] DataUpdateAgent agent, [Option('t')] StatusDataType statusDataType, DateTime from, DateTime to) { await agent.UpdateChangeSetByRangeAsync(statusDataType, from, to); }
private static async Task <ChangeSet[]> GetChangeSetsByBrowser(IChangeSetRepository changeSetRepository, StatusDataType type, DateTime date) { return((await Utility.Measure(() => changeSetRepository.GetChangeSetsRangeAsync(type, date.AddMonths(-6), date, take: 1))) .OrderByDescending(x => x.Date) .ToArray()); }
public StatusDataEntity(StatusDataType dataType, DateTime date, String content) { this.PartitionKey = "01_" + ((Int32)dataType).ToString(); this.RowKey = "01_" + (ConvertDateTimeToRowKey(date.Date)); // Date only this.DataType = (Int32)dataType; this.Date = date; this.DataCompression = (Int32)DataCompressionType.Bzip2; var splitSize = (1024 * 50); var compressedContent = Compress(Encoding.UTF8.GetBytes(content)); for (var i = 0; i < Math.Ceiling(compressedContent.Length / (double)splitSize); i++) { this.SplitCount = i; var tmpContent = new byte[Math.Min(splitSize, compressedContent.Length - (i * splitSize))]; Array.Copy(compressedContent, (i * splitSize), tmpContent, 0, tmpContent.Length); switch (i) { case 0: Content = tmpContent; break; case 1: Content1 = tmpContent; break; case 2: Content2 = tmpContent; break; case 3: Content3 = tmpContent; break; case 4: Content4 = tmpContent; break; default: throw new Exception("data size is too large.: " + compressedContent.Length); } } }
public async Task<PlatformStatuses[]> GetPlatformStatusesRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, Int32 take) { return new[] { new PlatformStatuses(DateTime.Parse("2015/1/1"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")), new PlatformStatuses(DateTime.Parse("2015/1/2"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")), new PlatformStatuses(DateTime.Parse("2015/1/3"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Under Consideration"",""iePrefixed"":"""",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")), new PlatformStatuses(DateTime.Parse("2015/1/4"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""In Development"",""iePrefixed"":"""",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")), new PlatformStatuses(DateTime.Parse("2015/1/5"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""99"",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")), }; }
public async Task InsertAsync(StatusDataType type, DateTime date, String jsonData) { // 何もしない }
public async Task InsertAsync(StatusDataType type, DateTime date, String jsonData) { var table = await CreateOrGetTable(); await table.ExecuteAsync(TableOperation.InsertOrReplace(new StatusDataEntity(type, date, jsonData))); }