public async Task <ChangeSet[]> GetChangeSetsRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, int take = 30)
{
if (fromDate > toDate)
{
throw new ArgumentException("fromDate parameter must be before toDate.");
}
var table = await CreateOrGetTableAsync();
var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_*
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (int)type)
);
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, "01_" + ChangeSetEntity.ConvertDateTimeToRowKey(fromDate.Date)) // RowKey <= fromDate.Date (Higher RowKey is older)
);
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, "01_" + ChangeSetEntity.ConvertDateTimeToRowKey(toDate.Date)) // RowKey >= toDate.Date (Higher RowKey is older)
);
var entities = await Utility.Measure(() => table.ExecuteQuerySegmentedAsync(new TableQuery <ChangeSetEntity>().Where(queryFilter).Take(take), null)).ConfigureAwait(false);
return(entities.Select(x => x.ToChangeSet()).ToArray());
}
public async Task <string> GetByDateAsync(StatusDataType dataType, DateTime date)
{
var container = await GetContainerAsync();
var typeNameV2 = dataType == StatusDataType.InternetExplorer ? "Edge" : dataType.ToString();
return(await container.GetBlockBlobReference($"{typeNameV2}/{date.ToString("yyyyMMdd")}.json").DownloadTextAsync().ConfigureAwait(false));
}
public async Task InsertAsync(StatusDataType dataType, DateTime date, string content)
{
var container = await GetContainerAsync();
var typeNameV2 = dataType == StatusDataType.InternetExplorer ? "Edge" : dataType.ToString();
var data = Encoding.UTF8.GetBytes(content);
await container.GetBlockBlobReference($"{typeNameV2}/{date.ToString("yyyyMMdd")}.json").UploadFromByteArrayAsync(data, 0, data.Length).ConfigureAwait(false);
}
public ChangeSetEntity(StatusDataType dataType, DateTime date, String changeSetContent, DateTime from)
{
PartitionKey = "01_" + ((int)dataType).ToString();
RowKey = "01_" + (ConvertDateTimeToRowKey(date.Date)); // Date only
DataType = (int)dataType;
Date = date.Date;
Content = LZ4Codec.Wrap(Encoding.UTF8.GetBytes(changeSetContent));
From = from;
UpdatedAt = DateTime.UtcNow;
}
private static async Task<ChangeSet[]> GetChangeSetsByBrowserAsync(IStatusDataRepository statusDataRepository, StatusDataType type)
{
var statusDataSet = (await statusDataRepository.GetPlatformStatusesRangeAsync(type, DateTime.UtcNow.AddMonths(-6), DateTime.UtcNow, take: 30))
.OrderByDescending(x => x.Date)
.ToArray();
if (statusDataSet.Length < 2)
{
return new ChangeSet[0];
}
return ChangeSet.GetChangeSetsFromPlatformStatuses(statusDataSet);
}
private static async Task<ChangeSet[]> GetChangeSetsByBrowser(IStatusDataRepository statusDataRepository, StatusDataType type, DateTime date)
{
var statusDataSet = (await Utility.Measure(() => statusDataRepository.GetPlatformStatusesRangeAsync(type, date.AddMonths(-6), date, take: 2)))
.OrderByDescending(x => x.Date)
.Take(2)
.ToArray();
if (statusDataSet.Length < 2)
{
return new ChangeSet[0];
}
return ChangeSet.GetChangeSetsFromPlatformStatuses(statusDataSet);
}
public async Task <ChangeSet> GetLatestChangeSetAsync(StatusDataType type)
{
var table = await CreateOrGetTableAsync();
var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_*
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (int)type)
);
return((await table.ExecuteQuerySegmentedAsync(new TableQuery <ChangeSetEntity>().Where(queryFilter).Take(1), null))
.Select(x => x.ToChangeSet())
.FirstOrDefault());
}
public static IPlatformStatus[] Deserialize(StatusDataType dataType, string jsonValue)
{
switch (dataType)
{
case StatusDataType.Chromium: return(DeserializeForChromiumStatus(jsonValue));
case StatusDataType.InternetExplorer: return(DeserializeForIeStatus(jsonValue));
case StatusDataType.WebKitWebCore: return(DeserializeForWebKitStatus(jsonValue));
case StatusDataType.WebKitJavaScriptCore: return(DeserializeForWebKitStatus(jsonValue));
case StatusDataType.Mozilla: return(DeserializeForMozillaStatus(jsonValue));
default: throw new NotSupportedException();
}
}
public async Task<PlatformStatuses[]> GetPlatformStatusesRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, Int32 take = 30)
{
if (fromDate > toDate)
throw new ArgumentException("fromDate parameter must be before toDate.");
var table = await CreateOrGetTable();
var queryFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "02_"); // 01_*
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "01_" + (Int32)type)
);
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, "01_" + ConvertDateTimeToRowKey(fromDate.Date)) // RowKey <= fromDate.Date (Higher RowKey is older)
);
queryFilter = TableQuery.CombineFilters(
queryFilter,
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, "01_" + ConvertDateTimeToRowKey(toDate.Date)) // RowKey >= toDate.Date (Higher RowKey is older)
);
var entities = await Utility.Measure(() => table.ExecuteQuerySegmentedAsync(new TableQuery<StatusDataEntity>().Where(queryFilter).Take(take), null));
var tasks = entities.Select(x =>
{
// return null if JSON data was broken.
try
{
return CreatePlatformStatusesFromEntityAsync(x);
}
catch (JsonReaderException)
{
return null;
}
})
.Where(x => x != null)
.ToArray();
return await Task.WhenAll(tasks);
}
public async Task InsertOrReplaceAsync(StatusDataType dataType, DateTime date, string changeSetContent, DateTime from)
{
var table = await CreateOrGetTableAsync();
await table.ExecuteAsync(TableOperation.InsertOrReplace(new ChangeSetEntity(dataType, date, changeSetContent, from))).ConfigureAwait(false);
}
private static async Task <ChangeSet[]> GetChangeSetsByBrowserAsync(IChangeSetRepository changeSetRepository, StatusDataType type)
{
return((await changeSetRepository.GetChangeSetsRangeAsync(type, DateTime.UtcNow.AddMonths(-6), DateTime.UtcNow, take: 30))
.OrderByDescending(x => x.Date)
.ToArray());
}
public async Task UpdateRange([FromService] DataUpdateAgent agent, [Option('t')] StatusDataType statusDataType, DateTime from, DateTime to)
{
await agent.UpdateChangeSetByRangeAsync(statusDataType, from, to);
}
private static async Task <ChangeSet[]> GetChangeSetsByBrowser(IChangeSetRepository changeSetRepository, StatusDataType type, DateTime date)
{
return((await Utility.Measure(() => changeSetRepository.GetChangeSetsRangeAsync(type, date.AddMonths(-6), date, take: 1)))
.OrderByDescending(x => x.Date)
.ToArray());
}
public StatusDataEntity(StatusDataType dataType, DateTime date, String content)
{
this.PartitionKey = "01_" + ((Int32)dataType).ToString();
this.RowKey = "01_" + (ConvertDateTimeToRowKey(date.Date)); // Date only
this.DataType = (Int32)dataType;
this.Date = date;
this.DataCompression = (Int32)DataCompressionType.Bzip2;
var splitSize = (1024 * 50);
var compressedContent = Compress(Encoding.UTF8.GetBytes(content));
for (var i = 0; i < Math.Ceiling(compressedContent.Length / (double)splitSize); i++)
{
this.SplitCount = i;
var tmpContent = new byte[Math.Min(splitSize, compressedContent.Length - (i * splitSize))];
Array.Copy(compressedContent, (i * splitSize), tmpContent, 0, tmpContent.Length);
switch (i)
{
case 0:
Content = tmpContent;
break;
case 1:
Content1 = tmpContent;
break;
case 2:
Content2 = tmpContent;
break;
case 3:
Content3 = tmpContent;
break;
case 4:
Content4 = tmpContent;
break;
default:
throw new Exception("data size is too large.: " + compressedContent.Length);
}
}
}
public async Task<PlatformStatuses[]> GetPlatformStatusesRangeAsync(StatusDataType type, DateTime fromDate, DateTime toDate, Int32 take)
{
return new[]
{
new PlatformStatuses(DateTime.Parse("2015/1/1"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")),
new PlatformStatuses(DateTime.Parse("2015/1/2"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")),
new PlatformStatuses(DateTime.Parse("2015/1/3"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Under Consideration"",""iePrefixed"":"""",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")),
new PlatformStatuses(DateTime.Parse("2015/1/4"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""In Development"",""iePrefixed"":"""",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")),
new PlatformStatuses(DateTime.Parse("2015/1/5"), PlatformStatuses.DeserializeForIeStatus(@"[{""name"":""A"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""10"",""ieUnprefixed"":""11""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}, {""name"":""B"",""category"":""JavaScript"",""link"":""http://www.example.com/"",""summary"":""a summary of the status"",""standardStatus"":""Editor's Draft"",""ieStatus"":{""text"":""Shipped"",""iePrefixed"":""99"",""ieUnprefixed"":""""},""msdn"":"""",""wpd"":"""",""demo"":"""",""id"":5681726336532480}]")),
};
}
public async Task InsertAsync(StatusDataType type, DateTime date, String jsonData)
{
// 何もしない
}
public async Task InsertAsync(StatusDataType type, DateTime date, String jsonData)
{
var table = await CreateOrGetTable();
await table.ExecuteAsync(TableOperation.InsertOrReplace(new StatusDataEntity(type, date, jsonData)));
}
