public static string[] GetLocalizationInfo(string repoName, Dictionary <string, LocRepoFromAmbientConfiguration> format)
{
string[] ret = new string[2];
string repository = "", locale = "";
//special case of wrong spelling
if (repoName.Equals("azure-content-zhcn-pb"))
{
repository = "azure-content-pr";
locale = "zhcn";
ret[0] = repository;
ret[1] = locale;
return(ret);
}
bool flag = false;
foreach (KeyValuePair <string, LocRepoFromAmbientConfiguration> item in format)
{
repository = item.Key;
LocRepoFromAmbientConfiguration value = item.Value;
if (value != null && !string.IsNullOrEmpty(value.repoNameRegex) && !string.IsNullOrEmpty(value.localeRegex) && value.localStartIndex >= 0 && value.localLength > 0)
{
Regex repoNameReg = new Regex(value.repoNameRegex);
if (repoName.Equals(repository, StringComparison.OrdinalIgnoreCase) || repoNameReg.IsMatch(repoName))
{
flag = true;
if (repoName.Equals(repository, StringComparison.OrdinalIgnoreCase))
{
locale = "en-us";
}
else
{
Regex localeReg = new Regex(value.localeRegex);
locale = localeReg.Match(repoName).Groups[0].Value.Substring(value.localStartIndex, value.localLength);
}
break;
}
}
}
//For repos not appearing in the ambient configuration file, we use dot to judge whether localization or not.
if (!flag)
{
if (new Regex("[\\S]*\\.[a-z]{2}-[a-z]{2}").IsMatch(repoName))
{
locale = repoName.ToLower().Substring(repoName.LastIndexOf('.') + 1);
repository = repoName.Substring(0, repoName.LastIndexOf('.'));
}
else
{
locale = "en-us";
repository = repoName;
}
}
ret[0] = repository;
ret[1] = locale;
return(ret);
}
public static string ComposeLocalizationRepoName(string repoName, string locale, Dictionary <string, LocRepoFromAmbientConfiguration> format)
{
LocRepoFromAmbientConfiguration value = format[repoName];
if (value.targetRepo == null)
{
return(null);
}
LocRepo targetRepo = value.targetRepo;
if (string.IsNullOrEmpty(targetRepo.Name))
{
return(repoName + "." + locale);
}
else
{
locale = locale.Replace("-", "");
return(targetRepo.Name.Replace("<target_locale_without_dash>", locale));
}
}
protected override void Load(object obj)
{
if (obj == null)
{
return;
}
List <GitHubRepository> repos = obj as List <GitHubRepository>;
Dictionary <string, LocRepoFromAmbientConfiguration> repoInfo = Util.GetRepoInfoFromAmbientConfiguration();
using (DataTable dt = new DataTable(), dt1 = new DataTable())
{
dt.Columns.Add("RepositoryId");
dt.Columns.Add("GitRepositoryAccount");
dt.Columns.Add("GitRepositoryName");
dt.Columns.Add("GitRepositoryUrl");
dt.Columns.Add("CreatedAt");
dt.Columns.Add("Repository");
dt.Columns.Add("IsLocalization");
dt.Columns.Add("Locale");
foreach (GitHubRepository repo in repos)
{
int isLocalization = 1;
string repository = "", locale = "";
string[] localeInfo = Util.GetLocalizationInfo(repo.RepositoryName, repoInfo);
if (localeInfo != null && localeInfo.Length == 2 && !string.IsNullOrEmpty(localeInfo[0]) && !string.IsNullOrEmpty(localeInfo[1]))
{
repository = localeInfo[0];
locale = localeInfo[1];
}
if (locale.Equals("en-us"))
{
isLocalization = 0;
repo.IsLocalization = false;
}
if (locale.Length == 4)
{
locale = locale.Insert(2, "-");
}
dt.Rows.Add(repo.PartitionKey, repo.Owner, repo.RepositoryName, repo.RepositoryUrl, repo.Timestamp, repository, isLocalization, locale);
}
//Add localization repositories that don't have en-us repository.
dt1.Columns.Add("RepositoryId");
dt1.Columns.Add("GitRepositoryAccount");
dt1.Columns.Add("GitRepositoryName");
dt1.Columns.Add("GitRepositoryUrl");
dt1.Columns.Add("CreatedAt");
dt1.Columns.Add("Repository");
dt1.Columns.Add("IsLocalization");
dt1.Columns.Add("Locale");
string token = Util.GetGithubToken("MIMDocs-pr.cs-cz");
string[] repositoryException = configManager.GetConfig("BackendJobs", "RepositoryException").Split(';');
InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr);
DataRow[] dataRow = InsightDBHelper.InsightDBHelper.ExecuteQuery(
string.Format("SELECT DISTINCT GitRepositoryName FROM OPS_Repositories_LocOnly WITH(NOLOCK)"), true);
HashSet <string> locRepoSet = new HashSet <string>();
if (dataRow != null && dataRow.Length > 0)
{
foreach (DataRow row in dataRow)
{
locRepoSet.Add(row.ItemArray[0].ToString());
}
}
foreach (string repo in repositoryException)
{
LocRepoFromAmbientConfiguration value = repoInfo[repo];
if (value.sourceRepo == null || value.targetRepo == null)
{
continue;
}
LocRepo sourceRepo = value.sourceRepo;
LocRepo targetRepo = value.targetRepo;
string[] items = Util.GetLocalizationList(repo, "master", token, repoInfo);
if (items == null || items.Length == 0)
{
continue;
}
foreach (string locale in items)
{
string localizatonRepoName = Util.ComposeLocalizationRepoName(repo, locale, repoInfo);
if (locRepoSet.Count == 0 || !locRepoSet.Contains(localizatonRepoName))
{
string repoId = System.Guid.NewGuid().ToString("D");
string remoteUrl = sourceRepo.RemoteUrl;
string repoUrl = remoteUrl.Substring(0, remoteUrl.LastIndexOf("/")) + "/" + localizatonRepoName;
dt1.Rows.Add(repoId, targetRepo.Owner, localizatonRepoName, repoUrl, "", repo, 1, locale);
}
}
}
InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr);
Dictionary <string, DataTable> paramDic1 = new Dictionary <string, DataTable>();
paramDic1.Add("GitRepoTableParam?GitRepoType", dt1);
InsightDBHelper.InsightDBHelper.ExecuteSP("SP_OPSStoreGitRepoLocOnly", paramDic1);
Dictionary <string, DataTable> paramDic = new Dictionary <string, DataTable>();
paramDic.Add("GitRepoTableParam?GitRepoType", dt);
InsightDBHelper.InsightDBHelper.ExecuteSP("SP_OPSStoreGitRepo", paramDic, true, 600);
}
// Get prod repo
InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr);
DataRow[] prodRepoRow = InsightDBHelper.InsightDBHelper.ExecuteQuery("SELECT DISTINCT RepositoryId FROM OPS_Repositories_live WITH (NOLOCK)", true);
if (prodRepoRow == null || prodRepoRow.Length == 0)
{
return;
}
List <DataRow> prodRepoList = prodRepoRow.ToList();
List <GitHubRepository> prod_all_tmp = repos.Where(v => prodRepoList.Exists(t => t.ItemArray[0].ToString().Equals(v.PartitionKey, StringComparison.OrdinalIgnoreCase))).ToList();
string testRepositoryIdString = configManager.GetConfig("BackendJobs", "TestRepositoryId");
string[] testIdList = testRepositoryIdString.Split(';');
//Add two test repositoryIds
prod_all_tmp.AddRange(repos.Where(v => Array.Exists(testIdList, t => t.Equals(v.PartitionKey, StringComparison.OrdinalIgnoreCase))).ToList());
SharedObject_Prod_All = prod_all_tmp;
SharedObject_Prod_GitHub = prod_all_tmp.Where(v => v.GitRepositoryType.Equals("GitHub", StringComparison.OrdinalIgnoreCase)).ToList();
SharedObject_Prod_VSO = prod_all_tmp.Where(v => v.GitRepositoryType.Equals("Vso", StringComparison.OrdinalIgnoreCase)).ToList();
return;
}
public static string[] GetLocalizationList(string repoName, string branch, string token, Dictionary <string, LocRepoFromAmbientConfiguration> format)
{
LocRepoFromAmbientConfiguration value = format[repoName];
if (value.sourceRepo == null || value.handbackRepo == null)
{
return(null);
}
LocRepo sourceRepo = value.sourceRepo;
LocRepo handbackRepo = value.handbackRepo;
//root folder of source repo
string localConfigUrl = "https://raw.githubusercontent.com/" + sourceRepo.Owner + "/" + repoName + "/" + branch + "/.localization-config";
JavaScriptSerializer j = new JavaScriptSerializer();
Regex reg = new Regex("\"[a-z]{2}-[a-z]{2}\"");
if (repoName.Equals("smbmadeira-content-pr", StringComparison.OrdinalIgnoreCase))
{
reg = new Regex("[a-z]{2}-[A-Z]{2}");
}
try
{
string jsonContent = Util.GetJsonContent(localConfigUrl, token);
//Many errors in json files of different repos, here we directly parser the string.
MatchCollection mc = reg.Matches(jsonContent);
string[] ret = new string[mc.Count];
for (int i = 0; i < ret.Length; i++)
{
ret[i] = mc[i].Value.Substring(1, 5);
}
return(ret.Distinct().ToArray());
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
try
{
//ol-config/<source-repo-owner>/<source-repo-name>/<source-repo-branch>/.localization-config
localConfigUrl = "https://raw.githubusercontent.com/" + handbackRepo.Owner + "/" + handbackRepo.Name + "/master/ol-config/"
+ sourceRepo.Owner + "/" + repoName + "/" + branch + "/.localization-config";
string jsonContent = Util.GetJsonContent(localConfigUrl, token);
//Many errors in json files of different repos, here we directly parser the string.
MatchCollection mc = reg.Matches(jsonContent);
string[] ret = new string[mc.Count];
for (int i = 0; i < ret.Length; i++)
{
if (repoName.Equals("smbmadeira-content-pr", StringComparison.OrdinalIgnoreCase))
{
ret[i] = mc[i].Value.ToLower();
}
else
{
ret[i] = mc[i].Value.Substring(1, 5);
}
}
return(ret.Distinct().ToArray());
}
catch (Exception)
{
return(null);
}
}
throw ex;
}
}
protected override object Extract()
{
List <GitRepoTopicLocalizationStatus> ret = new List <GitRepoTopicLocalizationStatus>();
InsightDBHelper.InsightDBHelper.ConnectDBWithConnectString(OPSDataSyncConnStr);
DataRow[] dataRow = InsightDBHelper.InsightDBHelper.ExecuteQuery("SELECT * FROM (SELECT RepositoryId, GitRepositoryName from OPS_Repositories_LocOnly WITH (NOLOCK)) Ex UNION "
+ "(SELECT R.RepositoryId, GitRepositoryName FROM OPS_Repositories R INNER JOIN OPS_Repositories_live L WITH (NOLOCK) "
+ "ON R.RepositoryId = L.RepositoryId WHERE IsLocalization = 1)", true);
//Key:Localization Repo Name; Value: Repo ID
Dictionary <string, string> dic = new Dictionary <string, string>();
Dictionary <string, LocRepoFromAmbientConfiguration> format = Util.GetRepoInfoFromAmbientConfiguration();
if (dataRow != null && dataRow.Length > 0)
{
foreach (DataRow row in dataRow)
{
if (!dic.ContainsKey(row.ItemArray[0].ToString()))
{
dic.Add(row.ItemArray[1].ToString(), row.ItemArray[0].ToString());
}
}
}
string token = Util.GetGithubToken("MIMDocs-pr.cs-cz");
foreach (KeyValuePair <string, LocRepoFromAmbientConfiguration> pair in format)
{
string repoName = pair.Key;
LocRepoFromAmbientConfiguration value = pair.Value;
if (value.sourceRepo == null || value.sourceRepo == null)
{
continue;
}
if (repoName.Equals("win-cpub-itpro-docs"))
{
token = Util.GetGithubToken("win-cpub-itpro-docs");
}
//Get locale list
string[] masterLocales = Util.GetLocalizationList(repoName, "master", token, format);
string[] liveLocales = Util.GetLocalizationList(repoName, "live", token, format);
//master branch
if (masterLocales != null && masterLocales.Length != 0)
{
foreach (string formatLocale in masterLocales)
{
string localizatonRepoName = Util.ComposeLocalizationRepoName(repoName, formatLocale, format);
if (dic.ContainsKey(localizatonRepoName))
{
string[] localizationInfo = Util.GetLocalizationInfo(localizatonRepoName, format);
if (localizationInfo == null || localizationInfo.Length != 2 || string.IsNullOrEmpty(localizationInfo[0]) || string.IsNullOrEmpty(localizationInfo[1]))
{
continue;
}
string locale = localizationInfo[1];
string repoId = dic[localizatonRepoName];
ret.AddRange(ParseOLData(repoId, localizatonRepoName, "master", locale, token, value));
}
}
}
//live branch
if (liveLocales != null && liveLocales.Length != 0)
{
foreach (string formatLocale in liveLocales)
{
string localizatonRepoName = Util.ComposeLocalizationRepoName(repoName, formatLocale, format);
if (dic.ContainsKey(localizatonRepoName))
{
string[] localizationInfo = Util.GetLocalizationInfo(localizatonRepoName, format);
if (localizationInfo == null || localizationInfo.Length != 2 || string.IsNullOrEmpty(localizationInfo[0]) || string.IsNullOrEmpty(localizationInfo[1]))
{
continue;
}
string locale = localizationInfo[1];
string repoId = dic[localizatonRepoName];
ret.AddRange(ParseOLData(repoId, localizatonRepoName, "live", locale, token, value));
}
}
}
}
return(ret);
}
private List <GitRepoTopicLocalizationStatus> ParseOLData(string repoId, string repoName, string branch, string locale, string token, LocRepoFromAmbientConfiguration value)
{
List <GitRepoTopicLocalizationStatus> ret = new List <GitRepoTopicLocalizationStatus>();
LocRepo targetRepo = value.targetRepo;
LocRepo handbackRepo = value.handbackRepo;
JavaScriptSerializer j = new JavaScriptSerializer();
j.MaxJsonLength = int.MaxValue;
string url = "https://raw.githubusercontent.com/" + targetRepo.Owner + "/" + repoName + "/" + branch + "/.translation-state";
try
{
string jsonContent = Util.GetJsonContent(url, token);
var pair = j.DeserializeObject(jsonContent) as IDictionary <string, object>;
ParseJsonContent(ret, pair, repoId, locale, branch);
return(ret);
//#region TO DO: Change to use get data API
//// 1. Get all commits for repositoty
//var modelSHAs = new List<string>();
//var commits = await github.Repository.Commits.GetAll(repository.Owner.Login, repository.Name);
//foreach (var commit in commits)
//{
// modelSHAs.Add(commit.Sha);
//}
//// 2. Get all references
//var refs = github.GitDatabase.Reference.GetAll(repository.Owner.Login, repository.Name).Result;
//if (null == refs || 0 == refs.Count)
// return ret;
//var currentRefSet = refs.Where(x => x.Ref.EndsWith(branch, StringComparison.OrdinalIgnoreCase));
//if (null == currentRefSet || 0 == currentRefSet.Count())
// return ret;
//var currentRef = currentRefSet.ElementAt(0).Object.Sha;
//// 3. Get content using get data API
//// currentRef is not correct in this scenario. We must find the SHA of the blob. Actually, the SHA will be returned in API github.Repository.Content.GetAllContents(string, string, string, string).
//var blobContent = github.GitDatabase.Blob.Get(repository.Owner.Login, repository.Name, currentRef).Result;
//var convertedContents = System.Text.Encoding.Default.GetString(Convert.FromBase64String(blobContent.Content));
//#endregion
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
try
{
//ol-handback/<taget-repo-owner>/<target-repo-name>/<target-repo-branch>/olsysfiles/.translation-state
url = "https://raw.githubusercontent.com/" + handbackRepo.Owner + "/" + handbackRepo.Name + "/" + branch + "/ol-handback/"
+ targetRepo.Owner + "/" + repoName + "/" + targetRepo.WorkingBranch + "/olsysfiles/.translation-state";
string jsonContent = Util.GetJsonContent(url, token);
var pair = j.DeserializeObject(jsonContent) as IDictionary <string, object>;
ParseJsonContent(ret, pair, repoId, locale, branch);
return(ret);
}
catch (Exception innerEx)
{
if (innerEx.Message.Contains("404"))
{
return(ret);
}
throw innerEx;
}
}
throw ex;
}
}