public static DtoAccount Obtain()
{
if (!bag.TryTake(out var obj))
{
obj = new DtoAccount();
}
return(obj);
}
public static void Release(DtoAccount obj)
{
obj.Reset();
bag.Add(obj);
}
// using low-level to catch wrong structure or properties
// ids and likes are internalized
public static bool Parse(ref JsonReader reader, DtoAccount dto, Storage store)
{
var count = 0;
if (reader.ReadIsNull() || !reader.ReadIsBeginObject())
{
return(false);
}
while (true)
{
var prop = reader.ReadPropertyNameSegmentRaw();
// id
if (prop.EqualTo(s_id))
{
if (reader.GetCurrentJsonToken() != JsonToken.Number)
{
return(false);
}
dto.id = Mapper.ExtIdToIntIdCreate(reader.ReadInt32());
dto.flags |= DtoFlags.Id;
}
else
// status
if (prop.EqualTo(s_status))
{
var st = new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape();
if (st == s_Free)
{
dto.status = STATUS_FREE;
}
else
if (st == s_Taken)
{
dto.status = STATUS_TAKEN;
}
else
if (st == s_Complicated)
{
dto.status = STATUS_COMPLICATED;
}
else
{
return(false);
}
dto.flags |= DtoFlags.Status;
}
else
// sex
if (prop.EqualTo(s_sex))
{
var st = reader.ReadStringSegmentRaw();
if (st.EqualTo(s_male))
{
dto.sex = SEX_MALE;
}
else
if (st.EqualTo(s_female))
{
dto.sex = SEX_FEMALE;
}
else
{
return(false);
}
dto.flags |= DtoFlags.Sex;
}
else
// email
if (prop.EqualTo(s_email))
{
dto.email = reader.ReadStringSegmentRaw();
dto.flags |= DtoFlags.Email;
}
else
// phone
if (prop.EqualTo(s_phone))
{
dto.phone = new AString(reader.ReadStringSegmentRaw()).Duplicate();
dto.flags |= DtoFlags.Phone;
}
else
// fname
if (prop.EqualTo(s_fname))
{
dto.fnameIdx = (short)store.Fnames.GetOrCreateRange(
new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index;
dto.flags |= DtoFlags.Fname;
}
else
// sname
if (prop.EqualTo(s_sname))
{
dto.snameIdx = (short)store.Snames.GetOrCreateRange(
new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index;
dto.flags |= DtoFlags.Sname;
}
else
// country
if (prop.EqualTo(s_country))
{
dto.countryIdx = (byte)store.Countries.GetOrCreateRange(
new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index;
dto.flags |= DtoFlags.Country;
}
else
// city
if (prop.EqualTo(s_city))
{
dto.cityIdx = (short)store.Cities.GetOrCreateRange(
new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index;
dto.flags |= DtoFlags.City;
}
else
// birth
if (prop.EqualTo(s_birth))
{
if (reader.GetCurrentJsonToken() != JsonToken.Number)
{
return(false);
}
dto.birth = reader.ReadInt32();
dto.flags |= DtoFlags.Birth;
}
else
// joined
if (prop.EqualTo(s_joined))
{
if (reader.GetCurrentJsonToken() != JsonToken.Number)
{
return(false);
}
dto.joined = reader.ReadInt32();
dto.flags |= DtoFlags.Joined;
}
else
// premium
if (prop.EqualTo(s_premium))
{
if (!DtoPremium.Parse(ref reader, ref dto.premium))
{
return(false);
}
dto.flags |= DtoFlags.Premium;
}
else
// interests
if (prop.EqualTo(s_interests))
{
if (!reader.ReadIsBeginArray())
{
return(false);
}
while (!reader.ReadIsEndArrayWithSkipValueSeparator(ref count))
{
dto.interests.Set(store.Interests.GetOrCreateRange(new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index);
}
dto.flags |= DtoFlags.Interests;
}
else
// likes
if (prop.EqualTo(s_likes))
{
if (!reader.ReadIsBeginArray())
{
return(false);
}
// read likes
if (reader.GetCurrentJsonToken() == JsonToken.BeginObject)
{
while (true)
{
Like like = new Like();
if (!reader.ReadIsBeginObject())
{
return(false);
}
while (true)
{
var propName = reader.ReadPropertyNameSegmentRaw();
if (propName[0] == (byte)'t') // ts
{
if (reader.GetCurrentJsonToken() != JsonToken.Number)
{
return(false);
}
like.ts = Like.ExtToIntTS(reader.ReadInt32());
}
else
if (propName[0] == (byte)'i') // id
{
if (reader.GetCurrentJsonToken() != JsonToken.Number)
{
return(false);
}
if (!Mapper.ExtIdToIntId(reader.ReadInt32(), out like.id))
{
return(false);
}
}
else // not ts nor id
{
return(false);
}
if (!reader.ReadIsValueSeparator())
{
break;
}
}
if (!reader.ReadIsEndObject())
{
return(false);
}
dto.likes.Add(like);
if (!reader.ReadIsValueSeparator())
{
break;
}
}
}
if (!reader.ReadIsEndArray())
{
return(false);
}
dto.flags |= DtoFlags.Likes;
}
else
{
return(false);
}
if (!reader.ReadIsValueSeparator())
{
break;
}
}
if (!reader.ReadIsEndObject())
{
return(false);
}
return(true);
}
public int ValidateUpdateAccount(DtoAccount dto)
{
var id = dto.id;
if (id == 17579)
{
}
// validate first, starting from fastest checks, and only after the validation create an account
if (id < 0 || id >= MAX_ACCOUNTS)
{
return(404);
}
// phone (if provided) contains area code
if (dto.flags.HasFlag(DtoFlags.Phone) && dto.phone != null)
{
var openBrace = dto.phone.IndexOf('(');
var closeBrace = dto.phone.IndexOf(')');
if (openBrace < 0 || closeBrace != openBrace + 4)
{
return(400);
}
}
// joined within range
if (dto.flags.HasFlag(DtoFlags.Joined) && (Utils.TimestampToDate(dto.joined) < MinJoined || Utils.TimestampToDate(dto.joined) > MaxJoined))
{
return(400);
}
// premium
if (dto.flags.HasFlag(DtoFlags.Premium))
{
if (dto.premium.start > 0 && Utils.TimestampToDate(dto.premium.start) < MinPremium ||
dto.premium.finish > 0 && Utils.TimestampToDate(dto.premium.finish) < MinPremium)
{
return(400);
}
}
// the rest requires locking
bool lockTaken = false;
try
{
updateLock.Enter(ref lockTaken);
// check if the account exists
if (!All[id])
{
return(404);
}
// likes
if (dto.flags.HasFlag(DtoFlags.Likes))
{
if (!verifyLikes(dto.likes))
{
return(400);
}
}
var acct = Accounts[id];
// update email and domain
if (dto.flags.HasFlag(DtoFlags.Email))
{
if (dto.email.IsEmpty || dto.email.Length > 100)
{
return(400);
}
if (!bufferFromEmail(dto.email, out var intEmail))
{
return(400);
}
// check for duplicates
if (Emails.Contains(intEmail) &&
!ByteArrayComparer.Instance.Equals(acct.Email, intEmail))
{
return(400); // such email exists and it's not ours
}
// unregister old email
Domains[acct.GetDomainIdx()].Exclude(id);
Emails.Remove(acct.Email);
// store and register new email
acct.Email = intEmail;
Emails.Add(acct.Email);
Domains[acct.GetDomainIdx()].Include(id);
}
// store new account info
Accounts[id] = acct;
}
finally
{
if (lockTaken)
{
updateLock.Exit();
}
}
return(202);
}
public void PostUpdateAccount(DtoAccount dto)
{
var id = dto.id;
if (id == 17579)
{
}
var acct = Accounts[id];
// sex
if (dto.flags.HasFlag(DtoFlags.Sex))
{
if (dto.sex)
{
acct.Flags |= Account.Male;
}
else
{
acct.Flags &= (byte)~Account.Male;
}
}
// status
if (dto.flags.HasFlag(DtoFlags.Status))
{
bool isFree = dto.status == DtoAccount.STATUS_FREE;
bool isTaken = dto.status == DtoAccount.STATUS_TAKEN;
bool isComplicated = dto.status == DtoAccount.STATUS_COMPLICATED;
// clear old flags
acct.Flags &= (byte)~(Account.Free | Account.Taken);
// add new flags
if (isFree)
{
acct.Flags |= Account.Free;
}
else
if (isTaken)
{
acct.Flags |= Account.Taken;
}
else
{
acct.Flags |= (byte)(Account.Free | Account.Taken);
}
}
// birth date
if (dto.flags.HasFlag(DtoFlags.Birth))
{
// exclude from old range
if (acct.BirthIdx > 0)
{
BirthYears[acct.BirthIdx].Exclude(id);
}
// store new birthday
acct.Birth = dto.birth;
// include into new range
acct.BirthIdx = (byte)BirthYears.GetOrCreateRangeThenInclude(
Utils.TimestampToDate(acct.Birth).Year, id).Index;
}
// joined date
if (dto.flags.HasFlag(DtoFlags.Joined))
{
// exclude from old range
if (acct.JoinedIdx > 0)
{
JoinYears[acct.JoinedIdx].Exclude(id);
}
// include into new range
acct.JoinedIdx = (byte)JoinYears.GetOrCreateRangeThenInclude(
Utils.TimestampToDate(dto.joined).Year, id).Index;
}
// premium
if (dto.flags.HasFlag(DtoFlags.Premium))
{
acct.PStart = dto.premium.start;
acct.PFinish = dto.premium.finish;
bool premiumNow = acct.PStart <= Now && acct.PFinish >= Now;
if (premiumNow)
{
acct.Flags |= Account.Premium;
}
else
{
acct.Flags &= (byte)~Account.Premium;
}
}
// fname
if (dto.flags.HasFlag(DtoFlags.Fname))
{
// exclude from old range
if (acct.FNameIdx > 0)
{
Fnames[acct.FNameIdx].Exclude(id);
}
acct.FNameIdx = dto.fnameIdx;
if (acct.FNameIdx > 0)
{
Fnames[acct.FNameIdx].Include(id);
}
}
// sname
if (dto.flags.HasFlag(DtoFlags.Sname))
{
// exclude from old range
if (acct.SNameIdx != 0)
{
var entry = Snames[acct.SNameIdx];
Snames2.GetOrCreateRange(entry.AName[0] + (entry.AName[1] << 8)).Exclude(id);
Snames2.GetOrCreateRange(entry.AName[0] + (entry.AName[1] << 8) + (entry.AName[2] << 16) + (entry.AName[3] << 24)).Exclude(id);
}
acct.SNameIdx = dto.snameIdx;
if (acct.SNameIdx > 0)
{
var entry = Snames[acct.SNameIdx];
Snames2.GetOrCreateRangeThenInclude(entry.AName[0] + (entry.AName[1] << 8), id);
Snames2.GetOrCreateRangeThenInclude(entry.AName[0] + (entry.AName[1] << 8) + (entry.AName[2] << 16) + (entry.AName[3] << 24), id);
}
}
// phone
if (dto.flags.HasFlag(DtoFlags.Phone))
{
// exclude from old range
var oldAreaCode = areaCodeFromPhone(new AString(acct.Phone));
if (!oldAreaCode.IsEmpty)
{
AreaCodes.GetOrCreateRange(oldAreaCode).Exclude(id);
}
// update the phone number
acct.Phone = dto.phone.Buffer;
// include into new range
var newAreaCode = areaCodeFromPhone(dto.phone);
if (!newAreaCode.IsEmpty)
{
AreaCodes.GetOrCreateRange(newAreaCode).Include(id);
}
}
// country
if (dto.flags.HasFlag(DtoFlags.Country))
{
// exclude from old range
if (acct.CountryIdx > 0)
{
Countries[acct.CountryIdx].Exclude(id);
}
acct.CountryIdx = dto.countryIdx;
// include into new range
if (acct.CountryIdx > 0)
{
Countries[acct.CountryIdx].Include(id);
}
}
// city
if (dto.flags.HasFlag(DtoFlags.City))
{
// exclude from old range
if (acct.CityIdx > 0)
{
Cities[acct.CityIdx].Exclude(id);
}
acct.CityIdx = dto.cityIdx;
// include into new range
if (acct.CityIdx > 0)
{
Cities[acct.CityIdx].Include(id);
}
}
// interests
if (dto.flags.HasFlag(DtoFlags.Interests))
{
// exclude old interests
for (int i = 1; i < BitMap96.MAX_BITS; i++)
{
if (acct.InterestMask.IsSet(i))
{
Interests[i].Exclude(id);
}
}
acct.InterestMask = dto.interests;
// include new interests
for (int i = 1; i < BitMap96.MAX_BITS; i++)
{
if (acct.InterestMask.IsSet(i))
{
Interests[i].Include(id);
}
}
}
// unlink old likes
if (dto.flags.HasFlag(DtoFlags.Likes))
{
if (acct.LikesCount > 0)
{
Log.Error("Replacing existing likes on update");
}
if (dto.likes != null)
{
addLikes(id, ref acct, dto.likes);
}
}
// uncount old record from cube groups
updateGroups(ref Accounts[id], false);
// store new account info
Accounts[id] = acct;
// count new record in cube groups
updateGroups(ref acct, true);
}
// synchronously process the request from requestBuffer, and return statusCode
public int Process(HttpCtx ctx, int extId)
{
var startTime = Stats.Watch.Elapsed;
// query sanity check
foreach (var query in ctx.Params)
{
var value = query.Value;
if (value.IsEmpty)
{
return(400);
}
if (query.Key != "query_id")
{
// all other parameters are invalid
return(400);
}
}
// translate to internal id and check for account existance
if (!Mapper.ExtIdToIntId(extId, out int id) || !store.All[id])
{
return(404);
}
// parse Json
var dto = DtoAccount.Obtain();
dto.id = id;
int statusCode = 400;
try
{
JsonReader reader = new JsonReader(ctx.Buffer, ctx.RequestBodyStart);
if (DtoAccount.Parse(ref reader, dto, store) && dto.flags != 0)
{
// update the account, register post-process action
statusCode = store.ValidateUpdateAccount(dto);
if (statusCode == 202)
{
ctx.PostAction = () =>
{
store.PostUpdateAccount(dto);
DtoAccount.Release(dto);
}
}
;
}
}
catch (Exception)
{
// fall through
}
// return the borrowed object
if (ctx.PostAction == null)
{
DtoAccount.Release(dto);
}
var stopTime = Stats.Watch.Elapsed;
ctx.ContextType = "PostUpdate";
return(statusCode);
}
// zip loader, called from Main() before the web server starts
public static void LoadFromZip(string path, Storage store)
{
// load options
var optionsFile = path + "/data/options.txt";
if (File.Exists(optionsFile))
{
Log.Info("Opening " + optionsFile);
using (StreamReader r = new StreamReader(File.OpenRead(optionsFile)))
{
store.Now = int.Parse(r.ReadLine());
store.IsRatingRun = int.Parse(r.ReadLine());
}
}
else
{
Log.Error(optionsFile + " is not found");
}
// load zipped data
var zipFile = path + "/data/data.zip";
if (File.Exists(zipFile))
{
Log.Info("Opening " + zipFile);
var totalAccounts = 0;
var errorAccounts = 0;
var fileBuffer = new byte[20000000];
using (var file = File.OpenRead(zipFile))
using (var zip = new ZipArchive(file, ZipArchiveMode.Read))
{
foreach (var entry in zip.Entries)
{
if (entry.Name == "options.txt")
{
Log.Info("Loading options.txt");
using (StreamReader r = new StreamReader(entry.Open()))
{
store.Now = int.Parse(r.ReadLine());
store.IsRatingRun = int.Parse(r.ReadLine());
}
}
else
{
//Log.Info("Loading " + entry.Name);
using (var stream = entry.Open())
{
Console.Write('.');
var ms = new MemoryStream(fileBuffer);
stream.CopyTo(ms);
JsonReader reader = new JsonReader(fileBuffer);
if (reader.ReadIsNull() || !reader.ReadIsBeginObject())
{
throw new Exception("Could not read the init json");
}
if (reader.ReadPropertyName() != "accounts")
{
throw new Exception("Unexpected object in init json");
}
// read array members
if (!reader.ReadIsBeginArray())
{
throw new Exception("Array of accounts not found");
}
var fileParseStart = Stats.Watch.Elapsed;
var dto = DtoAccount.Obtain();
while (true)
{
if (DtoAccount.Parse(ref reader, dto, store))
{
dto.flags = DtoFlags.Init;
// now, add to the storage the account with internal ids/likes
store.InitNewAccout(dto);
totalAccounts++;
dto.Reset();
}
else
{
break;
}
if (!reader.ReadIsValueSeparator())
{
break;
}
}
DtoAccount.Release(dto);
var fileParseEnd = Stats.Watch.Elapsed;
if ((fileParseEnd - fileParseStart).TotalSeconds > 1)
{
Garbage.CollectAll();
}
/*
* else
* Garbage.Collect0();*/
}
}
}
}
Log.Info("Total accounts loaded: " + totalAccounts + ", " + errorAccounts + " errors found");
fileBuffer = null;
}
else
{
Log.Error(zipFile + " not found");
}
}
// synchronously process the request from requestBuffer, and return statusCode
public int Process(HttpCtx ctx, int dummy)
{
var startTime = Stats.Watch.Elapsed;
// path sanity check
foreach (var query in ctx.Params)
{
var value = query.Value;
if (value.IsEmpty)
{
return(400);
}
if (query.Key != "query_id")
{
// all other parameters are invalid
return(400);
}
}
// load the body
if (ctx.RequestBodyStart == ctx.ResponseBodyStart)
{
return(400);
}
// parse Json and process dto
var dto = DtoAccount.Obtain();
int statusCode = 400;
try
{
JsonReader reader = new JsonReader(ctx.Buffer, ctx.RequestBodyStart);
if (DtoAccount.Parse(ref reader, dto, store) && dto.flags != 0)
{
// add the new account
statusCode = store.ValidateNewAccount(dto);
if (statusCode == 201)
{
ctx.PostAction = () =>
{
store.PostNewAccount(dto);
DtoAccount.Release(dto);
}
}
;
}
}
catch (Exception)
{
// fall through
}
// return the borrowed object
if (ctx.PostAction == null)
{
DtoAccount.Release(dto);
}
var stopTime = Stats.Watch.Elapsed;
ctx.ContextType = "PostNew";
return(statusCode);
}
