public static DtoAccount Obtain() { if (!bag.TryTake(out var obj)) { obj = new DtoAccount(); } return(obj); }
public static void Release(DtoAccount obj) { obj.Reset(); bag.Add(obj); }
// using low-level to catch wrong structure or properties // ids and likes are internalized public static bool Parse(ref JsonReader reader, DtoAccount dto, Storage store) { var count = 0; if (reader.ReadIsNull() || !reader.ReadIsBeginObject()) { return(false); } while (true) { var prop = reader.ReadPropertyNameSegmentRaw(); // id if (prop.EqualTo(s_id)) { if (reader.GetCurrentJsonToken() != JsonToken.Number) { return(false); } dto.id = Mapper.ExtIdToIntIdCreate(reader.ReadInt32()); dto.flags |= DtoFlags.Id; } else // status if (prop.EqualTo(s_status)) { var st = new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape(); if (st == s_Free) { dto.status = STATUS_FREE; } else if (st == s_Taken) { dto.status = STATUS_TAKEN; } else if (st == s_Complicated) { dto.status = STATUS_COMPLICATED; } else { return(false); } dto.flags |= DtoFlags.Status; } else // sex if (prop.EqualTo(s_sex)) { var st = reader.ReadStringSegmentRaw(); if (st.EqualTo(s_male)) { dto.sex = SEX_MALE; } else if (st.EqualTo(s_female)) { dto.sex = SEX_FEMALE; } else { return(false); } dto.flags |= DtoFlags.Sex; } else // email if (prop.EqualTo(s_email)) { dto.email = reader.ReadStringSegmentRaw(); dto.flags |= DtoFlags.Email; } else // phone if (prop.EqualTo(s_phone)) { dto.phone = new AString(reader.ReadStringSegmentRaw()).Duplicate(); dto.flags |= DtoFlags.Phone; } else // fname if (prop.EqualTo(s_fname)) { dto.fnameIdx = (short)store.Fnames.GetOrCreateRange( new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index; dto.flags |= DtoFlags.Fname; } else // sname if (prop.EqualTo(s_sname)) { dto.snameIdx = (short)store.Snames.GetOrCreateRange( new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index; dto.flags |= DtoFlags.Sname; } else // country if (prop.EqualTo(s_country)) { dto.countryIdx = (byte)store.Countries.GetOrCreateRange( new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index; dto.flags |= DtoFlags.Country; } else // city if (prop.EqualTo(s_city)) { dto.cityIdx = (short)store.Cities.GetOrCreateRange( new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index; dto.flags |= DtoFlags.City; } else // birth if (prop.EqualTo(s_birth)) { if (reader.GetCurrentJsonToken() != JsonToken.Number) { return(false); } dto.birth = reader.ReadInt32(); dto.flags |= DtoFlags.Birth; } else // joined if (prop.EqualTo(s_joined)) { if (reader.GetCurrentJsonToken() != JsonToken.Number) { return(false); } dto.joined = reader.ReadInt32(); dto.flags |= DtoFlags.Joined; } else // premium if (prop.EqualTo(s_premium)) { if (!DtoPremium.Parse(ref reader, ref dto.premium)) { return(false); } dto.flags |= DtoFlags.Premium; } else // interests if (prop.EqualTo(s_interests)) { if (!reader.ReadIsBeginArray()) { return(false); } while (!reader.ReadIsEndArrayWithSkipValueSeparator(ref count)) { dto.interests.Set(store.Interests.GetOrCreateRange(new AString(reader.ReadStringSegmentRaw()).InPlaceUnescape()).Index); } dto.flags |= DtoFlags.Interests; } else // likes if (prop.EqualTo(s_likes)) { if (!reader.ReadIsBeginArray()) { return(false); } // read likes if (reader.GetCurrentJsonToken() == JsonToken.BeginObject) { while (true) { Like like = new Like(); if (!reader.ReadIsBeginObject()) { return(false); } while (true) { var propName = reader.ReadPropertyNameSegmentRaw(); if (propName[0] == (byte)'t') // ts { if (reader.GetCurrentJsonToken() != JsonToken.Number) { return(false); } like.ts = Like.ExtToIntTS(reader.ReadInt32()); } else if (propName[0] == (byte)'i') // id { if (reader.GetCurrentJsonToken() != JsonToken.Number) { return(false); } if (!Mapper.ExtIdToIntId(reader.ReadInt32(), out like.id)) { return(false); } } else // not ts nor id { return(false); } if (!reader.ReadIsValueSeparator()) { break; } } if (!reader.ReadIsEndObject()) { return(false); } dto.likes.Add(like); if (!reader.ReadIsValueSeparator()) { break; } } } if (!reader.ReadIsEndArray()) { return(false); } dto.flags |= DtoFlags.Likes; } else { return(false); } if (!reader.ReadIsValueSeparator()) { break; } } if (!reader.ReadIsEndObject()) { return(false); } return(true); }
public int ValidateUpdateAccount(DtoAccount dto) { var id = dto.id; if (id == 17579) { } // validate first, starting from fastest checks, and only after the validation create an account if (id < 0 || id >= MAX_ACCOUNTS) { return(404); } // phone (if provided) contains area code if (dto.flags.HasFlag(DtoFlags.Phone) && dto.phone != null) { var openBrace = dto.phone.IndexOf('('); var closeBrace = dto.phone.IndexOf(')'); if (openBrace < 0 || closeBrace != openBrace + 4) { return(400); } } // joined within range if (dto.flags.HasFlag(DtoFlags.Joined) && (Utils.TimestampToDate(dto.joined) < MinJoined || Utils.TimestampToDate(dto.joined) > MaxJoined)) { return(400); } // premium if (dto.flags.HasFlag(DtoFlags.Premium)) { if (dto.premium.start > 0 && Utils.TimestampToDate(dto.premium.start) < MinPremium || dto.premium.finish > 0 && Utils.TimestampToDate(dto.premium.finish) < MinPremium) { return(400); } } // the rest requires locking bool lockTaken = false; try { updateLock.Enter(ref lockTaken); // check if the account exists if (!All[id]) { return(404); } // likes if (dto.flags.HasFlag(DtoFlags.Likes)) { if (!verifyLikes(dto.likes)) { return(400); } } var acct = Accounts[id]; // update email and domain if (dto.flags.HasFlag(DtoFlags.Email)) { if (dto.email.IsEmpty || dto.email.Length > 100) { return(400); } if (!bufferFromEmail(dto.email, out var intEmail)) { return(400); } // check for duplicates if (Emails.Contains(intEmail) && !ByteArrayComparer.Instance.Equals(acct.Email, intEmail)) { return(400); // such email exists and it's not ours } // unregister old email Domains[acct.GetDomainIdx()].Exclude(id); Emails.Remove(acct.Email); // store and register new email acct.Email = intEmail; Emails.Add(acct.Email); Domains[acct.GetDomainIdx()].Include(id); } // store new account info Accounts[id] = acct; } finally { if (lockTaken) { updateLock.Exit(); } } return(202); }
public void PostUpdateAccount(DtoAccount dto) { var id = dto.id; if (id == 17579) { } var acct = Accounts[id]; // sex if (dto.flags.HasFlag(DtoFlags.Sex)) { if (dto.sex) { acct.Flags |= Account.Male; } else { acct.Flags &= (byte)~Account.Male; } } // status if (dto.flags.HasFlag(DtoFlags.Status)) { bool isFree = dto.status == DtoAccount.STATUS_FREE; bool isTaken = dto.status == DtoAccount.STATUS_TAKEN; bool isComplicated = dto.status == DtoAccount.STATUS_COMPLICATED; // clear old flags acct.Flags &= (byte)~(Account.Free | Account.Taken); // add new flags if (isFree) { acct.Flags |= Account.Free; } else if (isTaken) { acct.Flags |= Account.Taken; } else { acct.Flags |= (byte)(Account.Free | Account.Taken); } } // birth date if (dto.flags.HasFlag(DtoFlags.Birth)) { // exclude from old range if (acct.BirthIdx > 0) { BirthYears[acct.BirthIdx].Exclude(id); } // store new birthday acct.Birth = dto.birth; // include into new range acct.BirthIdx = (byte)BirthYears.GetOrCreateRangeThenInclude( Utils.TimestampToDate(acct.Birth).Year, id).Index; } // joined date if (dto.flags.HasFlag(DtoFlags.Joined)) { // exclude from old range if (acct.JoinedIdx > 0) { JoinYears[acct.JoinedIdx].Exclude(id); } // include into new range acct.JoinedIdx = (byte)JoinYears.GetOrCreateRangeThenInclude( Utils.TimestampToDate(dto.joined).Year, id).Index; } // premium if (dto.flags.HasFlag(DtoFlags.Premium)) { acct.PStart = dto.premium.start; acct.PFinish = dto.premium.finish; bool premiumNow = acct.PStart <= Now && acct.PFinish >= Now; if (premiumNow) { acct.Flags |= Account.Premium; } else { acct.Flags &= (byte)~Account.Premium; } } // fname if (dto.flags.HasFlag(DtoFlags.Fname)) { // exclude from old range if (acct.FNameIdx > 0) { Fnames[acct.FNameIdx].Exclude(id); } acct.FNameIdx = dto.fnameIdx; if (acct.FNameIdx > 0) { Fnames[acct.FNameIdx].Include(id); } } // sname if (dto.flags.HasFlag(DtoFlags.Sname)) { // exclude from old range if (acct.SNameIdx != 0) { var entry = Snames[acct.SNameIdx]; Snames2.GetOrCreateRange(entry.AName[0] + (entry.AName[1] << 8)).Exclude(id); Snames2.GetOrCreateRange(entry.AName[0] + (entry.AName[1] << 8) + (entry.AName[2] << 16) + (entry.AName[3] << 24)).Exclude(id); } acct.SNameIdx = dto.snameIdx; if (acct.SNameIdx > 0) { var entry = Snames[acct.SNameIdx]; Snames2.GetOrCreateRangeThenInclude(entry.AName[0] + (entry.AName[1] << 8), id); Snames2.GetOrCreateRangeThenInclude(entry.AName[0] + (entry.AName[1] << 8) + (entry.AName[2] << 16) + (entry.AName[3] << 24), id); } } // phone if (dto.flags.HasFlag(DtoFlags.Phone)) { // exclude from old range var oldAreaCode = areaCodeFromPhone(new AString(acct.Phone)); if (!oldAreaCode.IsEmpty) { AreaCodes.GetOrCreateRange(oldAreaCode).Exclude(id); } // update the phone number acct.Phone = dto.phone.Buffer; // include into new range var newAreaCode = areaCodeFromPhone(dto.phone); if (!newAreaCode.IsEmpty) { AreaCodes.GetOrCreateRange(newAreaCode).Include(id); } } // country if (dto.flags.HasFlag(DtoFlags.Country)) { // exclude from old range if (acct.CountryIdx > 0) { Countries[acct.CountryIdx].Exclude(id); } acct.CountryIdx = dto.countryIdx; // include into new range if (acct.CountryIdx > 0) { Countries[acct.CountryIdx].Include(id); } } // city if (dto.flags.HasFlag(DtoFlags.City)) { // exclude from old range if (acct.CityIdx > 0) { Cities[acct.CityIdx].Exclude(id); } acct.CityIdx = dto.cityIdx; // include into new range if (acct.CityIdx > 0) { Cities[acct.CityIdx].Include(id); } } // interests if (dto.flags.HasFlag(DtoFlags.Interests)) { // exclude old interests for (int i = 1; i < BitMap96.MAX_BITS; i++) { if (acct.InterestMask.IsSet(i)) { Interests[i].Exclude(id); } } acct.InterestMask = dto.interests; // include new interests for (int i = 1; i < BitMap96.MAX_BITS; i++) { if (acct.InterestMask.IsSet(i)) { Interests[i].Include(id); } } } // unlink old likes if (dto.flags.HasFlag(DtoFlags.Likes)) { if (acct.LikesCount > 0) { Log.Error("Replacing existing likes on update"); } if (dto.likes != null) { addLikes(id, ref acct, dto.likes); } } // uncount old record from cube groups updateGroups(ref Accounts[id], false); // store new account info Accounts[id] = acct; // count new record in cube groups updateGroups(ref acct, true); }
// synchronously process the request from requestBuffer, and return statusCode public int Process(HttpCtx ctx, int extId) { var startTime = Stats.Watch.Elapsed; // query sanity check foreach (var query in ctx.Params) { var value = query.Value; if (value.IsEmpty) { return(400); } if (query.Key != "query_id") { // all other parameters are invalid return(400); } } // translate to internal id and check for account existance if (!Mapper.ExtIdToIntId(extId, out int id) || !store.All[id]) { return(404); } // parse Json var dto = DtoAccount.Obtain(); dto.id = id; int statusCode = 400; try { JsonReader reader = new JsonReader(ctx.Buffer, ctx.RequestBodyStart); if (DtoAccount.Parse(ref reader, dto, store) && dto.flags != 0) { // update the account, register post-process action statusCode = store.ValidateUpdateAccount(dto); if (statusCode == 202) { ctx.PostAction = () => { store.PostUpdateAccount(dto); DtoAccount.Release(dto); } } ; } } catch (Exception) { // fall through } // return the borrowed object if (ctx.PostAction == null) { DtoAccount.Release(dto); } var stopTime = Stats.Watch.Elapsed; ctx.ContextType = "PostUpdate"; return(statusCode); }
// zip loader, called from Main() before the web server starts public static void LoadFromZip(string path, Storage store) { // load options var optionsFile = path + "/data/options.txt"; if (File.Exists(optionsFile)) { Log.Info("Opening " + optionsFile); using (StreamReader r = new StreamReader(File.OpenRead(optionsFile))) { store.Now = int.Parse(r.ReadLine()); store.IsRatingRun = int.Parse(r.ReadLine()); } } else { Log.Error(optionsFile + " is not found"); } // load zipped data var zipFile = path + "/data/data.zip"; if (File.Exists(zipFile)) { Log.Info("Opening " + zipFile); var totalAccounts = 0; var errorAccounts = 0; var fileBuffer = new byte[20000000]; using (var file = File.OpenRead(zipFile)) using (var zip = new ZipArchive(file, ZipArchiveMode.Read)) { foreach (var entry in zip.Entries) { if (entry.Name == "options.txt") { Log.Info("Loading options.txt"); using (StreamReader r = new StreamReader(entry.Open())) { store.Now = int.Parse(r.ReadLine()); store.IsRatingRun = int.Parse(r.ReadLine()); } } else { //Log.Info("Loading " + entry.Name); using (var stream = entry.Open()) { Console.Write('.'); var ms = new MemoryStream(fileBuffer); stream.CopyTo(ms); JsonReader reader = new JsonReader(fileBuffer); if (reader.ReadIsNull() || !reader.ReadIsBeginObject()) { throw new Exception("Could not read the init json"); } if (reader.ReadPropertyName() != "accounts") { throw new Exception("Unexpected object in init json"); } // read array members if (!reader.ReadIsBeginArray()) { throw new Exception("Array of accounts not found"); } var fileParseStart = Stats.Watch.Elapsed; var dto = DtoAccount.Obtain(); while (true) { if (DtoAccount.Parse(ref reader, dto, store)) { dto.flags = DtoFlags.Init; // now, add to the storage the account with internal ids/likes store.InitNewAccout(dto); totalAccounts++; dto.Reset(); } else { break; } if (!reader.ReadIsValueSeparator()) { break; } } DtoAccount.Release(dto); var fileParseEnd = Stats.Watch.Elapsed; if ((fileParseEnd - fileParseStart).TotalSeconds > 1) { Garbage.CollectAll(); } /* * else * Garbage.Collect0();*/ } } } } Log.Info("Total accounts loaded: " + totalAccounts + ", " + errorAccounts + " errors found"); fileBuffer = null; } else { Log.Error(zipFile + " not found"); } }
// synchronously process the request from requestBuffer, and return statusCode public int Process(HttpCtx ctx, int dummy) { var startTime = Stats.Watch.Elapsed; // path sanity check foreach (var query in ctx.Params) { var value = query.Value; if (value.IsEmpty) { return(400); } if (query.Key != "query_id") { // all other parameters are invalid return(400); } } // load the body if (ctx.RequestBodyStart == ctx.ResponseBodyStart) { return(400); } // parse Json and process dto var dto = DtoAccount.Obtain(); int statusCode = 400; try { JsonReader reader = new JsonReader(ctx.Buffer, ctx.RequestBodyStart); if (DtoAccount.Parse(ref reader, dto, store) && dto.flags != 0) { // add the new account statusCode = store.ValidateNewAccount(dto); if (statusCode == 201) { ctx.PostAction = () => { store.PostNewAccount(dto); DtoAccount.Release(dto); } } ; } } catch (Exception) { // fall through } // return the borrowed object if (ctx.PostAction == null) { DtoAccount.Release(dto); } var stopTime = Stats.Watch.Elapsed; ctx.ContextType = "PostNew"; return(statusCode); }