public static void RefreshBasedOnRadarSignals_AndPublish(HashSet <Int32> considered) { // SendPostToLJ(); // SendPostToLJ("For <lj user=mcfnord>", "some kibble"); MMDB.MakeSureDBIsOpen(); // first trash watches entirely? so no need to drop people. // then populate watches entirely. build it up! readers of ljreader. not who ljreader reads! new MyNpgsqlCommand("delete from watches;", MMDB.DBConnection).ExecuteNonQuery(); // string fdataLJReader = FData.GetFData("ljfinder", true); // string fdataLJReader = FData.GetFData("mcfnord", true); // HashSet<Int32> whoReadsTheRobot = FData.IDsInTheyReadMeFData(fdataLJReader); considered.Remove(5731095); // no self! // ok, kill all adds that are oldder than 119 days. cuz i don't want to pblish offlines. or something. MMDB.ExecuteNonQuery(string.Format("delete from adds where daydetected < {0};", DateTime.Now.AddDays(-119).Subtract(Extras.TwoK).Days)); // MAYBE SOMEDAY I EXCLUDE SOME PEOPLE WHO READ THE ROBOT BECAUSE THEY SUCK ASS AND DON'T PICK UP THE SHIT. foreach (var jokerWhoReadsTheRobot in considered) { foreach (var someoneJokerReads in FData.IDsInIReadFData(FData.GetFData(IDMap.IDToName(jokerWhoReadsTheRobot)))) { MMDB.ExecuteNonQuery(string.Format("INSERT INTO watches (watcher, watched) Values({0}, {1});", jokerWhoReadsTheRobot, someoneJokerReads), false); } } // gather all actors in add events HashSet <Int32> watchers = new HashSet <int>(); NpgsqlCommand cmd = new NpgsqlCommand("select distinct(watcher) from watches where watched IN (select distinct(actor) from adds) ;", MMDB.DBConnection); cmd.CommandTimeout = 0; // forever! NpgsqlDataReader myReader = cmd.ExecuteReader(); while (myReader.Read()) { watchers.Add(myReader.GetInt32(0)); } myReader.Close(); HashSet <Int32> everyoneImmaUpdate = new HashSet <int>(); // I HATE RECOMMENDING RENAMES. I HATE IT SO MUCH THAT I WANT TO MAXIMALLY DETECT THEM IN A PRE-STEP. foreach (var watcher in watchers) { everyoneImmaUpdate.UnionWith(FData.IDsInIReadFData(FData.GetFData(IDMap.IDToName(watcher)))); } everyoneImmaUpdate.ExceptWith(watchers); foreach (var someParty in everyoneImmaUpdate) { if (false == FData.FDataConfirmedCurrentEnufOn(IDMap.IDToName(someParty))) { FData.GetFData(IDMap.IDToName(someParty)); } } everyoneImmaUpdate = null; // imma gonna order watchers into orderedWatchers, based on last-published-to date // but first it's ok i'll run this through... string sqlset = ""; foreach (var dude in watchers) { sqlset += dude.ToString() + ","; } sqlset = sqlset.Substring(0, sqlset.Length - 1); myReader = new NpgsqlCommand(string.Format("select id from nameidmap where id in ({0}) order by inbox_hit_week ;", sqlset), MMDB.DBConnection).ExecuteReader(); List <Int32> sortedWatchers = new List <int>(); while (myReader.Read()) { sortedWatchers.Add(myReader.GetInt32(0)); } myReader.Close(); watchers = null; foreach (var watcher in sortedWatchers) { if (watcher == 1179796) { Console.WriteLine("The bastardo."); } HashSet <Int32> whoWatcherReads = FData.IDsInIReadFData(FData.GetFData(IDMap.IDToName(watcher))); if (whoWatcherReads.Contains(watcher)) // ditch read-self! { whoWatcherReads.Remove(watcher); } // tell me who was added by all the parties we watch Dictionary <Int32, HashSet <Int32> > everyTarget = new Dictionary <Int32, HashSet <Int32> >(); string whoWatcherReadsInSQLSet = ""; foreach (var someone in whoWatcherReads) { whoWatcherReadsInSQLSet += someone.ToString() + ","; } whoWatcherReadsInSQLSet = whoWatcherReadsInSQLSet.Substring(0, whoWatcherReadsInSQLSet.Length - 1); // rule out anyone if i ever ever read them before. // my encyclopedic archive rules i guess! List <int> daysOfTheFData = FData.GetFDataDates(IDMap.IDToName(watcher)); HashSet <Int32> everyoneIEverRead = new HashSet <Int32>(); foreach (var day in daysOfTheFData) { string anFD = FData.FDataBy2kDay(IDMap.IDToName(watcher), day); everyoneIEverRead.UnionWith(FData.IDsInIReadFData(anFD)); } if (everyoneIEverRead.Contains(watcher)) { everyoneIEverRead.Remove(watcher); // remove watcher manually } if (watcher == 1) { if (everyoneIEverRead.Contains(578)) { Console.WriteLine("I read scribble before."); } } cmd = new NpgsqlCommand(string.Format("select target, actor from adds where actor IN ({0});", whoWatcherReadsInSQLSet), MMDB.DBConnection); myReader = cmd.ExecuteReader(); while (myReader.Read()) { Int32 iTarget = myReader.GetInt32(0); // skip self-whatevers. if (iTarget == watcher) { continue; } // skip ljfinder! if (iTarget == 5731095) { continue; } // detected as offline in last 120 days? NpgsqlDataReader innerReader = new MyNpgsqlCommand( string.Format("select offline_last_detected_on from nameidmap where id={0};", iTarget), MMDB.DBConnection).ExecuteReader(); innerReader.Read(); Int16?offline_detected_on = MMDB.MaybeNullInt16(innerReader, 0); // overload with a no-local-reader option! rolls royce db! innerReader.Close(); if (null != offline_detected_on) { if (offline_detected_on > DateTime.Now.AddDays(-120).Subtract(Extras.TwoK).Days) { continue; } } if (everyoneIEverRead.Contains(iTarget)) { continue; } // MORITORIUM ALSO ON ANYTHING THAT IS DEAD WTF // KILL IT WITH ANY F*****G FEATURE... UPDATEPPY? 404! // moritorium on any target that has been detected as a rename in the last 120 days innerReader = new MyNpgsqlCommand( string.Format("select made_by_rename_detected_on from nameidmap where id={0};", iTarget), MMDB.DBConnection).ExecuteReader(); innerReader.Read(); Int16?day_rename_detected = MMDB.MaybeNullInt16(innerReader, 0); // overload with a no-local-reader option! rolls royce db! innerReader.Close(); if (null != day_rename_detected) { if (day_rename_detected > DateTime.Now.AddDays(-120).Subtract(Extras.TwoK).Days) { continue; } } // create and add if (false == everyTarget.ContainsKey(iTarget)) { // if it's known offline i'm gonna hit this way too hard. i just need to remember somehow that i hit it. // Extras.CheckForRenameOrOffline(iTarget); everyTarget.Add(iTarget, new HashSet <Int32>()); } int iActor = myReader.GetInt32(1); ////////////////////// diagnose how actor == watcher, ever. // Console.WriteLine("Target: {0}, Actor: {1}, Watcher: {2}", IDMap.IDToName(iTarget), IDMap.IDToName(iActor), IDMap.IDToName(watcher)); ////////////////////// diagnose how actor == watcher, ever. System.Diagnostics.Debug.Assert(iActor != watcher); // watcher should not be in the IN set that does this query! everyTarget[iTarget].Add(iActor); // add the actor. this is why we can't just use a big IN sql statement. or can we? } myReader.Close(); // only publish if i've never published to this user, or barring that, at least one target must have more than one actor. cmd = new NpgsqlCommand(string.Format("select count(*) from radarpicks where userid={0} ;", watcher), MMDB.DBConnection); myReader = cmd.ExecuteReader(); myReader.Read(); Int64?any = MMDB.MaybeNullInt64(myReader, 0); myReader.Close(); if (everyTarget.Count > 0) // else we crash so f**k this situation. { if (any == 0 || everyTarget.Max(targ => targ.Value.Count) > 1) { List <HashSet <Int32> > actors = new List <HashSet <Int32> >(); List <HashSet <Int32> > targets = new List <HashSet <Int32> >(); var sortedGroups = (from g in everyTarget where g.Value.Count > ((any == 0) ? 0 : 1) // if we've never published anything, lower the standard to just 1 orderby g.Value.Count descending select g).Take(12); // fold the sorted groups into the two dictionaries. foreach (var thisG in sortedGroups) { // if these actors are already in my actor list, then we skip foreach (var existing in actors) { if (thisG.Value.SetEquals(existing)) // wha'ts the right way??? cuz it's not this. { goto ALREADY; } } // i know its key, so i can look for equal sets that aren't it. var sameActors = from x in everyTarget where (x.Key != thisG.Key && x.Value.SetEquals(thisG.Value)) // so same actors, different key, so excludes thisG! select x; // All of them are added as a same 'entry' slot into actors and targets sets. actors.Add(thisG.Value); System.Diagnostics.Debug.Assert(false == thisG.Value.Contains(watcher)); // watchers aren't actors in their deeds! HashSet <Int32> targs = new HashSet <int>(); targs.Add(thisG.Key); foreach (var same in sameActors) { targs.Add(same.Key); } targets.Add(targs); ALREADY: ; } // divide into two groups, based on whether i've published this name to this user before.] // builds a logic array of whether anyone in this set has been published before. List <bool> publishedBefore = new List <bool>(); foreach (var targSet in targets) { bool fPublishedBefore = true; foreach (var t in targSet) { cmd = new NpgsqlCommand(string.Format("select count(*) from radarpicks where userid={0} and userid_recommended={1};", watcher, t), MMDB.DBConnection); myReader = cmd.ExecuteReader(); myReader.Read(); try { fPublishedBefore = (myReader.GetInt64(0) > 0); if (false == fPublishedBefore) { break; // just one "news" in the set means NO, not published before! } } finally { myReader.Close(); } } publishedBefore.Add(fPublishedBefore); } //// TOTAL EXPERIMENT BEFORE THE PUBLISHED B4 CHECK. //// TOTAL EXPERIMENT BEFORE THE PUBLISHED B4 CHECK. /* * foreach (var set in actors) * { * List<List<Int32>> bunches = TCliquesClass.CustomGroupMain(watcher, set); // love this but need specialized "top set matters" timeout mode. * // that clique finder puts the watcher into the set. i take it out! * set.Remove(watcher); * * Console.WriteLine("Largest actor clique: "); * foreach (var party in bunches[0]) * { * Console.Write(IDMap.IDToName(party) + " "); * } * Console.WriteLine(); * } * */ //// TOTAL EXPERIMENT BEFORE THE PUBLISHED B4 CHECK. //// TOTAL EXPERIMENT BEFORE THE PUBLISHED B4 CHECK. // is there anything new to publish? if (any > 0) // if any == 0, then we publish cuz we never published b4 { if (false == publishedBefore.Contains(false)) { Console.WriteLine("Nothin new to publish, just stuff I already found."); continue; } } string title = "Hey <lj user="******">"; string archiveContent = ""; string newContent = ""; string names = ""; foreach (var t in targets) { foreach (var item in t) { names += IDMap.IDToName(item) + "|"; } } names = names.Substring(0, names.Length - 1); // in conclusion, a link to the visual // string content = "<ul><li><a href='http://ljmindmap.com/h.php?n=" + IDMap.IDToName(watcher) + "'>Тесен Мир / MindMap</a>"; string content = "<table><tr><td><center><a href='http://ljmindmap.com/h.php?n=" + IDMap.IDToName(watcher) + "'><img width='500' src='http://ljmindmap.com/s/?f=" + IDMap.IDToName(watcher) + ".gif'><br>Тесен Мир / MindMap</a></td></tr></table><br>"; /* * * * * <table><tr><td><center><a href="http://ljmindmap.com/h.php?n=micaturtle"><img width="500" src="http://ljmindmap.com/s/?f=micaturtle.gif"><br>Тесен Мир / MindMap</a></td></tr></table><br> */ // content += string.Format("<li><a href='http://ljmindmap.com/mass_add.php?a={0}'>Surf these LJs</a></ul>", names); content += "<table border='1'><tr><th><u>New LJs</u><th><u>Found by</u></tr>"; System.Diagnostics.Debug.Assert(actors.Count == targets.Count); for (int iPos = 0; iPos < actors.Count; iPos++) { string contentFragment = "<tr><td>"; foreach (var t in targets[iPos]) { contentFragment += "<lj user="******"> "; } contentFragment += "<td><b>"; foreach (var a in actors[iPos]) { System.Diagnostics.Debug.Assert(a != watcher); contentFragment += IDMap.IDToName(a) + " "; } contentFragment += "</b></tr>"; if (publishedBefore[iPos]) { archiveContent += contentFragment; } else { newContent += contentFragment; } } content += newContent; content += "</table>"; if (archiveContent.Length > 0) { content += "<br><h3>Archive</h3><table>"; content += archiveContent; content += "</table>"; } string url = AddOrUpdateLJPost(watcher, title, content); if (null == url) { // need to just re-post. } System.Diagnostics.Debug.Assert(url.Length > 0); Console.WriteLine(url); string emailcontent = "\r\nI've made your own custom MindMap and found some LJs your friends are adding. Yay!\r\n\r\n" + url + "\r\n\r\n- lil ljfinder"; NotifyViaLJInbox(watcher, emailcontent, url); NoteRecommendeds(watcher, targets); //NotifyViaLJBanner(watcher, emailcontent, url); } } } // still need to kill old events somewhere! at 110 days (rename prevention stops at 120) }
// ArrayList alAllLevels = ChurnAndBurnTribes(lju, twoWayReadership, seed); public static List <List <int> > TCliquesMain(string seed) { MMDB.MakeSureDBIsOpen(); int iUser = IDMap.NameToID(seed); string fd = FData.GetFData(seed, true); // absolutely new fdata. if (null == fd) { return(null); // f**k it } HashSet <int> whoIRead = FData.IDsInIReadFData(fd); if (false == whoIRead.Contains(iUser)) // i read my damn self ok { whoIRead.Add(iUser); } // what would i see if i scanned these for offlines? HashSet <int> whoIReadTrimmed = new HashSet <int>(); foreach (var v in whoIRead) { if (false == FData.KnownOffline(IDMap.IDToName(v))) { whoIReadTrimmed.Add(v); } } whoIRead = whoIReadTrimmed; // original list clobbered! whoIReadTrimmed = null; List <int> everyone = new List <int>(whoIRead); AGAIN_WITHOUT_THE_DINKS: // get an everyone List... by reading everyone's arrays? add-if-unique to a list of unique keys // HEY AM I IN THIS LIST??? Debug.Assert(everyone.IndexOf(iUser) != -1); // I WILL NEED TO ADD MYSELF TO THE EVERYONE LIST EVERY TIME OK? Dictionary <int, HashSet <int> > whoAllRead = new Dictionary <int, HashSet <int> >(); whoAllRead.Add(iUser, whoIRead); foreach (int iUserIRead in whoIRead) { // if i read myself, i was already added, skip if (iUserIRead == iUser) // meee! { continue; } string fd2 = FData.GetFData(IDMap.IDToName(iUserIRead)); if (fd2 == null) { continue; } HashSet <int> whoTheyRead = FData.IDsInIReadFData(fd2); // ReadArrayFromUserField(iUserIRead, "snapshot", whoIRead); if (null == whoTheyRead) { whoTheyRead = new HashSet <int> { } } ; // and for every unique id, we need to store their snapshot // so we can make the bitfield later whoAllRead.Add(iUserIRead, whoTheyRead); } // it's gonna blow my mind if core user is not in everyone set. Debug.Assert(-1 != everyone.IndexOf(iUser)); // i want to know every case where whoAllRead lacks a key that appears in everyone list. foreach (int i in everyone) { if (false == whoAllRead.ContainsKey(i)) { Console.WriteLine("whoAllRead has no key: " + i); // these are offline and so no fdata is provided for them. they still could be in everyone. fix that. } } // ATTENTION. EVERYONE I READ IS LOADED INTO whoAllRead. // For each, a sifted subset (containing only people I read) is constructed. // For too-large groups, I will now proceed to throw out parties who only read one other party in this set. // Maybe then I try throwing out two. // THIS INVOLVES CAREFUL WHATEVER... everyone is a slot-based scenario. // oh what the f***s please what the f***s ugh. // just cull the everyone list based on whoAllRead and GOTO up there to the whoAllRead. Console.WriteLine("of course i only want this for f*****g biggies not for me!"); if (whoAllRead.Count > 5000) { foreach (int iDude in whoAllRead.Keys) { HashSet <int> check = whoAllRead[iDude]; // two? this must iterate up until it's a confirmed manageable subset size! if (check.Count < 2) { everyone.Remove(iDude); } } Debug.Assert(false); //wtf is this? this is experimental two-wayer removal optimization that i never quite approved. goto AGAIN_WITHOUT_THE_DINKS; } // i better have a list who i read, right? // Debug.Assert(null != eachToOthers[iUser]); // POPULATE THE BITFIELDS, which are GRIDS of two-way readership realationships // RITE? // well i'm one of those bitfields // bit i think i am always in my own set of whoiRead (self) // i only include the set of those i read, i think. // DO I INCLUDE MYSELF???? i want a bitarray also!? // prep SortedList eachToOthers = new SortedList(); // the userid yields a bitarray foreach (int i in everyone) { eachToOthers.Add(i, new BitArray(everyone.Count())); } // that's the f*****g way to create the bit array! // what is the f*****g way to populate it? // per each bit array. foreach (int iKey in eachToOthers.Keys) { BitArray ba = (BitArray)eachToOthers[iKey]; // if whoALlRead[ikey] doesn't exist, emit a warning and skip it. can i f*****g do that? // somehow there's no entry for iKey in this whoAllRead, where do eachToOthers diverge in keys from whoAllRead? foreach (int iSomeUser in whoAllRead[iKey]) // whoAllRead[stooge] is a way of saying "who stooge reads" { // if i have never associated with them, they don't get in. if (everyone.IndexOf(iSomeUser) != -1) { ba.Set(everyone.IndexOf(iSomeUser), true);// all sets } } } // IS THERE A REQUIREMENT THAT I ONLY OR OPTIMALLY POPULATE TWO-WAY ONLY? List <List <int> > sl = CliquesFromBits(eachToOthers, iUser, everyone, null); // if i save this output, i can use it as hints later. could be valuable / essential. we shall see. // so this is a bit array of many groupings? everyone has a line in this bit array. return(sl); }
public static List <List <int> > CustomGroupMain(Int32 iSeed, HashSet <Int32> members) { MMDB.MakeSureDBIsOpen(); if (false == members.Contains(iSeed)) { Console.WriteLine("I crammed the seed back into the set."); members.Add(iSeed); } List <int> everyone = new List <int>(members); Dictionary <int, HashSet <int> > master = new Dictionary <int, HashSet <int> >(); foreach (int iUserIRead in members) { string fd2 = FData.GetFData(IDMap.IDToName(iUserIRead)); if (fd2 == null) { continue; } HashSet <int> whoTheyRead = FData.IDsInIReadFData(fd2); // ReadArrayFromUserField(iUserIRead, "snapshot", whoIRead); if (null == whoTheyRead) { whoTheyRead = new HashSet <int> { } } ; // and for every unique id, we need to store their snapshot // so we can make the bitfield later master.Add(iUserIRead, whoTheyRead); } // i want to know every case where whoAllRead lacks a key that appears in everyone list. foreach (int i in everyone) { if (false == master.ContainsKey(i)) { Console.WriteLine("whoAllRead has no key: " + i); // these are offline and so no fdata is provided for them. they still could be in everyone. fix that. } } // POPULATE THE BITFIELDS, which are GRIDS of two-way readership realationships // RITE? // well i'm one of those bitfields // bit i think i am always in my own set of whoiRead (self) // i only include the set of those i read, i think. // DO I INCLUDE MYSELF???? i want a bitarray also!? // prep SortedList eachToOthers = new SortedList(); // the userid yields a bitarray foreach (int i in everyone) { eachToOthers.Add(i, new BitArray(everyone.Count())); } // that's the f*****g way to create the bit array! // what is the f*****g way to populate it? // per each bit array. foreach (int iKey in eachToOthers.Keys) { BitArray ba = (BitArray)eachToOthers[iKey]; // if whoALlRead[ikey] doesn't exist, emit a warning and skip it. can i f*****g do that? // somehow there's no entry for iKey in this whoAllRead, where do eachToOthers diverge in keys from whoAllRead? foreach (int iSomeUser in master[iKey]) // whoAllRead[stooge] is a way of saying "who stooge reads" { // if i have never associated with them, they don't get in. if (everyone.IndexOf(iSomeUser) != -1) { ba.Set(everyone.IndexOf(iSomeUser), true);// all sets } } } // IS THERE A REQUIREMENT THAT I ONLY OR OPTIMALLY POPULATE TWO-WAY ONLY? List <List <int> > sl = CliquesFromBits(eachToOthers, iSeed, everyone, null); // if i save this output, i can use it as hints later. could be valuable / essential. we shall see. // so this is a bit array of many groupings? everyone has a line in this bit array. return(sl); } }
public static void KickUsAlong() { MMDB.MakeSureDBIsOpen(); HashSet <Int32> optedIn = FData.IDsInTheyReadMeFData(FData.GetFData("ljfinder", true)); optedIn.UnionWith(FData.IDsInTheyReadMeFData(FData.GetFData("ljmindmap", true))); DateTime TwoK = new DateTime(2000, 1, 1); int weeks = (int)(DateTime.Now.Subtract(TwoK).Days / 7.02); int months = (int)(DateTime.Now.Subtract(TwoK).Days / 30.4); string sqlset = ""; foreach (var dude in optedIn) { sqlset += dude.ToString() + ","; } sqlset = sqlset.Substring(0, sqlset.Length - 1); // any opt-ins who haven't been reached this week or last, for whom lastpublishedmonth is not this month, whose status is 'L' or 'N' or 'S'? // set them to status 'S' and set statuschanged=null. If any rows are effected, then I don't email anybody this time! // HEY f**k's sake i'm changing the value of week because emailing someone i emailed last week is too f*****g much. weeks = weeks - 1; // if mm lastpublishmonth isn't this month, is it status 'L' or 'N' ? // if so, then set it to status 'S' and set statuschanged=null, // and postpone any talk of notifications. NpgsqlCommand cmd = new NpgsqlCommand(string.Format( // "select count(*) from nameidmap where id in ({0}) and inbox_hit_week < {1} and lastpublishedmonth < {2} and status in ('L', 'N') ;", "select count(*) from nameidmap where id in ({0}) and inbox_hit_week < {1} and lastpublishedmonth < {2} and status in ('L', 'N', 'S') ;", sqlset, weeks, months), MMDB.DBConnection); NpgsqlDataReader myReader = cmd.ExecuteReader(); myReader.Read(); Int64?iResult = MMDB.MaybeNullInt64(myReader, 0); myReader.Close(); if (0 == iResult) { // go ahead and notify! // lots of optedIn are excluded because i've reached 'em too recently. HashSet <Int32> eligible = new HashSet <Int32>(); cmd = new NpgsqlCommand(string.Format( "select id from nameidmap where id in ({0}) and inbox_hit_week < {1} ;", sqlset, weeks, months), MMDB.DBConnection); myReader = cmd.ExecuteReader(); while (myReader.Read()) { eligible.Add(myReader.GetInt32(0)); } myReader.Close(); // eligible.Add(IDMap.NameToID("eclipsenacht")); // f**k eligible we're testing mcfnord // eligible.Clear(); // eligible.Add(IDMap.NameToID("mcfnord")); Radar2010.RefreshBasedOnRadarSignals_AndPublish(eligible); return; } Console.WriteLine("Waiting in {0} dudes.", iResult); MMDB.ExecuteNonQuery(string.Format( "update nameidmap set status='S', statuschanged=null where id in ({0}) and inbox_hit_week < {1} and lastpublishedmonth < {2} and status in ('L', 'N', 'S') ;", sqlset, weeks, months)); // kick along. }