示例#1
0
 /// <summary>
 /// Returns the those block hashes from the phone's database which do not match the existing ones.
 /// This version is threaded, allows for memory-based sqlite tables, and can process phone blocks
 /// that aren't actually in the database.
 /// </summary>
 /// <param name="phoneId">The unique identification for this phone in the database.</param>
 /// <param name="sql">The handler for the database of hashes.</param>
 /// <returns>A list of block hashes that did not match other existing ones in the database.</returns>
 private List<UnfilterdBlockResult> HashGetUnfilteredBlocks7a(int phoneId, SQLiteConnection sql)
 {
     // On 32-bit systems it's easy to run out of memory. Limit threads.
     int maxThreads = (Environment.Is64BitProcess) ? 8 : 2;
     const int minListLen = 100000;
     try {
         (new SQLiteCommand("PRAGMA temp_store=2", sql)).ExecuteNonQuery();
     } catch {
     }
     // Select the Run Id which is used to identify which hashes are ours.
     int hashRunId = -1;
     if (phoneId > (Int32.MaxValue - 100)) {
         // Special large phone id indicates the hashes weren't stored in the
         // DB. The unfiltered hashes should have been passed to us.
         if (_storedBlocks != null) {
             hashRunId = Int32.MaxValue - 1;  // dummy hashRunId
         }
     } else {
         try {
             const string stmt =
                 "SELECT hashRunId " +
                 "FROM tbl_HashRun " +
                 "WHERE slideAmount=@SA AND " +
                 "blockSizeBytes=@BS AND " +
                 "phoneId=@PID AND " +
                 "memoryId=@MID " +
                 "ORDER BY hashRunId DESC " +
                 "LIMIT 1";
             SQLiteCommand cmd = new SQLiteCommand(stmt, sql);
             cmd.Parameters.AddWithValue("@SA", _slideAmount);
             cmd.Parameters.AddWithValue("@BS", _blockSize);
             cmd.Parameters.AddWithValue("@PID", phoneId);
             cmd.Parameters.AddWithValue("@MID", _memoryId);
             SQLiteDataReader rdr = null;
             try {
                 rdr = cmd.ExecuteReader();
                 if (rdr.Read()) {
                     hashRunId = Convert.ToInt32(rdr["hashRunId"]);
                 }
             } finally {
                 if (rdr != null) {
                     rdr.Close();
                 }
             }
         } catch {
         }
     }
     if (hashRunId < 0) {
         return new List<UnfilterdBlockResult>();
     }
     try {
         // Use ArrayList rather than List so that the threaded version can
         // efficiently index into the list.
         ArrayList tempResults = new ArrayList();
         int dropped = 0;
         if (_storedBlocks != null) {
             // We were passed the blocks stored in the DB. No need to
             // read them out. Also, we can assume there are no duplicate
             // blocks.
             foreach (UnfilterdBlockResult r in _storedBlocks) {
                 tempResults.Add(r);
             }
             _storedBlocks.Clear();
             _storedBlocks = null;
         } else {
             // Tests have shown that it's quickest to get a list of hashes and their
             // block index first. Trying to perform filtering in the query tends to
             // be slower.
             SQLiteDataReader rdr = null;
             SQLiteCommand cmd = null;
             DateTime started = DateTime.Now;
             const string stmt2 =
                 "SELECT hash, blockIndex " +
                 "FROM tbl_Hash " +
                 "WHERE hashRunId=@HRID " +
                 "ORDER BY blockIndex ASC";
             cmd = new SQLiteCommand(stmt2, sql);
             cmd.Parameters.AddWithValue("@HRID", hashRunId);
             try {
                 HashSet<string> skip = new HashSet<string>();
                 rdr = cmd.ExecuteReader();
                 while (rdr.Read()) {
                     UnfilterdBlockResult result = new UnfilterdBlockResult()
                                                       {
                                                           hash = Convert.ToString(rdr["hash"]),
                                                           blockIndexFirst =
                                                               Convert.ToInt32(rdr["blockIndex"])
                                                       };
                     // Do not keep the block if it's got a hash that we've previously
                     // seen.
                     // TODO: If we don't save duplicate hashes then we don't need to
                     // perform this check. Dec0de behaves this way now, but we can't
                     // yet rely on the database not having duplicates.
                     if (!skip.Contains(result.hash)) {
                         tempResults.Add(result);
                         skip.Add(result.hash);
                     } else {
                         dropped++;
                     }
                 }
             } catch (Exception e) {
                 System.Console.WriteLine("SQL exception: " + e.Message);
                 return new List<UnfilterdBlockResult>();
             } finally {
                 if (rdr != null) {
                     rdr.Close();
                 }
             }
             DateTime finished = DateTime.Now;
             System.Console.WriteLine("It took " + (finished - started).TotalSeconds + " seconds to read hashes");
             System.Console.WriteLine("Skipped " + dropped + " blocks");
         }
         List<UnfilterdBlockResult> results = null;
         dropped = 0;
         if (_noFilter) {
             results = new List<UnfilterdBlockResult>();
             foreach (Object r in tempResults) {
                 results.Add((UnfilterdBlockResult) r);
             }
         } else {
             // Eliminate blocks whose hashes match those of other phones. Here we
             // assume constant block hashes are in tbl_Hashes rather than in the
             // deprecated tbl_Constants.
             // Our tests have shown that testing each individual hash is quicker
             // then doing it in a single query. The good news though is that we
             // can perform explicit threading.
             int nThreads = (tempResults.Count + minListLen - 1) / minListLen;
             if (nThreads > maxThreads) nThreads = maxThreads;
             if (nThreads <= 1) {
                 SQLiteDataReader rdr = null;
                 SQLiteCommand cmd = null;
                 // No need to create a thread.
                 results = new List<UnfilterdBlockResult>();
                 foreach (UnfilterdBlockResult result in tempResults) {
                     rdr = null;
                     try {
                         const string stmt =
                             "SELECT hash FROM tbl_Hash WHERE hashRunId<>@HRID AND hash=@HASH LIMIT 1";
                         cmd = new SQLiteCommand(stmt, sql);
                         cmd.Parameters.AddWithValue("@HRID", hashRunId);
                         cmd.Parameters.AddWithValue("@HASH", result.hash);
                         rdr = cmd.ExecuteReader();
                         if (!rdr.HasRows) {
                             results.Add(result);
                         } else {
                             dropped++;
                         }
                     } catch (Exception e) {
                         System.Console.WriteLine("SQL exception: " + e.Message);
                     } finally {
                         if (rdr != null) {
                             rdr.Close();
                         }
                     }
                 }
             } else {
                 // Perform the query in threads.
                 results = ThreadedBlockHashFilter(nThreads, hashRunId, tempResults, sql);
             }
             tempResults.Clear();
         }
         System.Console.WriteLine("Skipped " + dropped + " blocks");
         return results;
     } catch (Exception ex) {
         System.Console.WriteLine("SQL exception: " + ex.Message);
         return new List<UnfilterdBlockResult>();
     }
 }
示例#2
0
 private List<UnfilterdBlockResult> HashGetUnfilteredBlocks8(int phoneId, SQLiteConnection sql)
 {
     try {
         (new SQLiteCommand("PRAGMA temp_store=2", sql)).ExecuteNonQuery();
     } catch {
     }
     try {
         // Default is 2000 1K pages
         (new SQLiteCommand("PRAGMA cache_size=8000", sql)).ExecuteNonQuery();
     } catch {
     }
     int hashRunId = -1;
     try {
         const string stmt =
             "SELECT hashRunId " +
             "FROM tbl_HashRun " +
             "WHERE slideAmount=@SA AND " +
             "blockSizeBytes=@BS AND " +
             "phoneId=@PID AND " +
             "memoryId=@MID " +
             "ORDER BY hashRunId DESC " +
             "LIMIT 1";
         SQLiteCommand cmd = new SQLiteCommand(stmt, sql);
         cmd.Parameters.AddWithValue("@SA", _slideAmount);
         cmd.Parameters.AddWithValue("@BS", _blockSize);
         cmd.Parameters.AddWithValue("@PID", phoneId);
         cmd.Parameters.AddWithValue("@MID", _memoryId);
         SQLiteDataReader rdr = null;
         try {
             rdr = cmd.ExecuteReader();
             if (rdr.Read()) {
                 hashRunId = Convert.ToInt32(rdr["hashRunId"]);
             }
         } finally {
             if (rdr != null) {
                 rdr.Close();
             }
         }
     } catch {
     }
     if (hashRunId < 0) {
         return new List<UnfilterdBlockResult>();
     }
     try {
         DateTime started = DateTime.Now;
         SQLiteCommand cmd = null;
         const string stmt2 =
             "SELECT hash, MIN(blockIndex) AS blockIndexFirst " +
             "FROM tbl_Hash " +
             "WHERE hashRunId=@HRID " +
             "GROUP BY hash " +
             "ORDER BY blockIndexFirst ASC";
         cmd = new SQLiteCommand(stmt2, sql);
         cmd.Parameters.AddWithValue("@HRID", hashRunId);
         SQLiteDataReader rdr = null;
         List<UnfilterdBlockResult> tempResults = new List<UnfilterdBlockResult>();
         try {
             rdr = cmd.ExecuteReader();
             while (rdr.Read()) {
                 UnfilterdBlockResult result = new UnfilterdBlockResult()
                 {
                     hash = Convert.ToString(rdr["hash"]),
                     blockIndexFirst =
                         Convert.ToInt32(rdr["blockIndexFirst"])
                 };
                 tempResults.Add(result);
             }
         } catch (Exception e) {
             System.Console.WriteLine("SQL exception: " + e.Message);
             return new List<UnfilterdBlockResult>();
         } finally {
             if (rdr != null) {
                 rdr.Close();
             }
         }
         DateTime finished = DateTime.Now;
         System.Console.WriteLine("It took " + (finished - started).TotalSeconds + " seconds to read hashes");
         List<UnfilterdBlockResult> results = null;
         int dropped = 0;
         if (_noFilter) {
             results = tempResults;
         } else {
             results = new List<UnfilterdBlockResult>();
             foreach (UnfilterdBlockResult result in tempResults) {
                 rdr = null;
                 try {
                     const string stmt =
                         "SELECT hash FROM tbl_Hash WHERE hashRunId<>@HRID AND hash=@HASH LIMIT 1";
                     cmd = new SQLiteCommand(stmt, sql);
                     cmd.Parameters.AddWithValue("@HRID", hashRunId);
                     cmd.Parameters.AddWithValue("@HASH", result.hash);
                     rdr = cmd.ExecuteReader();
                     if (!rdr.HasRows) {
                         results.Add(result);
                     } else {
                         dropped++;
                     }
                 } catch (Exception e) {
                     System.Console.WriteLine("SQL exception: " + e.Message);
                 } finally {
                     if (rdr != null) {
                         rdr.Close();
                     }
                 }
             }
             tempResults.Clear();
         }
         System.Console.WriteLine("Skipped " + dropped + " blocks");
         return results;
     } catch (Exception ex) {
         System.Console.WriteLine("SQL exception: " + ex.Message);
         return new List<UnfilterdBlockResult>();
     }
 }
示例#3
0
 private List<UnfilterdBlockResult> HashGetUnfilteredBlocks6(int phoneId, SQLiteConnection sql)
 {
     try {
         // Default is 2000 1K pages
         (new SQLiteCommand("PRAGMA cache_size=8000", sql)).ExecuteNonQuery();
     } catch {
     }
     int hashRunId = -1;
     try {
         const string stmt =
             "SELECT hashRunId " +
             "FROM tbl_HashRun " +
             "WHERE slideAmount=@SA AND " +
             "blockSizeBytes=@BS AND " +
             "phoneId=@PID AND " +
             "memoryId=@MID " +
             "ORDER BY hashRunId DESC " +
             "LIMIT 1";
         SQLiteCommand cmd = new SQLiteCommand(stmt, sql);
         cmd.Parameters.AddWithValue("@SA", _slideAmount);
         cmd.Parameters.AddWithValue("@BS", _blockSize);
         cmd.Parameters.AddWithValue("@PID", phoneId);
         cmd.Parameters.AddWithValue("@MID", _memoryId);
         SQLiteDataReader rdr = null;
         try {
             rdr = cmd.ExecuteReader();
             if (rdr.Read()) {
                 hashRunId = Convert.ToInt32(rdr["hashRunId"]);
             }
         } finally {
             if (rdr != null) {
                 rdr.Close();
             }
         }
     } catch {
     }
     if (hashRunId < 0) {
         return new List<UnfilterdBlockResult>();
     }
     try {
         SQLiteCommand cmd = null;
         if (_noFilter) {
             const string stmt2 =
                 "SELECT hash, MIN(blockIndex) AS blockIndexFirst " +
                 "FROM tbl_Hash " +
                 "WHERE hashRunId=@HRID " +
                 "GROUP BY hash " +
                 "ORDER BY blockIndexFirst ASC";
             cmd = new SQLiteCommand(stmt2, sql);
             cmd.Parameters.AddWithValue("@HRID", hashRunId);
         } else {
             const string stmt2 =
                 "SELECT hash, MIN(blockIndex) AS blockIndexFirst " +
                 "FROM tbl_Hash " +
                 "WHERE hashRunId=@HRID1 " +
                 "AND hash NOT IN (SELECT hash FROM tbl_Hash WHERE  hashRunId<>@HRID2) " +
                 "GROUP BY hash " +
                 "ORDER BY blockIndexFirst ASC";
             cmd = new SQLiteCommand(stmt2, sql);
             cmd.Parameters.AddWithValue("@HRID1", hashRunId);
             cmd.Parameters.AddWithValue("@HRID2", hashRunId);
         }
         SQLiteDataReader rdr = null;
         List<UnfilterdBlockResult> results = new List<UnfilterdBlockResult>();
         try {
             rdr = cmd.ExecuteReader();
             while (rdr.Read()) {
                 UnfilterdBlockResult result = new UnfilterdBlockResult()
                 {
                     hash = Convert.ToString(rdr["hash"]),
                     blockIndexFirst =
                         Convert.ToInt32(rdr["blockIndexFirst"])
                 };
                 results.Add(result);
             }
             return results;
         } catch (Exception e) {
             System.Console.WriteLine("SQL exception: " + e.Message);
             return new List<UnfilterdBlockResult>();
         } finally {
             if (rdr != null) {
                 rdr.Close();
             }
         }
     } catch (Exception ex) {
         System.Console.WriteLine("SQL exception: " + ex.Message);
         return new List<UnfilterdBlockResult>();
     }
 }