You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

GBUdb.hpp 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. // GBUdb.hpp
  2. //
  3. // (C) Copyright 2006 - 2009 ARM Research Labs, LLC
  4. // See www.armresearch.com for the copyright terms.
  5. //
  6. // Good, Bad, Ugly, Ignore IP database engine.
  7. ////////////////////////////////////////////////////////////////////////////////
  8. // Include M_GBUdb Only Once
  9. #ifndef M_GBUdb
  10. #define M_GBUdb
  11. #include "CodeDweller/faults.hpp"
  12. #include "CodeDweller/threading.hpp"
  13. #include <cmath>
  14. #include <cctype>
  15. #include <string>
  16. #include <sstream>
  17. #include <list>
  18. #include <cstdlib>
  19. #include <ctime>
  20. namespace SNFMulti {
  21. const unsigned int GBUdbFlagsMask = 0xC0000000; // Top 2 bits are the flag.
  22. const unsigned int GBUdbIgnore = 0xC0000000; // Ignore is the 11 flag.
  23. const unsigned int GBUdbUgly = 0x00000000; // Ugly/Unknown is the 00 flag.
  24. const unsigned int GBUdbGood = 0x80000000; // Good is the 10 flag.
  25. const unsigned int GBUdbBad = 0x40000000; // Bad is the 01 flag.
  26. const unsigned int GBUdbGoodMask = 0x3FFF8000; // The good count is masked in this range.
  27. const unsigned int GBUdbBadMask = 0x00007FFF; // Tha bad count is masked here.
  28. const unsigned int GBUdbLimit = GBUdbBadMask; // When a count hits this, normalize in half.
  29. const unsigned int GBUdbGoodShift = 15; // Shift good counts this many bits.
  30. const unsigned int GBUdbMatchEntryBit = 0x80000000; // Match entry Index bit.
  31. const unsigned int GBUdbMatchUnusedBit = 0x40000000; // Unalocated Match entry Index bit.
  32. const unsigned int GBUdbMatchDataMask = 0x3fffffff; // IP Match data mask.
  33. enum GBUdbFlag { // A type for the GBUdb flag.
  34. Ignore = GBUdbIgnore, // Ignore
  35. Ugly = GBUdbUgly, // Ugly
  36. Good = GBUdbGood, // Good
  37. Bad = GBUdbBad // Bad
  38. };
  39. //// GBUdbLocking semantics
  40. //// When doForAllRecords() is called at the GBUdb level, we need to know how
  41. //// the GBUdb mutex should be handled.
  42. enum GBUdbLocking { // A type that describes locking semantics.
  43. Dataset, // Lock the through the entire operation.
  44. Record, // Lock and unlock for each record.
  45. None // Do not lock.
  46. };
  47. typedef unsigned int GBUdbIndex; // A type for Index values from records.
  48. const GBUdbIndex GBUdbUnknown = 0x00000000; // The unknown address.
  49. const int GBUdbRecordsPerNode = 256; // Records per node.
  50. const int GBUdbDefaultGrowNodes = 8192; // Default Nodes to grow.
  51. const int GBUdbDefaultArraySize = GBUdbRecordsPerNode * GBUdbDefaultGrowNodes; // Default initial Array size.
  52. const int GBUdbRootNodeOffset = 256; // First indexing node after node 0.
  53. const int GBUdbGrowthThreshold = 4; // Time to grow at this # free nodes.
  54. //// Node 0 is the go-nowhere node for when things fall off the index so it
  55. //// is coded to all GBUdbUnknown.
  56. //// The last node in the array is used for global statistics & allocation
  57. //// tables.
  58. const int GBUdbControlNodeOffset = -256; // Offset from end of data for control node.
  59. const int GBUdbNextFreeNodeOffset = GBUdbControlNodeOffset + 0; // Offset for next free node index.
  60. const int GBUdbMatchListOffset = GBUdbControlNodeOffset +1; // Offset for Match record allocation root.
  61. const int GBUdbIPCountOffset = GBUdbControlNodeOffset + 2; // Offset for count of IPs in GBUdb.
  62. // GBUdbRecord converts an ordinary unsigned long integer into a wealth of
  63. // useful information just by adding a collection of useful tools.
  64. class GBUdbRecord { // A GBUdb record is really just a
  65. public: // long integer, but it can be interpreted
  66. // lots of ways.
  67. unsigned int RawData; // The raw unsigned int goes here.
  68. GBUdbRecord(); // Initialize to zero.
  69. GBUdbFlag Flag(); // This returns the flag.
  70. GBUdbFlag Flag(GBUdbFlag f); // This sets and returns the flag.
  71. unsigned int Good(); // This returns the good count.
  72. unsigned int Good(unsigned int g); // This sets and returns the good count.
  73. unsigned int Bad(); // This returns the bad count.
  74. unsigned int Bad(unsigned int b); // This sets and returns the bad count.
  75. unsigned int addGood(unsigned int g = 1); // This increments the good count.
  76. unsigned int addBad(unsigned int b = 1); // This increments the bad count.
  77. GBUdbRecord& integrate(GBUdbRecord& A, int LocalWeight, int RemoteWeight); // This integrates another record.
  78. GBUdbIndex Index(); // This returns the record as an Index.
  79. GBUdbIndex Index(GBUdbIndex i); // This sets the record as an index.
  80. double Probability(); // Return +(bad) or -(good) probability.
  81. double Confidence(); // Return the confidence based on samples.
  82. };
  83. // Special events need to be recorded. For that job we have GBUdbAlerts
  84. const int UTCBufferSize = 16; // C string buffer size for UTC stamp.
  85. class GBUdbAlert {
  86. public:
  87. GBUdbAlert(); // Constructor sets timestamp & nulls.
  88. char UTC[UTCBufferSize]; // Time stamp for this alert.
  89. unsigned int IP; // IP for this alert.
  90. GBUdbRecord R; // GBUdbRecord for this alert.
  91. std::string toXML(); // Convert to an xml representation.
  92. };
  93. // Mass update kinds of operations are handled by providing a functor
  94. // of the type GBUdbOperator to the method doForAllRecords(). The functor is
  95. // called with every record in the GBUdb.
  96. //// Here is the virtual GBUdb Operator class.
  97. class GBUdbOperator {
  98. public:
  99. virtual GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) = 0;
  100. };
  101. // GBUdbDataset manages a large array of GBUdb records and nodes. Nodes are
  102. // simulated data structures -- essentially arrays of GBUdbRecords that are
  103. // interpreted as Indexes so that each byte of a particular IP can be used
  104. // to follow the index through the tree to the final record that actually
  105. // represents the IPs data.
  106. // The last few records in the array are used to keep track of some basic
  107. // statistics including where the next node will come from. As with the GBUdb
  108. // record itself, it's all in how the data is interpreted. Using this strategy
  109. // of converting plain-old integers into various data types on the fly allows
  110. // us to allocate the entire structure as a single block and avoid much
  111. // page swapping behind the scenes.
  112. class GBUdbDataset {
  113. private:
  114. GBUdbRecord* DataArray; // Array of GBUdbRecords, nodes, etc.
  115. int MyArraySize; // The size of the array in records.
  116. std::string MyFileName; // CString for the file name.
  117. GBUdbIndex ixIPCount(); // Index of the IP count for this db.
  118. GBUdbIndex ixNextFreeNode(); // Index of the Next Free Node Index.
  119. GBUdbIndex ixMatchListRoot(); // Index of the Match List Root Index.
  120. GBUdbIndex newMatchRecord(unsigned int IP); // Allocate a new Match record for IP.
  121. GBUdbIndex newMatchNodeRoot(); // Allocate a new Match node.
  122. GBUdbIndex newNodeRoot(); // Allocates a new node, returns offset.
  123. void deleteMatchAt(GBUdbIndex I); // Recall match record at I for reuse.
  124. // invokeAt() Handles invocation at each node/octet using and managing MatchRecords as needed.
  125. GBUdbIndex invokeAt(GBUdbRecord& R, unsigned int IP, int Octet, bool ExtendMatches);
  126. int increaseIPCount(); // When we add an IP to the db.
  127. int decreaseIPCount(); // When we drop an IP from the db.
  128. void increaseIPCountIfNew(GBUdbRecord& R); // If R is GBUdbUnknown, IncreaseIPCount.
  129. bool isMatch(GBUdbIndex I); // True if record at I is a match record.
  130. bool isMatch(GBUdbIndex I, unsigned int IP); // True if record at I is a match for IP.
  131. GBUdbRecord& MatchedData(GBUdbIndex I); // Returns the data for the match at I.
  132. unsigned int EncodedMatch(unsigned int IP); // Returns encoded raw dat for a Match.
  133. //// In order to support binmodal indexing we must make sure that
  134. //// no octet3 data is mapped to the root record in an octet3 node. If
  135. //// it were so mapped then an octet2 evaluation might misinterpret the
  136. //// GBUdbFlag fields as a MatchRecord indicator and cause the data to
  137. //// become corrupted. To solve this problem, any time an octet2 node
  138. //// maps to an octet3 node and NOT a MatchRecord, the 0 record in the
  139. //// octet3 node must have no flags. Since x.x.x.0 is presumed to be the
  140. //// network address, and x.x.x.255 is presumed to be a broadcast address
  141. //// we cause both to map to a single record (the 255 record) where the
  142. //// Class C, B, or A data can be recorded and modified in safety. Since
  143. //// there is no need to track the brodcast and network address cases.
  144. //// separately there is no inherent conflict in this approach. The
  145. //// remapIP00toFF method performs this transform as needed in the
  146. //// readRecord() and invokeRecord() methods.
  147. unsigned int remapIP00toFF(unsigned int IP); // Remaps final octet 00 to FF if needed.
  148. GBUdbRecord MySafeUnknownRecord; // Safe unknown record to return.
  149. GBUdbRecord& SafeUnknownRecord(); // Clears and returns the Safe record.
  150. // doForAllNodes does its job by launching a recursive search algorythm
  151. // which is embodied in doAllAtNode(). The doAllAtNode() method is called
  152. // for the root node by doForAllRecords and searches through the tree depth
  153. // first to locate each active record in the GBUdb and call the Operator.
  154. // updateWorkingIP() uses progressive input from eacn level to determine
  155. // the effective IP for the node under test.
  156. void updateWorkingIP(unsigned int& WIP, int OctetValue, int Level);
  157. void doAllAtNode(GBUdbIndex I, GBUdbOperator& O, int NodeLevel, unsigned int WorkingIP);
  158. public:
  159. ~GBUdbDataset(); // Flush & shutdown a dataset.
  160. GBUdbDataset(const char* SetFileName); // Create with a name or no name (NULL).
  161. GBUdbDataset(GBUdbDataset& Original); // Copy constructor.
  162. class CouldNotGrow {}; // Thrown when grow() fails.
  163. class NoFreeNodes {}; // Thrown when newNodeRoot() fails.
  164. class MatchAllocationCorrupted {}; // Thrown when newMatchRecord() fails.
  165. GBUdbRecord& readRecord(unsigned int IP); // Read only - find a GBUdb record.
  166. GBUdbRecord& invokeRecord(unsigned int IP); // Create and/or Find a GBUdb record.
  167. bool dropRecord(unsigned int IP); // Drop an IP record. (true if we did)
  168. int ArraySize(); // Array size.
  169. int FreeNodes(); // Number of free nodes remaining.
  170. int IPCount(); // Number of IPs stored.
  171. const char* FileName(const char* NewName); // Set new file name w/ cstring.
  172. const char* FileName(); // Return the name.
  173. void grow(int HowManyNodes = GBUdbDefaultGrowNodes); // Grow (by number of nodes).
  174. void save(); // Flush the dataset to disk.
  175. void load(); // Read the dataset from disk.
  176. void doForAllRecords(GBUdbOperator& O); // Calls O(IP, Record) W/ every record.
  177. };
  178. // The GBUdb ojbect manages access to the GBUdb. For example, it will grow the
  179. // dataset when that is required, report new events, and generally serve as the
  180. // main access point for a given GBUdb. It even serializes multiple threads.
  181. //// Here is the actual GBUdb class.
  182. class GBUdb {
  183. private:
  184. CodeDweller::Mutex MyMutex; // Data sync mutex.
  185. CodeDweller::Mutex AlertsMutex; // Mutex for the alerts list.
  186. GBUdbDataset* MyDataset; // Array of records.
  187. int PostsCounter; // Counts good/bad posts.
  188. std::list<GBUdbAlert> MyAlerts; // Allerts list.
  189. void recordAlertFor(unsigned int IP, GBUdbRecord& R, unsigned int C); // Append an alert record if needed.
  190. public:
  191. GBUdb(); // Open/Create w/ no name.
  192. GBUdb(const char* FileName); // Open/Create w/ cstring or NULL.
  193. ~GBUdb(); // Shutdown
  194. const char* FileName(const char* NewName); // Set/Change the file name.
  195. const char* FileName(); // Return the FileName.
  196. void save(); // Save the data.
  197. void load(); // Load the data.
  198. GBUdbRecord addGood(unsigned int IP, int i = 1); // Count an IP as good.
  199. GBUdbRecord addBad(unsigned int IP, int i = 1); // Count an IP as bad.
  200. GBUdbRecord setGood(unsigned int IP); // Set the flag to Good for this IP.
  201. GBUdbRecord setBad(unsigned int IP); // Set the flag to Bad for this IP.
  202. GBUdbRecord setUgly(unsigned int IP); // Set the flag to Ugly for this IP.
  203. GBUdbRecord setIgnore(unsigned int IP); // Set the flag to Ignore for this IP.
  204. bool dropRecord(unsigned int IP); // Drop an IP record. (true if we did)
  205. GBUdbRecord getRecord(unsigned int IP); // Retrieve an IP record.
  206. GBUdbRecord setRecord(unsigned int IP, GBUdbRecord& R); // Store an IP record.
  207. GBUdbRecord adjustCounts(unsigned int IP, GBUdbRecord& R); // Adds counts from R to record for IP.
  208. void doForAllRecords(GBUdbOperator& O, GBUdbLocking L = Dataset); // Call the Operator w/ All records.
  209. void saveSnapshot(); // Saves a snapshot of the current db.
  210. void reduce(); // Reduce all counts by half.
  211. void compress(); // Remove any unknown records (reduced to zero).
  212. int readIgnoreList(const char* FileName = "GBUdbIgnoreList.txt"); // setIgnore for a list of IPs
  213. void GetAlerts(std::list<GBUdbAlert>& ListToFill); // Get all current alerts & clear.
  214. void ImportAlerts(std::list<GBUdbAlert>& PeerAlerts); // Default log2 alert import function.
  215. int IPCount(); // Number of IPs stored.
  216. int Size(); // Size of GBUdb in bytes.
  217. double Utilization(); // Utilization (percent).
  218. int Posts(); // Number of posts since last save.
  219. };
  220. //// Include inline method definitions /////////////////////////////////////////
  221. #include "GBUdb.inline.hpp"
  222. } // namespace SNFMulti
  223. #endif
  224. // End of GBUdb Include Only Once
  225. ////////////////////////////////////////////////////////////////////////////////