Novacoin/CBlockStore.cs

   1 \feffusing System;
   2 using System.IO;
   3 using System.Linq;
   4 using System.Collections.Concurrent;
   5
   6 using SQLite.Net;
   7 using SQLite.Net.Attributes;
   8 using SQLite.Net.Interop;
   9 using SQLite.Net.Platform.Generic;
  10 using SQLiteNetExtensions.Attributes;
  11 using System.Collections.Generic;
  12
  13 namespace Novacoin
  14 {
  15     /// <summary>
  16     /// Block headers table
  17     /// </summary>
  18     [Table("BlockStorage")]
  19     class CBlockStoreItem
  20     {
  21         /// <summary>
  22         /// Item ID in the database
  23         /// </summary>
  24         [PrimaryKey, AutoIncrement]
  25         public int ItemID { get; set; }
  26
  27         /// <summary>
  28         /// PBKDF2+Salsa20 of block hash
  29         /// </summary>
  30         [Unique]
  31         public byte[] Hash { get; set; }
  32
  33         /// <summary>
  34         /// Version of block schema
  35         /// </summary>
  36         public uint nVersion { get; set; }
  37
  38         /// <summary>
  39         /// Previous block hash.
  40         /// </summary>
  41         public byte[] prevHash { get; set; }
  42
  43         /// <summary>
  44         /// Merkle root hash.
  45         /// </summary>
  46         public byte[] merkleRoot { get; set; }
  47
  48         /// <summary>
  49         /// Block timestamp.
  50         /// </summary>
  51         public uint nTime { get; set; }
  52
  53         /// <summary>
  54         /// Compressed difficulty representation.
  55         /// </summary>
  56         public uint nBits { get; set; }
  57
  58         /// <summary>
  59         /// Nonce counter.
  60         /// </summary>
  61         public uint nNonce { get; set; }
  62
  63         /// <summary>
  64         /// Block type flags
  65         /// </summary>
  66         public BlockType BlockTypeFlag { get; set; }
  67
  68         /// <summary>
  69         /// Stake modifier
  70         /// </summary>
  71         public long nStakeModifier { get; set; }
  72
  73         /// <summary>
  74         /// Stake entropy bit
  75         /// </summary>
  76         public byte nEntropyBit { get; set; }
  77
  78         /// <summary>
  79         /// Next block hash
  80         /// </summary>
  81         public byte[] NextHash { get; set; }
  82
  83         /// <summary>
  84         /// Block height
  85         /// </summary>
  86         public uint nHeight { get; set; }
  87
  88         /// <summary>
  89         /// Block position in file
  90         /// </summary>
  91         public long nBlockPos { get; set; }
  92
  93         /// <summary>
  94         /// Block size in bytes
  95         /// </summary>
  96         public int nBlockSize { get; set; }
  97
  98         /// <summary>
  99         /// Fill database item with data from given block header.
 100         /// </summary>
 101         /// <param name="header">Block header</param>
 102         /// <returns>Header hash</returns>
 103         public ScryptHash256 FillHeader(CBlockHeader header)
 104         {
 105             ScryptHash256 _hash;
 106             Hash = _hash = header.Hash;
 107
 108             nVersion = header.nVersion;
 109             prevHash = header.prevHash;
 110             merkleRoot = header.merkleRoot;
 111             nTime = header.nTime;
 112             nBits = header.nBits;
 113             nNonce = header.nNonce;
 114
 115             return _hash;
 116         }
 117
 118         /// <summary>
 119         /// Reconstruct block header from item data.
 120         /// </summary>
 121         public CBlockHeader BlockHeader
 122         {
 123             get
 124             {
 125                 CBlockHeader header = new CBlockHeader();
 126
 127                 header.nVersion = nVersion;
 128                 header.prevHash = new ScryptHash256(prevHash);
 129                 header.merkleRoot = new Hash256(merkleRoot);
 130                 header.nTime = nTime;
 131                 header.nBits = nBits;
 132                 header.nNonce = nNonce;
 133
 134                 return header;
 135             }
 136         }
 137
 138         /// <summary>
 139         /// Read block from file.
 140         /// </summary>
 141         /// <param name="reader">Stream with read access.</param>
 142         /// <param name="reader">CBlock reference.</param>
 143         /// <returns>Result</returns>
 144         public bool ReadFromFile(ref Stream reader, out CBlock block)
 145         {
 146             var buffer = new byte[nBlockSize];
 147             block = null;
 148
 149             try
 150             {
 151                 reader.Seek(nBlockPos, SeekOrigin.Begin);
 152
 153                 if (nBlockSize != reader.Read(buffer, 0, nBlockSize))
 154                 {
 155                     return false;
 156                 }
 157
 158                 block = new CBlock(buffer);
 159
 160                 return true;
 161             }
 162             catch (IOException)
 163             {
 164                 // I/O error
 165                 return false;
 166             }
 167             catch (BlockException)
 168             {
 169                 // Constructor exception
 170                 return false;
 171             }
 172         }
 173
 174         /// <summary>
 175         /// Writes given block to file and prepares cursor object for insertion into the database.
 176         /// </summary>
 177         /// <param name="writer">Stream with write access.</param>
 178         /// <param name="block">CBlock reference.</param>
 179         /// <returns>Result</returns>
 180         public bool WriteToFile(ref Stream writer, ref CBlock block)
 181         {
 182             try
 183             {
 184                 byte[] blockBytes = block;
 185
 186                 var magicBytes = BitConverter.GetBytes(CBlockStore.nMagicNumber);
 187                 var blkLenBytes = BitConverter.GetBytes(blockBytes.Length);
 188
 189                 // Seek to the end and then append magic bytes there.
 190                 writer.Seek(0, SeekOrigin.End);
 191                 writer.Write(magicBytes, 0, magicBytes.Length);
 192                 writer.Write(blkLenBytes, 0, blkLenBytes.Length);
 193
 194                 // Save block size and current position in the block cursor fields.
 195                 nBlockPos = writer.Position;
 196                 nBlockSize = blockBytes.Length;
 197
 198                 // Write block and flush the stream.
 199                 writer.Write(blockBytes, 0, blockBytes.Length);
 200                 writer.Flush();
 201
 202                 return true;
 203             }
 204             catch (IOException)
 205             {
 206                 // I/O error
 207                 return false;
 208             }
 209             catch (Exception)
 210             {
 211                 // Some serialization error
 212                 return false;
 213             }
 214         }
 215     }
 216
 217     /// <summary>
 218     /// Block type.
 219     /// </summary>
 220     public enum BlockType
 221     {
 222         PROOF_OF_WORK,
 223         PROOF_OF_WORK_MODIFIER,
 224         PROOF_OF_STAKE,
 225         PROOF_OF_STAKE_MODIFIER
 226     };
 227
 228     /// <summary>
 229     /// Transaction type.
 230     /// </summary>
 231     public enum TxType
 232     {
 233         TX_COINBASE,
 234         TX_COINSTAKE,
 235         TX_USER
 236     }
 237
 238     [Table("TransactionStorage")]
 239     public class CTransactionStoreItem
 240     {
 241         /// <summary>
 242         /// Transaction hash
 243         /// </summary>
 244         [PrimaryKey]
 245         public byte[] TransactionHash { get; set; }
 246
 247         /// <summary>
 248         /// Block hash
 249         /// </summary>
 250         [ForeignKey(typeof(CBlockStoreItem), Name = "Hash")]
 251         public byte[] BlockHash { get; set; }
 252
 253         /// <summary>
 254         /// Transaction type flag
 255         /// </summary>
 256         public TxType txType { get; set; }
 257
 258         /// <summary>
 259         /// Tx position in file
 260         /// </summary>
 261         public long nTxPos { get; set; }
 262
 263         /// <summary>
 264         /// Transaction size
 265         /// </summary>
 266         public int nTxSize { get; set; }
 267
 268         /// <summary>
 269         /// Read transaction from file.
 270         /// </summary>
 271         /// <param name="reader">Stream with read access.</param>
 272         /// <param name="tx">CTransaction reference.</param>
 273         /// <returns>Result</returns>
 274         public bool ReadFromFile(ref Stream reader, out CTransaction tx)
 275         {
 276             var buffer = new byte[250000]; // Max transaction size is 250kB
 277             tx = null;
 278
 279             try
 280             {
 281                 reader.Seek(nTxPos, SeekOrigin.Begin); // Seek to transaction offset
 282
 283                 if (nTxSize != reader.Read(buffer, 0, nTxSize))
 284                 {
 285                     return false;
 286                 }
 287
 288                 tx = new CTransaction(buffer);
 289
 290                 return true;
 291             }
 292             catch (IOException)
 293             {
 294                 // I/O error
 295                 return false;
 296             }
 297             catch (TransactionConstructorException)
 298             {
 299                 // Constructor error
 300                 return false;
 301             }
 302         }
 303     }
 304
 305     public class CBlockStore : IDisposable
 306     {
 307         public const uint nMagicNumber = 0xe5e9e8e4;
 308
 309         private bool disposed = false;
 310         private object LockObj = new object();
 311
 312         /// <summary>
 313         /// SQLite connection object.
 314         /// </summary>
 315         private SQLiteConnection dbConn;
 316
 317         /// <summary>
 318         /// Block file.
 319         /// </summary>
 320         private string strBlockFile;
 321
 322         /// <summary>
 323         /// Index database file.
 324         /// </summary>
 325         private string strDbFile;
 326
 327         /// <summary>
 328         /// Map of block tree nodes.
 329         /// </summary>
 330         private ConcurrentDictionary<ScryptHash256, CBlockStoreItem> blockMap = new ConcurrentDictionary<ScryptHash256, CBlockStoreItem>();
 331
 332         /// <summary>
 333         /// Orphaned blocks map.
 334         /// </summary>
 335         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMap = new ConcurrentDictionary<ScryptHash256, CBlock>();
 336         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMapByPrev = new ConcurrentDictionary<ScryptHash256, CBlock>();
 337
 338         /// <summary>
 339         /// Map of unspent items.
 340         /// </summary>
 341         private ConcurrentDictionary<Hash256, CTransactionStoreItem> txMap = new ConcurrentDictionary<Hash256, CTransactionStoreItem>();
 342
 343         private CBlock genesisBlock = new CBlock(
 344             Interop.HexToArray(
 345                 "01000000" + // nVersion=1
 346                 "0000000000000000000000000000000000000000000000000000000000000000" + // prevhash is zero
 347                 "7b0502ad2f9f675528183f83d6385794fbcaa914e6d385c6cb1d866a3b3bb34c" + // merkle root
 348                 "398e1151" + // nTime=1360105017
 349                 "ffff0f1e" + // nBits=0x1e0fffff
 350                 "d3091800" + // nNonce=1575379
 351                 "01" +       // nTxCount=1
 352                 "01000000" + // nVersion=1
 353                 "398e1151" + // nTime=1360105017
 354                 "01" +       // nInputs=1
 355                 "0000000000000000000000000000000000000000000000000000000000000000" + // input txid is zero
 356                 "ffffffff" + // n=uint.maxValue
 357                 "4d" +       // scriptSigLen=77
 358                 "04ffff001d020f274468747470733a2f2f626974636f696e74616c6b2e6f72672f696e6465782e7068703f746f7069633d3133343137392e6d736731353032313936236d736731353032313936" + // scriptSig
 359                 "ffffffff" + // nSequence=uint.maxValue
 360                 "01" +       // nOutputs=1
 361                 "0000000000000000" + // nValue=0
 362                 "00" +       // scriptPubkeyLen=0
 363                 "00000000" + // nLockTime=0
 364                 "00"         // sigLen=0
 365         ));
 366
 367         public static CBlockStore Instance;
 368
 369         /// <summary>
 370         /// Block file stream with read access
 371         /// </summary>
 372         private Stream reader;
 373
 374         /// <summary>
 375         /// Block file stream with write access
 376         /// </summary>
 377         private Stream writer;
 378
 379         /// <summary>
 380         /// Init the block storage manager.
 381         /// </summary>
 382         /// <param name="IndexDB">Path to index database</param>
 383         /// <param name="BlockFile">Path to block file</param>
 384         public CBlockStore(string IndexDB = "blockstore.dat", string BlockFile = "blk0001.dat")
 385         {
 386             strDbFile = IndexDB;
 387             strBlockFile = BlockFile;
 388
 389             bool firstInit = !File.Exists(strDbFile);
 390             dbConn = new SQLiteConnection(new SQLitePlatformGeneric(), strDbFile);
 391
 392             var fStreamReadWrite = File.Open(strBlockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
 393             reader = new BinaryReader(fStreamReadWrite).BaseStream;
 394             writer = new BinaryWriter(fStreamReadWrite).BaseStream;
 395
 396             if (firstInit)
 397             {
 398                 lock (LockObj)
 399                 {
 400                     // Create tables
 401                     dbConn.CreateTable<CBlockStoreItem>(CreateFlags.AutoIncPK);
 402                     dbConn.CreateTable<CTransactionStoreItem>(CreateFlags.ImplicitPK);
 403
 404                     // Write block to file.
 405                     var itemTemplate = new CBlockStoreItem()
 406                     {
 407                         nHeight = 0
 408                     };
 409
 410                     itemTemplate.FillHeader(genesisBlock.header);
 411
 412                     if (!AddItemToIndex(ref itemTemplate, ref genesisBlock))
 413                     {
 414                         throw new Exception("Unable to write genesis block");
 415                     }
 416                 }
 417             }
 418             else
 419             {
 420                 var blockTreeItems = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] order by [ItemId] asc");
 421
 422                 // Init list of block items
 423                 foreach (var item in blockTreeItems)
 424                 {
 425                     blockMap.TryAdd(new ScryptHash256(item.Hash), item);
 426                 }
 427             }
 428
 429             Instance = this;
 430         }
 431
 432         public bool GetTransaction(Hash256 TxID, ref CTransaction tx)
 433         {
 434             var QueryTx = dbConn.Query<CTransactionStoreItem>("select * from [TransactionStorage] where [TransactionHash] = ?", (byte[])TxID);
 435
 436             if (QueryTx.Count == 1)
 437             {
 438                 return QueryTx[0].ReadFromFile(ref reader, out tx);
 439             }
 440
 441             // Tx not found
 442
 443             return false;
 444         }
 445
 446         private bool AddItemToIndex(ref CBlockStoreItem itemTemplate, ref CBlock block)
 447         {
 448             var blockHash = new ScryptHash256(itemTemplate.Hash);
 449
 450             if (blockMap.ContainsKey(blockHash))
 451             {
 452                 // Already have this block.
 453                 return false;
 454             }
 455
 456             // TODO: compute chain trust, set stake entropy bit, record proof-of-stake hash value
 457
 458             // TODO: compute stake modifier
 459
 460             // Add to index
 461             itemTemplate.BlockTypeFlag = block.IsProofOfStake ? BlockType.PROOF_OF_STAKE : BlockType.PROOF_OF_WORK;
 462
 463             if (!itemTemplate.WriteToFile(ref writer, ref block))
 464             {
 465                 return false;
 466             }
 467
 468             dbConn.Insert(itemTemplate);
 469
 470             // We have no SetBestChain and ConnectBlock/Disconnect block yet, so adding these transactions manually.
 471             for (int i = 0; i < block.vtx.Length; i++)
 472             {
 473                 // Handle trasactions
 474
 475                 if (!block.vtx[i].VerifyScripts())
 476                 {
 477                     return false;
 478                 }
 479
 480                 var nTxOffset = itemTemplate.nBlockPos + block.GetTxOffset(i);
 481                 TxType txnType = TxType.TX_USER;
 482
 483                 if (block.vtx[i].IsCoinBase)
 484                 {
 485                     txnType = TxType.TX_COINBASE;
 486                 }
 487                 else if (block.vtx[i].IsCoinStake)
 488                 {
 489                     txnType = TxType.TX_COINSTAKE;
 490                 }
 491
 492                 var NewTxItem = new CTransactionStoreItem()
 493                 {
 494                     TransactionHash = block.vtx[i].Hash,
 495                     BlockHash = blockHash,
 496                     nTxPos = nTxOffset,
 497                     nTxSize = block.vtx[i].Size,
 498                     txType = txnType
 499                 };
 500
 501                 dbConn.Insert(NewTxItem);
 502             }
 503
 504             return blockMap.TryAdd(blockHash, itemTemplate);
 505         }
 506
 507         public bool AcceptBlock(ref CBlock block)
 508         {
 509             ScryptHash256 hash = block.header.Hash;
 510
 511             if (blockMap.ContainsKey(hash))
 512             {
 513                 // Already have this block.
 514                 return false;
 515             }
 516
 517             CBlockStoreItem prevBlockCursor = null;
 518             if (!blockMap.TryGetValue(block.header.prevHash, out prevBlockCursor))
 519             {
 520                 // Unable to get the cursor.
 521                 return false;
 522             }
 523
 524             var prevBlockHeader = prevBlockCursor.BlockHeader;
 525
 526             // TODO: proof-of-work/proof-of-stake verification
 527             uint nHeight = prevBlockCursor.nHeight + 1;
 528
 529             // Check timestamp against prev
 530             if (NetInfo.FutureDrift(block.header.nTime) < prevBlockHeader.nTime)
 531             {
 532                 // block's timestamp is too early
 533                 return false;
 534             }
 535
 536             // Check that all transactions are finalized
 537             foreach (var tx in block.vtx)
 538             {
 539                 if (!tx.IsFinal(nHeight, block.header.nTime))
 540                 {
 541                     return false;
 542                 }
 543             }
 544
 545             // TODO: Enforce rule that the coinbase starts with serialized block height
 546
 547             // Write block to file.
 548             var itemTemplate = new CBlockStoreItem()
 549             {
 550                 nHeight = nHeight
 551             };
 552
 553             itemTemplate.FillHeader(block.header);
 554
 555             if (!AddItemToIndex(ref itemTemplate, ref block))
 556             {
 557                 return false;
 558             }
 559
 560             return true;
 561         }
 562
 563         public bool GetBlock(ScryptHash256 blockHash, ref CBlock block)
 564         {
 565             var QueryBlock = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] where [Hash] = ?", (byte[])blockHash);
 566
 567             if (QueryBlock.Count == 1)
 568             {
 569                 return QueryBlock[0].ReadFromFile(ref reader, out block);
 570             }
 571
 572             // Block not found
 573
 574             return false;
 575         }
 576
 577         public bool ProcessBlock(ref CBlock block)
 578         {
 579             ScryptHash256 blockHash = block.header.Hash;
 580
 581             if (blockMap.ContainsKey(blockHash))
 582             {
 583                 // We already have this block.
 584                 return false;
 585             }
 586
 587             if (orphanMap.ContainsKey(blockHash))
 588             {
 589                 // We already have block in the list of orphans.
 590                 return false;
 591             }
 592
 593             // TODO: Limited duplicity on stake and reserialization of block signature
 594
 595             // Preliminary checks
 596             if (!block.CheckBlock(true, true, true))
 597             {
 598                 return true;
 599             }
 600
 601             if (block.IsProofOfStake)
 602             {
 603                 // TODO: proof-of-stake validation
 604             }
 605
 606             // TODO: difficulty verification
 607
 608             // If don't already have its previous block, shunt it off to holding area until we get it
 609             if (!blockMap.ContainsKey(block.header.prevHash))
 610             {
 611                 if (block.IsProofOfStake)
 612                 {
 613                     // TODO: limit duplicity on stake
 614                 }
 615
 616                 var block2 = new CBlock(block);
 617                 orphanMap.TryAdd(blockHash, block2);
 618                 orphanMapByPrev.TryAdd(blockHash, block2);
 619
 620                 return true;
 621             }
 622
 623             // Store block to disk
 624             if (!AcceptBlock(ref block))
 625             {
 626                 // Accept failed
 627                 return false;
 628             }
 629
 630             // Recursively process any orphan blocks that depended on this one
 631             var orphansQueue = new List<ScryptHash256>();
 632             orphansQueue.Add(blockHash);
 633
 634             for (int i = 0; i < orphansQueue.Count; i++)
 635             {
 636                 ScryptHash256 hashPrev = orphansQueue[i];
 637
 638                 foreach (var pair in orphanMap)
 639                 {
 640                     var orphanBlock = pair.Value;
 641
 642                     if (orphanBlock.header.prevHash == blockHash)
 643                     {
 644                         if (AcceptBlock(ref orphanBlock))
 645                         {
 646                             orphansQueue.Add(pair.Key);
 647                         }
 648
 649                         CBlock dummy1;
 650                         orphanMap.TryRemove(pair.Key, out dummy1);
 651                     }
 652                 }
 653
 654                 CBlock dummy2;
 655                 orphanMap.TryRemove(hashPrev, out dummy2);
 656             }
 657
 658             return true;
 659         }
 660
 661         public bool ParseBlockFile(string BlockFile = "bootstrap.dat")
 662         {
 663             // TODO: Rewrite completely.
 664
 665             var nOffset = 0L;
 666
 667             var buffer = new byte[1000000]; // Max block size is 1Mb
 668             var intBuffer = new byte[4];
 669
 670             var fStream2 = File.OpenRead(BlockFile);
 671             var readerForBlocks = new BinaryReader(fStream2).BaseStream;
 672
 673             readerForBlocks.Seek(nOffset, SeekOrigin.Begin); // Seek to previous offset + previous block length
 674
 675             dbConn.BeginTransaction();
 676
 677             while (readerForBlocks.Read(buffer, 0, 4) == 4) // Read magic number
 678             {
 679                 var nMagic = BitConverter.ToUInt32(buffer, 0);
 680                 if (nMagic != 0xe5e9e8e4)
 681                 {
 682                     throw new Exception("Incorrect magic number.");
 683                 }
 684
 685                 var nBytesRead = readerForBlocks.Read(buffer, 0, 4);
 686                 if (nBytesRead != 4)
 687                 {
 688                     throw new Exception("BLKSZ EOF");
 689                 }
 690
 691                 var nBlockSize = BitConverter.ToInt32(buffer, 0);
 692
 693                 nOffset = readerForBlocks.Position;
 694
 695                 nBytesRead = readerForBlocks.Read(buffer, 0, nBlockSize);
 696
 697                 if (nBytesRead == 0 || nBytesRead != nBlockSize)
 698                 {
 699                     throw new Exception("BLK EOF");
 700                 }
 701
 702                 var block = new CBlock(buffer);
 703                 var hash = block.header.Hash;
 704
 705                 if (blockMap.ContainsKey(hash))
 706                 {
 707                     continue;
 708                 }
 709
 710                 if (!ProcessBlock(ref block))
 711                 {
 712                     throw new Exception("Invalid block: " + block.header.Hash);
 713                 }
 714
 715                 int nCount = blockMap.Count;
 716                 Console.WriteLine("nCount={0}, Hash={1}, Time={2}", nCount, block.header.Hash, DateTime.Now); // Commit on each 100th block
 717
 718                 if (nCount % 100 == 0 && nCount != 0)
 719                 {
 720                     Console.WriteLine("Commit...");
 721                     dbConn.Commit();
 722                     dbConn.BeginTransaction();
 723                 }
 724             }
 725
 726             dbConn.Commit();
 727
 728             return true;
 729         }
 730
 731         ~CBlockStore()
 732         {
 733             Dispose(false);
 734         }
 735
 736         public void Dispose()
 737         {
 738             Dispose(true);
 739             GC.SuppressFinalize(this);
 740         }
 741
 742         protected virtual void Dispose(bool disposing)
 743         {
 744             if (!disposed)
 745             {
 746                 if (disposing)
 747                 {
 748                     // Free other state (managed objects).
 749
 750                     reader.Dispose();
 751                     reader.Dispose();
 752                 }
 753
 754                 if (dbConn != null)
 755                 {
 756                     dbConn.Close();
 757                     dbConn = null;
 758                 }
 759
 760                 disposed = true;
 761             }
 762         }
 763
 764     }
 765 }