Novacoin/CBlockStore.cs

   1 \feffusing System;
   2 using System.IO;
   3 using System.Linq;
   4 using System.Collections.Concurrent;
   5
   6 using SQLite.Net;
   7 using SQLite.Net.Attributes;
   8 using SQLite.Net.Interop;
   9 using SQLite.Net.Platform.Generic;
  10 using SQLiteNetExtensions.Attributes;
  11 using System.Collections.Generic;
  12
  13 namespace Novacoin
  14 {
  15     /// <summary>
  16     /// Block headers table
  17     /// </summary>
  18     [Table("BlockStorage")]
  19     class CBlockStoreItem
  20     {
  21         /// <summary>
  22         /// Item ID in the database
  23         /// </summary>
  24         [PrimaryKey, AutoIncrement]
  25         public int ItemID { get; set; }
  26
  27         /// <summary>
  28         /// PBKDF2+Salsa20 of block hash
  29         /// </summary>
  30         [Unique]
  31         public byte[] Hash { get; set; }
  32
  33         /// <summary>
  34         /// Version of block schema
  35         /// </summary>
  36         public uint nVersion { get; set; }
  37
  38         /// <summary>
  39         /// Previous block hash.
  40         /// </summary>
  41         public byte[] prevHash { get; set; }
  42
  43         /// <summary>
  44         /// Merkle root hash.
  45         /// </summary>
  46         public byte[] merkleRoot { get; set; }
  47
  48         /// <summary>
  49         /// Block timestamp.
  50         /// </summary>
  51         public uint nTime { get; set; }
  52
  53         /// <summary>
  54         /// Compressed difficulty representation.
  55         /// </summary>
  56         public uint nBits { get; set; }
  57
  58         /// <summary>
  59         /// Nonce counter.
  60         /// </summary>
  61         public uint nNonce { get; set; }
  62
  63         /// <summary>
  64         /// Block type flags
  65         /// </summary>
  66         public BlockType BlockTypeFlag { get; set; }
  67
  68         /// <summary>
  69         /// Stake modifier
  70         /// </summary>
  71         public long nStakeModifier { get; set; }
  72
  73         /// <summary>
  74         /// Stake entropy bit
  75         /// </summary>
  76         public byte nEntropyBit { get; set; }
  77
  78         /// <summary>
  79         /// Block height
  80         /// </summary>
  81         public uint nHeight { get; set; }
  82
  83         /// <summary>
  84         /// Block position in file
  85         /// </summary>
  86         public long nBlockPos { get; set; }
  87
  88         /// <summary>
  89         /// Block size in bytes
  90         /// </summary>
  91         public int nBlockSize { get; set; }
  92
  93         /// <summary>
  94         /// Fill database item with data from given block header.
  95         /// </summary>
  96         /// <param name="header">Block header</param>
  97         /// <returns>Header hash</returns>
  98         public uint256 FillHeader(CBlockHeader header)
  99         {
 100             uint256 _hash;
 101             Hash = _hash = header.Hash;
 102
 103             nVersion = header.nVersion;
 104             prevHash = header.prevHash;
 105             merkleRoot = header.merkleRoot;
 106             nTime = header.nTime;
 107             nBits = header.nBits;
 108             nNonce = header.nNonce;
 109
 110             return _hash;
 111         }
 112
 113         /// <summary>
 114         /// Reconstruct block header from item data.
 115         /// </summary>
 116         public CBlockHeader BlockHeader
 117         {
 118             get
 119             {
 120                 CBlockHeader header = new CBlockHeader();
 121
 122                 header.nVersion = nVersion;
 123                 header.prevHash = prevHash;
 124                 header.merkleRoot = merkleRoot;
 125                 header.nTime = nTime;
 126                 header.nBits = nBits;
 127                 header.nNonce = nNonce;
 128
 129                 return header;
 130             }
 131         }
 132
 133         /// <summary>
 134         /// Read block from file.
 135         /// </summary>
 136         /// <param name="reader">Stream with read access.</param>
 137         /// <param name="reader">CBlock reference.</param>
 138         /// <returns>Result</returns>
 139         public bool ReadFromFile(ref Stream reader, out CBlock block)
 140         {
 141             var buffer = new byte[nBlockSize];
 142             block = null;
 143
 144             try
 145             {
 146                 reader.Seek(nBlockPos, SeekOrigin.Begin);
 147
 148                 if (nBlockSize != reader.Read(buffer, 0, nBlockSize))
 149                 {
 150                     return false;
 151                 }
 152
 153                 block = new CBlock(buffer);
 154
 155                 return true;
 156             }
 157             catch (IOException)
 158             {
 159                 // I/O error
 160                 return false;
 161             }
 162             catch (BlockException)
 163             {
 164                 // Constructor exception
 165                 return false;
 166             }
 167         }
 168
 169         /// <summary>
 170         /// Writes given block to file and prepares cursor object for insertion into the database.
 171         /// </summary>
 172         /// <param name="writer">Stream with write access.</param>
 173         /// <param name="block">CBlock reference.</param>
 174         /// <returns>Result</returns>
 175         public bool WriteToFile(ref Stream writer, ref CBlock block)
 176         {
 177             try
 178             {
 179                 byte[] blockBytes = block;
 180
 181                 var magicBytes = BitConverter.GetBytes(CBlockStore.nMagicNumber);
 182                 var blkLenBytes = BitConverter.GetBytes(blockBytes.Length);
 183
 184                 // Seek to the end and then append magic bytes there.
 185                 writer.Seek(0, SeekOrigin.End);
 186                 writer.Write(magicBytes, 0, magicBytes.Length);
 187                 writer.Write(blkLenBytes, 0, blkLenBytes.Length);
 188
 189                 // Save block size and current position in the block cursor fields.
 190                 nBlockPos = writer.Position;
 191                 nBlockSize = blockBytes.Length;
 192
 193                 // Write block and flush the stream.
 194                 writer.Write(blockBytes, 0, blockBytes.Length);
 195                 writer.Flush();
 196
 197                 return true;
 198             }
 199             catch (IOException)
 200             {
 201                 // I/O error
 202                 return false;
 203             }
 204             catch (Exception)
 205             {
 206                 // Some serialization error
 207                 return false;
 208             }
 209         }
 210     }
 211
 212     /// <summary>
 213     /// Block type.
 214     /// </summary>
 215     public enum BlockType
 216     {
 217         PROOF_OF_WORK,
 218         PROOF_OF_WORK_MODIFIER,
 219         PROOF_OF_STAKE,
 220         PROOF_OF_STAKE_MODIFIER
 221     };
 222
 223     /// <summary>
 224     /// Transaction type.
 225     /// </summary>
 226     public enum TxType
 227     {
 228         TX_COINBASE,
 229         TX_COINSTAKE,
 230         TX_USER
 231     }
 232
 233     [Table("TransactionStorage")]
 234     public class CTransactionStoreItem
 235     {
 236         /// <summary>
 237         /// Transaction hash
 238         /// </summary>
 239         [PrimaryKey]
 240         public byte[] TransactionHash { get; set; }
 241
 242         /// <summary>
 243         /// Block hash
 244         /// </summary>
 245         [ForeignKey(typeof(CBlockStoreItem), Name = "Hash")]
 246         public byte[] BlockHash { get; set; }
 247
 248         /// <summary>
 249         /// Transaction type flag
 250         /// </summary>
 251         public TxType txType { get; set; }
 252
 253         /// <summary>
 254         /// Tx position in file
 255         /// </summary>
 256         public long nTxPos { get; set; }
 257
 258         /// <summary>
 259         /// Transaction size
 260         /// </summary>
 261         public int nTxSize { get; set; }
 262
 263         /// <summary>
 264         /// Read transaction from file.
 265         /// </summary>
 266         /// <param name="reader">Stream with read access.</param>
 267         /// <param name="tx">CTransaction reference.</param>
 268         /// <returns>Result</returns>
 269         public bool ReadFromFile(ref Stream reader, out CTransaction tx)
 270         {
 271             var buffer = new byte[CTransaction.nMaxTxSize];
 272             tx = null;
 273
 274             try
 275             {
 276                 reader.Seek(nTxPos, SeekOrigin.Begin); // Seek to transaction offset
 277
 278                 if (nTxSize != reader.Read(buffer, 0, nTxSize))
 279                 {
 280                     return false;
 281                 }
 282
 283                 tx = new CTransaction(buffer);
 284
 285                 return true;
 286             }
 287             catch (IOException)
 288             {
 289                 // I/O error
 290                 return false;
 291             }
 292             catch (TransactionConstructorException)
 293             {
 294                 // Constructor error
 295                 return false;
 296             }
 297         }
 298     }
 299
 300     public class CBlockStore : IDisposable
 301     {
 302         public const uint nMagicNumber = 0xe5e9e8e4;
 303
 304         private bool disposed = false;
 305         private object LockObj = new object();
 306
 307         /// <summary>
 308         /// SQLite connection object.
 309         /// </summary>
 310         private SQLiteConnection dbConn;
 311
 312         /// <summary>
 313         /// Block file.
 314         /// </summary>
 315         private string strBlockFile;
 316
 317         /// <summary>
 318         /// Index database file.
 319         /// </summary>
 320         private string strDbFile;
 321
 322         /// <summary>
 323         /// Map of block tree nodes.
 324         /// </summary>
 325         private ConcurrentDictionary<uint256, CBlockStoreItem> blockMap = new ConcurrentDictionary<uint256, CBlockStoreItem>();
 326
 327         /// <summary>
 328         /// Orphaned blocks map.
 329         /// </summary>
 330         private ConcurrentDictionary<uint256, CBlock> orphanMap = new ConcurrentDictionary<uint256, CBlock>();
 331         private ConcurrentDictionary<uint256, CBlock> orphanMapByPrev = new ConcurrentDictionary<uint256, CBlock>();
 332
 333         /// <summary>
 334         /// Map of unspent items.
 335         /// </summary>
 336         private ConcurrentDictionary<uint256, CTransactionStoreItem> txMap = new ConcurrentDictionary<uint256, CTransactionStoreItem>();
 337
 338         public static CBlockStore Instance;
 339
 340         /// <summary>
 341         /// Block file stream with read access
 342         /// </summary>
 343         private Stream fStreamReadWrite;
 344
 345         /// <summary>
 346         /// Init the block storage manager.
 347         /// </summary>
 348         /// <param name="IndexDB">Path to index database</param>
 349         /// <param name="BlockFile">Path to block file</param>
 350         public CBlockStore(string IndexDB = "blockstore.dat", string BlockFile = "blk0001.dat")
 351         {
 352             strDbFile = IndexDB;
 353             strBlockFile = BlockFile;
 354
 355             bool firstInit = !File.Exists(strDbFile);
 356             dbConn = new SQLiteConnection(new SQLitePlatformGeneric(), strDbFile);
 357
 358             fStreamReadWrite = File.Open(strBlockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite);
 359
 360             if (firstInit)
 361             {
 362                 lock (LockObj)
 363                 {
 364                     // Create tables
 365                     dbConn.CreateTable<CBlockStoreItem>(CreateFlags.AutoIncPK);
 366                     dbConn.CreateTable<CTransactionStoreItem>(CreateFlags.ImplicitPK);
 367
 368                     var genesisBlock = new CBlock(
 369                         Interop.HexToArray(
 370                             "01000000" + // nVersion=1
 371                             "0000000000000000000000000000000000000000000000000000000000000000" + // prevhash is zero
 372                             "7b0502ad2f9f675528183f83d6385794fbcaa914e6d385c6cb1d866a3b3bb34c" + // merkle root
 373                             "398e1151" + // nTime=1360105017
 374                             "ffff0f1e" + // nBits=0x1e0fffff
 375                             "d3091800" + // nNonce=1575379
 376                             "01" +       // nTxCount=1
 377                             "01000000" + // nVersion=1
 378                             "398e1151" + // nTime=1360105017
 379                             "01" +       // nInputs=1
 380                             "0000000000000000000000000000000000000000000000000000000000000000" + // input txid is zero
 381                             "ffffffff" + // n=uint.maxValue
 382                             "4d" +       // scriptSigLen=77
 383                             "04ffff001d020f274468747470733a2f2f626974636f696e74616c6b2e6f72672f696e6465782e7068703f746f7069633d3133343137392e6d736731353032313936236d736731353032313936" + // scriptSig
 384                             "ffffffff" + // nSequence=uint.maxValue
 385                             "01" +       // nOutputs=1
 386                             "0000000000000000" + // nValue=0
 387                             "00" +       // scriptPubkeyLen=0
 388                             "00000000" + // nLockTime=0
 389                             "00"         // sigLen=0
 390                     ));
 391
 392                     // Write block to file.
 393                     var itemTemplate = new CBlockStoreItem()
 394                     {
 395                         nHeight = 0
 396                     };
 397
 398                     itemTemplate.FillHeader(genesisBlock.header);
 399
 400                     if (!AddItemToIndex(ref itemTemplate, ref genesisBlock))
 401                     {
 402                         throw new Exception("Unable to write genesis block");
 403                     }
 404                 }
 405             }
 406             else
 407             {
 408                 var blockTreeItems = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] order by [ItemId] asc");
 409
 410                 // Init list of block items
 411                 foreach (var item in blockTreeItems)
 412                 {
 413                     blockMap.TryAdd(item.Hash, item);
 414                 }
 415             }
 416
 417             Instance = this;
 418         }
 419
 420         public bool GetTransaction(uint256 TxID, ref CTransaction tx)
 421         {
 422             var reader = new BinaryReader(fStreamReadWrite).BaseStream;
 423             var QueryTx = dbConn.Query<CTransactionStoreItem>("select * from [TransactionStorage] where [TransactionHash] = ?", (byte[])TxID);
 424
 425             if (QueryTx.Count == 1)
 426             {
 427                 return QueryTx[0].ReadFromFile(ref reader, out tx);
 428             }
 429
 430             // Tx not found
 431
 432             return false;
 433         }
 434
 435         private bool AddItemToIndex(ref CBlockStoreItem itemTemplate, ref CBlock block)
 436         {
 437             var writer = new BinaryWriter(fStreamReadWrite).BaseStream;
 438             uint256 blockHash = itemTemplate.Hash;
 439
 440             if (blockMap.ContainsKey(blockHash))
 441             {
 442                 // Already have this block.
 443                 return false;
 444             }
 445
 446             // TODO: compute chain trust, set stake entropy bit, record proof-of-stake hash value
 447
 448             // TODO: compute stake modifier
 449
 450             // Add to index
 451             itemTemplate.BlockTypeFlag = block.IsProofOfStake ? BlockType.PROOF_OF_STAKE : BlockType.PROOF_OF_WORK;
 452
 453             if (!itemTemplate.WriteToFile(ref writer, ref block))
 454             {
 455                 return false;
 456             }
 457
 458             dbConn.Insert(itemTemplate);
 459
 460             // We have no SetBestChain and ConnectBlock/Disconnect block yet, so adding these transactions manually.
 461             for (int i = 0; i < block.vtx.Length; i++)
 462             {
 463                 // Handle trasactions
 464
 465                 if (!block.vtx[i].VerifyScripts())
 466                 {
 467                     return false;
 468                 }
 469
 470                 var nTxOffset = itemTemplate.nBlockPos + block.GetTxOffset(i);
 471                 TxType txnType = TxType.TX_USER;
 472
 473                 if (block.vtx[i].IsCoinBase)
 474                 {
 475                     txnType = TxType.TX_COINBASE;
 476                 }
 477                 else if (block.vtx[i].IsCoinStake)
 478                 {
 479                     txnType = TxType.TX_COINSTAKE;
 480                 }
 481
 482                 var NewTxItem = new CTransactionStoreItem()
 483                 {
 484                     TransactionHash = block.vtx[i].Hash,
 485                     BlockHash = blockHash,
 486                     nTxPos = nTxOffset,
 487                     nTxSize = block.vtx[i].Size,
 488                     txType = txnType
 489                 };
 490
 491                 dbConn.Insert(NewTxItem);
 492             }
 493
 494             return blockMap.TryAdd(blockHash, itemTemplate);
 495         }
 496
 497         public bool AcceptBlock(ref CBlock block)
 498         {
 499             uint256 nHash = block.header.Hash;
 500
 501             if (blockMap.ContainsKey(nHash))
 502             {
 503                 // Already have this block.
 504                 return false;
 505             }
 506
 507             CBlockStoreItem prevBlockCursor = null;
 508             if (!blockMap.TryGetValue(block.header.prevHash, out prevBlockCursor))
 509             {
 510                 // Unable to get the cursor.
 511                 return false;
 512             }
 513
 514             var prevBlockHeader = prevBlockCursor.BlockHeader;
 515
 516             // TODO: proof-of-work/proof-of-stake verification
 517             uint nHeight = prevBlockCursor.nHeight + 1;
 518
 519             // Check timestamp against prev
 520             if (NetUtils.FutureDrift(block.header.nTime) < prevBlockHeader.nTime)
 521             {
 522                 // block's timestamp is too early
 523                 return false;
 524             }
 525
 526             // Check that all transactions are finalized
 527             foreach (var tx in block.vtx)
 528             {
 529                 if (!tx.IsFinal(nHeight, block.header.nTime))
 530                 {
 531                     return false;
 532                 }
 533             }
 534
 535             // TODO: Enforce rule that the coinbase starts with serialized block height
 536
 537             // Write block to file.
 538             var itemTemplate = new CBlockStoreItem()
 539             {
 540                 nHeight = nHeight,
 541                 nEntropyBit = Entropy.GetStakeEntropyBit(nHeight, nHash)
 542             };
 543
 544             itemTemplate.FillHeader(block.header);
 545
 546             if (!AddItemToIndex(ref itemTemplate, ref block))
 547             {
 548                 return false;
 549             }
 550
 551             return true;
 552         }
 553
 554         public bool GetBlock(uint256 blockHash, ref CBlock block)
 555         {
 556             var reader = new BinaryReader(fStreamReadWrite).BaseStream;
 557
 558             var QueryBlock = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] where [Hash] = ?", (byte[])blockHash);
 559
 560             if (QueryBlock.Count == 1)
 561             {
 562                 return QueryBlock[0].ReadFromFile(ref reader, out block);
 563             }
 564
 565             // Block not found
 566
 567             return false;
 568         }
 569
 570         public bool ProcessBlock(ref CBlock block)
 571         {
 572             var blockHash = block.header.Hash;
 573
 574             if (blockMap.ContainsKey(blockHash))
 575             {
 576                 // We already have this block.
 577                 return false;
 578             }
 579
 580             if (orphanMap.ContainsKey(blockHash))
 581             {
 582                 // We already have block in the list of orphans.
 583                 return false;
 584             }
 585
 586             // TODO: Limited duplicity on stake and reserialization of block signature
 587
 588             if (!block.CheckBlock(true, true, true))
 589             {
 590                 // Preliminary checks failure.
 591                 return false;
 592             }
 593
 594             if (block.IsProofOfStake)
 595             {
 596                 if (!block.SignatureOK || !block.vtx[1].VerifyScripts())
 597                 {
 598                     // Proof-of-Stake signature validation failure.
 599                     return false;
 600                 }
 601
 602                 // TODO: proof-of-stake validation
 603             }
 604
 605             // TODO: difficulty verification
 606
 607             // If don't already have its previous block, shunt it off to holding area until we get it
 608             if (!blockMap.ContainsKey(block.header.prevHash))
 609             {
 610                 if (block.IsProofOfStake)
 611                 {
 612                     // TODO: limit duplicity on stake
 613                 }
 614
 615                 var block2 = new CBlock(block);
 616                 orphanMap.TryAdd(blockHash, block2);
 617                 orphanMapByPrev.TryAdd(blockHash, block2);
 618
 619                 return true;
 620             }
 621
 622             // Store block to disk
 623             if (!AcceptBlock(ref block))
 624             {
 625                 // Accept failed
 626                 return false;
 627             }
 628
 629             // Recursively process any orphan blocks that depended on this one
 630             var orphansQueue = new List<uint256>();
 631             orphansQueue.Add(blockHash);
 632
 633             for (int i = 0; i < orphansQueue.Count; i++)
 634             {
 635                 var hashPrev = orphansQueue[i];
 636
 637                 foreach (var pair in orphanMap)
 638                 {
 639                     var orphanBlock = pair.Value;
 640
 641                     if (orphanBlock.header.prevHash == blockHash)
 642                     {
 643                         if (AcceptBlock(ref orphanBlock))
 644                         {
 645                             orphansQueue.Add(pair.Key);
 646                         }
 647
 648                         CBlock dummy1;
 649                         orphanMap.TryRemove(pair.Key, out dummy1);
 650                     }
 651                 }
 652
 653                 CBlock dummy2;
 654                 orphanMap.TryRemove(hashPrev, out dummy2);
 655             }
 656
 657             return true;
 658         }
 659
 660         public bool ParseBlockFile(string BlockFile = "bootstrap.dat")
 661         {
 662             // TODO: Rewrite completely.
 663
 664             var nOffset = 0L;
 665
 666             var buffer = new byte[CBlock.nMaxBlockSize]; // Max block size is 1Mb
 667             var intBuffer = new byte[4];
 668
 669             var fStream2 = File.OpenRead(BlockFile);
 670             var readerForBlocks = new BinaryReader(fStream2).BaseStream;
 671
 672             readerForBlocks.Seek(nOffset, SeekOrigin.Begin); // Seek to previous offset + previous block length
 673
 674             dbConn.BeginTransaction();
 675
 676             while (readerForBlocks.Read(buffer, 0, 4) == 4) // Read magic number
 677             {
 678                 var nMagic = BitConverter.ToUInt32(buffer, 0);
 679                 if (nMagic != 0xe5e9e8e4)
 680                 {
 681                     throw new Exception("Incorrect magic number.");
 682                 }
 683
 684                 var nBytesRead = readerForBlocks.Read(buffer, 0, 4);
 685                 if (nBytesRead != 4)
 686                 {
 687                     throw new Exception("BLKSZ EOF");
 688                 }
 689
 690                 var nBlockSize = BitConverter.ToInt32(buffer, 0);
 691
 692                 nOffset = readerForBlocks.Position;
 693
 694                 nBytesRead = readerForBlocks.Read(buffer, 0, nBlockSize);
 695
 696                 if (nBytesRead == 0 || nBytesRead != nBlockSize)
 697                 {
 698                     throw new Exception("BLK EOF");
 699                 }
 700
 701                 var block = new CBlock(buffer);
 702                 var hash = block.header.Hash;
 703
 704                 if (blockMap.ContainsKey(hash))
 705                 {
 706                     continue;
 707                 }
 708
 709                 if (!ProcessBlock(ref block))
 710                 {
 711                     throw new Exception("Invalid block: " + block.header.Hash);
 712                 }
 713
 714                 int nCount = blockMap.Count;
 715                 Console.WriteLine("nCount={0}, Hash={1}, Time={2}", nCount, block.header.Hash, DateTime.Now); // Commit on each 100th block
 716
 717                 if (nCount % 100 == 0 && nCount != 0)
 718                 {
 719                     Console.WriteLine("Commit...");
 720                     dbConn.Commit();
 721                     dbConn.BeginTransaction();
 722                 }
 723             }
 724
 725             dbConn.Commit();
 726
 727             return true;
 728         }
 729
 730         ~CBlockStore()
 731         {
 732             Dispose(false);
 733         }
 734
 735         public void Dispose()
 736         {
 737             Dispose(true);
 738             GC.SuppressFinalize(this);
 739         }
 740
 741         protected virtual void Dispose(bool disposing)
 742         {
 743             if (!disposed)
 744             {
 745                 if (disposing)
 746                 {
 747                     // Free other state (managed objects).
 748
 749                     fStreamReadWrite.Dispose();
 750                 }
 751
 752                 if (dbConn != null)
 753                 {
 754                     dbConn.Close();
 755                     dbConn = null;
 756                 }
 757
 758                 disposed = true;
 759             }
 760         }
 761
 762     }
 763 }