Novacoin/CBlockStore.cs

   1 \feffusing System;
   2 using System.IO;
   3 using System.Linq;
   4 using System.Collections.Concurrent;
   5
   6 using SQLite.Net;
   7 using SQLite.Net.Attributes;
   8 using SQLite.Net.Interop;
   9 using SQLite.Net.Platform.Generic;
  10 using SQLiteNetExtensions.Attributes;
  11 using System.Collections.Generic;
  12
  13 namespace Novacoin
  14 {
  15     /// <summary>
  16     /// Block headers table
  17     /// </summary>
  18     [Table("BlockStorage")]
  19     class CBlockStoreItem
  20     {
  21         /// <summary>
  22         /// Item ID in the database
  23         /// </summary>
  24         [PrimaryKey, AutoIncrement]
  25         public int ItemID { get; set; }
  26
  27         /// <summary>
  28         /// PBKDF2+Salsa20 of block hash
  29         /// </summary>
  30         [Unique]
  31         public byte[] Hash { get; set; }
  32
  33         /// <summary>
  34         /// Version of block schema
  35         /// </summary>
  36         public uint nVersion { get; set; }
  37
  38         /// <summary>
  39         /// Previous block hash.
  40         /// </summary>
  41         public byte[] prevHash { get; set; }
  42
  43         /// <summary>
  44         /// Merkle root hash.
  45         /// </summary>
  46         public byte[] merkleRoot { get; set; }
  47
  48         /// <summary>
  49         /// Block timestamp.
  50         /// </summary>
  51         public uint nTime { get; set; }
  52
  53         /// <summary>
  54         /// Compressed difficulty representation.
  55         /// </summary>
  56         public uint nBits { get; set; }
  57
  58         /// <summary>
  59         /// Nonce counter.
  60         /// </summary>
  61         public uint nNonce { get; set; }
  62
  63         /// <summary>
  64         /// Block type flags
  65         /// </summary>
  66         public BlockType BlockTypeFlag { get; set; }
  67
  68         /// <summary>
  69         /// Stake modifier
  70         /// </summary>
  71         public long nStakeModifier { get; set; }
  72
  73         /// <summary>
  74         /// Stake entropy bit
  75         /// </summary>
  76         public byte nEntropyBit { get; set; }
  77
  78         /// <summary>
  79         /// Next block hash
  80         /// </summary>
  81         public byte[] NextHash { get; set; }
  82
  83         /// <summary>
  84         /// Block height
  85         /// </summary>
  86         public uint nHeight { get; set; }
  87
  88         /// <summary>
  89         /// Block position in file
  90         /// </summary>
  91         public long nBlockPos { get; set; }
  92
  93         /// <summary>
  94         /// Block size in bytes
  95         /// </summary>
  96         public int nBlockSize { get; set; }
  97
  98         /// <summary>
  99         /// Fill database item with data from given block header.
 100         /// </summary>
 101         /// <param name="header">Block header</param>
 102         /// <returns>Header hash</returns>
 103         public ScryptHash256 FillHeader(CBlockHeader header)
 104         {
 105             ScryptHash256 _hash;
 106             Hash = _hash = header.Hash;
 107
 108             nVersion = header.nVersion;
 109             prevHash = header.prevHash;
 110             merkleRoot = header.merkleRoot;
 111             nTime = header.nTime;
 112             nBits = header.nBits;
 113             nNonce = header.nNonce;
 114
 115             return _hash;
 116         }
 117
 118         /// <summary>
 119         /// Reconstruct block header from item data.
 120         /// </summary>
 121         public CBlockHeader BlockHeader
 122         {
 123             get
 124             {
 125                 CBlockHeader header = new CBlockHeader();
 126
 127                 header.nVersion = nVersion;
 128                 header.prevHash = new ScryptHash256(prevHash);
 129                 header.merkleRoot = new Hash256(merkleRoot);
 130                 header.nTime = nTime;
 131                 header.nBits = nBits;
 132                 header.nNonce = nNonce;
 133
 134                 return header;
 135             }
 136         }
 137
 138         /// <summary>
 139         /// Read block from file.
 140         /// </summary>
 141         /// <param name="reader">Stream with read access.</param>
 142         /// <param name="reader">CBlock reference.</param>
 143         /// <returns>Result</returns>
 144         public bool ReadFromFile(ref Stream reader, out CBlock block)
 145         {
 146             var buffer = new byte[nBlockSize];
 147             block = null;
 148
 149             try
 150             {
 151                 reader.Seek(nBlockPos, SeekOrigin.Begin);
 152
 153                 if (nBlockSize != reader.Read(buffer, 0, nBlockSize))
 154                 {
 155                     return false;
 156                 }
 157
 158                 block = new CBlock(buffer);
 159
 160                 return true;
 161             }
 162             catch (IOException)
 163             {
 164                 // I/O error
 165                 return false;
 166             }
 167             catch (BlockException)
 168             {
 169                 // Constructor exception
 170                 return false;
 171             }
 172         }
 173
 174         /// <summary>
 175         /// Writes given block to file and prepares cursor object for insertion into the database.
 176         /// </summary>
 177         /// <param name="writer">Stream with write access.</param>
 178         /// <param name="block">CBlock reference.</param>
 179         /// <returns>Result</returns>
 180         public bool WriteToFile(ref Stream writer, ref CBlock block)
 181         {
 182             try
 183             {
 184                 byte[] blockBytes = block;
 185
 186                 var magicBytes = BitConverter.GetBytes(CBlockStore.nMagicNumber);
 187                 var blkLenBytes = BitConverter.GetBytes(blockBytes.Length);
 188
 189                 // Seek to the end and then append magic bytes there.
 190                 writer.Seek(0, SeekOrigin.End);
 191                 writer.Write(magicBytes, 0, magicBytes.Length);
 192
 193                 // Save block size and current position in the block cursor fields.
 194                 nBlockPos = writer.Position;
 195                 nBlockSize = blockBytes.Length;
 196
 197                 // Write block and flush the stream.
 198                 writer.Write(blkLenBytes, 0, blkLenBytes.Length);
 199                 writer.Write(blockBytes, 0, blockBytes.Length);
 200                 writer.Flush();
 201
 202                 return true;
 203             }
 204             catch (IOException)
 205             {
 206                 // I/O error
 207                 return false;
 208             }
 209             catch (Exception)
 210             {
 211                 // Some serialization error
 212                 return false;
 213             }
 214         }
 215     }
 216
 217     /// <summary>
 218     /// Block type.
 219     /// </summary>
 220     public enum BlockType
 221     {
 222         PROOF_OF_WORK,
 223         PROOF_OF_WORK_MODIFIER,
 224         PROOF_OF_STAKE,
 225         PROOF_OF_STAKE_MODIFIER
 226     };
 227
 228     /// <summary>
 229     /// Transaction type.
 230     /// </summary>
 231     public enum TxType
 232     {
 233         TX_COINBASE,
 234         TX_COINSTAKE,
 235         TX_USER
 236     }
 237
 238     [Table("TransactionStorage")]
 239     public class CTransactionStoreItem
 240     {
 241         /// <summary>
 242         /// Transaction hash
 243         /// </summary>
 244         [PrimaryKey]
 245         public byte[] TransactionHash { get; set; }
 246
 247         /// <summary>
 248         /// Block hash
 249         /// </summary>
 250         [ForeignKey(typeof(CBlockStoreItem), Name = "Hash")]
 251         public byte[] BlockHash { get; set; }
 252
 253         /// <summary>
 254         /// Transaction type flag
 255         /// </summary>
 256         public TxType txType { get; set; }
 257
 258         /// <summary>
 259         /// Tx position in file
 260         /// </summary>
 261         public long nTxPos { get; set; }
 262
 263         /// <summary>
 264         /// Transaction size
 265         /// </summary>
 266         public int nTxSize { get; set; }
 267
 268         /// <summary>
 269         /// Read transaction from file.
 270         /// </summary>
 271         /// <param name="reader">Stream with read access.</param>
 272         /// <param name="tx">CTransaction reference.</param>
 273         /// <returns>Result</returns>
 274         public bool ReadFromFile(ref Stream reader, out CTransaction tx)
 275         {
 276             var buffer = new byte[250000]; // Max transaction size is 250kB
 277             tx = null;
 278
 279             try
 280             {
 281                 reader.Seek(nTxPos, SeekOrigin.Begin); // Seek to transaction offset
 282
 283                 if (nTxSize != reader.Read(buffer, 0, nTxSize))
 284                 {
 285                     return false;
 286                 }
 287
 288                 tx = new CTransaction(buffer);
 289
 290                 return true;
 291             }
 292             catch (IOException)
 293             {
 294                 // I/O error
 295                 return false;
 296             }
 297             catch (TransactionConstructorException)
 298             {
 299                 // Constructor error
 300                 return false;
 301             }
 302         }
 303     }
 304
 305     public class CBlockStore : IDisposable
 306     {
 307         public const uint nMagicNumber = 0xe5e9e8e4;
 308
 309         private bool disposed = false;
 310         private object LockObj = new object();
 311
 312         /// <summary>
 313         /// SQLite connection object.
 314         /// </summary>
 315         private SQLiteConnection dbConn;
 316
 317         /// <summary>
 318         /// Block file.
 319         /// </summary>
 320         private string strBlockFile;
 321
 322         /// <summary>
 323         /// Index database file.
 324         /// </summary>
 325         private string strDbFile;
 326
 327         /// <summary>
 328         /// Map of block tree nodes.
 329         /// </summary>
 330         private ConcurrentDictionary<ScryptHash256, CBlockStoreItem> blockMap = new ConcurrentDictionary<ScryptHash256, CBlockStoreItem>();
 331
 332         /// <summary>
 333         /// Orphaned blocks map.
 334         /// </summary>
 335         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMap = new ConcurrentDictionary<ScryptHash256, CBlock>();
 336         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMapByPrev = new ConcurrentDictionary<ScryptHash256, CBlock>();
 337
 338         /// <summary>
 339         /// Map of unspent items.
 340         /// </summary>
 341         private ConcurrentDictionary<Hash256, CTransactionStoreItem> txMap = new ConcurrentDictionary<Hash256, CTransactionStoreItem>();
 342
 343         private CBlock genesisBlock = new CBlock(
 344             Interop.HexToArray(
 345                 "01000000" + // nVersion=1
 346                 "0000000000000000000000000000000000000000000000000000000000000000" + // prevhash is zero
 347                 "7b0502ad2f9f675528183f83d6385794fbcaa914e6d385c6cb1d866a3b3bb34c" + // merkle root
 348                 "398e1151" + // nTime=1360105017
 349                 "ffff0f1e" + // nBits=0x1e0fffff
 350                 "d3091800" + // nNonce=1575379
 351                 "01" +       // nTxCount=1
 352                 "01000000" + // nVersion=1
 353                 "398e1151" + // nTime=1360105017
 354                 "01" +       // nInputs=1
 355                 "0000000000000000000000000000000000000000000000000000000000000000" + // input txid is zero
 356                 "ffffffff" + // n=uint.maxValue
 357                 "4d" +       // scriptSigLen=77
 358                 "04ffff001d020f274468747470733a2f2f626974636f696e74616c6b2e6f72672f696e6465782e7068703f746f7069633d3133343137392e6d736731353032313936236d736731353032313936" + // scriptSig
 359                 "ffffffff" + // nSequence=uint.maxValue
 360                 "01" +       // nOutputs=1
 361                 "0000000000000000" + // nValue=0
 362                 "00" +       // scriptPubkeyLen=0
 363                 "00000000" + // nLockTime=0
 364                 "00"         // sigLen=0
 365         ));
 366
 367         public static CBlockStore Instance;
 368
 369         /// <summary>
 370         /// Block file stream with read access
 371         /// </summary>
 372         private Stream fStreamReadWrite;
 373
 374         /// <summary>
 375         /// Init the block storage manager.
 376         /// </summary>
 377         /// <param name="IndexDB">Path to index database</param>
 378         /// <param name="BlockFile">Path to block file</param>
 379         public CBlockStore(string IndexDB = "blockstore.dat", string BlockFile = "blk0001.dat")
 380         {
 381             strDbFile = IndexDB;
 382             strBlockFile = BlockFile;
 383
 384             bool firstInit = !File.Exists(strDbFile);
 385             dbConn = new SQLiteConnection(new SQLitePlatformGeneric(), strDbFile);
 386
 387             fStreamReadWrite = File.Open(strBlockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite);
 388
 389             if (firstInit)
 390             {
 391                 lock (LockObj)
 392                 {
 393                     // Create tables
 394                     dbConn.CreateTable<CBlockStoreItem>(CreateFlags.AutoIncPK);
 395                     dbConn.CreateTable<CTransactionStoreItem>(CreateFlags.ImplicitPK);
 396
 397                     // Write block to file.
 398                     var itemTemplate = new CBlockStoreItem()
 399                     {
 400                         nHeight = 0
 401                     };
 402
 403                     itemTemplate.FillHeader(genesisBlock.header);
 404
 405                     if (!AddItemToIndex(ref itemTemplate, ref genesisBlock))
 406                     {
 407                         throw new Exception("Unable to write genesis block");
 408                     }
 409                 }
 410             }
 411             else
 412             {
 413                 var blockTreeItems = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] order by [ItemId] asc");
 414
 415                 // Init list of block items
 416                 foreach (var item in blockTreeItems)
 417                 {
 418                     blockMap.TryAdd(new ScryptHash256(item.Hash), item);
 419                 }
 420             }
 421
 422             Instance = this;
 423         }
 424
 425         public bool GetTransaction(Hash256 TxID, ref CTransaction tx)
 426         {
 427             var reader = new BinaryReader(fStreamReadWrite).BaseStream;
 428             var QueryTx = dbConn.Query<CTransactionStoreItem>("select * from [TransactionStorage] where [TransactionHash] = ?", (byte[])TxID);
 429
 430             if (QueryTx.Count == 1)
 431             {
 432                 return QueryTx[0].ReadFromFile(ref reader, out tx);
 433             }
 434
 435             // Tx not found
 436
 437             return false;
 438         }
 439
 440         private bool AddItemToIndex(ref CBlockStoreItem itemTemplate, ref CBlock block)
 441         {
 442             var writer = new BinaryWriter(fStreamReadWrite).BaseStream;
 443             var blockHash = new ScryptHash256(itemTemplate.Hash);
 444
 445             if (blockMap.ContainsKey(blockHash))
 446             {
 447                 // Already have this block.
 448                 return false;
 449             }
 450
 451             // TODO: compute chain trust, set stake entropy bit, record proof-of-stake hash value
 452
 453             // TODO: compute stake modifier
 454
 455             // Add to index
 456             itemTemplate.BlockTypeFlag = block.IsProofOfStake ? BlockType.PROOF_OF_STAKE : BlockType.PROOF_OF_WORK;
 457
 458             if (!itemTemplate.WriteToFile(ref writer, ref block))
 459             {
 460                 return false;
 461             }
 462
 463             dbConn.Insert(itemTemplate);
 464
 465             // We have no SetBestChain and ConnectBlock/Disconnect block yet, so adding these transactions manually.
 466             for (int i = 0; i < block.vtx.Length; i++)
 467             {
 468                 // Handle trasactions
 469
 470                 if (!block.vtx[i].VerifyScripts())
 471                 {
 472                     return false;
 473                 }
 474
 475                 var nTxOffset = itemTemplate.nBlockPos + block.GetTxOffset(i);
 476                 TxType txnType = TxType.TX_USER;
 477
 478                 if (block.vtx[i].IsCoinBase)
 479                 {
 480                     txnType = TxType.TX_COINBASE;
 481                 }
 482                 else if (block.vtx[i].IsCoinStake)
 483                 {
 484                     txnType = TxType.TX_COINSTAKE;
 485                 }
 486
 487                 var NewTxItem = new CTransactionStoreItem()
 488                 {
 489                     TransactionHash = block.vtx[i].Hash,
 490                     BlockHash = blockHash,
 491                     nTxPos = nTxOffset,
 492                     nTxSize = block.vtx[i].Size,
 493                     txType = txnType
 494                 };
 495
 496                 dbConn.Insert(NewTxItem);
 497             }
 498
 499             return blockMap.TryAdd(blockHash, itemTemplate);
 500         }
 501
 502         public bool AcceptBlock(ref CBlock block)
 503         {
 504             ScryptHash256 hash = block.header.Hash;
 505
 506             if (blockMap.ContainsKey(hash))
 507             {
 508                 // Already have this block.
 509                 return false;
 510             }
 511
 512             CBlockStoreItem prevBlockCursor = null;
 513             if (!blockMap.TryGetValue(block.header.prevHash, out prevBlockCursor))
 514             {
 515                 // Unable to get the cursor.
 516                 return false;
 517             }
 518
 519             var prevBlockHeader = prevBlockCursor.BlockHeader;
 520
 521             // TODO: proof-of-work/proof-of-stake verification
 522             uint nHeight = prevBlockCursor.nHeight + 1;
 523
 524             // Check timestamp against prev
 525             if (NetUtils.FutureDrift(block.header.nTime) < prevBlockHeader.nTime)
 526             {
 527                 // block's timestamp is too early
 528                 return false;
 529             }
 530
 531             // Check that all transactions are finalized
 532             foreach (var tx in block.vtx)
 533             {
 534                 if (!tx.IsFinal(nHeight, block.header.nTime))
 535                 {
 536                     return false;
 537                 }
 538             }
 539
 540             // TODO: Enforce rule that the coinbase starts with serialized block height
 541
 542             // Write block to file.
 543             var itemTemplate = new CBlockStoreItem()
 544             {
 545                 nHeight = nHeight
 546             };
 547
 548             itemTemplate.FillHeader(block.header);
 549
 550             if (!AddItemToIndex(ref itemTemplate, ref block))
 551             {
 552                 return false;
 553             }
 554
 555             return true;
 556         }
 557
 558         public bool GetBlock(ScryptHash256 blockHash, ref CBlock block)
 559         {
 560             var reader = new BinaryReader(fStreamReadWrite).BaseStream;
 561
 562             var QueryBlock = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] where [Hash] = ?", (byte[])blockHash);
 563
 564             if (QueryBlock.Count == 1)
 565             {
 566                 return QueryBlock[0].ReadFromFile(ref reader, out block);
 567             }
 568
 569             // Block not found
 570
 571             return false;
 572         }
 573
 574         public bool ProcessBlock(ref CBlock block)
 575         {
 576             ScryptHash256 blockHash = block.header.Hash;
 577
 578             if (blockMap.ContainsKey(blockHash))
 579             {
 580                 // We already have this block.
 581                 return false;
 582             }
 583
 584             if (orphanMap.ContainsKey(blockHash))
 585             {
 586                 // We already have block in the list of orphans.
 587                 return false;
 588             }
 589
 590             // TODO: Limited duplicity on stake and reserialization of block signature
 591
 592             // Preliminary checks
 593             if (!block.CheckBlock(true, true, true))
 594             {
 595                 return true;
 596             }
 597
 598             if (block.IsProofOfStake)
 599             {
 600                 // TODO: proof-of-stake validation
 601             }
 602
 603             // TODO: difficulty verification
 604
 605             // If don't already have its previous block, shunt it off to holding area until we get it
 606             if (!blockMap.ContainsKey(block.header.prevHash))
 607             {
 608                 if (block.IsProofOfStake)
 609                 {
 610                     // TODO: limit duplicity on stake
 611                 }
 612
 613                 var block2 = new CBlock(block);
 614                 orphanMap.TryAdd(blockHash, block2);
 615                 orphanMapByPrev.TryAdd(blockHash, block2);
 616
 617                 return true;
 618             }
 619
 620             // Store block to disk
 621             if (!AcceptBlock(ref block))
 622             {
 623                 // Accept failed
 624                 return false;
 625             }
 626
 627             // Recursively process any orphan blocks that depended on this one
 628             var orphansQueue = new List<ScryptHash256>();
 629             orphansQueue.Add(blockHash);
 630
 631             for (int i = 0; i < orphansQueue.Count; i++)
 632             {
 633                 ScryptHash256 hashPrev = orphansQueue[i];
 634
 635                 foreach (var pair in orphanMap)
 636                 {
 637                     var orphanBlock = pair.Value;
 638
 639                     if (orphanBlock.header.prevHash == blockHash)
 640                     {
 641                         if (AcceptBlock(ref orphanBlock))
 642                         {
 643                             orphansQueue.Add(pair.Key);
 644                         }
 645
 646                         CBlock dummy1;
 647                         orphanMap.TryRemove(pair.Key, out dummy1);
 648                     }
 649                 }
 650
 651                 CBlock dummy2;
 652                 orphanMap.TryRemove(hashPrev, out dummy2);
 653             }
 654
 655             return true;
 656         }
 657
 658         public bool ParseBlockFile(string BlockFile = "bootstrap.dat")
 659         {
 660             // TODO: Rewrite completely.
 661
 662             var nOffset = 0L;
 663
 664             var buffer = new byte[1000000]; // Max block size is 1Mb
 665             var intBuffer = new byte[4];
 666
 667             var fStream2 = File.OpenRead(BlockFile);
 668             var readerForBlocks = new BinaryReader(fStream2).BaseStream;
 669
 670             readerForBlocks.Seek(nOffset, SeekOrigin.Begin); // Seek to previous offset + previous block length
 671
 672             dbConn.BeginTransaction();
 673
 674             while (readerForBlocks.Read(buffer, 0, 4) == 4) // Read magic number
 675             {
 676                 var nMagic = BitConverter.ToUInt32(buffer, 0);
 677                 if (nMagic != 0xe5e9e8e4)
 678                 {
 679                     throw new Exception("Incorrect magic number.");
 680                 }
 681
 682                 var nBytesRead = readerForBlocks.Read(buffer, 0, 4);
 683                 if (nBytesRead != 4)
 684                 {
 685                     throw new Exception("BLKSZ EOF");
 686                 }
 687
 688                 var nBlockSize = BitConverter.ToInt32(buffer, 0);
 689
 690                 nOffset = readerForBlocks.Position;
 691
 692                 nBytesRead = readerForBlocks.Read(buffer, 0, nBlockSize);
 693
 694                 if (nBytesRead == 0 || nBytesRead != nBlockSize)
 695                 {
 696                     throw new Exception("BLK EOF");
 697                 }
 698
 699                 var block = new CBlock(buffer);
 700                 var hash = block.header.Hash;
 701
 702                 if (blockMap.ContainsKey(hash))
 703                 {
 704                     continue;
 705                 }
 706
 707                 if (!ProcessBlock(ref block))
 708                 {
 709                     throw new Exception("Invalid block: " + block.header.Hash);
 710                 }
 711
 712                 int nCount = blockMap.Count;
 713                 Console.WriteLine("nCount={0}, Hash={1}, Time={2}", nCount, block.header.Hash, DateTime.Now); // Commit on each 100th block
 714
 715                 if (nCount % 100 == 0 && nCount != 0)
 716                 {
 717                     Console.WriteLine("Commit...");
 718                     dbConn.Commit();
 719                     dbConn.BeginTransaction();
 720                 }
 721             }
 722
 723             dbConn.Commit();
 724
 725             return true;
 726         }
 727
 728         ~CBlockStore()
 729         {
 730             Dispose(false);
 731         }
 732
 733         public void Dispose()
 734         {
 735             Dispose(true);
 736             GC.SuppressFinalize(this);
 737         }
 738
 739         protected virtual void Dispose(bool disposing)
 740         {
 741             if (!disposed)
 742             {
 743                 if (disposing)
 744                 {
 745                     // Free other state (managed objects).
 746
 747                     fStreamReadWrite.Dispose();
 748                 }
 749
 750                 if (dbConn != null)
 751                 {
 752                     dbConn.Close();
 753                     dbConn = null;
 754                 }
 755
 756                 disposed = true;
 757             }
 758         }
 759
 760     }
 761 }