Novacoin/CBlockStore.cs

   1 \feffusing System;
   2 using System.IO;
   3 using System.Linq;
   4 using System.Collections.Concurrent;
   5
   6 using SQLite.Net;
   7 using SQLite.Net.Attributes;
   8 using SQLite.Net.Interop;
   9 using SQLite.Net.Platform.Generic;
  10 using SQLiteNetExtensions.Attributes;
  11 using System.Collections.Generic;
  12
  13 namespace Novacoin
  14 {
  15     /// <summary>
  16     /// Block headers table
  17     /// </summary>
  18     [Table("BlockStorage")]
  19     class CBlockStoreItem
  20     {
  21         /// <summary>
  22         /// Item ID in the database
  23         /// </summary>
  24         [PrimaryKey, AutoIncrement]
  25         public int ItemID { get; set; }
  26
  27         /// <summary>
  28         /// PBKDF2+Salsa20 of block hash
  29         /// </summary>
  30         [Unique]
  31         public byte[] Hash { get; set; }
  32
  33         /// <summary>
  34         /// Version of block schema
  35         /// </summary>
  36         public uint nVersion { get; set; }
  37
  38         /// <summary>
  39         /// Previous block hash.
  40         /// </summary>
  41         public byte[] prevHash { get; set; }
  42
  43         /// <summary>
  44         /// Merkle root hash.
  45         /// </summary>
  46         public byte[] merkleRoot { get; set; }
  47
  48         /// <summary>
  49         /// Block timestamp.
  50         /// </summary>
  51         public uint nTime { get; set; }
  52
  53         /// <summary>
  54         /// Compressed difficulty representation.
  55         /// </summary>
  56         public uint nBits { get; set; }
  57
  58         /// <summary>
  59         /// Nonce counter.
  60         /// </summary>
  61         public uint nNonce { get; set; }
  62
  63         /// <summary>
  64         /// Block type flags
  65         /// </summary>
  66         public BlockType BlockTypeFlag { get; set; }
  67
  68         /// <summary>
  69         /// Stake modifier
  70         /// </summary>
  71         public long nStakeModifier { get; set; }
  72
  73         /// <summary>
  74         /// Stake entropy bit
  75         /// </summary>
  76         public byte nEntropyBit { get; set; }
  77
  78         /// <summary>
  79         /// Next block hash
  80         /// </summary>
  81         public byte[] NextHash { get; set; }
  82
  83         /// <summary>
  84         /// Block height
  85         /// </summary>
  86         public uint nHeight { get; set; }
  87
  88         /// <summary>
  89         /// Block position in file
  90         /// </summary>
  91         public long nBlockPos { get; set; }
  92
  93         /// <summary>
  94         /// Block size in bytes
  95         /// </summary>
  96         public int nBlockSize { get; set; }
  97
  98         /// <summary>
  99         /// Fill database item with data from given block header.
 100         /// </summary>
 101         /// <param name="header">Block header</param>
 102         /// <returns>Header hash</returns>
 103         public ScryptHash256 FillHeader(CBlockHeader header)
 104         {
 105             ScryptHash256 _hash;
 106             Hash = _hash = header.Hash;
 107
 108             nVersion = header.nVersion;
 109             prevHash = header.prevHash;
 110             merkleRoot = header.merkleRoot;
 111             nTime = header.nTime;
 112             nBits = header.nBits;
 113             nNonce = header.nNonce;
 114
 115             return _hash;
 116         }
 117
 118         /// <summary>
 119         /// Reconstruct block header from item data.
 120         /// </summary>
 121         public CBlockHeader BlockHeader
 122         {
 123             get
 124             {
 125                 CBlockHeader header = new CBlockHeader();
 126
 127                 header.nVersion = nVersion;
 128                 header.prevHash = new ScryptHash256(prevHash);
 129                 header.merkleRoot = new Hash256(merkleRoot);
 130                 header.nTime = nTime;
 131                 header.nBits = nBits;
 132                 header.nNonce = nNonce;
 133
 134                 return header;
 135             }
 136         }
 137
 138         /// <summary>
 139         /// Read block from file.
 140         /// </summary>
 141         /// <param name="reader">Stream with read access.</param>
 142         /// <param name="reader">CBlock reference.</param>
 143         /// <returns>Result</returns>
 144         public bool ReadFromFile(ref Stream reader, out CBlock block)
 145         {
 146             var buffer = new byte[nBlockSize];
 147             block = null;
 148
 149             try
 150             {
 151                 reader.Seek(nBlockPos, SeekOrigin.Begin);
 152
 153                 if (nBlockSize != reader.Read(buffer, 0, nBlockSize))
 154                 {
 155                     return false;
 156                 }
 157
 158                 block = new CBlock(buffer);
 159
 160                 return true;
 161             }
 162             catch (IOException)
 163             {
 164                 // I/O error
 165                 return false;
 166             }
 167             catch (BlockException)
 168             {
 169                 // Constructor exception
 170                 return false;
 171             }
 172         }
 173
 174         /// <summary>
 175         /// Writes given block to file and prepares cursor object for insertion into the database.
 176         /// </summary>
 177         /// <param name="writer">Stream with write access.</param>
 178         /// <param name="block">CBlock reference.</param>
 179         /// <returns>Result</returns>
 180         public bool WriteToFile(ref Stream writer, ref CBlock block)
 181         {
 182             try
 183             {
 184                 byte[] blockBytes = block;
 185
 186                 var magicBytes = BitConverter.GetBytes(CBlockStore.nMagicNumber);
 187                 var blkLenBytes = BitConverter.GetBytes(blockBytes.Length);
 188
 189                 // Seek to the end and then append magic bytes there.
 190                 writer.Seek(0, SeekOrigin.End);
 191                 writer.Write(magicBytes, 0, magicBytes.Length);
 192
 193                 // Save block size and current position in the block cursor fields.
 194                 nBlockPos = writer.Position;
 195                 nBlockSize = blockBytes.Length;
 196
 197                 // Write block and flush the stream.
 198                 writer.Write(blkLenBytes, 0, blkLenBytes.Length);
 199                 writer.Write(blockBytes, 0, blockBytes.Length);
 200                 writer.Flush();
 201
 202                 return true;
 203             }
 204             catch (IOException)
 205             {
 206                 // I/O error
 207                 return false;
 208             }
 209             catch (Exception)
 210             {
 211                 // Some serialization error
 212                 return false;
 213             }
 214         }
 215     }
 216
 217     /// <summary>
 218     /// Block type.
 219     /// </summary>
 220     public enum BlockType
 221     {
 222         PROOF_OF_WORK,
 223         PROOF_OF_WORK_MODIFIER,
 224         PROOF_OF_STAKE,
 225         PROOF_OF_STAKE_MODIFIER
 226     };
 227
 228     /// <summary>
 229     /// Transaction type.
 230     /// </summary>
 231     public enum TxType
 232     {
 233         TX_COINBASE,
 234         TX_COINSTAKE,
 235         TX_USER
 236     }
 237
 238     [Table("TransactionStorage")]
 239     public class CTransactionStoreItem
 240     {
 241         /// <summary>
 242         /// Transaction hash
 243         /// </summary>
 244         [PrimaryKey]
 245         public byte[] TransactionHash { get; set; }
 246
 247         /// <summary>
 248         /// Block hash
 249         /// </summary>
 250         [ForeignKey(typeof(CBlockStoreItem), Name = "Hash")]
 251         public byte[] BlockHash { get; set; }
 252
 253         /// <summary>
 254         /// Transaction type flag
 255         /// </summary>
 256         public TxType txType { get; set; }
 257
 258         /// <summary>
 259         /// Tx position in file
 260         /// </summary>
 261         public long nTxPos { get; set; }
 262
 263         /// <summary>
 264         /// Transaction size
 265         /// </summary>
 266         public int nTxSize { get; set; }
 267
 268         /// <summary>
 269         /// Read transaction from file.
 270         /// </summary>
 271         /// <param name="reader">Stream with read access.</param>
 272         /// <param name="tx">CTransaction reference.</param>
 273         /// <returns>Result</returns>
 274         public bool ReadFromFile(ref Stream reader, out CTransaction tx)
 275         {
 276             var buffer = new byte[250000]; // Max transaction size is 250kB
 277             tx = null;
 278
 279             try
 280             {
 281                 reader.Seek(nTxPos, SeekOrigin.Begin); // Seek to transaction offset
 282
 283                 if (nTxSize != reader.Read(buffer, 0, nTxSize))
 284                 {
 285                     return false;
 286                 }
 287
 288                 tx = new CTransaction(buffer);
 289
 290                 return true;
 291             }
 292             catch (IOException)
 293             {
 294                 // I/O error
 295                 return false;
 296             }
 297             catch (TransactionConstructorException)
 298             {
 299                 // Constructor error
 300                 return false;
 301             }
 302         }
 303     }
 304
 305     public class CBlockStore : IDisposable
 306     {
 307         public const uint nMagicNumber = 0xe5e9e8e4;
 308
 309         private bool disposed = false;
 310         private object LockObj = new object();
 311
 312         /// <summary>
 313         /// SQLite connection object.
 314         /// </summary>
 315         private SQLiteConnection dbConn;
 316
 317         /// <summary>
 318         /// Block file.
 319         /// </summary>
 320         private string strBlockFile;
 321
 322         /// <summary>
 323         /// Index database file.
 324         /// </summary>
 325         private string strDbFile;
 326
 327         /// <summary>
 328         /// Map of block tree nodes.
 329         /// </summary>
 330         private ConcurrentDictionary<ScryptHash256, CBlockStoreItem> blockMap = new ConcurrentDictionary<ScryptHash256, CBlockStoreItem>();
 331
 332         /// <summary>
 333         /// Orphaned blocks map.
 334         /// </summary>
 335         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMap = new ConcurrentDictionary<ScryptHash256, CBlock>();
 336         private ConcurrentDictionary<ScryptHash256, CBlock> orphanMapByPrev = new ConcurrentDictionary<ScryptHash256, CBlock>();
 337
 338         /// <summary>
 339         /// Map of unspent items.
 340         /// </summary>
 341         private ConcurrentDictionary<Hash256, CTransactionStoreItem> txMap = new ConcurrentDictionary<Hash256, CTransactionStoreItem>();
 342
 343         private CBlock genesisBlock = new CBlock(
 344             Interop.HexToArray(
 345                 "01000000" + // nVersion=1
 346                 "0000000000000000000000000000000000000000000000000000000000000000" + // prevhash is zero
 347                 "7b0502ad2f9f675528183f83d6385794fbcaa914e6d385c6cb1d866a3b3bb34c" + // merkle root
 348                 "398e1151" + // nTime=1360105017
 349                 "ffff0f1e" + // nBits=0x1e0fffff
 350                 "d3091800" + // nNonce=1575379
 351                 "01" +       // nTxCount=1
 352                 "01000000" + // nVersion=1
 353                 "398e1151" + // nTime=1360105017
 354                 "01" +       // nInputs=1
 355                 "0000000000000000000000000000000000000000000000000000000000000000" + // input txid is zero
 356                 "ffffffff" + // n=uint.maxValue
 357                 "4d" +       // scriptSigLen=77
 358                 "04ffff001d020f274468747470733a2f2f626974636f696e74616c6b2e6f72672f696e6465782e7068703f746f7069633d3133343137392e6d736731353032313936236d736731353032313936" + // scriptSig
 359                 "ffffffff" + // nSequence=uint.maxValue
 360                 "01" +       // nOutputs=1
 361                 "0000000000000000" + // nValue=0
 362                 "00" +       // scriptPubkeyLen=0
 363                 "00000000" + // nLockTime=0
 364                 "00"         // sigLen=0
 365         ));
 366
 367         public static CBlockStore Instance;
 368
 369         /// <summary>
 370         /// Block file stream with read access
 371         /// </summary>
 372         private Stream reader;
 373
 374         /// <summary>
 375         /// Block file stream with write access
 376         /// </summary>
 377         private Stream writer;
 378
 379         /// <summary>
 380         /// Init the block storage manager.
 381         /// </summary>
 382         /// <param name="IndexDB">Path to index database</param>
 383         /// <param name="BlockFile">Path to block file</param>
 384         public CBlockStore(string IndexDB = "blockstore.dat", string BlockFile = "blk0001.dat")
 385         {
 386             strDbFile = IndexDB;
 387             strBlockFile = BlockFile;
 388
 389             bool firstInit = !File.Exists(strDbFile);
 390             dbConn = new SQLiteConnection(new SQLitePlatformGeneric(), strDbFile);
 391
 392             var fStreamReadWrite = File.Open(strBlockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite);
 393             writer = new BinaryWriter(fStreamReadWrite).BaseStream;
 394             reader = new BinaryReader(fStreamReadWrite).BaseStream;
 395
 396             if (firstInit)
 397             {
 398                 lock (LockObj)
 399                 {
 400                     // Create tables
 401                     dbConn.CreateTable<CBlockStoreItem>(CreateFlags.AutoIncPK);
 402                     dbConn.CreateTable<CTransactionStoreItem>(CreateFlags.ImplicitPK);
 403
 404                     // Write block to file.
 405                     var itemTemplate = new CBlockStoreItem()
 406                     {
 407                         nHeight = 0
 408                     };
 409
 410                     itemTemplate.FillHeader(genesisBlock.header);
 411
 412                     if (!AddItemToIndex(ref itemTemplate, ref genesisBlock))
 413                     {
 414                         throw new Exception("Unable to write genesis block");
 415                     }
 416                 }
 417             }
 418             else
 419             {
 420                 var blockTreeItems = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] order by [ItemId] asc");
 421
 422                 // Init list of block items
 423                 foreach (var item in blockTreeItems)
 424                 {
 425                     blockMap.TryAdd(new ScryptHash256(item.Hash), item);
 426                 }
 427             }
 428
 429             Instance = this;
 430         }
 431
 432         public bool GetTransaction(Hash256 TxID, ref CTransaction tx)
 433         {
 434             var QueryTx = dbConn.Query<CTransactionStoreItem>("select * from [TransactionStorage] where [TransactionHash] = ?", (byte[])TxID);
 435
 436             if (QueryTx.Count == 1)
 437             {
 438                 return QueryTx[0].ReadFromFile(ref reader, out tx);
 439             }
 440
 441             // Tx not found
 442
 443             return false;
 444         }
 445
 446         private bool AddItemToIndex(ref CBlockStoreItem itemTemplate, ref CBlock block)
 447         {
 448             var blockHash = new ScryptHash256(itemTemplate.Hash);
 449
 450             if (blockMap.ContainsKey(blockHash))
 451             {
 452                 // Already have this block.
 453                 return false;
 454             }
 455
 456             // TODO: compute chain trust, set stake entropy bit, record proof-of-stake hash value
 457
 458             // TODO: compute stake modifier
 459
 460             // Add to index
 461             itemTemplate.BlockTypeFlag = block.IsProofOfStake ? BlockType.PROOF_OF_STAKE : BlockType.PROOF_OF_WORK;
 462
 463             if (!itemTemplate.WriteToFile(ref writer, ref block))
 464             {
 465                 return false;
 466             }
 467
 468             dbConn.Insert(itemTemplate);
 469
 470             // We have no SetBestChain and ConnectBlock/Disconnect block yet, so adding these transactions manually.
 471             for (int i = 0; i < block.vtx.Length; i++)
 472             {
 473                 // Handle trasactions
 474
 475                 var nTxOffset = itemTemplate.nBlockPos + block.GetTxOffset(i);
 476                 TxType txnType = TxType.TX_USER;
 477
 478                 if (block.vtx[i].IsCoinBase)
 479                 {
 480                     txnType = TxType.TX_COINBASE;
 481                 }
 482                 else if (block.vtx[i].IsCoinStake)
 483                 {
 484                     txnType = TxType.TX_COINSTAKE;
 485                 }
 486
 487                 var NewTxItem = new CTransactionStoreItem()
 488                 {
 489                     TransactionHash = block.vtx[i].Hash,
 490                     BlockHash = blockHash,
 491                     nTxPos = nTxOffset,
 492                     nTxSize = block.vtx[i].Size,
 493                     txType = txnType
 494                 };
 495
 496                 dbConn.Insert(NewTxItem);
 497             }
 498
 499             return blockMap.TryAdd(blockHash, itemTemplate);
 500         }
 501
 502         public bool AcceptBlock(ref CBlock block)
 503         {
 504             ScryptHash256 hash = block.header.Hash;
 505
 506             if (blockMap.ContainsKey(hash))
 507             {
 508                 // Already have this block.
 509                 return false;
 510             }
 511
 512             CBlockStoreItem prevBlockCursor = null;
 513             if (!blockMap.TryGetValue(block.header.prevHash, out prevBlockCursor))
 514             {
 515                 // Unable to get the cursor.
 516                 return false;
 517             }
 518
 519             var prevBlockHeader = prevBlockCursor.BlockHeader;
 520
 521             // TODO: proof-of-work/proof-of-stake verification
 522             uint nHeight = prevBlockCursor.nHeight + 1;
 523
 524             // Check timestamp against prev
 525             if (NetUtils.FutureDrift(block.header.nTime) < prevBlockHeader.nTime)
 526             {
 527                 // block's timestamp is too early
 528                 return false;
 529             }
 530
 531             // Check that all transactions are finalized
 532             foreach (var tx in block.vtx)
 533             {
 534                 if (!tx.IsFinal(nHeight, block.header.nTime))
 535                 {
 536                     return false;
 537                 }
 538             }
 539
 540             // TODO: Enforce rule that the coinbase starts with serialized block height
 541
 542             // Write block to file.
 543             var itemTemplate = new CBlockStoreItem()
 544             {
 545                 nHeight = nHeight
 546             };
 547
 548             itemTemplate.FillHeader(block.header);
 549
 550             if (!AddItemToIndex(ref itemTemplate, ref block))
 551             {
 552                 return false;
 553             }
 554
 555             return true;
 556         }
 557
 558         public bool GetBlock(ScryptHash256 blockHash, ref CBlock block)
 559         {
 560             var QueryBlock = dbConn.Query<CBlockStoreItem>("select * from [BlockStorage] where [Hash] = ?", (byte[])blockHash);
 561
 562             if (QueryBlock.Count == 1)
 563             {
 564                 return QueryBlock[0].ReadFromFile(ref reader, out block);
 565             }
 566
 567             // Block not found
 568
 569             return false;
 570         }
 571
 572         public bool ProcessBlock(ref CBlock block)
 573         {
 574             ScryptHash256 blockHash = block.header.Hash;
 575
 576             if (blockMap.ContainsKey(blockHash))
 577             {
 578                 // We already have this block.
 579                 return false;
 580             }
 581
 582             if (orphanMap.ContainsKey(blockHash))
 583             {
 584                 // We already have block in the list of orphans.
 585                 return false;
 586             }
 587
 588             // TODO: Limited duplicity on stake and reserialization of block signature
 589
 590             // Preliminary checks
 591             if (!block.CheckBlock(true, true, true))
 592             {
 593                 return true;
 594             }
 595
 596             if (block.IsProofOfStake)
 597             {
 598                 // TODO: proof-of-stake validation
 599             }
 600
 601             // TODO: difficulty verification
 602
 603             // If don't already have its previous block, shunt it off to holding area until we get it
 604             if (!blockMap.ContainsKey(block.header.prevHash))
 605             {
 606                 if (block.IsProofOfStake)
 607                 {
 608                     // TODO: limit duplicity on stake
 609                 }
 610
 611                 var block2 = new CBlock(block);
 612                 orphanMap.TryAdd(blockHash, block2);
 613                 orphanMapByPrev.TryAdd(blockHash, block2);
 614
 615                 return true;
 616             }
 617
 618             // Store block to disk
 619             if (!AcceptBlock(ref block))
 620             {
 621                 // Accept failed
 622                 return false;
 623             }
 624
 625             // Recursively process any orphan blocks that depended on this one
 626             var orphansQueue = new List<ScryptHash256>();
 627             orphansQueue.Add(blockHash);
 628
 629             for (int i = 0; i < orphansQueue.Count; i++)
 630             {
 631                 ScryptHash256 hashPrev = orphansQueue[i];
 632
 633                 foreach (var pair in orphanMap)
 634                 {
 635                     var orphanBlock = pair.Value;
 636
 637                     if (orphanBlock.header.prevHash == blockHash)
 638                     {
 639                         if (AcceptBlock(ref orphanBlock))
 640                         {
 641                             orphansQueue.Add(pair.Key);
 642                         }
 643
 644                         CBlock dummy1;
 645                         orphanMap.TryRemove(pair.Key, out dummy1);
 646                     }
 647                 }
 648
 649                 CBlock dummy2;
 650                 orphanMap.TryRemove(hashPrev, out dummy2);
 651             }
 652
 653             return true;
 654         }
 655
 656         public bool ParseBlockFile(string BlockFile = "bootstrap.dat")
 657         {
 658             // TODO: Rewrite completely.
 659
 660             var nOffset = 0L;
 661
 662             var buffer = new byte[1000000]; // Max block size is 1Mb
 663             var intBuffer = new byte[4];
 664
 665             var fStream2 = File.OpenRead(BlockFile);
 666             var readerForBlocks = new BinaryReader(fStream2).BaseStream;
 667
 668             readerForBlocks.Seek(nOffset, SeekOrigin.Begin); // Seek to previous offset + previous block length
 669
 670             dbConn.BeginTransaction();
 671
 672             while (readerForBlocks.Read(buffer, 0, 4) == 4) // Read magic number
 673             {
 674                 var nMagic = BitConverter.ToUInt32(buffer, 0);
 675                 if (nMagic != 0xe5e9e8e4)
 676                 {
 677                     throw new Exception("Incorrect magic number.");
 678                 }
 679
 680                 var nBytesRead = readerForBlocks.Read(buffer, 0, 4);
 681                 if (nBytesRead != 4)
 682                 {
 683                     throw new Exception("BLKSZ EOF");
 684                 }
 685
 686                 var nBlockSize = BitConverter.ToInt32(buffer, 0);
 687
 688                 nOffset = readerForBlocks.Position;
 689
 690                 nBytesRead = readerForBlocks.Read(buffer, 0, nBlockSize);
 691
 692                 if (nBytesRead == 0 || nBytesRead != nBlockSize)
 693                 {
 694                     throw new Exception("BLK EOF");
 695                 }
 696
 697                 var block = new CBlock(buffer);
 698                 var hash = block.header.Hash;
 699
 700                 if (blockMap.ContainsKey(hash))
 701                 {
 702                     continue;
 703                 }
 704
 705                 if (!ProcessBlock(ref block))
 706                 {
 707                     throw new Exception("Invalid block: " + block.header.Hash);
 708                 }
 709
 710                 int nCount = blockMap.Count;
 711                 Console.WriteLine("nCount={0}, Hash={1}, Time={2}", nCount, block.header.Hash, DateTime.Now); // Commit on each 100th block
 712
 713                 if (nCount % 100 == 0 && nCount != 0)
 714                 {
 715                     Console.WriteLine("Commit...");
 716                     dbConn.Commit();
 717                     dbConn.BeginTransaction();
 718                 }
 719             }
 720
 721             dbConn.Commit();
 722
 723             return true;
 724         }
 725
 726         ~CBlockStore()
 727         {
 728             Dispose(false);
 729         }
 730
 731         public void Dispose()
 732         {
 733             Dispose(true);
 734             GC.SuppressFinalize(this);
 735         }
 736
 737         protected virtual void Dispose(bool disposing)
 738         {
 739             if (!disposed)
 740             {
 741                 if (disposing)
 742                 {
 743                     // Free other state (managed objects).
 744
 745                     reader.Dispose();
 746                     writer.Dispose();
 747                 }
 748
 749                 if (dbConn != null)
 750                 {
 751                     dbConn.Close();
 752                     dbConn = null;
 753                 }
 754
 755                 disposed = true;
 756             }
 757         }
 758
 759     }
 760 }