From: ThomasV <thomasv@gitorious>
Date: Mon, 19 Nov 2012 15:45:00 +0000 (+0400)
Subject: store hashes in the database, to optimize space. use util.py
X-Git-Url: https://git.novaco.in/?p=electrum-server.git;a=commitdiff_plain;h=fdc576873627546b8677b7acb560918fac9cafd0

store hashes in the database, to optimize space. use util.py
---

diff --git a/backends/bitcoind/blockchain_processor.py b/backends/bitcoind/blockchain_processor.py
index 784cd98..7891f68 100644
--- a/backends/bitcoind/blockchain_processor.py
+++ b/backends/bitcoind/blockchain_processor.py
@@ -6,47 +6,8 @@ from Queue import Queue
 import traceback, sys, os, random
 
 
-
-Hash = lambda x: hashlib.sha256(hashlib.sha256(x).digest()).digest()
-hash_encode = lambda x: x[::-1].encode('hex')
-hash_decode = lambda x: x.decode('hex')[::-1]
-
-
-
-def rev_hex(s):
-    return s.decode('hex')[::-1].encode('hex')
-
-
-def int_to_hex(i, length=1):
-    s = hex(i)[2:].rstrip('L')
-    s = "0"*(2*length - len(s)) + s
-    return rev_hex(s)
-
-def header_to_string(res):
-    pbh = res.get('prev_block_hash')
-    if pbh is None: pbh = '0'*64
-    s = int_to_hex(res.get('version'),4) \
-        + rev_hex(pbh) \
-        + rev_hex(res.get('merkle_root')) \
-        + int_to_hex(int(res.get('timestamp')),4) \
-        + int_to_hex(int(res.get('bits')),4) \
-        + int_to_hex(int(res.get('nonce')),4)
-    return s
-
-def header_from_string( s):
-    hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
-    h = {}
-    h['version'] = hex_to_int(s[0:4])
-    h['prev_block_hash'] = hash_encode(s[4:36])
-    h['merkle_root'] = hash_encode(s[36:68])
-    h['timestamp'] = hex_to_int(s[68:72])
-    h['bits'] = hex_to_int(s[72:76])
-    h['nonce'] = hex_to_int(s[76:80])
-    return h
-
-
-
-
+from util import Hash, hash_encode, hash_decode, rev_hex, int_to_hex
+from util import bc_address_to_hash_160, hash_160_to_bc_address, header_to_string, header_from_string
 from processor import Processor, print_log
 
 class BlockchainProcessor(Processor):
@@ -258,7 +219,8 @@ class BlockchainProcessor(Processor):
 
         with self.dblock:
             try:
-                hist = self.deserialize(self.db.Get(addr))
+                hash_160 = bc_address_to_hash_160(addr)
+                hist = self.deserialize(self.db.Get(hash_160))
                 is_known = True
             except: 
                 hist = []
@@ -437,12 +399,11 @@ class BlockchainProcessor(Processor):
                     txo = (txid + int_to_hex(x.get('index'), 4)).decode('hex')
                     block_outputs.append(txo)
             
-
-
         # read histories of addresses
         for txid, tx in txdict.items():
             for x in tx.get('outputs'):
-                addr_to_read.append(x.get('address'))
+                hash_160 = bc_address_to_hash_160(x.get('address'))
+                addr_to_read.append(hash_160)
 
         addr_to_read.sort()
         for addr in addr_to_read:
@@ -454,7 +415,7 @@ class BlockchainProcessor(Processor):
 
         if revert: 
             undo_info = self.get_undo_info(block_height)
-            print "undo", block_height, undo_info
+            # print "undo", block_height, undo_info
         else: undo_info = {}
 
         # process
@@ -472,11 +433,13 @@ class BlockchainProcessor(Processor):
                 undo_info[txid] = undo
 
                 for x in tx.get('outputs'):
-                    self.add_to_history( x.get('address'), txid, x.get('index'), block_height)
+                    hash_160 = bc_address_to_hash_160(x.get('address'))
+                    self.add_to_history( hash_160, txid, x.get('index'), block_height)
                     
             else:
                 for x in tx.get('outputs'):
-                    self.remove_from_history( x.get('address'), txid, x.get('index'))
+                    hash_160 = bc_address_to_hash_160(x.get('address'))
+                    self.remove_from_history( hash_160, txid, x.get('index'))
 
                 i = 0
                 for x in tx.get('inputs'):
@@ -489,7 +452,7 @@ class BlockchainProcessor(Processor):
 
                     # re-add them to the history
                     self.add_to_history( prevout_addr, x.get('prevout_hash'), x.get('prevout_n'), prevout_height)
-                    print "new hist for", prevout_addr, self.deserialize(self.batch_list[prevout_addr])
+                    print_log( "new hist for", hash_160_to_bc_address(prevout_addr), self.deserialize(self.batch_list[prevout_addr]) )
 
         # write
         max_len = 0
@@ -675,7 +638,9 @@ class BlockchainProcessor(Processor):
             next_block_hash = self.bitcoind('getblockhash', [self.height+1])
             next_block = self.bitcoind('getblock', [next_block_hash, 1])
 
-            revert = (random.randint(1, 10)==1) if self.is_test else False
+            # fixme: this is unsafe, if we revert when the undo info is not yet written 
+            revert = (random.randint(1, 100)==1) if self.is_test else False        
+
             if (next_block.get('previousblockhash') == self.last_hash) and not revert:
 
                 self.import_block(next_block, next_block_hash, self.height+1, sync)
diff --git a/backends/bitcoind/util.py b/backends/bitcoind/util.py
new file mode 100644
index 0000000..2f33d11
--- /dev/null
+++ b/backends/bitcoind/util.py
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+#
+# Electrum - lightweight Bitcoin client
+# Copyright (C) 2011 thomasv@gitorious
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import hashlib, base64, re
+
+
+def rev_hex(s):
+    return s.decode('hex')[::-1].encode('hex')
+
+def int_to_hex(i, length=1):
+    s = hex(i)[2:].rstrip('L')
+    s = "0"*(2*length - len(s)) + s
+    return rev_hex(s)
+
+def var_int(i):
+    if i<0xfd:
+        return int_to_hex(i)
+    elif i<=0xffff:
+        return "fd"+int_to_hex(i,2)
+    elif i<=0xffffffff:
+        return "fe"+int_to_hex(i,4)
+    else:
+        return "ff"+int_to_hex(i,8)
+
+
+Hash = lambda x: hashlib.sha256(hashlib.sha256(x).digest()).digest()
+hash_encode = lambda x: x[::-1].encode('hex')
+hash_decode = lambda x: x.decode('hex')[::-1]
+
+
+def header_to_string(res):
+    pbh = res.get('prev_block_hash')
+    if pbh is None: pbh = '0'*64
+    s = int_to_hex(res.get('version'),4) \
+        + rev_hex(pbh) \
+        + rev_hex(res.get('merkle_root')) \
+        + int_to_hex(int(res.get('timestamp')),4) \
+        + int_to_hex(int(res.get('bits')),4) \
+        + int_to_hex(int(res.get('nonce')),4)
+    return s
+
+def header_from_string( s):
+    hex_to_int = lambda s: eval('0x' + s[::-1].encode('hex'))
+    h = {}
+    h['version'] = hex_to_int(s[0:4])
+    h['prev_block_hash'] = hash_encode(s[4:36])
+    h['merkle_root'] = hash_encode(s[36:68])
+    h['timestamp'] = hex_to_int(s[68:72])
+    h['bits'] = hex_to_int(s[72:76])
+    h['nonce'] = hex_to_int(s[76:80])
+    return h
+
+
+############ functions from pywallet ##################### 
+
+addrtype = 0
+
+def hash_160(public_key):
+    try:
+        md = hashlib.new('ripemd160')
+        md.update(hashlib.sha256(public_key).digest())
+        return md.digest()
+    except:
+        import ripemd
+        md = ripemd.new(hashlib.sha256(public_key).digest())
+        return md.digest()
+
+
+def public_key_to_bc_address(public_key):
+    h160 = hash_160(public_key)
+    return hash_160_to_bc_address(h160)
+
+def hash_160_to_bc_address(h160):
+    vh160 = chr(addrtype) + h160
+    h = Hash(vh160)
+    addr = vh160 + h[0:4]
+    return b58encode(addr)
+
+def bc_address_to_hash_160(addr):
+    bytes = b58decode(addr, 25)
+    return bytes[1:21]
+
+
+__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+__b58base = len(__b58chars)
+
+def b58encode(v):
+    """ encode v, which is a string of bytes, to base58."""
+
+    long_value = 0L
+    for (i, c) in enumerate(v[::-1]):
+        long_value += (256**i) * ord(c)
+
+    result = ''
+    while long_value >= __b58base:
+        div, mod = divmod(long_value, __b58base)
+        result = __b58chars[mod] + result
+        long_value = div
+    result = __b58chars[long_value] + result
+
+    # Bitcoin does a little leading-zero-compression:
+    # leading 0-bytes in the input become leading-1s
+    nPad = 0
+    for c in v:
+        if c == '\0': nPad += 1
+        else: break
+
+    return (__b58chars[0]*nPad) + result
+
+def b58decode(v, length):
+    """ decode v into a string of len bytes."""
+    long_value = 0L
+    for (i, c) in enumerate(v[::-1]):
+        long_value += __b58chars.find(c) * (__b58base**i)
+
+    result = ''
+    while long_value >= 256:
+        div, mod = divmod(long_value, 256)
+        result = chr(mod) + result
+        long_value = div
+    result = chr(long_value) + result
+
+    nPad = 0
+    for c in v:
+        if c == __b58chars[0]: nPad += 1
+        else: break
+
+    result = chr(0)*nPad + result
+    if length is not None and len(result) != length:
+        return None
+
+    return result
+
+
+def EncodeBase58Check(vchIn):
+    hash = Hash(vchIn)
+    return b58encode(vchIn + hash[0:4])
+
+def DecodeBase58Check(psz):
+    vchRet = b58decode(psz, None)
+    key = vchRet[0:-4]
+    csum = vchRet[-4:]
+    hash = Hash(key)
+    cs32 = hash[0:4]
+    if cs32 != csum:
+        return None
+    else:
+        return key
+
+def PrivKeyToSecret(privkey):
+    return privkey[9:9+32]
+
+def SecretToASecret(secret):
+    vchIn = chr(addrtype+128) + secret
+    return EncodeBase58Check(vchIn)
+
+def ASecretToSecret(key):
+    vch = DecodeBase58Check(key)
+    if vch and vch[0] == chr(addrtype+128):
+        return vch[1:]
+    else:
+        return False
+
+########### end pywallet functions #######################
+