New bootstraper db format with ipv6 support

plugins/disabled-Bootstrapper/BootstrapperDb.py

@@ -10,7 +10,7 @@ from util import helper
 
 class BootstrapperDb(Db):
     def __init__(self):
-        self.version = 6
+        self.version = 7
         self.hash_ids = {}  # hash -> id cache
         super(BootstrapperDb, self).__init__({"db_name": "Bootstrapper"}, "%s/bootstrapper.db" % config.data_dir)
         self.foreign_keys = True
@@ -20,7 +20,7 @@ class BootstrapperDb(Db):
 
     def cleanup(self):
         while 1:
-            time.sleep(4*60)
+            time.sleep(4 * 60)
             timeout = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time() - 60 * 40))
             self.execute("DELETE FROM peer WHERE date_announced < ?", [timeout])
 
@@ -47,14 +47,15 @@ class BootstrapperDb(Db):
         # Create new tables
         self.execute("""
             CREATE TABLE peer (
-                peer_id        INTEGER  PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE,
+                peer_id        INTEGER PRIMARY KEY ASC AUTOINCREMENT NOT NULL UNIQUE,
+                type           TEXT,
+                address        TEXT,
                 port           INTEGER NOT NULL,
-                ip4            TEXT,
-                onion          TEXT UNIQUE,
                 date_added     DATETIME DEFAULT (CURRENT_TIMESTAMP),
                 date_announced DATETIME DEFAULT (CURRENT_TIMESTAMP)
             );
         """)
+        self.execute("CREATE UNIQUE INDEX peer_key ON peer (address, port);")
 
         self.execute("""
             CREATE TABLE peer_to_hash (
@@ -82,19 +83,13 @@ class BootstrapperDb(Db):
             self.hash_ids[hash] = self.cur.cursor.lastrowid
         return self.hash_ids[hash]
 
-    def peerAnnounce(self, ip4=None, onion=None, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False):
+    def peerAnnounce(self, ip_type, address, port=None, hashes=[], onion_signed=False, delete_missing_hashes=False):
         hashes_ids_announced = []
         for hash in hashes:
             hashes_ids_announced.append(self.getHashId(hash))
 
-        if not ip4 and not onion:
-            return 0
-
         # Check user
-        if onion:
-            res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"onion": onion})
-        else:
-            res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"ip4": ip4, "port": port})
+        res = self.execute("SELECT peer_id FROM peer WHERE ? LIMIT 1", {"address": address, "port": port})
 
         user_row = res.fetchone()
         now = time.strftime("%Y-%m-%d %H:%M:%S")
@@ -102,10 +97,10 @@ class BootstrapperDb(Db):
             peer_id = user_row["peer_id"]
             self.execute("UPDATE peer SET date_announced = ? WHERE peer_id = ?", (now, peer_id))
         else:
-            self.log.debug("New peer: %s %s signed: %s" % (ip4, onion, onion_signed))
-            if onion and not onion_signed:
+            self.log.debug("New peer: %s signed: %s" % (address, onion_signed))
+            if ip_type == "onion" and not onion_signed:
                 return len(hashes)
-            self.execute("INSERT INTO peer ?", {"ip4": ip4, "onion": onion, "port": port, "date_announced": now})
+            self.execute("INSERT INTO peer ?", {"type": ip_type, "address": address, "port": port, "date_announced": now})
             peer_id = self.cur.cursor.lastrowid
 
         # Check user's hashes
@@ -114,7 +109,7 @@ class BootstrapperDb(Db):
         if hash_ids_db != hashes_ids_announced:
             hash_ids_added = set(hashes_ids_announced) - set(hash_ids_db)
             hash_ids_removed = set(hash_ids_db) - set(hashes_ids_announced)
-            if not onion or onion_signed:
+            if ip_type != "onion" or onion_signed:
                 for hash_id in hash_ids_added:
                     self.execute("INSERT INTO peer_to_hash ?", {"peer_id": peer_id, "hash_id": hash_id})
                 if hash_ids_removed and delete_missing_hashes:
@@ -124,10 +119,10 @@ class BootstrapperDb(Db):
         else:
             return 0
 
-    def peerList(self, hash, ip4=None, onions=[], port=None, limit=30, need_types=["ip4", "onion"], order=True):
-        hash_peers = {"ip4": [], "onion": []}
+    def peerList(self, hash, address=None, onions=[], port=None, limit=30, need_types=["ipv4", "onion"], order=True):
+        back = {"ipv4": [], "ipv6": [], "onion": []}
         if limit == 0:
-            return hash_peers
+            return back
         hashid = self.getHashId(hash)
 
         if order:
@@ -137,27 +132,25 @@ class BootstrapperDb(Db):
         where_sql = "hash_id = :hashid"
         if onions:
             onions_escaped = ["'%s'" % re.sub("[^a-z0-9,]", "", onion) for onion in onions if type(onion) is str]
-            where_sql += " AND (onion NOT IN (%s) OR onion IS NULL)" % ",".join(onions_escaped)
-        elif ip4:
-            where_sql += " AND (NOT (ip4 = :ip4 AND port = :port) OR ip4 IS NULL)"
+            where_sql += " AND address NOT IN (%s)" % ",".join(onions_escaped)
+        elif address:
+            where_sql += " AND NOT (address = :address AND port = :port)"
 
         query = """
-            SELECT ip4, port, onion
+            SELECT type, address, port
             FROM peer_to_hash
             LEFT JOIN peer USING (peer_id)
             WHERE %s
             %s
             LIMIT :limit
         """ % (where_sql, order_sql)
-        res = self.execute(query, {"hashid": hashid, "ip4": ip4, "onions": onions, "port": port, "limit": limit})
+        res = self.execute(query, {"hashid": hashid, "address": address, "port": port, "limit": limit})
 
         for row in res:
-            if row["ip4"] and "ip4" in need_types:
-                hash_peers["ip4"].append(
-                    helper.packAddress(row["ip4"], row["port"])
-                )
-            if row["onion"] and "onion" in need_types:
-                hash_peers["onion"].append(
-                    helper.packOnionAddress(row["onion"], row["port"])
-                )
-        return hash_peers
+            if row["type"] in need_types:
+                if row["type"] == "onion":
+                    packed = helper.packOnionAddress(row["address"], row["port"])
+                else:
+                    packed = helper.packAddress(str(row["address"]), row["port"])
+                back[row["type"]].append(packed)
+        return back

plugins/disabled-Bootstrapper/BootstrapperPlugin.py

@@ -1,8 +1,11 @@
 import time
 
+from util import helper
+
 from Plugin import PluginManager
 from BootstrapperDb import BootstrapperDb
 from Crypt import CryptRsa
+from Config import config
 
 if "db" not in locals().keys():  # Share during reloads
     db = BootstrapperDb()
@@ -10,39 +13,50 @@ if "db" not in locals().keys():  # Share during reloads
 
 @PluginManager.registerTo("FileRequest")
 class FileRequestPlugin(object):
+    def checkOnionSigns(self, onions, onion_signs, onion_sign_this):
+        if not onion_signs or len(onion_signs) != len(set(onions)):
+            return False
+
+        if time.time() - float(onion_sign_this) > 3 * 60:
+            return False  # Signed out of allowed 3 minutes
+
+        onions_signed = []
+        # Check onion signs
+        for onion_publickey, onion_sign in onion_signs.items():
+            if CryptRsa.verify(onion_sign_this, onion_publickey, onion_sign):
+                onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey))
+            else:
+                break
+
+        # Check if the same onion addresses signed as the announced onces
+        if sorted(onions_signed) == sorted(set(onions)):
+            return True
+        else:
+            return False
+
     def actionAnnounce(self, params):
         time_started = time.time()
         s = time.time()
+        # Backward compatibility
+        if "ip4" in params["add"]:
+            params["add"].append("ipv4")
+        if "ip4" in params["need_types"]:
+            params["need_types"].append("ipv4")
+
         hashes = params["hashes"]
 
-        if "onion_signs" in params and len(params["onion_signs"]) == len(set(params["onions"])):
-            # Check if all sign is correct
-            if time.time() - float(params["onion_sign_this"]) < 3 * 60:  # Peer has 3 minute to sign the message
-                onions_signed = []
-                # Check onion signs
-                for onion_publickey, onion_sign in params["onion_signs"].items():
-                    if CryptRsa.verify(params["onion_sign_this"], onion_publickey, onion_sign):
-                        onions_signed.append(CryptRsa.publickeyToOnion(onion_publickey))
-                    else:
-                        break
-                # Check if the same onion addresses signed as the announced onces
-                if sorted(onions_signed) == sorted(set(params["onions"])):
-                    all_onions_signed = True
-                else:
-                    all_onions_signed = False
-            else:
-                # Onion sign this out of 3 minute
-                all_onions_signed = False
-        else:
-            # Incorrect signs number
-            all_onions_signed = False
+        all_onions_signed = self.checkOnionSigns(params.get("onions", []), params.get("onion_signs"), params.get("onion_sign_this"))
 
         time_onion_check = time.time() - s
 
-        if "ip4" in params["add"] and self.connection.ip != "127.0.0.1" and not self.connection.ip.endswith(".onion"):
-            ip4 = self.connection.ip
+        ip_type = helper.getIpType(self.connection.ip)
+
+        if ip_type == "onion" or self.connection.ip in config.ip_local:
+            is_port_open = False
+        elif ip_type in params["add"]:
+            is_port_open = True
         else:
-            ip4 = None
+            is_port_open = False
 
         s = time.time()
         # Separatley add onions to sites or at once if no onions present
@@ -58,7 +72,8 @@ class FileRequestPlugin(object):
         db.execute("BEGIN")
         for onion, onion_hashes in onion_to_hash.iteritems():
             hashes_changed += db.peerAnnounce(
-                onion=onion,
+                ip_type="onion",
+                address=onion,
                 port=params["port"],
                 hashes=onion_hashes,
                 onion_signed=all_onions_signed
@@ -67,15 +82,16 @@ class FileRequestPlugin(object):
         time_db_onion = time.time() - s
 
         s = time.time()
-        # Announce all sites if ip4 defined
-        if ip4:
+
+        if is_port_open:
             hashes_changed += db.peerAnnounce(
-                ip4=ip4,
+                ip_type=ip_type,
+                address=self.connection.ip,
                 port=params["port"],
                 hashes=hashes,
                 delete_missing_hashes=params.get("delete")
             )
-        time_db_ip4 = time.time() - s
+        time_db_ip = time.time() - s
 
         s = time.time()
         # Query sites
@@ -97,16 +113,19 @@ class FileRequestPlugin(object):
 
             hash_peers = db.peerList(
                 hash,
-                ip4=self.connection.ip, onions=onion_to_hash.keys(), port=params["port"],
+                address=self.connection.ip, onions=onion_to_hash.keys(), port=params["port"],
                 limit=min(limit, params["need_num"]), need_types=params["need_types"], order=order
             )
+            if "ip4" in params["need_types"]:  # Backward compatibility
+                hash_peers["ip4"] = hash_peers["ipv4"]
+                del(hash_peers["ipv4"])
             peers.append(hash_peers)
         time_peerlist = time.time() - s
 
         back["peers"] = peers
         self.connection.log(
-            "Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip4: %.3fs, peerlist: %.3fs, limit: %s)" %
-            (len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip4, time_peerlist, limit)
+            "Announce %s sites (onions: %s, onion_check: %.3fs, db_onion: %.3fs, db_ip: %.3fs, peerlist: %.3fs, limit: %s)" %
+            (len(hashes), len(onion_to_hash), time_onion_check, time_db_onion, time_db_ip, time_peerlist, limit)
         )
         self.response(back)