bbc1.core.data_handler — BBc1 1.2 documentation

# Source code for bbc1.core.data_handler

    # -*- coding: utf-8 -*-
    """
    Copyright (c) 2017 beyond-blockchain.org.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
    """

    import traceback
    import binascii
    import os
    import sys
    sys.path.extend(["../../", os.path.abspath(os.path.dirname(__file__))])
    from bbc1.core import bbclib
    import bbc1.core.libs.bbclib_config as bbclib_config
    from bbc1.core.message_key_types import to_2byte, PayloadType, KeyType, InfraMessageCategory
    from bbc1.core import logger

    transaction_tbl_definition = [
        ["transaction_id", "BLOB"], ["transaction_data", "BLOB"],
    ]

    asset_info_definition = [
        ["id", "INTEGER"],
        ["transaction_id", "BLOB"], ["asset_group_id", "BLOB"], ["asset_id", "BLOB"], ["user_id", "BLOB"],
    ]

    topology_info_definition = [
        ["id", "INTEGER"], ["base", "BLOB"], ["point_to", "BLOB"]
    ]

    cross_ref_tbl_definition = [
        ["id", "INTEGER"], ["transaction_id", "BLOB"], ["outer_domain_id", "BLOB"], ["txid_having_cross_ref", "BLOB"],
    ]

    #--- for anchoring ethereum/bitcoin blockchain ---
    merkle_branch_db_definition = [
        ["digest", "BLOB"], ["leaf_left", "BLOB"], ["leaf_right", "BLOB"],
    ]

    merkle_leaf_db_definition = [
        ["digest", "BLOB"], ["leaf_left", "BLOB"], ["leaf_right", "BLOB"], ["prev", "BLOB"],
    ]

    merkle_root_db_definition = [
        ["root", "BLOB"], ["spec", "BLOB"],
    ]

    class DataHandler:
        """DB and storage handler"""
        REPLICATION_ALL = 0
        REPLICATION_P2P = 1
        REPLICATION_EXT = 2
        REQUEST_REPLICATION_INSERT = to_2byte(0)
        RESPONSE_REPLICATION_INSERT = to_2byte(1)
        REQUEST_SEARCH = to_2byte(2)
        RESPONSE_SEARCH = to_2byte(3)
        NOTIFY_INSERTED = to_2byte(4)
        REPAIR_TRANSACTION_DATA = to_2byte(5)
        REPLICATION_CROSS_REF = to_2byte(6)

        def __init__(self, networking=None, config=None, workingdir=None, domain_id=None, loglevel="all", logname=None):
            self.networking = networking
            self.core = networking.core
            self.stats = networking.core.stats
            self.logger = logger.get_logger(key="data_handler", level=loglevel, logname=logname)
            self.domain_id = domain_id
            self.domain_id_str = bbclib.convert_id_to_string(domain_id)
            self.config = config
            self.working_dir = workingdir
            self.storage_root = os.path.join(self.working_dir, self.domain_id_str)
            if not os.path.exists(self.storage_root):
                os.makedirs(self.storage_root, exist_ok=True)
            self.use_external_storage = self._storage_setup()
            self.replication_strategy = DataHandler.REPLICATION_ALL
            self.db_adaptors = list()
            self.dbs = list()
            self._db_setup()

        def _db_setup(self):
            """Setup DB"""
            dbconf = self.config['db']
            if dbconf['replication_strategy'] == 'all':
                self.replication_strategy = DataHandler.REPLICATION_ALL
            elif dbconf['replication_strategy'] == 'p2p':
                self.replication_strategy = DataHandler.REPLICATION_P2P
            else:
                self.replication_strategy = DataHandler.REPLICATION_EXT
            db_type = dbconf.get("db_type", "sqlite")
            db_name = dbconf.get("db_name", "bbc1_db.sqlite")
            if db_type == "sqlite":
                self.db_adaptors.append(SqliteAdaptor(self, db_name=os.path.join(self.storage_root, db_name)))
            elif db_type == "mysql":
                count = 0
                for c in dbconf['db_servers']:
                    db_addr = c.get("db_addr", "127.0.0.1")
                    db_port = c.get("db_port", 3306)
                    db_user = c.get("db_user", "user")
                    db_pass = c.get("db_pass", "password")
                    self.db_adaptors.append(MysqlAdaptor(self, db_name=db_name, db_num=count,
                                                         server_info=(db_addr, db_port, db_user, db_pass)))
                    count += 1

            for db in self.db_adaptors:
                db.open_db()
                db.create_table('transaction_table', transaction_tbl_definition, primary_key=0, indices=[0])
                db.create_table('asset_info_table', asset_info_definition, primary_key=0, indices=[0, 1, 2, 3, 4])
                db.create_table('topology_table', topology_info_definition, primary_key=0, indices=[0, 1, 2])
                db.create_table('cross_ref_table', cross_ref_tbl_definition, primary_key=0, indices=[1])
                db.create_table('merkle_branch_table', merkle_branch_db_definition, primary_key=0, indices=[1, 2])
                db.create_table('merkle_leaf_table', merkle_leaf_db_definition, primary_key=0, indices=[1, 2])
                db.create_table('merkle_root_table', merkle_root_db_definition, primary_key=0, indices=[0])

        def _storage_setup(self):
            """Setup storage"""
            if self.config['storage']['type'] == "external":
                return True
            if 'root' in self.config['storage'] and self.config['storage']['root'].startswith("/"):
                self.storage_root = os.path.join(self.config['storage']['root'], self.domain_id_str)
            else:
                self.storage_root = os.path.join(self.working_dir, self.domain_id_str)
            os.makedirs(self.storage_root, exist_ok=True)
            return False

        def exec_sql(self, db_num=0, sql=None, args=(), commit=False, fetch_one=False):
            """Execute sql sentence

            Args:
                db_num (int): index of DB if multiple DBs are used
                sql (str): SQL string
                args (list): Args for the SQL
                commit (bool): If True, commit is performed
                fetch_one (bool): If True, fetch just one record
            Returns:
                list: list of records
            """
            self.stats.update_stats_increment("data_handler", "exec_sql", 1)
            #print("sql=", sql)
            #if len(args) > 0:
            #    print("args=", args)
            try:
                db_num = 0 if db_num >= len(self.db_adaptors) else db_num
                if len(args) > 0:
                    self.db_adaptors[db_num].db_cur.execute(sql, args)
                else:
                    self.db_adaptors[db_num].db_cur.execute(sql)
                if commit:
                    self.db_adaptors[db_num].db.commit()
                    ret = None
                else:
                    if fetch_one:
                        ret = self.db_adaptors[db_num].db_cur.fetchone()
                    else:
                        ret = self.db_adaptors[db_num].db_cur.fetchall()
            except:
                self.logger.error(traceback.format_exc())
                traceback.print_exc()
                self.stats.update_stats_increment("data_handler", "fail_exec_sql", 1)
                return None
            if ret is None:
                return []
            else:
                return list(ret)

        def get_asset_info(self, txobj):
            """Retrieve asset information from transaction object

            Args:
                txobj (BBcTransaction): transaction object to analyze
            Returns:
                list: list of list [asset_group_id, asset_id, user_id, file_size, file_digest]
            """
            info = list()
            for idx, evt in enumerate(txobj.events):
                ast = evt.asset
                if ast is not None:
                    info.append((evt.asset_group_id, ast.asset_id, ast.user_id, ast.asset_file_size>0,
                                 ast.asset_file_digest))
            for idx, rtn in enumerate(txobj.relations):
                ast = rtn.asset
                if rtn.asset is not None:
                    info.append((rtn.asset_group_id, ast.asset_id, ast.user_id, ast.asset_file_size>0,
                                 ast.asset_file_digest))
            return info

        def _get_topology_info(self, txobj):
            """Retrieve topology information from transaction object

            This method returns (from, to) list that describe the topology of transactions

            Args:
                txobj (BBcTransaction): transaction object to analyze
            Returns:
                list: list of tuple (base transaction_id, pointing transaction_id)
            """
            info = list()
            for reference in txobj.references:
                info.append((txobj.transaction_id, reference.transaction_id))  # (base, point_to)
            for idx, rtn in enumerate(txobj.relations):
                for pt in rtn.pointers:
                    info.append((txobj.transaction_id, pt.transaction_id))  # (base, point_to)
            return info

        def insert_transaction(self, txdata, txobj=None, fmt_type=bbclib_config.DEFAULT_BBC_FORMAT, asset_files=None, no_replication=False):
            """Insert transaction data and its asset files

            Either txdata or txobj must be given to insert the transaction.

            Args:
                txdata (bytes): serialized transaction data
                txobj (BBcTransaction): transaction object to insert
                fmt_type (int): 2-byte value of BBcFormat type
                asset_files (dict): asset files in the transaction
            Returns:
                set: set of asset_group_ids in the transaction
            """
            self.stats.update_stats_increment("data_handler", "insert_transaction", 1)
            if txobj is None:
                txobj, fmt_type = self.core.validate_transaction(txdata, asset_files=asset_files)
                if txobj is None:
                    return None

            inserted_count = 0
            for i in range(len(self.db_adaptors)):
                if self._insert_transaction_into_a_db(i, txobj, fmt_type):
                    inserted_count += 1
            if inserted_count == 0:
                return None

            asset_group_ids = self._store_asset_files(txobj, asset_files)

            if not no_replication and self.replication_strategy != DataHandler.REPLICATION_EXT:
                self._send_replication_to_other_cores(txdata, asset_files)

            if self.networking.domain0manager is not None:
                self.networking.domain0manager.distribute_cross_ref_in_domain0(domain_id=self.domain_id,
                                                                               transaction_id=txobj.transaction_id)
                if txobj.cross_ref is not None:
                    self.networking.domain0manager.cross_ref_registered(domain_id=self.domain_id,
                                                                        transaction_id=txobj.transaction_id,
                                                                        cross_ref=(txobj.cross_ref.domain_id,
                                                                                   txobj.cross_ref.transaction_id))

            return asset_group_ids

        def _insert_transaction_into_a_db(self, db_num, txobj, fmt_type=bbclib_config.DEFAULT_BBC_FORMAT):
            """Insert transaction data into the transaction table of the specified DB

            Args:
                db_num (int): index of DB if multiple DBs are used
                txobj (BBcTransaction): transaction object to insert
                fmt_type (int): 2-byte value of BBcFormat type
            Returns:
                bool: True if successful
            """
            #print("_insert_transaction_into_a_db: for txid =", txobj.transaction_id.hex())
            txdata = bbclib.serialize(txobj, format_type=fmt_type)
            ret = self.exec_sql(db_num=db_num,
                                sql="INSERT INTO transaction_table VALUES (%s,%s)" % (self.db_adaptors[0].placeholder,
                                                                                      self.db_adaptors[0].placeholder),
                                args=(txobj.transaction_id, txdata), commit=True)
            if ret is None:
                return False

            for asset_group_id, asset_id, user_id, fileflag, filedigest in self.get_asset_info(txobj):
                self.exec_sql(db_num=db_num,
                              sql="INSERT INTO asset_info_table(transaction_id, asset_group_id, asset_id, user_id) "
                                  "VALUES (%s, %s, %s, %s)" % (
                                  self.db_adaptors[0].placeholder, self.db_adaptors[0].placeholder,
                                  self.db_adaptors[0].placeholder, self.db_adaptors[0].placeholder),
                              args=(txobj.transaction_id, asset_group_id, asset_id, user_id), commit=True)
            for base, point_to in self._get_topology_info(txobj):
                self.exec_sql(db_num=db_num,
                              sql="INSERT INTO topology_table(base, point_to) VALUES (%s, %s)" %
                                  (self.db_adaptors[0].placeholder, self.db_adaptors[0].placeholder),
                              args=(base, point_to), commit=True)
                #print("topology: base:%s, point_to:%s" % (base.hex(), point_to.hex()))
            return True

        def insert_cross_ref(self, transaction_id, outer_domain_id, txid_having_cross_ref, no_replication=False):
            """Insert cross_ref information into cross_ref_table

            Args:
                transaction_id (bytes): target transaction_id
                outer_domain_id (bytes): domain_id that holds cross_ref about the transaction_id
                txid_having_cross_ref (bytes): transaction_id in the outer_domain that includes the cross_ref
                no_replication (bool): If False, the replication is sent to other nodes in the domain
            """
            self.stats.update_stats_increment("data_handler", "insert_cross_ref", 1)
            sql = "INSERT INTO cross_ref_table (transaction_id, outer_domain_id, txid_having_cross_ref) " + \
                  "VALUES (%s, %s, %s)" % (self.db_adaptors[0].placeholder, self.db_adaptors[0].placeholder,
                                           self.db_adaptors[0].placeholder)
            for i in range(len(self.db_adaptors)):
                self.exec_sql(db_num=i, sql=sql, args=(transaction_id, outer_domain_id, txid_having_cross_ref), commit=True)

            if not no_replication:
                self._send_cross_ref_replication_to_other_cores(transaction_id, outer_domain_id, txid_having_cross_ref)

        def count_domain_in_cross_ref(self, outer_domain_id):
            """Count the number of domains in the cross_ref table"""
            # TODO: need to consider registered_time
            sql = "SELECT count(*) FROM cross_ref_table WHERE outer_domain = %s" % self.db_adaptors[0].placeholder
            ret = self.exec_sql(sql=sql, args=(outer_domain_id,))
            return ret

        def search_domain_having_cross_ref(self, transaction_id=None):
            """Search domain_id that holds cross_ref about the specified transaction_id

            Args:
                transaction_id (bytes): target transaction_id
            Returns:
                list: records of cross_ref_tables ["id","transaction_id", "outer_domain_id", "txid_having_cross_ref"]
            """
            if transaction_id is not None:
                sql = "SELECT * FROM cross_ref_table WHERE transaction_id = %s" % self.db_adaptors[0].placeholder
                return self.exec_sql(sql=sql, args=(transaction_id,))
            else:
                return self.exec_sql(sql="SELECT * FROM cross_ref_table")

        def _store_asset_files(self, txobj, asset_files):
            """Store all asset_files related to the transaction_object

            Args:
                txobj (BBcTransaction): transaction object to insert
                asset_files (dict): dictionary of {asset_id: content} for the transaction
            Returns:
                set: set of asset_group_ids in the transaction
            """
            #print("_store_asset_files: for txid =", txobj.transaction_id.hex())
            asset_group_ids = set()
            for asset_group_id, asset_id, user_id, fileflag, filedigest in self.get_asset_info(txobj):
                asset_group_ids.add(asset_group_id)
                if not self.use_external_storage and asset_files is not None and asset_id in asset_files:
                    self.store_in_storage(asset_group_id, asset_id, asset_files[asset_id])
            return asset_group_ids

        def restore_transaction_data(self, db_num, transaction_id, txobj):
            """Remove and insert a transaction"""
            if txobj is not None:
                self.remove(transaction_id, txobj=txobj, db_num=db_num)
                self._insert_transaction_into_a_db(db_num=db_num, txobj=txobj)

        def _send_replication_to_other_cores(self, txdata, asset_files=None):
            """Broadcast replication of transaction data"""
            msg = {
                KeyType.domain_id: self.domain_id,
                KeyType.infra_msg_type: InfraMessageCategory.CATEGORY_DATA,
                KeyType.infra_command: DataHandler.REQUEST_REPLICATION_INSERT,
                KeyType.transaction_data: txdata,
            }
            if asset_files is not None:
                msg[KeyType.all_asset_files] = asset_files
            if self.replication_strategy == DataHandler.REPLICATION_ALL:
                self.networking.broadcast_message_in_network(domain_id=self.domain_id,
                                                             payload_type=PayloadType.Type_any, msg=msg)
            elif self.replication_strategy == DataHandler.REPLICATION_P2P:
                pass  # TODO: implement (destinations determined by TopologyManager)

        def _send_cross_ref_replication_to_other_cores(self, transaction_id, outer_domain_id, txid_having_cross_ref):
            """Broadcast replication of cross_ref

            Args:
                transaction_id (bytes): target transaction_id
                outer_domain_id (bytes): domain_id that holds cross_ref about the transaction_id
                txid_having_cross_ref (bytes): transaction_id in the outer_domain that includes the cross_ref
            """
            msg = {
                KeyType.domain_id: self.domain_id,
                KeyType.infra_msg_type: InfraMessageCategory.CATEGORY_DATA,
                KeyType.infra_command: DataHandler.REPLICATION_CROSS_REF,
                KeyType.transaction_id: transaction_id,
                KeyType.outer_domain_id: outer_domain_id,
                KeyType.txid_having_cross_ref: txid_having_cross_ref,
            }
            if self.replication_strategy == DataHandler.REPLICATION_ALL:
                self.networking.broadcast_message_in_network(domain_id=self.domain_id,
                                                             payload_type=PayloadType.Type_any, msg=msg)
            elif self.replication_strategy == DataHandler.REPLICATION_P2P:
                pass  # TODO: implement (destinations determined by TopologyManager)

        def remove(self, transaction_id, txobj=None, db_num=-1):
            """Delete all data regarding the specified transaction_id

            This method requires either transaction_id or txobj.

            Args:
                transaction_id (bytes): target transaction_id
                txobj (BBcTransaction): transaction object to remove
                db_num (int): index of DB if multiple DBs are used
            """
            if transaction_id is None:
                return
            if txobj is None:
                txdata = self.exec_sql(sql="SELECT * FROM transaction_table WHERE transaction_id = %s" %
                                       self.db_adaptors[0].placeholder, args=(transaction_id,))
                txobj, fmt_type = bbclib.deserialize(txdata[0][1])
            elif txobj.transaction_id != transaction_id:
                return

            if db_num == -1 or db_num >= len(self.db_adaptors):
                for i in range(len(self.db_adaptors)):
                    self._remove_transaction(txobj, i)
            else:
                self._remove_transaction(txobj, db_num)

        def _remove_transaction(self, txobj, db_num):
            """Remove transaction from DB"""
            #print("_remove_transaction: for txid =", txobj.transaction_id.hex())
            self.exec_sql(
                db_num=db_num,
                sql="DELETE FROM transaction_table WHERE transaction_id = %s" % self.db_adaptors[0].placeholder,
                args=(txobj.transaction_id,), commit=True)
            for base, point_to in self._get_topology_info(txobj):
                self.exec_sql(
                    db_num=db_num,
                    sql="DELETE FROM topology_table WHERE base = %s AND point_to = %s" %
                        (self.db_adaptors[0].placeholder,self.db_adaptors[0].placeholder),
                    args=(base, point_to), commit=True)

        def _remove_asset_files(self, txobj, asset_files=None):
            """Remove all asset files related to the transaction

            If asset_files is given, only asset files in given param are removed

            Args:
                txobj (BBcTransaction): transaction object that includes the asset to be removed
                asset_files (dict): dictionary of {asset_id: content} for the transaction
            """
            #print("_remove_asset_files: for txid =", txobj.transaction_id.hex())
            if self.use_external_storage:
                return
            for asset_group_id, asset_id, user_id, fileflag, filedigest in self.get_asset_info(txobj):
                if asset_files is not None:
                    if asset_id in asset_files:
                        self._remove_in_storage(asset_group_id, asset_id)
                else:
                    self._remove_in_storage(asset_group_id, asset_id)

        def search_transaction(self, transaction_id=None, asset_group_id=None, asset_id=None, user_id=None,
                               direction=0, count=1, db_num=0):
            """Search transaction data

            When Multiple conditions are given, they are considered as AND condition.

            Args:
                transaction_id (bytes): target transaction_id
                asset_group_id (bytes): asset_group_id that target transactions should have
                asset_id (bytes): asset_id that target transactions should have
                user_id (bytes): user_id that target transactions should have
                direction (int): 0: descend, 1: ascend
                count (int): The maximum number of transactions to retrieve
                db_num (int): index of DB if multiple DBs are used
            Returns:
                dict: mapping from transaction_id to serialized transaction data
                dict: dictionary of {asset_id: content} for the transaction
            """
            if transaction_id is not None:
                txinfo = self.exec_sql(
                    db_num=db_num,
                    sql="SELECT * FROM transaction_table WHERE transaction_id = %s" % self.db_adaptors[0].placeholder,
                    args=(transaction_id,))
                if len(txinfo) == 0:
                    return None, None
            else:
                dire = "DESC"
                if direction == 1:
                    dire = "ASC"
                sql = "SELECT * from asset_info_table WHERE "
                conditions = list()
                if asset_group_id is not None:
                    conditions.append("asset_group_id = %s " % self.db_adaptors[0].placeholder)
                if asset_id is not None:
                    conditions.append("asset_id = %s " % self.db_adaptors[0].placeholder)
                if user_id is not None:
                    conditions.append("user_id = %s " % self.db_adaptors[0].placeholder)
                sql += "AND ".join(conditions) + "ORDER BY id %s" % dire
                if count > 0:
                    sql += " limit %d" % count
                sql += ";"
                args = list(filter(lambda a: a is not None, (asset_group_id, asset_id, user_id)))
                ret = self.exec_sql(db_num=db_num, sql=sql, args=args)
                txinfo = list()
                for record in ret:
                    tx = self.exec_sql(
                        db_num=db_num,
                        sql="SELECT * FROM transaction_table WHERE transaction_id = %s" % self.db_adaptors[0].placeholder,
                        args=(record[1],))
                    if tx is not None and len(tx) == 1:
                        txinfo.append(tx[0])

            result_txobj = dict()
            result_asset_files = dict()
            for txid, txdata in txinfo:
                txobj, fmt_type = bbclib.deserialize(txdata)
                result_txobj[txid] = txobj
                for asset_group_id, asset_id, user_id, fileflag, filedigest in self.get_asset_info(txobj):
                    if fileflag:
                        result_asset_files[asset_id] = self.get_in_storage(asset_group_id, asset_id)
            return result_txobj, result_asset_files

        def count_transactions(self, asset_group_id=None, asset_id=None, user_id=None, db_num=0):
            """Count transactions that matches the given conditions

            When Multiple conditions are given, they are considered as AND condition.

            Args:
                asset_group_id (bytes): asset_group_id that target transactions should have
                asset_id (bytes): asset_id that target transactions should have
                user_id (bytes): user_id that target transactions should have
                db_num (int): index of DB if multiple DBs are used
            Returns:
                int: the number of transactions
            """
            sql = "SELECT count( DISTINCT transaction_id ) from asset_info_table WHERE "
            conditions = list()
            if asset_group_id is not None:
                conditions.append("asset_group_id = %s " % self.db_adaptors[0].placeholder)
            if asset_id is not None:
                conditions.append("asset_id = %s " % self.db_adaptors[0].placeholder)
            if user_id is not None:
                conditions.append("user_id = %s " % self.db_adaptors[0].placeholder)
            sql += "AND ".join(conditions)
            args = list(filter(lambda a: a is not None, (asset_group_id, asset_id, user_id)))
            ret = self.exec_sql(db_num=db_num, sql=sql, args=args)
            return ret[0][0]

        def search_transaction_topology(self, transaction_id, traverse_to_past=True):
            """Search in topology info

            Args:
                transaction_id (bytes): base transaction_id
                traverse_to_past (bool): True: search backward (to past), False: search forward (to future)
            Returns:
                list: list of records of topology table
            """
            if transaction_id is None:
                return None
            if traverse_to_past:
                return self.exec_sql(sql="SELECT * FROM topology_table WHERE base = %s" %
                                     self.db_adaptors[0].placeholder, args=(transaction_id,))

            else:
                return self.exec_sql(sql="SELECT * FROM topology_table WHERE point_to = %s" %
                                     self.db_adaptors[0].placeholder, args=(transaction_id,))

        def store_in_storage(self, asset_group_id, asset_id, content, do_overwrite=False):
            """Store asset file in local storage

            Args:
                asset_group_id (bytes): asset_group_id of the asset
                asset_id (bytes): asset_id of the asset
                content (bytes): the content of the asset file
                do_overwrite (bool): If True, file is overwritten
            Returns:
                bool: True if successful
            """
            #print("store_in_storage: for asset_id =", asset_id.hex())
            self.stats.update_stats_increment("data_handler", "store_in_storage", 1)
            asset_group_id_str = binascii.b2a_hex(asset_group_id).decode('utf-8')
            storage_path = os.path.join(self.storage_root, asset_group_id_str)
            if not os.path.exists(storage_path):
                os.makedirs(storage_path, exist_ok=True)
            path = os.path.join(storage_path, binascii.b2a_hex(asset_id).decode('utf-8'))
            if not do_overwrite and os.path.exists(path):
                return False
            with open(path, 'wb') as f:
                try:
                    f.write(content)
                except:
                    return False
            return os.path.exists(path)

        def get_in_storage(self, asset_group_id, asset_id):
            """Get the asset file with the asset_id from local storage

            Args:
                asset_group_id (bytes): asset_group_id of the asset
                asset_id (bytes): asset_id of the asset
            Returns:
                bytes or None: the file content
            """
            asset_group_id_str = binascii.b2a_hex(asset_group_id).decode('utf-8')
            storage_path = os.path.join(self.storage_root, asset_group_id_str)
            if not os.path.exists(storage_path):
                return None
            path = os.path.join(storage_path, binascii.b2a_hex(asset_id).decode('utf-8'))
            if not os.path.exists(path):
                return None
            try:
                with open(path, 'rb') as f:
                    content = f.read()
                return content
            except:
                self.logger.error(traceback.format_exc())
                return None

        def _remove_in_storage(self, asset_group_id, asset_id):
            """Delete asset file

            Args:
                asset_group_id (bytes): asset_group_id of the asset
                asset_id (bytes): asset_id of the asset
            """
            #print("_remove_in_storage: for asset_id =", asset_id.hex())
            asset_group_id_str = binascii.b2a_hex(asset_group_id).decode('utf-8')
            storage_path = os.path.join(self.storage_root, asset_group_id_str)
            if not os.path.exists(storage_path):
                return
            path = os.path.join(storage_path, binascii.b2a_hex(asset_id).decode('utf-8'))
            if not os.path.exists(path):
                return
            os.remove(path)

        def process_message(self, msg):
            """Process received message

            Args:
                msg (dict): received message
            """
            if KeyType.infra_command not in msg:
                return

            if msg[KeyType.infra_command] == DataHandler.REQUEST_REPLICATION_INSERT:
                self.stats.update_stats_increment("data_handler", "REQUEST_REPLICATION_INSERT", 1)
                self.insert_transaction(msg[KeyType.transaction_data],
                                        asset_files=msg.get(KeyType.all_asset_files, None), no_replication=True)

            elif msg[KeyType.infra_command] == DataHandler.RESPONSE_REPLICATION_INSERT:
                self.stats.update_stats_increment("data_handler", "RESPONSE_REPLICATION_INSERT", 1)
                pass

            elif msg[KeyType.infra_command] == DataHandler.REQUEST_SEARCH:
                self.stats.update_stats_increment("data_handler", "REQUEST_SEARCH", 1)
                ret = self.search_transaction(msg[KeyType.transaction_id])
                msg[KeyType.infra_command] = DataHandler.RESPONSE_SEARCH
                if ret is None or len(ret) == 0:
                    msg[KeyType.result] = False
                    msg[KeyType.reason] = "Not found"
                else:
                    msg[KeyType.result] = True
                    msg[KeyType.transaction_data] = ret[0][1]
                self.networking.send_message_in_network(nodeinfo=None, payload_type=PayloadType.Type_any,
                                                        domain_id=self.domain_id, msg=msg)

            elif msg[KeyType.infra_command] == DataHandler.RESPONSE_SEARCH:
                self.stats.update_stats_increment("data_handler", "RESPONSE_SEARCH", 1)
                if msg[KeyType.result]:
                    self.insert_transaction(msg[KeyType.transaction_data])

            elif msg[KeyType.infra_command] == DataHandler.NOTIFY_INSERTED:
                self.stats.update_stats_increment("data_handler", "NOTIFY_INSERTED", 1)
                if KeyType.transaction_id not in msg or KeyType.asset_group_ids not in msg:
                    return
                transaction_id = msg[KeyType.transaction_id]
                asset_group_ids = msg[KeyType.asset_group_ids]
                self.core.send_inserted_notification(self.domain_id, asset_group_ids, transaction_id,
                                                     only_registered_user=True)

            elif msg[KeyType.infra_command] == DataHandler.REPAIR_TRANSACTION_DATA:
                self.networking.domains[self.domain_id]['repair'].put_message(msg)

            elif msg[KeyType.infra_command] == DataHandler.REPLICATION_CROSS_REF:
                transaction_id = msg[KeyType.transaction_id]
                outer_domain_id = msg[KeyType.outer_domain_id]
                txid_having_cross_ref = msg[KeyType.txid_having_cross_ref]
                self.insert_cross_ref(transaction_id, outer_domain_id, txid_having_cross_ref, no_replication=True)

    class DataHandlerDomain0(DataHandler):
        """Data handler for domain_global_0"""
        def __init__(self, networking=None, config=None, workingdir=None, domain_id=None, loglevel="all", logname=None):
            pass

        def exec_sql(self, sql, *args):
            pass

        def get_asset_info(self, txobj):
            pass

        def _get_topology_info(self, txobj):
            pass

        def insert_transaction(self, txdata, txobj=None, asset_files=None, no_replication=False):
            return True

        def _send_replication_to_other_cores(self, txdata, asset_files=None):
            pass

        def remove(self, transaction_id):
            pass

        def search_transaction(self, transaction_id=None, asset_group_id=None, asset_id=None, user_id=None, count=1):
            return None, None

        def search_transaction_topology(self, transaction_id, reverse_link=False):
            return None

        def store_in_storage(self, asset_group_id, asset_id, content):
            return True

        def get_in_storage(self, asset_group_id, asset_id):
            return None

        def _remove_in_storage(self, asset_group_id, asset_id):
            pass

        def process_message(self, msg):
            pass

    class DbAdaptor:
        """Base class for DB adaptor"""
        def __init__(self, handler=None, db_name=None, db_num=0, loglevel="all", logname=None):
            self.handler = handler
            self.db = None
            self.db_cur = None
            self.db_name = db_name
            self.db_num = db_num
            self.placeholder = ""
            self.logger = logger.get_logger(key="db_adaptor", level=loglevel, logname=logname)

        def open_db(self):
            """Open the DB"""
            pass

        def create_table(self, tbl, tbl_definition, primary_key=0, indices=[]):
            """Create a table"""
            pass

        def check_table_existence(self, tblname):
            """Check whether the table exists or not"""
            pass

    class SqliteAdaptor(DbAdaptor):
        """DB adaptor for SQLite3"""
        def __init__(self, handler=None, db_name=None, loglevel="all", logname=None):
            super(SqliteAdaptor, self).__init__(handler=handler, db_name=db_name, loglevel=loglevel, logname=logname)
            self.placeholder = "?"

        def open_db(self):
            """Open the DB (create DB file if not exists)"""
            import sqlite3
            self.db = sqlite3.connect(self.db_name, isolation_level=None)
            self.db_cur = self.db.cursor()

        def create_table(self, tbl, tbl_definition, primary_key=0, indices=[]):
            """Create a table

            Args:
                tbl (str): table name
                tbl_definition (list): schema of the table [["column_name", "data type"],["colmun_name", "data type"],,]
                primary_key (int): index (column) of the primary key of the table
                indices (list): list of indices to create index
            """
            if len(self.check_table_existence(tbl)) > 0:
                return
            sql = "CREATE TABLE %s " % tbl
            sql += "("
            sql += ", ".join(["%s %s" % (d[0],d[1]) for d in tbl_definition])
            sql += ", PRIMARY KEY ("+tbl_definition[primary_key][0]+")"
            sql += ");"
            self.handler.exec_sql(sql=sql, commit=True)
            for idx in indices:
                self.handler.exec_sql(sql="CREATE INDEX %s_idx_%d ON %s (%s);" % (tbl, idx, tbl, tbl_definition[idx][0]),
                                      commit=True)

        def check_table_existence(self, tblname):
            """Check whether the table exists or not"""
            return self.handler.exec_sql(sql="SELECT * FROM sqlite_master WHERE type='table' AND name=?", args=(tblname,))

    class MysqlAdaptor(DbAdaptor):
        """DB adaptor for MySQL"""
        def __init__(self, handler=None, db_name=None, db_num=None, server_info=None, loglevel="all", logname=None):
            super(MysqlAdaptor, self).__init__(handler, db_name, db_num, loglevel, logname)
            self.placeholder = "%s"
            self.db_addr = server_info[0]
            self.db_port = server_info[1]
            self.db_user = server_info[2]
            self.db_pass = server_info[3]

        def open_db(self):
            """Open the DB"""
            import mysql.connector
            self.db = mysql.connector.connect(
                host=self.db_addr,
                port=self.db_port,
                db=self.db_name,
                user=self.db_user,
                password=self.db_pass,
                charset='utf8'
            )
            self.db_cur = self.db.cursor(buffered=True)

        def create_table(self, tbl, tbl_definition, primary_key=0, indices=[]):
            """Create a table

            Args:
                tbl (str): table name
                tbl_definition (list): schema of the table [["column_name", "data type"],["colmun_name", "data type"],,]
                primary_key (int): index (column) of the primary key of the table
                indices (list): list of indices to create index
            """
            if len(self.check_table_existence(tbl)) == 1:
                return
            sql = "CREATE TABLE %s " % tbl
            sql += "("
            defs = list()
            for d in tbl_definition:
                if d[0] == "id":
                    defs.append("%s %s AUTO_INCREMENT NOT NULL" % (d[0], d[1]))
                else:
                    defs.append("%s %s" % (d[0], d[1]))
            sql += ",".join(defs)
            if tbl_definition[primary_key][1] in ["BLOB", "TEXT"]:
                sql += ", PRIMARY KEY (%s(32))" % tbl_definition[primary_key][0]
            else:
                sql += ", PRIMARY KEY (%s)" % tbl_definition[primary_key][0]
            sql += ") CHARSET=utf8;"
            self.handler.exec_sql(db_num=self.db_num, sql=sql, commit=True)
            for idx in indices:
                if tbl_definition[idx][1] in ["BLOB", "TEXT"]:
                    self.handler.exec_sql(db_num=self.db_num, sql="ALTER TABLE %s ADD INDEX (%s(32));"
                                                                  % (tbl, tbl_definition[idx][0]), commit=True)
                else:
                    self.handler.exec_sql(db_num=self.db_num, sql="ALTER TABLE %s ADD INDEX (%s);"
                                                                  % (tbl, tbl_definition[idx][0]), commit=True)

        def check_table_existence(self, tblname):
            """Check whether the table exists or not"""
            sql = "show tables from %s like '%s';" % (self.db_name, tblname)
            return self.handler.exec_sql(db_num=self.db_num, sql=sql)