Quick look at JSON-RPC batching

[dreaded369]

This post follows an issue I posted in truckblocks-docker regarding rpc connections to erigon breaking, and further discussion in the Erigon discord. In an effort to overcome the issues I explored batcing requests- which is part of the JSON-RPC spec (should work in go-ethereum and Erigon). The below code is in Python and was adapted from code from https://github.com/ethstorage

Batching requests is as simple as passing an array values with the call.

To confuse things I've Im also using a function BATCHED (native to Python 3.12, imported for <3.12), which breaks up values into smaller groups to pass with the JSON-RPC call.

`

batchBlocks = 100
rpcBatchLimit = 1000

# Batch get_blocks 
for i in batched(remaining_blocks, batchBlocks):
    batch = [
        {
        "jsonrpc":"2.0",
        "method":"eth_getBlockByNumber",
        "params":[
            block, 
            True
        ],
        "id":id
        }
        for id, block in enumerate(i)
    ]
    try:
        blockResponse = requests.post(nodeUrl, json=batch).json()
    except HTTPError as e:
        print(e.response.text)
    except Exception as e:
        print(e)
    if "error" in blockResponse:
                    raise Exception(transactionsResponse["error"])     
            
    # Batch transactions receipts per block
    items = []
    for block in blockResponse:  
        time = fr = to = value = gas = gasprice = blockid = txhash = input = contract_to = contract_value = status = ''
        block = block['result']
        blockid = int(block['number'], base=16)
        time = int(block['timestamp'], base=16)

        if len(block['transactions']) < 1:
            cur.execute('INSERT INTO public.ethtxs(time, block) VALUES (%s, %s )',(time, blockid))
        else:
            items = []
            transactionsResponse = []
            batch = [
                {
                    "jsonrpc":"2.0",
                    "method":"eth_getTransactionReceipt",
                    "params":[
                    transaction['hash']
                    ],
                    "id":id
                }
                for id, transaction in enumerate(block['transactions'])
            ]
            try:
                for i in batched(batch, rpcBatchLimit):
                    transactionsResponse.extend(requests.post(nodeUrl, json=i).json())
            except HTTPError as e:
                print(e.response.text)
            except Exception as e:
                print(e)
            if "error" in transactionsResponse:
                    raise Exception(transactionsResponse["error"]) 
                 
        for txNumber in range(0, len(block['transactions'])):          
            trans = block['transactions'][txNumber]
            transReceipt = transactionsResponse[txNumber]['result']
            fr = trans['from']
            to = trans['to']
            value = int(trans['value'], base=16)
            gas =  int(trans['gas'], base=16)
            gasprice =  int(trans['gasPrice'], base=16)
            input = trans['input']    
            txhash = trans['hash'] # eth-storage uses .hex() here but I think that's because web3 converts to type hexbytes
            status = transReceipt['status']
            status = bool(transReceipt['status'])
            
            values = [time, fr, to, value, gas, gasprice, blockid, txhash, input, status]
            items.append(values)
                
        sql = """INSERT INTO  public.ethtxs(time, txfrom, txto, value, gas, gasprice, block, txhash, input, status) 
                 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) 
              """
        cur.executemany(sql, items)

`