Slow transfer speeds for the last chunk of data transferred from Python ASGI app

Question

While attempting to analyse the network transfer speeds using a hypercorn ASGI app and requests API, there is a significant drop in speed for the last chunk.

Hypercorn ASGI app

from typing import Any

ONE_KB = 1024
ONE_MB = ONE_KB * ONE_KB
ONE_GB = ONE_MB * ONE_KB

chunk_size = 256 * ONE_KB

dump = bytes(ONE_GB)  # 1GB


async def app(scope: Any, receive: Any, send: Any) -> None:
    '''
    Run: `hypercorn app:app --bind localhost:7001`
    '''
    assert scope["type"] == "http"

    await send(
        {
            "type": "http.response.start",
            "status": 200,
            "headers": [[b"Content-Type", b"application/octet-stream"], [b"Content-Length", str(len(dump)).encode()]],
        }
    )

    chunks = len(dump) // chunk_size

    for i in range(chunks):
        await send(
            {
                "type": "http.response.body",
                "body": dump[i * chunk_size : (i + 1) * chunk_size],
                "more_body": i != chunks - 1,
            }
        )

test.py

import time
import requests

ONE_MB = 2**20

data_size_mb = 1000


async def test_speed():
    # Parse the URL to get host and path
    path = "/"

    start_time = time.time()

    # Create a TCP socket
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        # Connect to the server
        s.connect((host, port))

        # Send HTTP GET request
        request = f"GET {path} HTTP/1.1\r\nHost: {host}\r\n\r\n"
        s.sendall(request.encode())

        # Create a BufferedReader from the socket
        bufferedReader = io.BufferedReader(s.makefile("rb"))

        try:
            # Read response headers
            headers = bufferedReader.readline()
            while headers.strip():
                headers = bufferedReader.readline()
            bufferedReader.flush()

            # Receive response in chunks and write to file
            recv_size = 0
            unit = 8
            chunks = []
            for i in range(unit):
                st = time.time()
                for _ in range(20 * 1000):
                    data = bufferedReader.read(ONE_MB // unit)
                    recv_size += len(data)
                    # chunks.append(data)

                chunk_time = time.time() - st
                spd = data_size_mb // (unit * chunk_time)
                print("Chunk #{}: Time {:.6f} seconds, Speed {:.2f}".format(i + 1, chunk_time, spd))
        finally:
            bufferedReader.close()

    recv_data_mb = recv_size / ONE_MB
    print("[*] Received data ", recv_size / ONE_MB, "MB")
    print("[*] Chunks size {}".format(str(len(chunks))))

    end_time = time.time()

    recv_time = end_time - start_time
    recv_speed = recv_data_mb / recv_time

    print("===========================================")
    print("Data size: {:.2f} MB".format(data_size_mb))
    print("Recv data size: {:.2f} MB".format(recv_data_mb))
    print("Recv time: {:.6f} seconds".format(recv_time))
    print("Recv speed: {:.2f} mbps".format(recv_speed))
    print("===========================================")

Output:

$ python test.py
Chunk #1: Time 0.943959 seconds, Speed 2648.00
Chunk #2: Time 0.998727 seconds, Speed 2503.00
Chunk #3: Time 0.937339 seconds, Speed 2667.00
Chunk #4: Time 0.947274 seconds, Speed 2639.00
Chunk #5: Time 0.941455 seconds, Speed 2655.00
Chunk #6: Time 0.951273 seconds, Speed 2628.00
Chunk #7: Time 0.980101 seconds, Speed 2550.00
Chunk #8: Time 5.945689 seconds, Speed 420.00
[*] Received data  20000.0 MB
[*] Chunks size 0
===========================================
Data size: 20000.00 MB
Recv data size: 20000.00 MB
Recv time: 14.233940 seconds
Recv speed: 1405.09 mbps
===========================================

Expectation: The transfer speed must be identical for all the chunks.

Actual: The transfer speed for the last chunk is significantly slower.

Env

• The machine has over 1 TB of RAM and sufficient CPU.
• Python version: 3.10.12