Nodejs - Fire multiple API calls while limiting the rate and wait until they are all done

Question

My issues

• Launch 1000+ online API that limits the number of API calls to 10 calls/sec.
• Wait for all the API calls to give back a result (or retry), it can take 5 sec before the API sends it data
• Use the combined data in the rest of my app

What I have tried while looking at a lot of different questions and answers here on the site

Use promise to wait for one API request

const https = require("https");

function myRequest(param) {
  const options = {
    host: "api.xxx.io",
    port: 443,
    path: "/custom/path/"+param,
    method: "GET"
  }

  return new Promise(function(resolve, reject) {
    https.request(options, function(result) {
      let str = "";
      result.on('data', function(chunk) {str += chunk;});
      result.on('end', function() {resolve(JSON.parse(str));});
      result.on('error', function(err) {console.log("Error: ", err);});
    }).end();
  });
};

Use Promise.all to do all the requests and wait for them to finish

const params = [{item: "param0"}, ... , {item: "param1000+"}]; // imagine 1000+ items

const promises = [];
base.map(function(params){
  promises.push(myRequest(params.item));
});

result = Promise.all(promises).then(function(data) {
  // doing some funky stuff with dat
});

So far so good, sort of

It works when I limit the number of API requests to a maximum of 10 because then the rate limiter kicks in. When I console.log(promises), it gives back an array of 'request'.

I have tried to add setTimeout in different places, like:

...
base.map(function(params){
  promises.push(setTimeout(function() {
    myRequest(params.item);
  }, 100));
});
...

But that does not seem to work. When I console.log(promises), it gives back an array of 'function'

My questions

• Now I am stuck ... any ideas?
• How do I build in retries when the API gives an error

Thank you for reading up to hear, you are already a hero in my book!

Answer 1

When you have a complicated control-flow using async/await helps a lot to clarify the logic of the flow.

Let's start with the following simple algorithm to limit everything to 10 requests per second:

make 10 requests

wait 1 second

repeat until no more requests

For this the following simple implementation will work:

async function rateLimitedRequests (params) {
    let results = [];

    while (params.length > 0) {
        let batch = [];

        for (i=0; i<10; i++) {
            let thisParam = params.pop();
            if (thisParam) {                          // use shift instead 
              batch.push(myRequest(thisParam.item));  // of pop if you want
            }                                         // to process in the
                                                      // original order.
        }

        results = results.concat(await Promise.all(batch));

        await delayOneSecond();
    }

    return results;
}

Now we just need to implement the one second delay. We can simply promisify setTimeout for this:

function delayOneSecond() {
    return new Promise(ok => setTimeout(ok, 1000));
}

This will definitely give you a rate limiter of just 10 requests each second. In fact it performs somewhat slower than that because each batch will execute in request time + one second. This is perfectly fine and already meet your original intent but we can improve this to squeeze a few more requests to get as close as possible to exactly 10 requests per second.

We can try the following algorithm:

remember the start time

make 10 requests

compare end time with start time

delay one second minus request time

repeat until no more requests

Again, we can use almost exactly the same logic as the simple code above but just tweak it to do time calculations:

const ONE_SECOND = 1000;

async function rateLimitedRequests (params) {
    let results = [];

    while (params.length > 0) {
        let batch = [];
        let startTime = Date.now();

        for (i=0; i<10; i++) {
            let thisParam = params.pop();
            if (thisParam) {
                batch.push(myRequest(thisParam.item));
            }
        }

        results = results.concat(await Promise.all(batch));

        let endTime = Date.now();
        let requestTime = endTime - startTime;
        let delayTime = ONE_SECOND - requestTime;

        if (delayTime > 0) {
            await delay(delayTime);
        }
    }

    return results;
}

Now instead of hardcoding the one second delay function we can write one that accept a delay period:

function delay(milliseconds) {
    return new Promise(ok => setTimeout(ok, milliseconds));
}

We have here a simple, easy to understand function that will rate limit as close as possible to 10 requests per second. It is rather bursty in that it makes 10 parallel requests at the beginning of each one second period but it works. We can of course keep implementing more complicated algorithms to smooth out the request pattern etc. but I leave that to your creativity and as homework for the reader.