Uploading multiple files to Backblaze; my code doesn't wait for the promise to resolve

Question

The code works and uploads images to Backblaze but it's not waiting for the for loop to finish running and the query to finish before calling next. If I console log the file ids it will list the ids 3 or four seconds after next() has been called!

import fs from "fs";
import B2 from "backblaze-b2";
import path from "path";
import dotenv from "dotenv";

dotenv.config();

export const uploadToBackBlaze = async (req, res, next) => {
    console.log("I'm here uploading to Backblaze");

    const b2 = new B2({
        applicationKeyId: process.env.BACKBLAZE_ACCOUNT_ID,
        applicationKey: process.env.BACKBLAZE_MASTER_APPLICATION_ID,
    });

    const __dirname = path.resolve();
    let tempDir = path.join(__dirname, "chapterTemp");

    const imageIds = [];
    try {
        b2.authorize()
            .then(async () => {
                const bucketId = process.env.BACKBLAZE_BUCKET_ID;

                fs.readdir(tempDir, async function (err, files) {
                    if (err) {
                        console.error(err);
                        res.sendStatus(500).json({ message: err.message });
                        return;
                    }
                    for (const file of files) {
                        const fileData = fs.readFileSync(path.join(tempDir, file));
                        const uploadFileName = path.join(file);
                        const uploadUrl = await b2.getUploadUrl(bucketId);
                        const response = await b2.uploadFile({
                            uploadUrl: uploadUrl.data.uploadUrl,
                            uploadAuthToken: uploadUrl.data.authorizationToken,
                            filename: uploadFileName,
                            data: fileData,
                            mime: "image/png" || "image/jpg" || "image/jpeg" || "image/webp", // replace with the appropriate MIME type for your files
                        });
                        console.log(response.data.fileId);
                        imageIds.push(response.data.fileId);
                    }
                });
            })
            .then(() => {
                console.log(imageIds);
                req.imageIds = imageIds;
                next();
            });
    } catch (error) {
        console.log(error);
        return res.status(500).json({ message: error.message });
    }
};

I want it to finish waiting for the for loop and push the ids into the array before calling next, that way I can store the Ids in the database!

Answer 1

The root cause of your problem is that, as @jfriend00 says in a comment, fs.readdir() is a plain callback async function. Therefore, execution proceeds to the then() containing next() while fs.readdir() is still running. As @jfriend00 also mentions, mixing then() and await makes the code more complex and difficult to write correctly.

Here's a version that works. It uses fs.readdirSync(), which simply returns the array of files, and replaces then() with await to make the code easier to understand.

import fs from "fs";
import B2 from "backblaze-b2";
import path from "path";
import dotenv from "dotenv";

dotenv.config();

export const uploadToBackBlaze = async (req, res, next) => {
  console.log("I'm here uploading to Backblaze");

  const b2 = new B2({
    applicationKeyId: process.env.BACKBLAZE_ACCOUNT_ID,
    applicationKey: process.env.BACKBLAZE_MASTER_APPLICATION_ID,
  });

  const __dirname = path.resolve();
  let tempDir = path.join(__dirname, "chapterTemp");

  const imageIds = [];
  try {
    await b2.authorize();

    const bucketId = process.env.BACKBLAZE_BUCKET_ID;

    const files = fs.readdirSync(tempDir);
    for (const file of files) {
      const fileData = fs.readFileSync(path.join(tempDir, file));
      const uploadFileName = path.join(file);
      const uploadUrl = await b2.getUploadUrl(bucketId);
      const response = await b2.uploadFile({
        uploadUrl: uploadUrl.data.uploadUrl,
        uploadAuthToken: uploadUrl.data.authorizationToken,
        filename: uploadFileName,
        data: fileData,
        mime: "image/png" || "image/jpg" || "image/jpeg" || "image/webp", // replace with the appropriate MIME type for your files
      });
      console.log(response.data.fileId);
      imageIds.push(response.data.fileId);
    }

    console.log(imageIds);
    req.imageIds = imageIds;
    next();
  } catch (error) {
    console.log(error);
    return res.status(500).json({ message: error.message });
  }
};

Output:

I'm here uploading to Backblaze
4_z0145cfc9e3f5ec0f74ed0c1b_f117d3fad62e89e20_d20230402_m183653_c004_v0402012_t0031_u01680460613262
4_z0145cfc9e3f5ec0f74ed0c1b_f106c056c78a74bfd_d20230402_m183653_c004_v0402006_t0016_u01680460613601
4_z0145cfc9e3f5ec0f74ed0c1b_f10983156bddd6e1d_d20230402_m183654_c004_v0402001_t0002_u01680460614026
4_z0145cfc9e3f5ec0f74ed0c1b_f110cce66726eafc9_d20230402_m183654_c004_v0402002_t0004_u01680460614244
4_z0145cfc9e3f5ec0f74ed0c1b_f11877e4fecf8da12_d20230402_m183654_c004_v0402012_t0026_u01680460614505
4_z0145cfc9e3f5ec0f74ed0c1b_f107e976b612ad7d0_d20230402_m183654_c004_v0402005_t0057_u01680460614840
[
  '4_z0145cfc9e3f5ec0f74ed0c1b_f117d3fad62e89e20_d20230402_m183653_c004_v0402012_t0031_u01680460613262',
  '4_z0145cfc9e3f5ec0f74ed0c1b_f106c056c78a74bfd_d20230402_m183653_c004_v0402006_t0016_u01680460613601',
  '4_z0145cfc9e3f5ec0f74ed0c1b_f10983156bddd6e1d_d20230402_m183654_c004_v0402001_t0002_u01680460614026',
  '4_z0145cfc9e3f5ec0f74ed0c1b_f110cce66726eafc9_d20230402_m183654_c004_v0402002_t0004_u01680460614244',
  '4_z0145cfc9e3f5ec0f74ed0c1b_f11877e4fecf8da12_d20230402_m183654_c004_v0402012_t0026_u01680460614505',
  '4_z0145cfc9e3f5ec0f74ed0c1b_f107e976b612ad7d0_d20230402_m183654_c004_v0402005_t0057_u01680460614840'
]