Training OpenCV model on multiple pictures based on color, not on shape AKA "Banana ripeness detector"

Question

So long story short, I'm looking for an OpenCV model that does training on multiple image sets based on color, not on shape. Like a FaceRegonition algorithm (Fisherfacerecognizer, EigenFaces, Lbphfacerecognizer etc) but instead of looking for facials features, it looks for colors and then, using an input image, makes a prediction based on previous training.

From what I understood, SVM (Support Vector Machine) would be the preferable approach. However, I have no idea on how should I train it. I'll leave you with some code and a concrete case:

I'm doing a "Banana Ripeness Detector" using OpenCV & C++ where you load a picture of a banana and it tells you how "ripe" it is. For this purpose, I have a training_models folder with 3 subfolders "unripe_bananas", "ripe_bananas" and "overripe_bananas", each with multiple pictures of bananas.

Then I have a CSV in order to read all this pictures inside the program inside a vector called "images". The number after semicolon represents the label (unripe_bananas -> label 1, ripe_bananas -> label 2, overripe_bananas -> label 3)

resources/training_bananas/unripe_bananas/banana1.jpg;1
resources/training_bananas/unripe_bananas/banana2.jpg;1
resources/training_bananas/unripe_bananas/banana3.jpg;1
resources/training_bananas/unripe_bananas/banana4.jpg;1
resources/training_bananas/unripe_bananas/banana5.jpg;1
resources/training_bananas/unripe_bananas/banana6.jpg;1
resources/training_bananas/unripe_bananas/banana7.jpg;1
resources/training_bananas/ripe_bananas/banana8.jpg;2
resources/training_bananas/ripe_bananas/banana9.jpg;2
resources/training_bananas/ripe_bananas/banana10.jpg;2
resources/training_bananas/ripe_bananas/banana11.jpg;2
...

And this is the code. The banana detection part in a picture is already handled using "banana_detect_cascade.xml" (works similary to the haarcascade_frontalface_alt.xml cascade)

#include "opencv2/objdetect/objdetect.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/ml.hpp"
#include <iostream>
#include <stdio.h>
#include <fstream>

using namespace std;
using namespace cv;
using namespace ml;

void detectAndDisplayBanana(Mat frame);
/** Global variables */
String banana_cascade_name = "resources//banana_detect_cascade.xml";
CascadeClassifier banana_cascade;
string window_name = "Capture - Banana Ripe Detection";

// Reading the CSV containing banana pictures for training
// Numbers after semicolon (;) are labels

static void read_csv(const string filename, vector<Mat>& images, vector<int>& labels, char separator = ';')
{
    ifstream file(filename, ios::in);
    if (file)
    {
        string line, path, classLabel;
        while (getline(file, line))
        {
            stringstream liness(line);
            getline(liness, path, separator);
            getline(liness, classLabel);
            if (!path.empty() && !classLabel.empty())
            {
                images.push_back(imread(path, ImreadModes::IMREAD_COLOR));
                labels.push_back(atoi(classLabel.c_str()));
            }
        }
    }
}
int im_width;
int im_height;
Ptr<SVM> model;




int main(int argc, const char** argv)
{

    Mat frame;
    //-- 1. Load the cascades
    if (!banana_cascade.load(banana_cascade_name)) { printf("--(!)Error loading\n"); return -1; };
    string fileName = string("resources//csv.ext");

    
    vector<Mat> images; // the vector with the training images
    vector<int> labels; // the vector with the labels for the training images
    read_csv(fileName, images, labels);

    im_width = images[0].cols;
    im_height = images[0].rows;
    model = SVM::create();
    

    ///////// .... HOW TO TRAIN ?????


    //-- 3. Read the image

    while (true)
    {
        frame = imread("banana_example.jpg");
        //-- 3. Apply the classifier to the frame
        if (!frame.empty())
        {
            detectAndDisplayBanana(frame);
        }
        else
        {
            printf("No banana image!"); break;
        }
        int c = waitKey(10);
        if ((char)c == 'c') { break; }
    }

    return 0;
}


void detectAndDisplayBanana(Mat frame)
{
    // This functions detect bananas in a picture - already handled

    string ripeStage;
    std::vector<Rect> bananas;
    Mat frame_gray;
    cvtColor(frame, frame_gray, COLOR_BGR2GRAY);
    equalizeHist(frame_gray, frame_gray);
    //-- Detect bananas


    banana_cascade.detectMultiScale(frame_gray, bananas, 1.1, 2, 0 | 1, Size(30, 30));
    for (size_t i = 0; i < bananas.size(); i++)
    {
        Rect banana_i = bananas[i];
        Mat banana = frame_gray(banana_i);
        Mat banana_resized;
    cv:resize(banana, banana_resized, Size(im_width, im_height), 1.0, 1.0, InterpolationFlags::INTER_CUBIC);
        
        int ripeLabel = model->predict(banana_resized);

        switch (ripeLabel)
        {
        case 1:
            ripeStage = "Unripe";
            break;
        case 2:
            ripeStage = "Ripe";
            break;
        case 3:
            ripeStage = "Overripe";
            break;
        default:
            ripeStage = "Unknown";
        }



        
        
        string box_text = format("%s %f", ripeStage.c_str());
        cout << box_text << endl;
        
    }
    //-- Show what you got
    imshow(window_name, frame);
}