General sports ball tracker and calculus integrated into _AI.

/*
    Copyright(C) 2025 Tyler Crockett | Macdaddy4sure.ai

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissionsand
    limitations under the License.
*/

#include "AugmentedIntelligence.hpp"
#include "Ball.hpp"
#include "Working-Memory.hpp"
#include "Short-Term Memory.hpp"
#include "Long-Term Memory.hpp"
#include "Settings.hpp"
#include "Sound.hpp"
#include "Vision.hpp"

using namespace std;

// Ball is life... there will be no other. Sports are life; was it me?
// Should ball have its own detector? Should I leave ball tracking to Vision.cpp?
// 1. Sports Ball Detector Neural Network
// 2. Sports Ball Distance
// 3. Sports Ball IoU
// 4. Sports Ball Velocity
// 5. Sports Ball Accelleration
// 6. Sports Ball Path Prediction

void _Ball::BallHeuristicInit()
{
    Object my_car;
    Object current_object;
    Object prev_object;

    if (vision_object_detection)
    {
        TF_Status* status = TF_NewStatus();
        TF_Graph* graph = TF_NewGraph();
        TF_SessionOptions* options = TF_NewSessionOptions();
        TF_Buffer* run_opts = nullptr;
        const char* tags = "serve";
        TF_Session* session = TF_LoadSessionFromSavedModel(options, nullptr, tensorflow_model.c_str(), &tags, 1, graph, nullptr, status);

        for (;;)
        {
            cv::Mat frame;

            // Find the last image and use that image for object detection
            for (int x = 999; x >= 0; x--)
            {
                if (!stm_vision_path_camera1[x][0].empty())
                {
                    frame = cv::imread(stm_vision_path_camera1[x][0]);

                    // Check the hash of the image?

                    // Store the current image inside the ball memory array
                    for (int y = 0; y < 999; y++)
                    {
                        if (wm_ball[y][0].empty())
                        {
                            ////lock_guard<mutex> lock(//mtx_stm_vision_path_camera1[x][0]);
                            ////lock_guard<mutex> lock1(//mtx_stm_vision_path_camera1[x][1]);
                            wm_ball[y][0] = stm_vision_path_camera1[x][0];
                            wm_ball[y][1] = stm_vision_path_camera1[x][1];
                        }
                    }
                    break;
                }
            }

            vector<vector<string>> coco_ball = _Ball::ObjectDetection::ObjectDetectionBall(status, graph, options, run_opts, tags, session, frame);

            // Read the values from the coco_ball array and store them into memory
            for (int x = 0; x <= coco_ball.size(); x++)
            {
                if (coco_ball[x][1] == "sports ball")
                {
                    current_object.unique_id = "object " + to_string(x);
                    current_object.class_name = coco_ball[x][1];
                    current_object.x = stoi(coco_ball[x][2]);
                    current_object.box_width = stoi(coco_ball[x][3]);
                    current_object.y = stoi(coco_ball[x][4]);
                    current_object.box_height = stoi(coco_ball[x][5]);
                    current_object.distance = _Ball::Math::getBallDistance(current_object, prev_object);
                }
            }

            if (current_object.class_name == prev_object.class_name)
            {
                double IoU = _Ball::Math::CalculateIoU(current_object, prev_object);

                if (IoU >= iou_min && IoU <= iou_max)
                {
                    current_object.velocity = _Ball::Math::CalculateObjectVelocity(current_object, prev_object, 1 / camera1_fps);
                    current_object.acceleration = _Ball::Math::CalculateAcceleration(current_object, prev_object, 1 / camera1_fps);

                    // Get the probability of collision from this data
                    double relativeVelocity = _Ball::Math::calculateRelativeVelocity(my_car, current_object);

                    cv::Rect currentRect(current_object.x, current_object.y, current_object.box_width, current_object.box_height);

                    cv::rectangle(frame, currentRect, cv::Scalar(0, 255, 0), 2);
                    std::string label = "Velocity: " + std::to_string((int)current_object.velocity) + " mph";
                    cv::putText(frame, label, cv::Point(currentRect.x, currentRect.y - 10),
                        cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 2);
                }
            }

            // Save the coordinates to the current array
            // How to save individual vehicles into memory and access that memory
            // Find the first empty position in the array
            // Get the previous object's data
            prev_object.unique_id = current_object.unique_id;
            prev_object.class_name = current_object.class_name;
            prev_object.x = current_object.x;
            prev_object.box_width = current_object.box_width;
            prev_object.y = current_object.y;
            prev_object.box_height = current_object.box_height;
            prev_object.distance = current_object.distance;
            prev_object.velocity[0] = current_object.velocity[0];
            prev_object.velocity[1] = current_object.velocity[1];
            prev_object.acceleration[0] = current_object.acceleration[0];
            prev_object.acceleration[1] = current_object.acceleration[1];
        }
    }
    else
    {
        vision_object_detection = false;
    }
}

vector<vector<string>> _Ball::ObjectDetection::ObjectDetectionBall(TF_Status* status, TF_Graph* graph, TF_SessionOptions* options, TF_Buffer* run_opts, const char* tags, TF_Session* session, Mat frame)
{
    string current_date;
    ostringstream oss;
    auto entry = time(nullptr);
    auto tm1 = *localtime(&entry);

    oss << put_time(&tm1, "%d-%m-%Y_%H-%M-%S");
    current_date = oss.str();

    //std::string imagePath = "D:/_test/saved_sequence2/";
    // Debug
    //std::string model_path = "D:/_AugmentedIntelligence/_src/tensorflow_models/coco_object_detection/saved_model/";
    //std::string labelsPath = "D:/_AugmentedIntelligence/_src/tensorflow_models/coco_object_detection/coco.names";

    // Convert image to tensor
    TF_Tensor* inputTensor = _Vision::MatToTensor(frame);

    // Define input operation
    TF_Output input_op = { TF_GraphOperationByName(graph, "serving_default_input_tensor"), 0 };

    // Define output operations
    std::vector<TF_Output> output_ops{
        { TF_GraphOperationByName(graph, "StatefulPartitionedCall"), 1 },
        { TF_GraphOperationByName(graph, "StatefulPartitionedCall"), 2 },
        { TF_GraphOperationByName(graph, "StatefulPartitionedCall"), 4 },
    };

    std::vector<TF_Tensor*> input_tensors = { inputTensor };
    TF_Tensor* output_tensors[] = { NULL, NULL, NULL };

    if (TF_GetCode(status) != TF_OK)
    {
        fprintf(stderr, "Error: %s\n", TF_Message(status));
        // Handle error appropriately
    }

    // Run the session
    TF_SessionRun(session, nullptr,
        &input_op, input_tensors.data(), input_tensors.size(),
        output_ops.data(), output_tensors, output_ops.size(),
        nullptr, 0, nullptr, status);

    if (TF_GetCode(status) == TF_OK)
    {
        printf("Session run successfully\n");
    }
    else
    {
        fprintf(stderr, "Session run error: %s\n", TF_Message(status));
    }

    // Debug
    cout << "output_tensors[0]: " << output_tensors[0] << endl;
    cout << "output_tensors[1]: " << output_tensors[1] << endl;
    cout << "output_tensors[2]: " << output_tensors[2] << endl;
    //cin.get();

    // Process the output tensors to extract boxes, scores, and class IDs
    auto boxes = _Vision::ExtractBoxes(output_tensors[0], frame.size()); // Needs implementation
    auto classIds = _Vision::ExtractClassIds(output_tensors[1]); // Needs implementation
    auto scores = _Vision::ExtractScores(output_tensors[2]); // Needs implementation
    std::vector<string> classLabels = _Vision::LoadLabels(tensorflow_labels);
    cout << "classIds: " << classIds[0] << endl;

    _Vision::DrawBoundingBoxes(frame, boxes, classIds, scores, classLabels);

    // Debug
    std::string temp = vision_memory_directory;
    temp += "/camera1/driving/object_detection/";
    temp += current_date.c_str();
    temp += "_object_detection_camera1.png";
    std::cout << "Writing image: " << temp << std::endl;
    cv::imwrite(temp.c_str(), frame);

    vector<vector<string>> return_array;

    // Create the return array
    for (int i = 0; i < boxes.size(); i++)
    {
        const auto& box = boxes[i];
        int classId = classIds[i];
        float score = scores[i];

        if (score >= 0.90)
        {
            vector<string> row(7);
            row[0] = temp;
            cout << row[0] << endl;
            row[1] = classLabels[classId - 1]; // Use the actual class ID instead of x
            cout << row[1] << endl;
            row[2] = to_string(box.x);
            cout << row[2] << endl;
            row[3] = to_string(box.width);
            cout << row[3] << endl;
            row[4] = to_string(box.y);
            cout << row[4] << endl;
            row[5] = to_string(box.height);
            cout << row[5] << endl;
            row[6] = to_string(score);
            cout << row[6] << endl;

            return_array.push_back(row);
        }
    }

    // Cleanup
    //TF_DeleteTensor(inputTensor);
    //TF_DeleteTensor(output_tensors[0]);
    //TF_DeleteTensor(output_tensors[1]);
    //TF_DeleteTensor(output_tensors[2]);
    //TF_DeleteSession(session, status);
    //TF_DeleteSessionOptions(options);
    //TF_DeleteStatus(status);

    return return_array;
}

---

Filed under: Uncategorized - @ March 6, 2026 3:21 pm