OpenVINO C++ 多请求 异步处理

OpenVINO C++支持多请求异步处理。多请求可以并行处理，从而提高推理速度。

以下是一个简单的示例，展示如何使用OpenVINO C++进行异步处理：

#include <inference_engine.hpp>
#include <iostream>
#include <string>
#include <vector>
#include <chrono>

using namespace InferenceEngine;

int main() {
    // Load the model
    Core ie;
    CNNNetwork network = ie.ReadNetwork("model.xml", "model.bin");
    ExecutableNetwork executable_network = ie.LoadNetwork(network, "CPU");

    // Create multiple infer requests
    std::vector<InferRequest> requests;
    for (int i = 0; i < 3; i++) {
        requests.push_back(executable_network.CreateInferRequest());
    }

    // Prepare the input data
    std::vector<float> input_data = {1.0, 2.0, 3.0, 4.0};
    Blob::Ptr input_blob = requests[0].GetBlob("input");
    MemoryBlob::Ptr input_mem_blob = as<MemoryBlob>(input_blob);
    if (!input_mem_blob) {
        std::cerr << "Failed to cast input blob to memory blob." << std::endl;
        return 1;
    }
    auto input_mem_ptr = input_mem_blob->rwmap();
    std::memcpy(input_mem_ptr.as<float*>(), input_data.data(), input_data.size() * sizeof(float));

    // Start async inference
    for (int i = 0; i < requests.size(); i++) {
        requests[i].StartAsync();
    }

    // Wait for async inference to complete
    for (int i = 0; i < requests.size(); i++) {
        requests[i].Wait(InferRequest::WaitMode::RESULT_READY);
        Blob::Ptr output_blob = requests[i].GetBlob("output");
        MemoryBlob::Ptr output_mem_blob = as<MemoryBlob>(output_blob);
        if (!output_mem_blob) {
            std::cerr << "Failed to cast output blob to memory blob." << std::endl;
            return 1;
        }
        auto output_mem_ptr = output_mem_blob->rmap();
        std::cout << "Request " << i << " output: ";
        for (int j = 0; j < output_blob->size(); j++) {
            std::cout << output_mem_ptr.as<float*>()[j] << " ";
        }
        std::cout << std::endl;
    }

    return 0;
}

该示例加载一个模型，并创建三个InferRequest对象。在准备输入数据后，它会启动异步推理，然后等待所有请求完成。最后，它会打印每个请求的输出。注意，每个请求的输出顺序可能不同，因为它们是并行处理的。