前言

最近在学习yolov5模型，然后用onnxruntime在linux平台上搭建一个GPU推理环境，运行一下，顺带记录一下环境搭建的整体过程，记录一下踩坑经历，造福后来人，也避免自己忘记了，可以回来再看看。其实onnxruntime + OCR的三个模型在linux + GPU环境的部署，去年玩过一次，没想到这次搭建yolov5，居然花费了将近两天时间，就是因为没有写文章记录的原因，肯定是的。

代码和模型

首先还是把代码和模型先贴上来吧，这个我也是在GitHub上找了蛮久，有的代码和模型没有匹配，跑不起来，也很麻烦，代码如下：

#include 
#include 
#include 
#include 
#include 
#include 
//#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include #include 
#include 
#include 
#include 
#include using namespace std;
using namespace cv;
using namespace Ort;struct Net_config
{float confThreshold; // Confidence thresholdfloat nmsThreshold;  // Non-maximum suppression thresholdfloat objThreshold;  //Object Confidence thresholdstring modelpath;
};typedef struct BoxInfo
{float x1;float y1;float x2;float y2;float score;int label;
} BoxInfo;int endsWith(string s, string sub) {return s.rfind(sub) == (s.length() - sub.length()) ? 1 : 0;
}const float anchors_640[3][6] = { {10.0,  13.0, 16.0,  30.0,  33.0,  23.0},{30.0,  61.0, 62.0,  45.0,  59.0,  119.0},{116.0, 90.0, 156.0, 198.0, 373.0, 326.0} };const float anchors_1280[4][6] = { {19, 27, 44, 40, 38, 94},{96, 68, 86, 152, 180, 137},{140, 301, 303, 264, 238, 542},{436, 615, 739, 380, 925, 792} };class YOLO
{
public:YOLO(Net_config config);void detect(vector& frames, int batch_size);
private:float* anchors;int num_stride;int inpWidth;int inpHeight;int nout;int num_proposal;vector class_names;int num_class;int seg_num_class;float confThreshold;float nmsThreshold;float objThreshold;const bool keep_ratio = true;void normalize_(Mat img, vector& input_image_);void nms(vector& input_boxes);Mat resize_image(Mat srcimg, int* newh, int* neww, int* top, int* left);Ort::Env env = Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR, "test");Ort::Session* ort_session = nullptr;Ort::SessionOptions sessionOptions = SessionOptions();    vector input_names;vector output_names;vector> input_node_dims; // >=1 outputsvector> output_node_dims; // >=1 outputs
};YOLO::YOLO(Net_config config)
{this->confThreshold = config.confThreshold;this->nmsThreshold = config.nmsThreshold;this->objThreshold = config.objThreshold;string classesFile = "coco.names";string model_path = "yolov5s.onnx";OrtCUDAProviderOptions cuda_options{};sessionOptions.AppendExecutionProvider_CUDA(cuda_options);ort_session = new Session(env, model_path.c_str(), sessionOptions);size_t numInputNodes = ort_session->GetInputCount();size_t numOutputNodes = ort_session->GetOutputCount();AllocatorWithDefaultOptions allocator;for (int i = 0; i < numInputNodes; i++){input_names.push_back(ort_session->GetInputName(i, allocator));Ort::TypeInfo input_type_info = ort_session->GetInputTypeInfo(i);auto input_tensor_info = input_type_info.GetTensorTypeAndShapeInfo();auto input_dims = input_tensor_info.GetShape();input_node_dims.push_back(input_dims);}for (int i = 0; i < numOutputNodes; i++){output_names.push_back(ort_session->GetOutputName(i, allocator));Ort::TypeInfo output_type_info = ort_session->GetOutputTypeInfo(i);auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo();auto output_dims = output_tensor_info.GetShape();output_node_dims.push_back(output_dims);}this->inpHeight = input_node_dims[0][2];this->inpWidth = input_node_dims[0][3];this->nout = output_node_dims[0][2];this->num_proposal = output_node_dims[0][1];ifstream ifs(classesFile.c_str());string line;while (getline(ifs, line)) this->class_names.push_back(line);this->num_class = class_names.size();if (endsWith(config.modelpath, "6.onnx")){anchors = (float*)anchors_1280;this->num_stride = 4;}else{anchors = (float*)anchors_640;this->num_stride = 3;}
}Mat YOLO::resize_image(Mat srcimg, int* newh, int* neww, int* top, int* left)
{int srch = srcimg.rows, srcw = srcimg.cols;*newh = this->inpHeight;*neww = this->inpWidth;Mat dstimg;if (this->keep_ratio && srch != srcw) {float hw_scale = (float)srch / srcw;if (hw_scale > 1) {*newh = this->inpHeight;*neww = int(this->inpWidth / hw_scale);resize(srcimg, dstimg, Size(*neww, *newh), INTER_AREA);*left = int((this->inpWidth - *neww) * 0.5);copyMakeBorder(dstimg, dstimg, 0, 0, *left, this->inpWidth - *neww - *left, BORDER_CONSTANT, 114);}else {*newh = (int)this->inpHeight * hw_scale;*neww = this->inpWidth;resize(srcimg, dstimg, Size(*neww, *newh), INTER_AREA);*top = (int)(this->inpHeight - *newh) * 0.5;copyMakeBorder(dstimg, dstimg, *top, this->inpHeight - *newh - *top, 0, 0, BORDER_CONSTANT, 114);}}else {resize(srcimg, dstimg, Size(*neww, *newh), INTER_AREA);}return dstimg;
}void YOLO::normalize_(Mat img, vector& input_image_)
{//    img.convertTo(img, CV_32F);int row = img.rows;int col = img.cols;input_image_.resize(row * col * img.channels());for (int c = 0; c < 3; c++){for (int i = 0; i < row; i++){for (int j = 0; j < col; j++){float pix = img.ptr(i)[j * 3 + 2 - c];input_image_[c * row * col + i * col + j] = pix / 255.0;}}}
}void YOLO::nms(vector& input_boxes)
{sort(input_boxes.begin(), input_boxes.end(), [](BoxInfo a, BoxInfo b) { return a.score > b.score; });vector vArea(input_boxes.size());for (int i = 0; i < int(input_boxes.size()); ++i){vArea[i] = (input_boxes.at(i).x2 - input_boxes.at(i).x1 + 1)* (input_boxes.at(i).y2 - input_boxes.at(i).y1 + 1);}vector isSuppressed(input_boxes.size(), false);for (int i = 0; i < int(input_boxes.size()); ++i){if (isSuppressed[i]) { continue; }for (int j = i + 1; j < int(input_boxes.size()); ++j){if (isSuppressed[j]) { continue; }float xx1 = (max)(input_boxes[i].x1, input_boxes[j].x1);float yy1 = (max)(input_boxes[i].y1, input_boxes[j].y1);float xx2 = (min)(input_boxes[i].x2, input_boxes[j].x2);float yy2 = (min)(input_boxes[i].y2, input_boxes[j].y2);float w = (max)(float(0), xx2 - xx1 + 1);float h = (max)(float(0), yy2 - yy1 + 1);float inter = w * h;float ovr = inter / (vArea[i] + vArea[j] - inter);if (ovr >= this->nmsThreshold){isSuppressed[j] = true;}}}// return post_nms;int idx_t = 0;input_boxes.erase(remove_if(input_boxes.begin(), input_boxes.end(), [&idx_t, &isSuppressed](const BoxInfo& f) { return isSuppressed[idx_t++]; }), input_boxes.end());
}void YOLO::detect(vector& frames, int batch_size)
{vectornewhs, newws, padhs, padws;auto allocator_info = MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);array input_shape_{ 1, 3, this->inpHeight, this->inpWidth };std::vector inputBatchTensors;for (int b_idx = 0; b_idx < batch_size; b_idx++) {vector input_image_;int newh = 0, neww = 0, padh = 0, padw = 0;Mat dstimg = this->resize_image(frames[b_idx], &newh, &neww, &padh, &padw);this->normalize_(dstimg, input_image_);newhs.push_back(newh);newws.push_back(neww);padhs.push_back(padh);padws.push_back(padw);inputBatchTensors.push_back(Value::CreateTensor(allocator_info, input_image_.data(),input_image_.size(), input_shape_.data(), input_shape_.size()));}std::vector inputBatchNames, outputBatchNames;for (size_t i = 0; i < batch_size; i++) {inputBatchNames.insert(inputBatchNames.end(), input_names.begin(), input_names.end());outputBatchNames.insert(outputBatchNames.end(), output_names.begin(), output_names.end());}// 开始推理vector ort_outputs = ort_session->Run(RunOptions{ nullptr }, inputBatchNames.data(), inputBatchTensors.data(), batch_size, outputBatchNames.data(), outputBatchNames.size());   // 开始推理for(int pos_idx = 0; pos_idx < batch_size; pos_idx++){/generate proposalsvector generate_boxes;float ratioh = (float)frames[pos_idx].rows / newhs[pos_idx], ratiow = (float)frames[pos_idx].cols / newws[pos_idx];int n = 0, q = 0, i = 0, j = 0, row_ind = 0, k = 0; ///xmin,ymin,xamx,ymax,box_score, class_scoreconst float* pdata = ort_outputs[pos_idx].GetTensorMutableData();for (n = 0; n < this->num_stride; n++)   ///特征图尺度{const float stride = pow(2, n + 3);int num_grid_x = (int)ceil((this->inpWidth / stride));int num_grid_y = (int)ceil((this->inpHeight / stride));for (q = 0; q < 3; q++)    ///anchor{const float anchor_w = this->anchors[n * 6 + q * 2];const float anchor_h = this->anchors[n * 6 + q * 2 + 1];for (i = 0; i < num_grid_y; i++){for (j = 0; j < num_grid_x; j++){float box_score = pdata[4];if (box_score > this->objThreshold){int max_ind = 0;float max_class_socre = 0;for (k = 0; k < num_class; k++){if (pdata[k + 5] > max_class_socre){max_class_socre = pdata[k + 5];max_ind = k;}}max_class_socre *= box_score;if (max_class_socre > this->confThreshold){float cx = (pdata[0] * 2.f - 0.5f + j) * stride;  ///cxfloat cy = (pdata[1] * 2.f - 0.5f + i) * stride;   ///cyfloat w = powf(pdata[2] * 2.f, 2.f) * anchor_w;   ///wfloat h = powf(pdata[3] * 2.f, 2.f) * anchor_h;  ///hfloat xmin = (cx - padws[pos_idx] - 0.5 * w) * ratiow;float ymin = (cy - padhs[pos_idx] - 0.5 * h) * ratioh;float xmax = (cx - padws[pos_idx] + 0.5 * w) * ratiow;float ymax = (cy - padhs[pos_idx] + 0.5 * h) * ratioh;generate_boxes.push_back(BoxInfo{ xmin, ymin, xmax, ymax, max_class_socre, max_ind });}}row_ind++;pdata += nout;}}}}// Perform non maximum suppression to eliminate redundant overlapping boxes with// lower confidencesnms(generate_boxes);for (size_t i = 0; i < generate_boxes.size(); ++i){int xmin = int(generate_boxes[i].x1);int ymin = int(generate_boxes[i].y1);rectangle(frames[pos_idx], Point(xmin, ymin), Point(int(generate_boxes[i].x2), int(generate_boxes[i].y2)), Scalar(0, 0, 255), 2);string label = format("%.2f", generate_boxes[i].score);label = this->class_names[generate_boxes[i].label] + ":" + label;putText(frames[pos_idx], label, Point(xmin, ymin - 5), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0, 255, 0), 1);}}
}int main()
{Net_config yolo_nets = { 0.3, 0.5, 0.3,"./yolov5s.onnx" };YOLO yolo_model(yolo_nets);string imgpath = "./bus.jpg";Mat srcimg = imread(imgpath);int batchsize = 2048;vector frames;for (int k = 0; k < batchsize; k++) {frames.push_back(srcimg);}clock_t start = clock();for (int i = 0; i < 10; i++) {yolo_model.detect(frames, batchsize);std::cout << " --------all step is 10000  now is--- " << i << std::endl;}clock_t end = clock();std::cout << "10000 inference  takes " << (double)(end - start) /( CLOCKS_PER_SEC) << " s" << std::endl;cv::imwrite("result.jpg",srcimg);return 0;
}

代码里面是用batch去做推理，但是模型推理前后处理都是串行的，其实这种方式的并行，感觉也快不了多少。

模型的话，就放在笔者的资源里面，需要的小伙伴们可以去下载。

环境搭建

0、环境版本如下：

cuda——11.1
cudnn——8.0
opencv——4.5.2
onnxruntime-gpu——1.8.0
cmake———3.20

1、安装cuda和cudnn

这个网上的教程资料都很多，比如Linux系统CUDA10.2+CUDNN安装教程 ，该篇作者写的很详细。这篇教程也很不错，还写了如何在一台服务器上安装多个版本的cuda和cudnn

2、安装cmake

cmake首先需要需要用系统命令 cmake-version查一下，看看cmake 的版本。一般来说，cmake需要用到3.12以上的版本，这样后面编译opencv才不会出问题。安装cmake也很简单，一种就是直接用系统命令安装，apt-get install cmake，还有一种方式就是直接下载特定版本的cmake包，然后放到 /usr/local/目录下或者添加软连接即可，具体可以参考Linux安装CMake

3、安装opencv

安装opencv，网上资料也很多，但是在安装opencv之前，需要注意升级cmake的版本。具体安装opencv版本的方法可以参考Ubuntu20.04下安装opencv for C++，编译opencv时要加上-DOPENCV_GENERATE_PKGCONFIG=ON这样一个编译条件，这样编译代码的时候，就可以很快速的查找到opencv这个库的相关依赖了。

4、安装onnxruntime

这个其实不需要安装，只需要在onnxruntime官网下载对应的版本，然后进行解压即可，后面cmakelist文件中链接对应的路径就行了。千万不要蠢到自己的去编译源码，因为这里有太多的坑要填了，看看onnxruntime官网的issue就知道了。

CmakeList文件

这个文件可以参考，需要注意的是，不需要包含libonnxruntime_providers_cuda.so，包含之后，出现了两个问题，始终没有解决。

cmake_minimum_required(VERSION 3.10)project(yolov5_test_batch VERSION 0.0.1 LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 14)#onnxruntime
set(ONNXRUNTIME_ROOT_PATH /root/onnxruntime-linux-x64-gpu-1.8.1)
set(ONNXRUNTIME_INCLUDE_DIRS ${ONNXRUNTIME_ROOT_PATH}/include)
set(ONNXRUNTIME_LIB ${ONNXRUNTIME_ROOT_PATH}/lib/libonnxruntime.so)
include_directories(${ONNXRUNTIME_INCLUDE_DIRS})#cuda  cudnn
set(DEPS ${DEPS} "/usr/local/cuda/lib64/libcudart.so")
set(DEPS ${DEPS} "/usr/local/cudnn/lib64/libcudnn.so" )
#set(DEPS ${DEPS} "/usr/local/cuda/lib64/libcudnn.so" )find_package(OpenCV REQUIRED)
if (OpenCV_FOUND)message(STATUS "OpenCV_LIBS: ${OpenCV_LIBS}")message(STATUS "OpenCV_INCLUDE_DIRS: ${OpenCV_INCLUDE_DIRS}")
else ()message(FATAL_ERROR "opencv Not Found!")
endif (OpenCV_FOUND)add_executable(yolov5_test_batch example_code_batch.cc)target_link_libraries(yolov5_test_batch PRIVATE ${ONNXRUNTIME_LIB} ${DEPS}${OpenCV_LIBS})

参考

Linux系统CUDA10.2+CUDNN安装教程
Linux安装CMake
Ubuntu20.04下安装opencv for C++