RV1126 frame reading CPU usage

emreharbutoglu

Hi, I am using the firmware above. I wrote a C++ code that reads frames from a USB camera with ffmpeg library. But the problem is CPU usage. The process is consuming CPU around between %50-60. This is too much for a frame reading task. What should I do for this?

1. #include <iostream>
   
2. #include <chrono>
   
3. #include <cstddef>
   
4. #include <unistd.h>
   
5. #include <string.h>
   
6. 
   
7. extern "C" {
   
8. #include <libavformat/avformat.h>
   
9. #include <libavcodec/avcodec.h>
   
10. #include <libavdevice/avdevice.h>
    
11. #include <libavutil/avutil.h>
    
12. #include <libavutil/pixdesc.h>
    
13. #include <libswscale/swscale.h>
    
14. }
    
15. 
    
16. #include "opencv2/opencv.hpp"
    
17. #include "zmq.hpp"
    
18. #include "nlohmann/json.hpp"
    
19. 
    
20. #include <opencv2/highgui.hpp>
    
21. 
    
22. using json = nlohmann::json;
    
23. 
    
24. #define WIDTH (640)
    
25. #define HEIGHT (480)
    
26. #define VIDM_COEFF (7)
    
27. 
    
28. void int2array(u_char* output_array, uint64_t num, uint8_t digit);
    
29. void combine_arrays(u_char* main_array, int id_size, int data_size, u_char* array_1, u_char* array_2);
    
30. 
    
31. int main(){
    
32.     std::ifstream cfg_file("config.json");
    
33.     json cfg = json::parse(cfg_file);
    
34. 
    
35.     zmq::context_t ctx;
    
36.     zmq::socket_t publisher(ctx, ZMQ_PUB);
    
37.     publisher.bind("ipc:///frames_socket.ipc");
    
38.     const char* infile = "/dev/video25";
    
39. 
    
40.     // initialize FFmpeg library
    
41.     avcodec_register_all();
    
42.     avdevice_register_all();
    
43.     av_register_all();
    
44. //  av_log_set_level(AV_LOG_DEBUG);
    
45. 
    
46.     AVInputFormat *inputFormat =av_find_input_format("v4l2");
    
47.     AVDictionary *options = NULL;
    
48.     av_dict_set(&options, "input_format", "mjpeg", 0);
    
49.     av_dict_set(&options, "framerate", "15", 0);
    
50.     av_dict_set(&options, "aspect", "4:3", 0);
    
51.     av_dict_set(&options, "video_size", "640x480", 0);
    
52. 
    
53.     //AVFormatContext *pAVFormatContext = NULL;
    
54. 
    
55.     int ret;
    
56. 
    
57.     // open input file context
    
58.     AVFormatContext* inctx = nullptr;
    
59.     ret = avformat_open_input(&inctx, infile, inputFormat, &options);
    
60.     if (ret < 0) {
    
61.         std::cerr << "fail to avforamt_open_input(\"" << infile << "\"): ret=" << ret;
    
62.         return 2;
    
63.     }
    
64.     // retrive input stream information
    
65.     ret = avformat_find_stream_info(inctx, nullptr);
    
66.     if (ret < 0) {
    
67.         std::cerr << "fail to avformat_find_stream_info: ret=" << ret;
    
68.         return 2;
    
69.     }
    
70. 
    
71.     // find primary video stream
    
72.     AVCodec* vcodec = nullptr;
    
73.     ret = av_find_best_stream(inctx, AVMEDIA_TYPE_VIDEO, -1, -1, &vcodec, 0);
    
74.     if (ret < 0) {
    
75.         std::cerr << "fail to av_find_best_stream: ret=" << ret;
    
76.         return 2;
    
77.     }
    
78.     const int vstrm_idx = ret;
    
79.     AVStream* vstrm = inctx->streams[vstrm_idx];
    
80. 
    
81.     // open video decoder context
    
82.     ret = avcodec_open2(vstrm->codec, vcodec, nullptr);
    
83.     if (ret < 0) {
    
84.         std::cerr << "fail to avcodec_open2: ret=" << ret;
    
85.         return 2;
    
86.     }
    
87. 
    
88.     const int dst_width = vstrm->codec->width;
    
89.     const int dst_height = vstrm->codec->height;
    
90.     const AVPixelFormat dst_pix_fmt = AV_PIX_FMT_BGR24;
    
91.     SwsContext* swsctx = sws_getCachedContext(
    
92.         nullptr, vstrm->codec->width, vstrm->codec->height, vstrm->codec->pix_fmt,
    
93.         dst_width, dst_height, dst_pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr);
    
94.     if (!swsctx) {
    
95.         std::cerr << "fail to sws_getCachedContext";
    
96.         return 2;
    
97.     }
    
98. 
    
99.     AVFrame* frame = av_frame_alloc();
    
100.     std::vector<uint8_t> framebuf(avpicture_get_size(dst_pix_fmt, dst_width, dst_height));
     
101.     avpicture_fill(reinterpret_cast<AVPicture*>(frame), framebuf.data(), dst_pix_fmt, dst_width, dst_height);
     
102. 
     
103.     AVFrame* decframe = av_frame_alloc();
     
104.     bool end_of_stream = false;
     
105.     int got_pic = 0;
     
106.     AVPacket pkt;
     
107. 
     
108.     auto rpm = std::chrono::high_resolution_clock::now();
     
109.     uint16_t fps_counter = 0;
     
110. 
     
111.     do {
     
112.         if ((std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - rpm)).count() >= 1000000) {
     
113.             std::cout << "FPS_COUNTER: " << fps_counter << std::endl;
     
114.             rpm = std::chrono::high_resolution_clock::now();
     
115.             fps_counter = 0;
     
116.         }
     
117.         auto fps_time_start = std::chrono::high_resolution_clock::now();
     
118.         usleep(50000); //35000 for cam
     
119.         //while(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - fps_time_start).count() <= 100000);
     
120.         if (!end_of_stream) {
     
121.             ret = av_read_frame(inctx, &pkt);
     
122.             if (ret < 0 && ret != AVERROR_EOF) {
     
123.                 std::cerr << "fail to av_read_frame: ret=" << ret;
     
124.                 return 2;
     
125.             }
     
126.             if (ret == 0 && pkt.stream_index != vstrm_idx)
     
127.                 goto next_packet;
     
128.             end_of_stream = (ret == AVERROR_EOF);
     
129.         }
     
130.         if (end_of_stream) {
     
131.             // null packet for bumping process
     
132.             av_init_packet(&pkt);
     
133.             pkt.data = nullptr;
     
134.             pkt.size = 0;
     
135.         }
     
136.         // decode video frame
     
137.         avcodec_decode_video2(vstrm->codec, decframe, &got_pic, &pkt);
     
138.         if (!got_pic) {
     
139.             goto next_packet;
     
140.         }
     
141.         // convert frame to OpenCV matrix
     
142.         sws_scale(swsctx, decframe->data, decframe->linesize, 0, decframe->height, frame->data, frame->linesize);
     
143.         
     
144.         {
     
145.         cv::Mat image(dst_height, dst_width, CV_8UC3, framebuf.data(), frame->linesize[0]);
     
146.         //cv::imwrite("frame_test.png", image);
     
147.         //cv::resize(image, image, cv::Size(640, 640), cv::INTER_LINEAR);
     
148.       
     
149.         //auto fps_duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - fps_time_start);
     
150.         //
     
151.         //std::cout << "fps_duration" << 1000000/fps_duration.count() << std::endl;
     
152.         
     
153.         uint64_t frame_id = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
     
154.         u_char b_frame_id[16];
     
155.         int2array(b_frame_id, frame_id, 16);
     
156.         auto pub_time_start = std::chrono::high_resolution_clock::now();
     
157.         u_char main_array[16+image.total()*image.elemSize()];
     
158.         combine_arrays(main_array, 16, image.total()*image.elemSize(), b_frame_id, image.data);
     
159.         zmq::message_t msg(main_array, 16+image.total()*image.elemSize());
     
160.         //zmq::message_t msg_frame_id(b_frame_id, 16);
     
161.         //zmq::message_t msg_frame(image.data, image.total()*image.elemSize());
     
162.         //publisher.send(msg_frame_id, zmq::send_flags::sndmore);
     
163.         //publisher.send(msg_frame, zmq::send_flags::none);
     
164.         publisher.send(msg, zmq::send_flags::none);
     
165. 
     
166.         auto pub_duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - pub_time_start);
     
167.         auto fps_duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - fps_time_start);
     
168.         std::cout << "ZMQ publish took: " << pub_duration.count() << " microseconds " << 1000000/fps_duration.count() << " fps" << std::endl;
     
169.         time_t now = time(0);
     
170.         fps_counter++;
     
171.         std::cout << "[INFO] Frame Published, frame_id: " << frame_id << ", datetime: " << std::ctime(&now);   
     
172.         }
     
173.       
     
174. next_packet:
     
175.         av_free_packet(&pkt);
     
176.     } while (!end_of_stream || got_pic);
     
177.     av_frame_free(&decframe);
     
178.     av_frame_free(&frame);
     
179.     avcodec_close(vstrm->codec);
     
180.     avformat_close_input(&inctx);
     
181.     return 0;  
     
182. }
     
183. 
     
184. void combine_arrays(u_char* main_array, int id_size, int data_size, u_char* array_1, u_char* array_2){
     
185.     for(int i = 0; i < id_size; i++){
     
186.         main_array[i] = array_1[i];
     
187.     }
     
188.     for(int i = 0; i < data_size; i++){
     
189.         main_array[i+16] = array_2[i];
     
190.     }
     
191. }
     
192. 
     
193. void int2array(u_char* output_array, uint64_t num, uint8_t digit){
     
194.     for (int i = digit-1-3; i >= 0; i--)
     
195.         {
     
196.             output_array[i] = (num % 10);
     
197.             num /= 10;
     
198.         }
     
199.         output_array[13] = '=';
     
200.         output_array[14] = '=';
     
201.         output_array[15] = '=';
     
202. }
     

Copy the code

Also I have found a solution but it's CPU usage around between %20-30. I executing a shell command that reads frames with ffmpeg command. Not an effective solution.
My goal is reducing the CPU usage to between %5-10.

emreharbutoglu

BTW FPS is 15