feat: 完善架构优化性能

- 调整音视频架构,提升 RKMPP 编码 MJPEG-->H264 性能,同时解决丢帧马赛克问题;
- 删除多用户逻辑,只保留单用户,支持设置 web 单会话;
- 修复删除体验不好的的回退逻辑,前端页面菜单位置微调;
- 增加 OTG USB 设备动态调整功能;
- 修复 mdns 问题,webrtc 视频切换更顺畅。
This commit is contained in:
mofeng
2026-01-25 16:04:29 +08:00
parent 01e01430da
commit 1786b7689d
66 changed files with 4225 additions and 2936 deletions

View File

@@ -98,6 +98,7 @@ mod ffmpeg {
link_os();
build_ffmpeg_ram(builder);
build_ffmpeg_hw(builder);
}
/// Link system FFmpeg using pkg-config or custom path
@@ -373,4 +374,57 @@ mod ffmpeg {
);
}
}
fn build_ffmpeg_hw(builder: &mut Build) {
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let ffmpeg_hw_dir = manifest_dir.join("cpp").join("ffmpeg_hw");
let ffi_header = ffmpeg_hw_dir
.join("ffmpeg_hw_ffi.h")
.to_string_lossy()
.to_string();
bindgen::builder()
.header(ffi_header)
.rustified_enum("*")
.generate()
.unwrap()
.write_to_file(Path::new(&env::var_os("OUT_DIR").unwrap()).join("ffmpeg_hw_ffi.rs"))
.unwrap();
let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
let enable_rkmpp = matches!(target_arch.as_str(), "aarch64" | "arm")
|| std::env::var_os("CARGO_FEATURE_RKMPP").is_some();
if enable_rkmpp {
// Include RGA headers for NV16->NV12 conversion (RGA im2d API)
let rga_sys_dirs = [
Path::new("/usr/aarch64-linux-gnu/include/rga"),
Path::new("/usr/include/rga"),
];
let mut added = false;
for dir in rga_sys_dirs.iter() {
if dir.exists() {
builder.include(dir);
added = true;
}
}
if !added {
// Fallback to repo-local rkrga headers if present
let repo_root = manifest_dir
.parent()
.and_then(|p| p.parent())
.map(|p| p.to_path_buf())
.unwrap_or_else(|| manifest_dir.clone());
let rkrga_dir = repo_root.join("ffmpeg").join("rkrga");
if rkrga_dir.exists() {
builder.include(rkrga_dir.join("include"));
builder.include(rkrga_dir.join("im2d_api"));
}
}
builder.file(ffmpeg_hw_dir.join("ffmpeg_hw_mjpeg_h264.cpp"));
} else {
println!(
"cargo:info=Skipping ffmpeg_hw_mjpeg_h264.cpp (RKMPP) for arch {}",
target_arch
);
}
}
}

View File

@@ -0,0 +1,42 @@
#pragma once
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct FfmpegHwMjpegH264 FfmpegHwMjpegH264;
FfmpegHwMjpegH264* ffmpeg_hw_mjpeg_h264_new(const char* dec_name,
const char* enc_name,
int width,
int height,
int fps,
int bitrate_kbps,
int gop,
int thread_count);
int ffmpeg_hw_mjpeg_h264_encode(FfmpegHwMjpegH264* ctx,
const uint8_t* data,
int len,
int64_t pts_ms,
uint8_t** out_data,
int* out_len,
int* out_keyframe);
int ffmpeg_hw_mjpeg_h264_reconfigure(FfmpegHwMjpegH264* ctx,
int bitrate_kbps,
int gop);
int ffmpeg_hw_mjpeg_h264_request_keyframe(FfmpegHwMjpegH264* ctx);
void ffmpeg_hw_mjpeg_h264_free(FfmpegHwMjpegH264* ctx);
void ffmpeg_hw_packet_free(uint8_t* data);
const char* ffmpeg_hw_last_error(void);
#ifdef __cplusplus
}
#endif

View File

@@ -0,0 +1,444 @@
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/avutil.h>
#include <libavutil/error.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>
#include <libavutil/opt.h>
}
#include <string>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#define LOG_MODULE "FFMPEG_HW"
#include "../common/log.h"
#include "ffmpeg_hw_ffi.h"
namespace {
thread_local std::string g_last_error;
static void set_last_error(const std::string &msg) {
g_last_error = msg;
LOG_ERROR(msg);
}
static std::string make_err(const std::string &ctx, int err) {
return ctx + " (ret=" + std::to_string(err) + "): " + av_err2str(err);
}
static const char* pix_fmt_name(AVPixelFormat fmt) {
const char *name = av_get_pix_fmt_name(fmt);
return name ? name : "unknown";
}
struct FfmpegHwMjpegH264Ctx {
AVCodecContext *dec_ctx = nullptr;
AVCodecContext *enc_ctx = nullptr;
AVPacket *dec_pkt = nullptr;
AVFrame *dec_frame = nullptr;
AVPacket *enc_pkt = nullptr;
AVBufferRef *hw_device_ctx = nullptr;
AVBufferRef *hw_frames_ctx = nullptr;
AVPixelFormat hw_pixfmt = AV_PIX_FMT_NONE;
std::string dec_name;
std::string enc_name;
int width = 0;
int height = 0;
int fps = 30;
int bitrate_kbps = 2000;
int gop = 60;
int thread_count = 1;
bool force_keyframe = false;
};
static enum AVPixelFormat get_hw_format(AVCodecContext *ctx,
const enum AVPixelFormat *pix_fmts) {
auto *self = reinterpret_cast<FfmpegHwMjpegH264Ctx *>(ctx->opaque);
if (self && self->hw_pixfmt != AV_PIX_FMT_NONE) {
const enum AVPixelFormat *p;
for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
if (*p == self->hw_pixfmt) {
return *p;
}
}
}
return pix_fmts[0];
}
static int init_decoder(FfmpegHwMjpegH264Ctx *ctx) {
const AVCodec *dec = avcodec_find_decoder_by_name(ctx->dec_name.c_str());
if (!dec) {
set_last_error("Decoder not found: " + ctx->dec_name);
return -1;
}
ctx->dec_ctx = avcodec_alloc_context3(dec);
if (!ctx->dec_ctx) {
set_last_error("Failed to allocate decoder context");
return -1;
}
ctx->dec_ctx->width = ctx->width;
ctx->dec_ctx->height = ctx->height;
ctx->dec_ctx->thread_count = ctx->thread_count > 0 ? ctx->thread_count : 1;
ctx->dec_ctx->opaque = ctx;
// Pick HW pixfmt for RKMPP
const AVCodecHWConfig *cfg = nullptr;
for (int i = 0; (cfg = avcodec_get_hw_config(dec, i)); i++) {
if (cfg->device_type == AV_HWDEVICE_TYPE_RKMPP) {
ctx->hw_pixfmt = cfg->pix_fmt;
break;
}
}
if (ctx->hw_pixfmt == AV_PIX_FMT_NONE) {
set_last_error("No RKMPP hw pixfmt for decoder");
return -1;
}
int ret = av_hwdevice_ctx_create(&ctx->hw_device_ctx,
AV_HWDEVICE_TYPE_RKMPP, NULL, NULL, 0);
if (ret < 0) {
set_last_error(make_err("av_hwdevice_ctx_create failed", ret));
return -1;
}
ctx->dec_ctx->hw_device_ctx = av_buffer_ref(ctx->hw_device_ctx);
ctx->dec_ctx->get_format = get_hw_format;
ret = avcodec_open2(ctx->dec_ctx, dec, NULL);
if (ret < 0) {
set_last_error(make_err("avcodec_open2 decoder failed", ret));
return -1;
}
ctx->dec_pkt = av_packet_alloc();
ctx->dec_frame = av_frame_alloc();
ctx->enc_pkt = av_packet_alloc();
if (!ctx->dec_pkt || !ctx->dec_frame || !ctx->enc_pkt) {
set_last_error("Failed to allocate packet/frame");
return -1;
}
return 0;
}
static int init_encoder(FfmpegHwMjpegH264Ctx *ctx, AVBufferRef *frames_ctx) {
const AVCodec *enc = avcodec_find_encoder_by_name(ctx->enc_name.c_str());
if (!enc) {
set_last_error("Encoder not found: " + ctx->enc_name);
return -1;
}
ctx->enc_ctx = avcodec_alloc_context3(enc);
if (!ctx->enc_ctx) {
set_last_error("Failed to allocate encoder context");
return -1;
}
ctx->enc_ctx->width = ctx->width;
ctx->enc_ctx->height = ctx->height;
ctx->enc_ctx->time_base = AVRational{1, 1000};
ctx->enc_ctx->framerate = AVRational{ctx->fps, 1};
ctx->enc_ctx->bit_rate = (int64_t)ctx->bitrate_kbps * 1000;
ctx->enc_ctx->gop_size = ctx->gop > 0 ? ctx->gop : ctx->fps;
ctx->enc_ctx->max_b_frames = 0;
ctx->enc_ctx->pix_fmt = AV_PIX_FMT_DRM_PRIME;
ctx->enc_ctx->sw_pix_fmt = AV_PIX_FMT_NV12;
if (frames_ctx) {
AVHWFramesContext *hwfc = reinterpret_cast<AVHWFramesContext *>(frames_ctx->data);
if (hwfc) {
ctx->enc_ctx->pix_fmt = static_cast<AVPixelFormat>(hwfc->format);
ctx->enc_ctx->sw_pix_fmt = static_cast<AVPixelFormat>(hwfc->sw_format);
if (hwfc->width > 0) ctx->enc_ctx->width = hwfc->width;
if (hwfc->height > 0) ctx->enc_ctx->height = hwfc->height;
}
ctx->hw_frames_ctx = av_buffer_ref(frames_ctx);
ctx->enc_ctx->hw_frames_ctx = av_buffer_ref(frames_ctx);
}
if (ctx->hw_device_ctx) {
ctx->enc_ctx->hw_device_ctx = av_buffer_ref(ctx->hw_device_ctx);
}
AVDictionary *opts = nullptr;
av_dict_set(&opts, "rc_mode", "CBR", 0);
av_dict_set(&opts, "profile", "high", 0);
av_dict_set_int(&opts, "qp_init", 23, 0);
av_dict_set_int(&opts, "qp_max", 48, 0);
av_dict_set_int(&opts, "qp_min", 0, 0);
av_dict_set_int(&opts, "qp_max_i", 48, 0);
av_dict_set_int(&opts, "qp_min_i", 0, 0);
int ret = avcodec_open2(ctx->enc_ctx, enc, &opts);
av_dict_free(&opts);
if (ret < 0) {
std::string detail = "avcodec_open2 encoder failed: ";
detail += ctx->enc_name;
detail += " fmt=" + std::string(pix_fmt_name(ctx->enc_ctx->pix_fmt));
detail += " sw=" + std::string(pix_fmt_name(ctx->enc_ctx->sw_pix_fmt));
detail += " size=" + std::to_string(ctx->enc_ctx->width) + "x" + std::to_string(ctx->enc_ctx->height);
detail += " fps=" + std::to_string(ctx->fps);
set_last_error(make_err(detail, ret));
avcodec_free_context(&ctx->enc_ctx);
ctx->enc_ctx = nullptr;
if (ctx->hw_frames_ctx) {
av_buffer_unref(&ctx->hw_frames_ctx);
ctx->hw_frames_ctx = nullptr;
}
return -1;
}
return 0;
}
static void free_encoder(FfmpegHwMjpegH264Ctx *ctx) {
if (ctx->enc_ctx) {
avcodec_free_context(&ctx->enc_ctx);
ctx->enc_ctx = nullptr;
}
if (ctx->hw_frames_ctx) {
av_buffer_unref(&ctx->hw_frames_ctx);
ctx->hw_frames_ctx = nullptr;
}
}
} // namespace
extern "C" FfmpegHwMjpegH264* ffmpeg_hw_mjpeg_h264_new(const char* dec_name,
const char* enc_name,
int width,
int height,
int fps,
int bitrate_kbps,
int gop,
int thread_count) {
if (!dec_name || !enc_name || width <= 0 || height <= 0) {
set_last_error("Invalid parameters for ffmpeg_hw_mjpeg_h264_new");
return nullptr;
}
auto *ctx = new FfmpegHwMjpegH264Ctx();
ctx->dec_name = dec_name;
ctx->enc_name = enc_name;
ctx->width = width;
ctx->height = height;
ctx->fps = fps > 0 ? fps : 30;
ctx->bitrate_kbps = bitrate_kbps > 0 ? bitrate_kbps : 2000;
ctx->gop = gop > 0 ? gop : ctx->fps;
ctx->thread_count = thread_count > 0 ? thread_count : 1;
if (init_decoder(ctx) != 0) {
ffmpeg_hw_mjpeg_h264_free(reinterpret_cast<FfmpegHwMjpegH264*>(ctx));
return nullptr;
}
return reinterpret_cast<FfmpegHwMjpegH264*>(ctx);
}
extern "C" int ffmpeg_hw_mjpeg_h264_encode(FfmpegHwMjpegH264* handle,
const uint8_t* data,
int len,
int64_t pts_ms,
uint8_t** out_data,
int* out_len,
int* out_keyframe) {
if (!handle || !data || len <= 0 || !out_data || !out_len || !out_keyframe) {
set_last_error("Invalid parameters for encode");
return -1;
}
auto *ctx = reinterpret_cast<FfmpegHwMjpegH264Ctx*>(handle);
*out_data = nullptr;
*out_len = 0;
*out_keyframe = 0;
av_packet_unref(ctx->dec_pkt);
int ret = av_new_packet(ctx->dec_pkt, len);
if (ret < 0) {
set_last_error(make_err("av_new_packet failed", ret));
return -1;
}
memcpy(ctx->dec_pkt->data, data, len);
ctx->dec_pkt->size = len;
ret = avcodec_send_packet(ctx->dec_ctx, ctx->dec_pkt);
if (ret < 0) {
set_last_error(make_err("avcodec_send_packet failed", ret));
return -1;
}
while (true) {
ret = avcodec_receive_frame(ctx->dec_ctx, ctx->dec_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
return 0;
}
if (ret < 0) {
set_last_error(make_err("avcodec_receive_frame failed", ret));
return -1;
}
if (ctx->dec_frame->format != AV_PIX_FMT_DRM_PRIME) {
set_last_error("Decoder output is not DRM_PRIME");
av_frame_unref(ctx->dec_frame);
return -1;
}
if (!ctx->enc_ctx) {
if (!ctx->dec_frame->hw_frames_ctx) {
set_last_error("Decoder returned frame without hw_frames_ctx");
av_frame_unref(ctx->dec_frame);
return -1;
}
if (init_encoder(ctx, ctx->dec_frame->hw_frames_ctx) != 0) {
av_frame_unref(ctx->dec_frame);
return -1;
}
}
AVFrame *send_frame = ctx->dec_frame;
AVFrame *tmp = nullptr;
if (ctx->force_keyframe) {
tmp = av_frame_clone(send_frame);
if (tmp) {
tmp->pict_type = AV_PICTURE_TYPE_I;
send_frame = tmp;
}
ctx->force_keyframe = false;
}
send_frame->pts = pts_ms; // time_base is ms
ret = avcodec_send_frame(ctx->enc_ctx, send_frame);
if (tmp) {
av_frame_free(&tmp);
}
if (ret < 0) {
std::string detail = "avcodec_send_frame failed";
if (send_frame) {
detail += " frame_fmt=";
detail += pix_fmt_name(static_cast<AVPixelFormat>(send_frame->format));
detail += " w=" + std::to_string(send_frame->width);
detail += " h=" + std::to_string(send_frame->height);
if (send_frame->format == AV_PIX_FMT_DRM_PRIME && send_frame->data[0]) {
const AVDRMFrameDescriptor *drm =
reinterpret_cast<const AVDRMFrameDescriptor *>(send_frame->data[0]);
if (drm && drm->layers[0].format) {
detail += " drm_fmt=0x";
char buf[9];
snprintf(buf, sizeof(buf), "%08x", drm->layers[0].format);
detail += buf;
}
if (drm && drm->objects[0].format_modifier) {
detail += " drm_mod=0x";
char buf[17];
snprintf(buf, sizeof(buf), "%016llx",
(unsigned long long)drm->objects[0].format_modifier);
detail += buf;
}
}
}
set_last_error(make_err(detail, ret));
av_frame_unref(ctx->dec_frame);
return -1;
}
av_packet_unref(ctx->enc_pkt);
ret = avcodec_receive_packet(ctx->enc_ctx, ctx->enc_pkt);
if (ret == AVERROR(EAGAIN)) {
av_frame_unref(ctx->dec_frame);
return 0;
}
if (ret < 0) {
set_last_error(make_err("avcodec_receive_packet failed", ret));
av_frame_unref(ctx->dec_frame);
return -1;
}
if (ctx->enc_pkt->size > 0) {
uint8_t *buf = (uint8_t*)malloc(ctx->enc_pkt->size);
if (!buf) {
set_last_error("malloc for output packet failed");
av_packet_unref(ctx->enc_pkt);
av_frame_unref(ctx->dec_frame);
return -1;
}
memcpy(buf, ctx->enc_pkt->data, ctx->enc_pkt->size);
*out_data = buf;
*out_len = ctx->enc_pkt->size;
*out_keyframe = (ctx->enc_pkt->flags & AV_PKT_FLAG_KEY) ? 1 : 0;
av_packet_unref(ctx->enc_pkt);
av_frame_unref(ctx->dec_frame);
return 1;
}
av_frame_unref(ctx->dec_frame);
}
}
extern "C" int ffmpeg_hw_mjpeg_h264_reconfigure(FfmpegHwMjpegH264* handle,
int bitrate_kbps,
int gop) {
if (!handle) {
set_last_error("Invalid handle for reconfigure");
return -1;
}
auto *ctx = reinterpret_cast<FfmpegHwMjpegH264Ctx*>(handle);
if (!ctx->enc_ctx || !ctx->hw_frames_ctx) {
set_last_error("Encoder not initialized for reconfigure");
return -1;
}
ctx->bitrate_kbps = bitrate_kbps > 0 ? bitrate_kbps : ctx->bitrate_kbps;
ctx->gop = gop > 0 ? gop : ctx->gop;
AVBufferRef *frames_ref = ctx->hw_frames_ctx ? av_buffer_ref(ctx->hw_frames_ctx) : nullptr;
free_encoder(ctx);
if (init_encoder(ctx, frames_ref) != 0) {
if (frames_ref) av_buffer_unref(&frames_ref);
return -1;
}
if (frames_ref) av_buffer_unref(&frames_ref);
return 0;
}
extern "C" int ffmpeg_hw_mjpeg_h264_request_keyframe(FfmpegHwMjpegH264* handle) {
if (!handle) {
set_last_error("Invalid handle for request_keyframe");
return -1;
}
auto *ctx = reinterpret_cast<FfmpegHwMjpegH264Ctx*>(handle);
ctx->force_keyframe = true;
return 0;
}
extern "C" void ffmpeg_hw_mjpeg_h264_free(FfmpegHwMjpegH264* handle) {
auto *ctx = reinterpret_cast<FfmpegHwMjpegH264Ctx*>(handle);
if (!ctx) return;
if (ctx->dec_pkt) av_packet_free(&ctx->dec_pkt);
if (ctx->dec_frame) av_frame_free(&ctx->dec_frame);
if (ctx->enc_pkt) av_packet_free(&ctx->enc_pkt);
if (ctx->dec_ctx) avcodec_free_context(&ctx->dec_ctx);
free_encoder(ctx);
if (ctx->hw_device_ctx) av_buffer_unref(&ctx->hw_device_ctx);
delete ctx;
}
extern "C" void ffmpeg_hw_packet_free(uint8_t* data) {
if (data) {
free(data);
}
}
extern "C" const char* ffmpeg_hw_last_error(void) {
return g_last_error.c_str();
}

View File

@@ -0,0 +1,118 @@
#![allow(non_upper_case_globals)]
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
use std::{
ffi::{CStr, CString},
os::raw::c_int,
};
include!(concat!(env!("OUT_DIR"), "/ffmpeg_hw_ffi.rs"));
#[derive(Debug, Clone)]
pub struct HwMjpegH264Config {
pub decoder: String,
pub encoder: String,
pub width: i32,
pub height: i32,
pub fps: i32,
pub bitrate_kbps: i32,
pub gop: i32,
pub thread_count: i32,
}
pub struct HwMjpegH264Pipeline {
ctx: *mut FfmpegHwMjpegH264,
config: HwMjpegH264Config,
}
unsafe impl Send for HwMjpegH264Pipeline {}
impl HwMjpegH264Pipeline {
pub fn new(config: HwMjpegH264Config) -> Result<Self, String> {
unsafe {
let dec = CString::new(config.decoder.as_str()).map_err(|_| "decoder name invalid".to_string())?;
let enc = CString::new(config.encoder.as_str()).map_err(|_| "encoder name invalid".to_string())?;
let ctx = ffmpeg_hw_mjpeg_h264_new(
dec.as_ptr(),
enc.as_ptr(),
config.width,
config.height,
config.fps,
config.bitrate_kbps,
config.gop,
config.thread_count,
);
if ctx.is_null() {
return Err(last_error_message());
}
Ok(Self { ctx, config })
}
}
pub fn encode(&mut self, data: &[u8], pts_ms: i64) -> Result<Option<(Vec<u8>, bool)>, String> {
unsafe {
let mut out_data: *mut u8 = std::ptr::null_mut();
let mut out_len: c_int = 0;
let mut out_key: c_int = 0;
let ret = ffmpeg_hw_mjpeg_h264_encode(
self.ctx,
data.as_ptr(),
data.len() as c_int,
pts_ms,
&mut out_data,
&mut out_len,
&mut out_key,
);
if ret < 0 {
return Err(last_error_message());
}
if out_data.is_null() || out_len == 0 {
return Ok(None);
}
let slice = std::slice::from_raw_parts(out_data, out_len as usize);
let mut vec = Vec::with_capacity(slice.len());
vec.extend_from_slice(slice);
ffmpeg_hw_packet_free(out_data);
Ok(Some((vec, out_key != 0)))
}
}
pub fn reconfigure(&mut self, bitrate_kbps: i32, gop: i32) -> Result<(), String> {
unsafe {
let ret = ffmpeg_hw_mjpeg_h264_reconfigure(self.ctx, bitrate_kbps, gop);
if ret != 0 {
return Err(last_error_message());
}
self.config.bitrate_kbps = bitrate_kbps;
self.config.gop = gop;
Ok(())
}
}
pub fn request_keyframe(&mut self) {
unsafe {
let _ = ffmpeg_hw_mjpeg_h264_request_keyframe(self.ctx);
}
}
}
impl Drop for HwMjpegH264Pipeline {
fn drop(&mut self) {
unsafe {
ffmpeg_hw_mjpeg_h264_free(self.ctx);
}
self.ctx = std::ptr::null_mut();
}
}
pub fn last_error_message() -> String {
unsafe {
let ptr = ffmpeg_hw_last_error();
if ptr.is_null() {
return String::new();
}
let cstr = CStr::from_ptr(ptr);
cstr.to_string_lossy().to_string()
}
}

View File

@@ -1,5 +1,7 @@
pub mod common;
pub mod ffmpeg;
#[cfg(any(target_arch = "aarch64", target_arch = "arm", feature = "rkmpp"))]
pub mod ffmpeg_hw;
pub mod ffmpeg_ram;
#[no_mangle]