虚拟摄像头之 十二 v4l2loopback虚拟机摄像头移植实战(完结篇)
本篇记录在 android8 的 IMX8QM 平台移植 v4l2loopback 虚拟摄像头实战过程记录;其中主旨是记录整个过程、已经期间出现的踩坑过程，后期虚拟摄像驱动还有移植到多核ARM平台上,此文以作备忘。
·
前言
本篇记录在 android8 的 IMX8QM 平台移植 v4l2loopback 虚拟摄像头实战过程记录;其中主旨是记录整个过程、已经期间出现的踩坑过程，后期虚拟摄像驱动还有移植到多核ARM平台上,此文以作备忘。
重构 v4l2_camera_hal 驱动
IMX8QM 平台NXP厂家提供的 Android8 中包含着摄像头HAL驱动和通用Camera接口内容,需要把 NXP 的Camera HAL 移除并添加虚拟摄像头的HAL驱动;
camera.provider@2.4-impl库生成规制如下:
@hardware/interface/camera/provider/2.4/defualt/Android.bp
cc_library_shared {
name: "android.hardware.camera.provider@2.4-impl",
defaults: ["hidl_defaults"],
proprietary: true,
relative_install_path: "hw",
srcs: ["CameraProvider.cpp"],
shared_libs: [
"libhidlbase",
"libhidltransport",
"libutils",
"libcutils",
"android.hardware.camera.device@1.0",
"android.hardware.camera.device@3.2",
"android.hardware.camera.device@3.3",
"camera.device@1.0-impl",
"camera.device@3.2-impl",
"camera.device@3.3-impl",
"android.hardware.camera.provider@2.4",
"android.hardware.camera.common@1.0",
"android.hardware.graphics.mapper@2.0",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
"liblog",
"libhardware",
"libcamera_metadata"
],
static_libs: [
"android.hardware.camera.common@1.0-helper"
]
}
cc_binary {
name: "android.hardware.camera.provider@2.4-service",
defaults: ["hidl_defaults"],
proprietary: true,
relative_install_path: "hw",
srcs: ["service.cpp"],
compile_multilib: "32",
init_rc: ["android.hardware.camera.provider@2.4-service.rc"],
shared_libs: [
"libhidlbase",
"libhidltransport",
"libbinder",
"liblog",
"libutils",
"android.hardware.camera.device@1.0",
"android.hardware.camera.device@3.2",
"android.hardware.camera.device@3.3",
"android.hardware.camera.provider@2.4",
"android.hardware.camera.common@1.0",
],
}
在《虚拟摄像头之四: 谁在调用 v4l2_camera_HAL 摄像头驱动》文章中清晰梳理 CameraProvider 作为物理相机管理服务开机启动，CameraProviderManager 通过 'legacy/0’服务名称,
获取该服务、并用 Binder 与之通讯;我们接下来的任务是把 Camera.imx8.so 库替换为 v4l2llpback.so 库,把虚拟摄像HAL驱动添加到 android系统中。
替换 CameraHAL 驱动
修改 imx8q的BoardConfigCommon.mk 内容如下:
@device/fsl/imx8q/BoardConfigCommon.mk
# 关闭 NXP 原厂的 CameraHAL 的驱动
#BOARD_HAVE_IMX_CAMERA := true
# 开启 CAMERA_V4L2_HAL 的驱动,虚拟相机驱动
USE_CAMERA_V4L2_HAL := true
BOARD_HAVE_USB_CAMERA := false
修改第二部分内容。
@device/fsl/imx8q/ProductConfigCommon.mk
# camera
ifneq ($(PRODUCT_IMX_CAR),true)
PRODUCT_PACKAGES += \
android.hardware.camera.provider@2.4-impl \
android.hardware.camera.provider@2.4-service \
camera.device@1.0-impl \
camera.device@3.2-impl \
camera.imx8
#camera.v4l2
endif
修改 V4L2 Camera HAL 的 LOCAL_MODULE
@hardware/libhardware/modules/camera/3_4/Android.mk
# V4L2 Camera HAL.
# ==============================================================================
include $(CLEAR_VARS)
LOCAL_MODULE := camera.imx8
LOCAL_MODULE_RELATIVE_PATH := hw
LOCAL_VENDOR_MODULE := true
增加开机自动加载 v4l2loopback.ko 库
@device/fsl/imx8q/mek_8q/early.init.cfg
insmod vendor/lib/modules/wlan.ko
# 增加开机加载项
insmod vendor/lib/modules/v4l2loopback.ko
系统在启动调用的 hw_get_module(CAMERA_HARDWARE_MODULE_ID, (const hw_module_t **)&rawModule) 就是 @hardware/libhardware/modules/camera/3.4/v4l2_camera_hal.cpp 的驱动内容。
moudules 装载错误
# dmesg
[ 46.420434] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2000[v4l2_loopback_open]
[ 46.433328] v4l2-loopback[2026]: opened dev:ffff8008f03d8000 with image: (null)
[ 46.441759] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2027[v4l2_loopback_open]
[ 46.455045] v4l2-loopback[774]: cap->capabilities=0x85008003
[ 46.462542] v4l2-loopback[777]: cap->capabilities=0x85008003
[ 46.468765] v4l2-loopback[796]: cap->capabilities=0x85008003
[ 46.474685] video4: VIDIOC_QUERYCAP: driver=v4l2 loopback, card=v4l2loopback video device (0x00, bus=platform:v4l2loopback-000, version=0x00040e62, capabilities=0x85208003, device_caps=0x05208003
[ 46.614770] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2036[v4l2_loopback_close]
[ 46.626969] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2153[try_free_buffers]
[ 46.639130] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2136[free_buffers]
[ 46.650688] v4l2-loopback[2137]: freeing image@ (null) for dev:ffff8008f03d8000
[ 46.658828] /home/robot/android_build/vendor/nxp-opensource/kernel_imx/drivers/virtual_camera/v4l2loopback.c:2058[v4l2_loopback_close]
@hardware/libhardware/hardware.c
int hw_get_module_by_class(const char *class_id, const char *inst,
const struct hw_module_t **module)
{
int i = 0;
char prop[PATH_MAX] = {0};
char path[PATH_MAX] = {0};
char name[PATH_MAX] = {0};
char prop_name[PATH_MAX] = {0};
if (inst)
snprintf(name, PATH_MAX, "%s.%s", class_id, inst);
else
strlcpy(name, class_id, PATH_MAX);
/*
* Here we rely on the fact that calling dlopen multiple times on
* the same .so will simply increment a refcount (and not load
* a new copy of the library).
* We also assume that dlopen() is thread-safe.
*/
/* First try a property specific to the class and possibly instance */
snprintf(prop_name, sizeof(prop_name), "ro.hardware.%s", name);
if (property_get(prop_name, prop, NULL) > 0) {
if (hw_module_exists(path, sizeof(path), name, prop) == 0) {
goto found;
}
}
/* Loop through the configuration variants looking for a module */
for (i=0 ; i<HAL_VARIANT_KEYS_COUNT; i++) {
if (property_get(variant_keys[i], prop, NULL) == 0) {
continue;
}
if (hw_module_exists(path, sizeof(path), name, prop) == 0) {
goto found;
}
}
/* Nothing found, try the default */
if (hw_module_exists(path, sizeof(path), name, "default") == 0) {
goto found;
}
return -ENOENT;
found:
/* load the module, if this fails, we're doomed, and we should not try
* to load a different variant. */
return load(class_id, path, module);
}
修改 v4l2_camera_hal 驱动、增加虚拟相机
ubuntu 上装载驱动
下载 v4l2loopback 源码编译安装
make clean
make
sudo make install
sudo depmod -a
sudo insmod ./v4l2loopback.ko devices=2
使用 ffmpeg 和 ffplay 验证驱动。
v4l2loopback 喂数据
需要先制作YUV420格式的视频数据,然后通过 yuv420_infiniteloop 工具把数据写入虚拟摄像头中;
ffmpeg 制作 mp4
ffmpeg -f x11grab -s 640*480 -framerate 30 -i :0.0+10,00 out640_480.mp4
ffmpeg -f x11grab -s 640*480 -framerate 25 -i :0.0+0,0 -vcodec rawvideo -pix_fmt yuv420p out64_480_420p.yuv
停止录制，按q键即可，或者Ctrl + C ;
视频旋转 90 度
ffmpeg -i out640_480.mp4 -metadata:s:v rotate="90" -codec copy rotate640_480.mp4
mp4(yuv444) to yuv420P(yuyv422)
ffmpeg -i out640_480.mp4 -vcodec rawvideo -an -pix_fmt yuv420P -s 640*480 -y out640_480_420.yuv
ffmpeg -i out640_480.mp4 -vcodec rawvideo -an -pix_fmt nv12 -s 640*480 -y out64_48_nv12.yuv
查看视频源方法
ffplay -f rawvideo -video_size 640x480 -pix_fmt yuv420p out640_480.yuv
mp4 to mpeg/mjpeg
FFMPEG -i out640_480.mp4 out640_480.mpeg
FFMPEG -i out640_480.mp4 out640_480.mjpeg
mp4 to H264
FFMPEG -i out640_480.mp4 out640_480.h264
mp4 to yu12
ffmpeg -i out480_640.mp4 -vcodec rawvideo -an -pix_fmt yu12 -s 480*640 -y out480_640_yu12.yuv
yuv420_infiniteloop 喂视频数据
``cpp
surface size
yuv420_infiniteloop /dev/video4 /sdcard/Movies/out640_480.yuv 1920 1080 30
关于 ffmpeg 视频转换方法参考下面链接:
https://blog.csdn.net/kl1411/article/details/121701003
# 虚拟摄像头预览
笔者采用 v4l2loopback.ko 驱动, 在测试中发现相机画面闪烁,通过调整喂视频流帧速率把30帧调整1帧,
可以清晰看到一帧正常显示、一帧是绿屏。通过logcat和 dmesg对比发现，Android frameworks 中申请流
的数量为3路，而v4l2_loopback.ko 库限制为2路流。 在函数 vidioc_reqbufs 中会判断申请 buffer 的
数量是否大于设备流最大数量,如下:
```cpp
static int vidioc_reqbufs(struct file *file, void *fh,
struct v4l2_requestbuffers *b)
{
struct v4l2_loopback_device *dev;
struct v4l2_loopback_opener *opener;
int i;
MARK();
dev = v4l2loopback_getdevice(file);
opener = fh_to_opener(fh);
dprintk("reqbufs: memory=%d, count=%d, number=%d\n", b->memory, b->count, dev->buffers_number);
if (opener->timeout_image_io) {
if (b->memory != V4L2_MEMORY_MMAP)
return -EINVAL;
b->count = 1;
return 0;
}
init_buffers(dev);
switch (b->memory) {
case V4L2_MEMORY_MMAP:
/* do nothing here, buffers are always allocated */
if (b->count < 1 || dev->buffers_number < 1)
return 0;
if (b->count > dev->buffers_number) //> 此处限制流的数量不能超出最大数量, 在 android frameworks中申请数量为 3 路，
b->count = dev->buffers_number; //> v4l2_loopback.ko 中最大值为 2 路流，导致摄像头显示缺失帧。
/* make sure that outbufs_list contains buffers from 0 to used_buffers-1
* actually, it will have been already populated via v4l2_loopback_init()
* at this point */
if (list_empty(&dev->outbufs_list)) {
for (i = 0; i < dev->used_buffers; ++i)
list_add_tail(&dev->buffers[i].list_head,
&dev->outbufs_list);
}
/* also, if dev->used_buffers is going to be decreased, we should remove
* out-of-range buffers from outbufs_list, and fix bufpos2index mapping */
if (b->count < dev->used_buffers) {
struct v4l2l_buffer *pos, *n;
list_for_each_entry_safe (pos, n, &dev->outbufs_list,
list_head) {
if (pos->buffer.index >= b->count)
list_del(&pos->list_head);
}
/* after we update dev->used_buffers, buffers in outbufs_list will
* correspond to dev->write_position + [0;b->count-1] range */
i = dev->write_position;
list_for_each_entry (pos, &dev->outbufs_list,
list_head) {
dev->bufpos2index[i % b->count] =
pos->buffer.index;
++i;
}
}
opener->buffers_number = b->count;
if (opener->buffers_number < dev->used_buffers)
dev->used_buffers = opener->buffers_number;
dprintk("reqbufs: dev->used_buffers=%d, count=%d\n", dev->used_buffers, b->count);
return 0;
default:
return -EINVAL;
}
}
哪如何调整 dev->buffers_number 参数缺省配置值呢，可在 v4l2_loopback.c 中检索 V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 宏定义内容如下:
/* module parameters */
static int debug = 3; //> default = 0
module_param(debug, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(debug, "debugging level (higher values == more verbose)");
//#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 2
#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 4
static int max_buffers = V4L2LOOPBACK_DEFAULT_MAX_BUFFERS;
module_param(max_buffers, int, S_IRUGO);
MODULE_PARM_DESC(max_buffers,
"how many buffers should be allocated [DEFAULT: " STRINGIFY2(
V4L2LOOPBACK_DEFAULT_MAX_BUFFERS) "]");
可以看到笔者修改流的数量最大为 4 路流,此处问题解决，摄像头隔帧绿屏另有原因，是NXP的libg2d.ko 闭源GPU驱动问题，采用opencl库摄像头显示
就正常、没有绿屏现象;因此问题专用文章描述解决此bug的过程,所有简单描述之。
虚拟摄像头拍照
笔者使用的是 google 开源的 Camera2BasicFragement 工程,源码地址 https://github.com/googlearchive/android-Camera2Basic.git, 拍照回调函数中
有判断自动聚焦状态、由于虚拟摄像头暂时未实现聚焦功能、调整一下代码就能够实现拍照。
/**
* A {@link CameraCaptureSession.CaptureCallback} that handles events related to JPEG capture.
*/
private CameraCaptureSession.CaptureCallback mCaptureCallback
= new CameraCaptureSession.CaptureCallback() {
private void process(CaptureResult result) {
switch (mState) {
case STATE_PREVIEW: {
// We have nothing to do when the camera preview is working normally.
break;
}
case STATE_WAITING_LOCK: {
Integer afState = result.get(CaptureResult.CONTROL_AF_STATE);
if (afState == null) {
captureStillPicture();
} else if (CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED == afState ||
CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED == afState) {
// CONTROL_AE_STATE can be null on some devices
Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
if (aeState == null ||
aeState == CaptureResult.CONTROL_AE_STATE_CONVERGED) {
mState = STATE_PICTURE_TAKEN;
captureStillPicture();
} else {
runPrecaptureSequence();
}
} else { //> don't use autofocus,增加创建拍照 request
captureStillPicture();
}
break;
}
case STATE_WAITING_PRECAPTURE: {
// CONTROL_AE_STATE can be null on some devices
Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
if (aeState == null ||
aeState == CaptureResult.CONTROL_AE_STATE_PRECAPTURE ||
aeState == CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) {
mState = STATE_WAITING_NON_PRECAPTURE;
}
break;
}
case STATE_WAITING_NON_PRECAPTURE: {
// CONTROL_AE_STATE can be null on some devices
Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
if (aeState == null || aeState != CaptureResult.CONTROL_AE_STATE_PRECAPTURE) {
mState = STATE_PICTURE_TAKEN;
captureStillPicture();
}
break;
}
}
}
@Override
public void onCaptureProgressed(@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request,
@NonNull CaptureResult partialResult) {
process(partialResult);
}
@Override
public void onCaptureCompleted(@NonNull CameraCaptureSession session,
@NonNull CaptureRequest request,
@NonNull TotalCaptureResult result) {
process(result);
}
};
远程视频数据送到虚拟摄像头驱动评估
笔者通过ffmpeg制作如下格式的视频数据流文件,从数据大小和图像质量上对比;
robot@ubuntu:~/camera_test$ ls -l
-rw-rw-r-- 1 robot robot 2311538 Aug 29 12:13 out640_480.h264
-rw-rw-r-- 1 robot robot 24084256 Aug 29 13:17 out640_480.mjpeg
-rw-rw-r-- 1 robot robot 2520783 Aug 25 09:37 out640_480.mp4
-rw-rw-r-- 1 robot robot 2990080 Aug 29 12:08 out640_480.mpeg
-rw-rw-r-- 1 robot robot 894566400 Aug 29 10:38 out640_480.yuv
从图像质量和大小权衡的话、h264格式是非常理想的选择。
linux 内核 v4l2 驱动详解
https://www.codenong.com/cs105578727/
华为开发者空间，是为全球开发者打造的专属开发空间，汇聚了华为优质开发资源及工具，致力于让每一位开发者拥有一台云主机，基于华为根生态开发、创新。
更多推荐
6
0- 0
已为社区贡献19条内容
所有评论(0)