Skip to content

fix: Fix a crash AV1 encoder #1093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

fix: Fix a crash AV1 encoder #1093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6683281
add patches
karasusan Aug 19, 2025
f6c7cc2
fix
karasusan Aug 19, 2025
9907b8a
Add Vulkan validation layer when debug build.
karasusan Aug 19, 2025
3b95e53
format
karasusan Aug 19, 2025
f8084e7
Merge branch 'main' into fix/crash-av1-encoder
karasusan Aug 20, 2025
bc97779
Update libwebrtc build scripts
karasusan Aug 20, 2025
68bedc5
format
karasusan Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions BuildScripts~/build_libwebrtc_android.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ patch -N "src/build/android/gyp/turbine.py" < "$COMMAND_DIR/patches/downgradeJDK
# Fix SetRawImagePlanes() in LibvpxVp8Encoder
patch -N "src/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp8_encoder.patch"

patch -N "src/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp9_encoder.patch"

patch -N "src/modules/video_coding/codecs/av1/libaom_av1_encoder.cc" < "$COMMAND_DIR/patches/libaom_av1_encoder.patch"


pushd src
# Fix AdaptedVideoTrackSource::video_adapter()
patch -p1 < "$COMMAND_DIR/patches/fix_adaptedvideotracksource.patch"
Expand Down
4 changes: 4 additions & 0 deletions BuildScripts~/build_libwebrtc_ios.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ patch -N "src/sdk/BUILD.gn" < "$COMMAND_DIR/patches/add_objc_deps.patch"
# Fix SetRawImagePlanes() in LibvpxVp8Encoder
patch -N "src/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp8_encoder.patch"

patch -N "src/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp9_encoder.patch"

patch -N "src/modules/video_coding/codecs/av1/libaom_av1_encoder.cc" < "$COMMAND_DIR/patches/libaom_av1_encoder.patch"

# use included python
export PATH="$(pwd)/depot_tools/bootstrap-3.8.0.chromium.8_bin/python/bin:$PATH"

Expand Down
4 changes: 4 additions & 0 deletions BuildScripts~/build_libwebrtc_linux.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ patch -N "src/BUILD.gn" < "$COMMAND_DIR/patches/add_jsoncpp.patch"
# Fix SetRawImagePlanes() in LibvpxVp8Encoder
patch -N "src/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp8_encoder.patch"

patch -N "src/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp9_encoder.patch"

patch -N "src/modules/video_coding/codecs/av1/libaom_av1_encoder.cc" < "$COMMAND_DIR/patches/libaom_av1_encoder.patch"

mkdir -p "$ARTIFACTS_DIR/lib"

for target_cpu in "x64"
Expand Down
4 changes: 4 additions & 0 deletions BuildScripts~/build_libwebrtc_macos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ patch -N "src/sdk/BUILD.gn" < "$COMMAND_DIR/patches/add_objc_deps.patch"
# Fix SetRawImagePlanes() in LibvpxVp8Encoder
patch -N "src/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp8_encoder.patch"

patch -N "src/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc" < "$COMMAND_DIR/patches/libvpx_vp9_encoder.patch"

patch -N "src/modules/video_coding/codecs/av1/libaom_av1_encoder.cc" < "$COMMAND_DIR/patches/libaom_av1_encoder.patch"

mkdir -p "$ARTIFACTS_DIR/lib"

for is_debug in "true" "false"
Expand Down
4 changes: 4 additions & 0 deletions BuildScripts~/build_libwebrtc_win.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ patch -N "src\api\task_queue\task_queue_base.h" < "%COMMAND_DIR%\patches\fix_tas
rem fix SetRawImagePlanes() in LibvpxVp8Encoder
patch -N "src\modules\video_coding\codecs\vp8\libvpx_vp8_encoder.cc" < "%COMMAND_DIR%\patches\libvpx_vp8_encoder.patch"

patch -N "src\modules\video_coding\codecs\vp9\libvpx_vp9_encoder.cc" < "%COMMAND_DIR%\patches\libvpx_vp9_encoder.patch"

patch -N "src\modules\video_coding\codecs\av1\libaom_av1_encoder.cc" < "%COMMAND_DIR%\patches\libaom_av1_encoder.patch"

mkdir "%ARTIFACTS_DIR%\lib"

setlocal enabledelayedexpansion
Expand Down
15 changes: 15 additions & 0 deletions BuildScripts~/patches/libaom_av1_encoder.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
index 28a8e5f846..817826d248 100644
--- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
+++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
@@ -603,7 +603,9 @@ int32_t LibaomAv1Encoder::Encode(
// Set frame_for_encode_ data pointers and strides.
MaybeRewrapImgWithFormat(AOM_IMG_FMT_I420);
auto i420_buffer = mapped_buffer->GetI420();
- RTC_DCHECK(i420_buffer);
+ if (!i420_buffer) {
+ return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
+ }
frame_for_encode_->planes[AOM_PLANE_Y] =
const_cast<unsigned char*>(i420_buffer->DataY());
frame_for_encode_->planes[AOM_PLANE_U] =
15 changes: 15 additions & 0 deletions BuildScripts~/patches/libvpx_vp9_encoder.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
index e460625aea..4194e550df 100644
--- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
+++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc
@@ -2090,7 +2090,9 @@ rtc::scoped_refptr<VideoFrameBuffer> LibvpxVp9Encoder::PrepareBufferForProfile0(
case VideoFrameBuffer::Type::kI420A: {
MaybeRewrapRawWithFormat(VPX_IMG_FMT_I420);
const I420BufferInterface* i420_buffer = mapped_buffer->GetI420();
- RTC_DCHECK(i420_buffer);
+ if(!i420_buffer) {
+ return {};
+ }
raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
2 changes: 2 additions & 0 deletions Plugin~/WebRTCPlugin/GpuMemoryBuffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,10 @@ namespace webrtc
{
// One texture cannot map CUDA memory and CPU memory simultaneously.
// Believe there is still room for improvement.
#if CUDA_PLATFORM
if (!device_->CopyResourceFromNativeV(texture_.get(), ptr))
return false;
#endif
if (!device_->CopyResourceFromNativeV(textureCpuRead_.get(), ptr))
return false;
return true;
Expand Down
8 changes: 8 additions & 0 deletions Plugin~/WebRTCPlugin/GraphicsDevice/Vulkan/VulkanGraphicsDevice.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "pch.h"

#include <chrono>
#include <third_party/libyuv/include/libyuv/convert.h>

#include "GraphicsDevice/GraphicsUtility.h"
Expand Down Expand Up @@ -107,6 +108,7 @@ namespace webrtc
kUnityVulkanResourceAccess_PipelineBarrier,
unityVulkanImage.get()))
{
RTC_LOG(LS_ERROR) << "UnityGraphicsVulkan::AccessTexture failed.";
return nullptr;
}
return unityVulkanImage;
Expand Down Expand Up @@ -463,6 +465,12 @@ namespace webrtc
m_LastStateCond.wait_until(lock, std::chrono::system_clock::now() + m_syncTimeout, [vulkanTexture, this] {
return vulkanTexture->currentFrameNumber <= m_LastState.safeFrameNumber;
});
if (!ret)
{
RTC_LOG(LS_ERROR) << "WaitSync timeout. "
Copy link

Copilot AI Aug 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing space after 'timeout.' - should be 'timeout. ' to properly separate from the next line.

Copilot uses AI. Check for mistakes.
<< "texture: " << texture << "currentFrameNumber: " << vulkanTexture->currentFrameNumber
<< ", safeFrameNumber: " << m_LastState.safeFrameNumber;
}
return ret;
}

Expand Down