8313105: Improved media framing

Backport-of: 0a52a4c

Author: Alexander Matveev

modules/javafx.media/src/main/native/jfxmedia/PipelineManagement/VideoFrame.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,50 +23,51 @@
  * questions.
  */
 
+// For UINT_MAX, we cannot use GLib here, since it is shared code between
+// GStreamer and AVFoundation.
+#include <limits.h>
+
 #include "VideoFrame.h"
 #include <Common/VSMemory.h>
 
 //*************************************************************************************************
 //********** class CVideoFrame
 //*************************************************************************************************
 CVideoFrame::CVideoFrame()
-:   m_iWidth(0),
-    m_iHeight(0),
-    m_iEncodedWidth(0),
-    m_iEncodedHeight(0),
+:   m_uiWidth(0),
+    m_uiHeight(0),
+    m_uiEncodedWidth(0),
+    m_uiEncodedHeight(0),
     m_typeFrame(UNKNOWN),
     m_bHasAlpha(false),
     m_dTime(0.0),
-    m_FrameDirty(false),
-    m_iPlaneCount(1)
+    m_FrameDirty(false)
 {
-    m_piPlaneStrides[0] = m_piPlaneStrides[1] = m_piPlaneStrides[2] = m_piPlaneStrides[3] = 0;
-    m_pulPlaneSize[0] = m_pulPlaneSize[1] = m_pulPlaneSize[2] = m_pulPlaneSize[3] = 0;
-    m_pvPlaneData[0] = m_pvPlaneData[1] = m_pvPlaneData[2] = m_pvPlaneData[3] = NULL;
+    Reset();
 }
 
 CVideoFrame::~CVideoFrame()
 {
 }
 
-int CVideoFrame::GetWidth()
+unsigned int CVideoFrame::GetWidth()
 {
-    return m_iWidth;
+    return m_uiWidth;
 }
 
-int CVideoFrame::GetHeight()
+unsigned int CVideoFrame::GetHeight()
 {
-    return m_iHeight;
+    return m_uiHeight;
 }
 
-int CVideoFrame::GetEncodedWidth()
+unsigned int CVideoFrame::GetEncodedWidth()
 {
-    return m_iEncodedWidth;
+    return m_uiEncodedWidth;
 }
 
-int CVideoFrame::GetEncodedHeight()
+unsigned int CVideoFrame::GetEncodedHeight()
 {
-    return m_iEncodedHeight;
+    return m_uiEncodedHeight;
 }
 
 CVideoFrame::FrameType CVideoFrame::GetType()
@@ -84,31 +85,41 @@ double CVideoFrame::GetTime()
     return m_dTime;
 }
 
-int CVideoFrame::GetPlaneCount()
+unsigned int CVideoFrame::GetPlaneCount()
 {
-    return m_iPlaneCount;
+    return m_uiPlaneCount;
+}
+
+void CVideoFrame::SetPlaneCount(unsigned int count)
+{
+    if (count <= MAX_PLANE_COUNT) {
+        m_uiPlaneCount = count;
+    } else {
+        // Should never happen
+        m_uiPlaneCount = MAX_PLANE_COUNT;
+    }
 }
 
-void* CVideoFrame::GetDataForPlane(int planeIndex)
+void* CVideoFrame::GetDataForPlane(unsigned int planeIndex)
 {
-    if (planeIndex < 4 && planeIndex >= 0) {
+    if (planeIndex < MAX_PLANE_COUNT) {
         return m_pvPlaneData[planeIndex];
     }
     return NULL;
 }
 
-unsigned long CVideoFrame::GetSizeForPlane(int planeIndex)
+unsigned long CVideoFrame::GetSizeForPlane(unsigned int planeIndex)
 {
-    if (planeIndex < 4 && planeIndex >= 0) {
+    if (planeIndex < MAX_PLANE_COUNT) {
         return m_pulPlaneSize[planeIndex];
     }
     return 0;
 }
 
-int CVideoFrame::GetStrideForPlane(int planeIndex)
+unsigned int CVideoFrame::GetStrideForPlane(unsigned int planeIndex)
 {
-    if (planeIndex < 4 && planeIndex >= 0) {
-        return m_piPlaneStrides[planeIndex];
+    if (planeIndex < MAX_PLANE_COUNT) {
+        return m_puiPlaneStrides[planeIndex];
     }
     return 0;
 }
@@ -118,12 +129,22 @@ CVideoFrame *CVideoFrame::ConvertToFormat(FrameType type)
     return NULL;
 }
 
-void CVideoFrame::SwapPlanes(int aa, int bb)
+void CVideoFrame::Reset()
+{
+    m_uiPlaneCount = 0;
+    for (int i = 0; i < MAX_PLANE_COUNT; i++) {
+        m_puiPlaneStrides[i] = 0;
+        m_pulPlaneSize[i] = 0;
+        m_pvPlaneData[i] = NULL;
+    }
+}
+
+void CVideoFrame::SwapPlanes(unsigned int aa, unsigned int bb)
 {
-    if (aa != bb && aa >= 0 && aa < m_iPlaneCount && bb >= 0 && bb < m_iPlaneCount) {
-        int stride = m_piPlaneStrides[aa];
-        m_piPlaneStrides[aa] = m_piPlaneStrides[bb];
-        m_piPlaneStrides[bb] = stride;
+    if (aa != bb && aa < m_uiPlaneCount && bb < m_uiPlaneCount) {
+        unsigned int stride = m_puiPlaneStrides[aa];
+        m_puiPlaneStrides[aa] = m_puiPlaneStrides[bb];
+        m_puiPlaneStrides[bb] = stride;
 
         unsigned long size = m_pulPlaneSize[aa];
         m_pulPlaneSize[aa] = m_pulPlaneSize[bb];
@@ -134,3 +155,53 @@ void CVideoFrame::SwapPlanes(int aa, int bb)
         m_pvPlaneData[bb] = vptr;
     }
 }
+
+unsigned long CVideoFrame::CalcSize(unsigned int a, unsigned int b, bool *pbValid)
+{
+    if (pbValid == NULL || *(pbValid) == false) {
+        return 0;
+    }
+
+    if (b > 0 && a <= (UINT_MAX / b)) {
+        return (a * b);
+    }
+
+    *(pbValid) = false;
+    return 0;
+}
+
+unsigned long CVideoFrame::AddSize(unsigned long a, unsigned long b, bool *pbValid)
+{
+    if (pbValid == NULL || *(pbValid) == false) {
+        return 0;
+    }
+
+    // unsigned long can be 32-bit or 64-bit, make sure it is no more then UINT_MAX
+    if (a <= UINT_MAX && b <= UINT_MAX && a <= (UINT_MAX - b)) {
+        return (a + b);
+    }
+
+    *(pbValid) = false;
+    return 0;
+}
+
+void* CVideoFrame::CalcPlanePointer(intptr_t baseAddress, unsigned int offset,
+                                    unsigned long planeSize, unsigned long baseSize,
+                                    bool *pbValid)
+{
+    if (pbValid == NULL || *(pbValid) == false) {
+        return NULL;
+    }
+
+    // We will read planeSize bytes from baseAddress starting with offset, so
+    // make sure we do not read pass baseSize.
+    unsigned long endOfPlane = AddSize(offset, planeSize, pbValid);
+    if (*(pbValid)) { // Make sure AddSize() did not failed.
+        if (endOfPlane <= baseSize) {
+            return (void*)(baseAddress + offset);
+        }
+    }
+
+    *(pbValid) = false;
+    return NULL;
+}

modules/javafx.media/src/main/native/jfxmedia/PipelineManagement/VideoFrame.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,9 @@
 #define _VIDEO_FRAME_H_
 
 #include <stdlib.h>
+#include <stdint.h>
+
+#define MAX_PLANE_COUNT 4
 
 /**
  * class CVideoFrame
@@ -56,41 +59,61 @@ class CVideoFrame
 
     double              GetTime();
 
-    int                 GetWidth();
-    int                 GetHeight();
-    int                 GetEncodedWidth();
-    int                 GetEncodedHeight();
+    unsigned int        GetWidth();
+    unsigned int        GetHeight();
+    unsigned int        GetEncodedWidth();
+    unsigned int        GetEncodedHeight();
 
     FrameType           GetType();
     bool                HasAlpha();
 
-    int                 GetPlaneCount();
-    void*               GetDataForPlane(int planeIndex);
-    unsigned long       GetSizeForPlane(int planeIndex);
-    int                 GetStrideForPlane(int planeIndex);
+    unsigned int        GetPlaneCount();
+    void                SetPlaneCount(unsigned int count);
+    void*               GetDataForPlane(unsigned int planeIndex);
+    unsigned long       GetSizeForPlane(unsigned int planeIndex);
+    unsigned int        GetStrideForPlane(unsigned int planeIndex);
 
     virtual CVideoFrame *ConvertToFormat(FrameType type);
 
     bool                GetFrameDirty() { return m_FrameDirty; }
     void                SetFrameDirty(bool dirty) { m_FrameDirty = dirty; }
 
 protected:
-    int                 m_iWidth;
-    int                 m_iHeight;
-    int                 m_iEncodedWidth;
-    int                 m_iEncodedHeight;
+    unsigned int        m_uiWidth;
+    unsigned int        m_uiHeight;
+    unsigned int        m_uiEncodedWidth;
+    unsigned int        m_uiEncodedHeight;
     FrameType           m_typeFrame;
     bool                m_bHasAlpha;
     double              m_dTime;
     bool                m_FrameDirty;
 
     // frame data buffers
-    int                 m_iPlaneCount;
-    void*               m_pvPlaneData[4];
-    unsigned long       m_pulPlaneSize[4];
-    int                 m_piPlaneStrides[4];
+    void*               m_pvPlaneData[MAX_PLANE_COUNT];
+    unsigned long       m_pulPlaneSize[MAX_PLANE_COUNT];
+    unsigned int        m_puiPlaneStrides[MAX_PLANE_COUNT];
+
+    void Reset();
+    void SwapPlanes(unsigned int aa, unsigned int bb);
+
+    // CalcSize(), AddSize(), CalcPlanePointer() requires bValid to be set to
+    // true initially, if bValid is false these functions do nothing. It is
+    // implemented this way, so all these functions can be chain called without
+    // checking bValid after each call. bValid will be set to false only if
+    // calculation failed and will never be set to true.
+    // Multiplies a and b, bValid set to false if integer overflow detected.
+    unsigned long CalcSize(unsigned int a, unsigned int b, bool *pbValid);
+    // Adds a and b, bValid set to false if integer overflow detected.
+    unsigned long AddSize(unsigned long a, unsigned long b, bool *pbValid);
+    // Calculates plane pointer (baseAddress + offset) and checks that calculated
+    // pointer within buffer. Returns NULL and sets bValid to false if calculated
+    // pointer is invalid.
+    void* CalcPlanePointer(intptr_t baseAddress, unsigned int offset,
+                           unsigned long planeSize, unsigned long baseSize,
+                           bool *pbValid);
 
-    void SwapPlanes(int aa, int bb);
+private:
+    unsigned int        m_uiPlaneCount;
 };
 
 #endif  //_VIDEO_FRAME_H_

modules/javafx.media/src/main/native/jfxmedia/jni/NativeVideoBuffer.cpp

@@ -71,8 +71,8 @@ JNIEXPORT jobject JNICALL Java_com_sun_media_jfxmediaimpl_NativeVideoBuffer_nati
 {
     CVideoFrame *frame = (CVideoFrame*)jlong_to_ptr(nativeHandle);
     if (frame) {
-        void *dataPtr = frame->GetDataForPlane((int)plane);
-        jlong capacity = (jlong)frame->GetSizeForPlane((int)plane);
+        void *dataPtr = frame->GetDataForPlane((unsigned int)plane);
+        jlong capacity = (jlong)frame->GetSizeForPlane((unsigned int)plane);
         jobject buffer = env->NewDirectByteBuffer(dataPtr, capacity);
         if (env->ExceptionCheck()) {
             env->ExceptionClear();
@@ -215,7 +215,7 @@ JNIEXPORT jintArray JNICALL Java_com_sun_media_jfxmediaimpl_NativeVideoBuffer_na
             return NULL;
         }
 
-        for (int ii=0; ii < count; ii++) {
+        for (unsigned int ii=0; ii < count; ii++) {
             strideArray[ii] = frame->GetStrideForPlane(ii);
         }