Major buffer producer refactor, see details:

- Introduced NativeHandle (display/types.h)
- Introduced NvGraphicBuffer (nvidia/graphic_buffer.h)
- Renamed BqQueueBufferInput to BqBufferInput
- Renamed BqQueueBufferOutput to BqBufferOutput
- BqGraphicBuffer is now defined and marshalled in a way that matches
  official sw more closely, with a pointer to a NativeHandle instead of
  inline flattened data and other miscellaneous junk
- Const correctness fixes
- bqSetPreallocatedBuffer now has flattening logic for BqGraphicBuffer
- bqRequestBuffer doesn't have this logic for now, passing anything other
  than NULL will fail
- gfx.c updated to use the refactored buffer producer

nx/include/switch.h

@@ -89,6 +89,7 @@ extern "C" {
 
 #include "switch/nvidia/ioctl.h"
 #include "switch/nvidia/map.h"
+#include "switch/nvidia/graphic_buffer.h"
 #include "switch/nvidia/address_space.h"
 #include "switch/nvidia/channel.h"
 #include "switch/nvidia/info.h"

nx/include/switch/display/buffer_producer.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "types.h"
+#include "binder.h"
 #include "../nvidia/fence.h"
 
 typedef struct {
@@ -19,69 +20,30 @@ typedef struct {
     u32 unk;
     u32 swapInterval;
     NvMultiFence fence;
-} BqQueueBufferInput;
+} BqBufferInput;
 
 typedef struct {
     u32 width;
     u32 height;
     u32 transformHint;
     u32 numPendingBuffers;
-} BqQueueBufferOutput;
+} BqBufferOutput;
 
 typedef struct {
-    u32 magic;
     u32 width;
     u32 height;
     u32 stride;
     u32 format;
     u32 usage;
-
+    NativeHandle* native_handle;
-    u32 pid;
-    u32 refcount;
-
-    u32 numFds;
-    u32 numInts;
-
-    struct { // Actual size is numFds*4 + numInts*4.
-        u32 unk_x0;
-        u32 nvmap_handle0;
-        u32 unk_x8;
-        u32 unk_xc;
-        u32 unk_x10;
-        u32 unk_x14;
-        u32 unk_x18;
-        u32 unk_x1c;
-        u32 unk_x20;
-        u32 width_unk0;
-        u32 buffer_size0;
-        u32 unk_x2c;
-        u32 unk_x30;
-        u32 width_unk1;
-        u32 height_unk;
-        u32 flags;
-        u32 unk_x40;
-        u32 unk_x44;
-        u32 byte_stride;
-        u32 nvmap_handle1;
-        u32 buffer_offset;
-        u32 unk_x54;
-        u32 unk_x58;
-        u32 unk_x5c;
-        u32 unk_x60;
-        u32 unk_x64;
-        u32 unk_x68;
-        u32 buffer_size1;
-        u32 unk_x70[0x33]; // Normally all-zero.
-        struct { u64 timestamp; } PACKED; // unused
-    } data;
 } BqGraphicBuffer;
 
 Result bqRequestBuffer(Binder *b, s32 bufferIdx, BqGraphicBuffer *buf);
 Result bqDequeueBuffer(Binder *b, bool async, u32 width, u32 height, s32 format, u32 usage, s32 *buf, NvMultiFence *fence);
 Result bqDetachBuffer(Binder *b, s32 slot);
-Result bqQueueBuffer(Binder *b, s32 buf, BqQueueBufferInput *input, BqQueueBufferOutput *output);
+Result bqQueueBuffer(Binder *b, s32 buf, const BqBufferInput *input, BqBufferOutput *output);
 Result bqCancelBuffer(Binder *b, s32 buf, NvMultiFence *fence);
 Result bqQuery(Binder *b, s32 what, s32* value);
-Result bqConnect(Binder *b, s32 api, bool producerControlledByApp, BqQueueBufferOutput *output);
+Result bqConnect(Binder *b, s32 api, bool producerControlledByApp, BqBufferOutput *output);
 Result bqDisconnect(Binder *b, s32 api);
-Result bqSetPreallocatedBuffer(Binder *b, s32 buf, BqGraphicBuffer *input);
+Result bqSetPreallocatedBuffer(Binder *b, s32 buf, const BqGraphicBuffer *input);

nx/include/switch/display/types.h

@@ -133,3 +133,10 @@ enum {
     /* rotate source image 270 degrees clock-wise */
     NATIVE_WINDOW_TRANSFORM_ROT_270 = HAL_TRANSFORM_ROT_270,
 };
+
+// From Android native_handle.h.
+typedef struct {
+    int version;
+    int num_fds;
+    int num_ints;
+} NativeHandle;

nx/include/switch/nvidia/graphic_buffer.h

@@ -0,0 +1,39 @@
+#pragma once
+#include "../display/types.h"
+#include "types.h"
+
+typedef struct {
+    u32 width;
+    u32 height;
+    u64 color_format;
+    NvLayout layout;
+    u32 pitch;
+    u32 unused; // usually this field contains the nvmap handle, but it's completely unused/overwritten during marshalling
+    u32 offset;
+    NvKind kind;
+    u32 block_height_log2;
+    NvDisplayScanFormat scan;
+    u32 second_field_offset;
+    u64 flags;
+    u64 size;
+    u32 unk[6]; // compression related
+} NvSurface;
+
+typedef struct {
+    NativeHandle header;
+    s32 unk0;       // -1
+    s32 nvmap_id;   // nvmap object id
+    u32 unk2;       // 0
+    u32 magic;      // 0xDAFFCAFF
+    u32 pid;        // 42
+    u32 type;       // ?
+    u32 usage;      // GRALLOC_USAGE_* bitmask
+    u32 format;     // PIXEL_FORMAT_*
+    u32 ext_format; // copy of the above (in most cases)
+    u32 stride;     // in pixels!
+    u32 total_size; // in bytes
+    u32 num_planes; // usually 1
+    u32 unk12;      // 0
+    NvSurface layers[3];
+    u64 unused;     // official sw writes a pointer to bookkeeping data here, but it's otherwise completely unused/overwritten during marshalling
+} NvGraphicBuffer;

nx/source/display/buffer_producer.c

@@ -51,11 +51,11 @@ Result bqRequestBuffer(Binder *b, s32 bufferIdx, BqGraphicBuffer *buf)
             void* tmp_ptr;
 
             tmp_ptr = parcelReadFlattenedObject(&parcel_reply, &tmp_size);
-            if (tmp_ptr == NULL || tmp_size != sizeof(BqGraphicBuffer))
+            if (!tmp_ptr)
                 return MAKERESULT(Module_Libnx, LibnxError_BadInput);
 
             if (buf)
-                memcpy(buf, tmp_ptr, sizeof(BqGraphicBuffer));
+                return MAKERESULT(Module_Libnx, LibnxError_BadInput); // not implemented
         }
 
         rc = binderConvertErrorCode(parcelReadInt32(&parcel_reply));
@@ -123,7 +123,7 @@ Result bqDetachBuffer(Binder *b, s32 slot)
     return rc;
 }
 
-Result bqQueueBuffer(Binder *b, s32 buf, BqQueueBufferInput *input, BqQueueBufferOutput *output)
+Result bqQueueBuffer(Binder *b, s32 buf, const BqBufferInput *input, BqBufferOutput *output)
 {
     Result rc;
     Parcel parcel, parcel_reply;
@@ -187,7 +187,7 @@ Result bqQuery(Binder *b, s32 what, s32* value)
     return rc;
 }
 
-Result bqConnect(Binder *b, s32 api, bool producerControlledByApp, BqQueueBufferOutput *output)
+Result bqConnect(Binder *b, s32 api, bool producerControlledByApp, BqBufferOutput *output)
 {
     Result rc;
     Parcel parcel, parcel_reply;
@@ -234,11 +234,17 @@ Result bqDisconnect(Binder *b, s32 api)
     return rc;
 }
 
-Result bqSetPreallocatedBuffer(Binder *b, s32 buf, BqGraphicBuffer *input)
+Result bqSetPreallocatedBuffer(Binder *b, s32 buf, const BqGraphicBuffer *input)
 {
     Result rc;
     Parcel parcel, parcel_reply;
-    bool flag = 0;
+    bool hasInput = false;
+
+    if (input) {
+        hasInput = true;
+        if (!input->native_handle || input->native_handle->num_fds || input->native_handle->num_ints > 0x80)
+            return MAKERESULT(Module_Libnx, LibnxError_BadInput);
+    }
 
     parcelCreate(&parcel);
     parcelCreate(&parcel_reply);
@@ -246,12 +252,40 @@ Result bqSetPreallocatedBuffer(Binder *b, s32 buf, BqGraphicBuffer *input)
     parcelWriteInterfaceToken(&parcel, g_bq_InterfaceDescriptor);
     parcelWriteInt32(&parcel, buf);
 
-    if (input != NULL)
+    parcelWriteInt32(&parcel, hasInput);
-        flag = 1;
+    if (hasInput) {
+        struct {
+            u32 magic;
+            u32 width;
+            u32 height;
+            u32 stride;
+            u32 format;
+            u32 usage;
 
-    parcelWriteInt32(&parcel, flag);
+            u32 pid;
-    if (flag)
+            u32 refcount;
-        parcelWriteFlattenedObject(&parcel, input, sizeof(BqGraphicBuffer));
+
+            u32 numFds;
+            u32 numInts;
+
+            u32 ints[input->native_handle->num_ints];
+        } buf;
+
+        // Serialize the buffer
+        buf.magic = 0x47424652; // GBFR (Graphic Buffer)
+        buf.width = input->width;
+        buf.height = input->height;
+        buf.stride = input->stride;
+        buf.format = input->format;
+        buf.usage = input->usage;
+        buf.pid = 42; // Official sw sets this to the value of getpid(), which is hardcoded to return 42.
+        buf.refcount = 0; // Official sw sets this to the output of android_atomic_inc(). We instead don't care and set it to zero since it is ignored during marshalling.
+        buf.numFds = 0;
+        buf.numInts = input->native_handle->num_ints;
+        memcpy(buf.ints, input->native_handle+1, sizeof(buf.ints));
+
+        parcelWriteFlattenedObject(&parcel, &buf, sizeof(buf));
+    }
 
     rc = parcelTransact(b, SET_PREALLOCATED_BUFFER, &parcel, &parcel_reply);
     // Reply parcel has no content

nx/source/display/gfx.c

@@ -11,6 +11,7 @@
 #include "display/buffer_producer.h"
 #include "display/gfx.h"
 #include "nvidia/map.h"
+#include "nvidia/graphic_buffer.h"
 
 __attribute__((weak)) ViServiceType __nx_gfx_vi_service_type = ViServiceType_Default;
 
@@ -26,8 +27,8 @@ static bool g_gfx_ProducerConnected = 0;
 static u32 g_gfx_ProducerSlotsRequested = 0;
 static u8 *g_gfxFramebuf;
 static size_t g_gfxFramebufSize;
-static BqQueueBufferOutput g_gfx_Connect_QueueBufferOutput;
+static BqBufferOutput g_gfx_Connect_QueueBufferOutput;
-static BqQueueBufferOutput g_gfx_QueueBuffer_QueueBufferOutput;
+static BqBufferOutput g_gfx_QueueBuffer_QueueBufferOutput;
 
 static GfxMode g_gfxMode = GfxMode_LinearDouble;
 
@@ -55,7 +56,7 @@ static NvMap g_nvmap_obj;
 //static Result _gfxGetDisplayResolution(u64 *width, u64 *height);
 
 // TODO: Let the user configure some of this?
-static BqQueueBufferInput g_gfxQueueBufferData = {
+static BqBufferInput g_gfxQueueBufferData = {
     .timestamp = 0x0,
     .isAutoTimestamp = 0x1,
     .crop = {0x0, 0x0, 0x0, 0x0}, //Official apps which use multiple resolutions configure this for the currently used resolution, depending on the current appletOperationMode.
@@ -67,37 +68,33 @@ static BqQueueBufferInput g_gfxQueueBufferData = {
     .fence = {0},
 };
 
-// Some of this struct is based on tegra_dc_ext_flip_windowattr.
+static NvGraphicBuffer g_gfx_GraphicBuffer = {
-static BqGraphicBuffer g_gfx_BufferInitData = {
+    .header = {
-    .magic = 0x47424652,//"RFBG"/'GBFR'
+        .num_ints = (sizeof(NvGraphicBuffer) - sizeof(NativeHandle)) / 4,
-    .format = 0x1,
+    },
+    .unk0 = -1,
+    .magic = 0xDAFFCAFF,
+    .pid = 42,
     .usage = GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_TEXTURE,
-
+    .format = PIXEL_FORMAT_RGBA_8888,
-    .pid = 0x2a, //Official sw sets this to the output of "getpid()", which calls a func which is hard-coded for returning 0x2a.
+    .ext_format = PIXEL_FORMAT_RGBA_8888,
-    .refcount = 0x0,  //Official sw sets this to the output of "android_atomic_inc()".
+    .num_planes = 1,
-
+    .layers = {
-    .numFds = 0x0,
+        {
-    .numInts = sizeof(g_gfx_BufferInitData.data)>>2,//0x51
+            .color_format = 0x100532120UL, // this is 'A8B8G8R8' according to symbols in official sw
-
+            .layout = NvLayout_BlockLinear,
-    .data = {
+            .kind = NvKind_Generic_16BX2,
-        .unk_x0 = 0xffffffff,
+            .block_height_log2 = 4, // i.e. block height is 16 which is the preferred value according to TRM
-        .unk_x8 = 0x0,
+        }
-        .unk_xc = 0xdaffcaff,
-        .unk_x10 = 0x2a,
-        .unk_x14 = 0,
-        .unk_x18 = 0xb00,
-        .unk_x1c = 0x1,
-        .unk_x20 = 0x1,
-        .unk_x2c = 0x1,
-        .unk_x30 = 0,
-        .flags = 0x532120,
-        .unk_x40 = 0x1,
-        .unk_x44 = 0x3,
-        .unk_x54 = 0xfe,
-        .unk_x58 = 0x4,
     }
 };
 
+static BqGraphicBuffer g_gfx_BufferInitData = {
+    .format = PIXEL_FORMAT_RGBA_8888,
+    .usage = GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_TEXTURE,
+    .native_handle = &g_gfx_GraphicBuffer.header,
+};
+
 static Result _gfxDequeueBuffer(void) {
     if (g_gfxCurrentProducerBuffer >= 0)
         return 0;
@@ -206,14 +203,13 @@ Result gfxInitDefault(void) {
     g_gfx_BufferInitData.height = g_gfx_framebuf_height;
     g_gfx_BufferInitData.stride = g_gfx_framebuf_aligned_width;
 
-    g_gfx_BufferInitData.data.width_unk0 = g_gfx_framebuf_width;
+    g_gfx_GraphicBuffer.stride = g_gfx_framebuf_width;
-    g_gfx_BufferInitData.data.width_unk1 = g_gfx_framebuf_width;
+    g_gfx_GraphicBuffer.total_size = g_gfx_singleframebuf_size;
-    g_gfx_BufferInitData.data.height_unk = g_gfx_framebuf_height;
 
-    g_gfx_BufferInitData.data.byte_stride = g_gfx_framebuf_aligned_width*4;
+    g_gfx_GraphicBuffer.layers[0].width = g_gfx_framebuf_width;
-
+    g_gfx_GraphicBuffer.layers[0].height = g_gfx_framebuf_height;
-    g_gfx_BufferInitData.data.buffer_size0 = g_gfx_singleframebuf_size;
+    g_gfx_GraphicBuffer.layers[0].pitch = g_gfx_framebuf_aligned_width*4;
-    g_gfx_BufferInitData.data.buffer_size1 = g_gfx_singleframebuf_size;
+    g_gfx_GraphicBuffer.layers[0].size = g_gfx_singleframebuf_size;
 
     g_gfxFramebufLinear = memalign(0x1000, g_gfx_singleframebuf_linear_size);
     if (g_gfxFramebufLinear) {
@@ -263,12 +259,9 @@ Result gfxInitDefault(void) {
     if (R_SUCCEEDED(rc)) rc = nvMapCreate(&g_nvmap_obj, g_gfxFramebuf, g_gfxFramebufSize, 0x20000, NvKind_Pitch, true);
 
     if (R_SUCCEEDED(rc)) {
+        g_gfx_GraphicBuffer.nvmap_id = nvMapGetId(&g_nvmap_obj);
         for (s32 i = 0; i < g_nvgfx_totalframebufs; i ++) {
-            g_gfx_BufferInitData.refcount = i;
+            g_gfx_GraphicBuffer.layers[0].offset = g_gfx_singleframebuf_size*i;
-            g_gfx_BufferInitData.data.nvmap_handle0 = nvMapGetId(&g_nvmap_obj);
-            g_gfx_BufferInitData.data.nvmap_handle1 = nvMapGetHandle(&g_nvmap_obj);
-            g_gfx_BufferInitData.data.buffer_offset = g_gfx_singleframebuf_size*i;
-            //g_gfx_BufferInitData.data.timestamp = svcGetSystemTick();
             rc = bqSetPreallocatedBuffer(&g_gfxBinderSession, i, &g_gfx_BufferInitData);
             if (R_FAILED(rc))
                 break;