I’ve attempted to implement a simple demo, however, it only draw the all-time classic black image onto the window.
main.c:
#define _GNU_SOURCE
#include <assert.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>
#include <SDL3/SDL.h>
#include <gst/gst.h>
#include <gst/app/app.h>
#include <gst/vulkan/vulkan.h>
typedef struct {
GMutex mutex;
VkInstance instance;
VkPhysicalDevice physical_device;
VkDevice device;
GMainLoop *loop;
GQueue queue;
GstPipeline *pipeline;
SDL_Window *window;
SDL_Renderer *renderer;
SDL_Texture *texture;
GstVulkanImageMemory *vk_memory;
} AppData;
static void end_stream_cb(GstBus *bus, GstMessage *msg, AppData *appdata) {
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS:
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "End of stream");
g_main_loop_quit(appdata->loop);
break;
default:
break;
}
}
static SDL_Renderer *create_renderer(AppData *appdata) {
SDL_PropertiesID props = SDL_CreateProperties();
SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_WINDOW_POINTER, appdata->window);
SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_VULKAN_INSTANCE_POINTER, appdata->instance);
SDL_SetPointerProperty(props, SDL_PROP_RENDERER_CREATE_VULKAN_PHYSICAL_DEVICE_POINTER, appdata->physical_device);
SDL_Renderer *renderer = SDL_CreateRendererWithProperties(props);
SDL_DestroyProperties(props);
if (renderer == NULL) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to create renderer: %s", SDL_GetError());
} else {
SDL_ShowWindow(appdata->window);
}
return renderer;
}
static SDL_Texture *create_texture(AppData *appdata) {
SDL_PropertiesID props = SDL_CreateProperties();
SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_WIDTH_NUMBER, gst_vulkan_image_memory_get_width(appdata->vk_memory));
SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_HEIGHT_NUMBER, gst_vulkan_image_memory_get_height(appdata->vk_memory));
SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, SDL_PIXELFORMAT_ABGR8888);
SDL_SetNumberProperty(props, SDL_PROP_TEXTURE_CREATE_VULKAN_TEXTURE_NUMBER, (Sint64) appdata->vk_memory->image);
SDL_Texture *texture = SDL_CreateTextureWithProperties(appdata->renderer, props);
SDL_DestroyProperties(props);
if (texture == NULL) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to create texture: %s", SDL_GetError());
}
return texture;
}
static const char *vk_phys_dev_types[] = {
[VK_PHYSICAL_DEVICE_TYPE_OTHER] = "WTF",
[VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU] = "iGPU",
[VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU] = "dGPU",
[VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU] = "vGPU",
[VK_PHYSICAL_DEVICE_TYPE_CPU] = "CPU",
};
static void sync_bus_call(GstBus *bus, GstMessage *msg, gpointer data) {
AppData *appdata = data;
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_NEED_CONTEXT:
const gchar *context_type = NULL;
gst_message_parse_context_type(msg, &context_type);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Requested context type %s", context_type);
break;
case GST_MESSAGE_HAVE_CONTEXT:
GstContext *context;
gst_message_parse_have_context(msg, &context);
context_type = gst_context_get_context_type(context);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Have context type %s", context_type);
if (g_strcmp0(context_type, GST_VULKAN_INSTANCE_CONTEXT_TYPE_STR) == 0) {
GstVulkanInstance *instance = NULL;
gst_context_get_vulkan_instance(context, &instance);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkInstance: %p", instance->instance);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevices: %p", instance->physical_devices);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] n_physical_devices: %u", instance->n_physical_devices);
assert(instance->n_physical_devices > 0);
VkPhysicalDeviceProperties props = {};
vkGetPhysicalDeviceProperties(instance->physical_devices[0], &props);
assert(props.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU);
for (size_t i = 0; i < instance->n_physical_devices; i++) {
VkPhysicalDeviceProperties props = {};
vkGetPhysicalDeviceProperties(instance->physical_devices[i], &props);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevice[%zu]: name %s", i, props.deviceName);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[GST] VkPhysicalDevice[%zu]: type %s", i, vk_phys_dev_types[props.deviceType]);
}
appdata->instance = instance->instance;
appdata->physical_device = instance->physical_devices[0];
appdata->renderer = create_renderer(appdata);
}
gst_context_unref (context);
break;
default:
break;
}
}
#define NANOS_PER_SEC (1000*1000*1000)
uint64_t nanos_since_unspecified_epoch(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return NANOS_PER_SEC * ts.tv_sec + ts.tv_nsec;
}
#define check(what, msg) \
if (!(what)) { \
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, msg ": %s", SDL_GetError()); \
g_main_loop_quit(appdata->loop); \
return G_SOURCE_REMOVE; \
}
static gboolean handle_input_and_draw_unlocked(AppData *appdata) {
char THREAD_NAME[128] = {}; pthread_getname_np(pthread_self(), THREAD_NAME, sizeof(THREAD_NAME));
if (appdata->renderer == NULL) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "GST VK is not initialized yet...");
return G_SOURCE_CONTINUE;
}
SDL_Event event;
while (SDL_PollEvent(&event)) {
if (event.type == SDL_EVENT_QUIT) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "SDL_EVENT_QUIT");
// gst_element_send_event(GST_ELEMENT(appdata->pipeline), gst_event_new_eos()); // this hangs
g_main_loop_quit(appdata->loop);
return G_SOURCE_REMOVE;
}
}
GstVulkanImageMemory *next_memory = g_queue_pop_head(&appdata->queue);
if (next_memory != NULL) {
if (appdata->texture != NULL) {
SDL_DestroyTexture(appdata->texture);
appdata->texture = NULL;
}
if (appdata->vk_memory != NULL) {
gst_memory_unref(GST_MEMORY_CAST(appdata->vk_memory));
appdata->vk_memory = NULL;
}
appdata->vk_memory = next_memory;
appdata->texture = create_texture(appdata);
} else if (appdata->texture == NULL) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[%s] Empty queue and no texture yet", THREAD_NAME);
}
if (appdata->texture == NULL) {
check(SDL_SetRenderDrawColor(appdata->renderer, 0xFF, 0x00, 0x00, 0xFF), "Failed to set color");
check(SDL_RenderClear(appdata->renderer), "Failed to clear with color");
} else {
check(SDL_RenderTexture(appdata->renderer, appdata->texture, NULL, NULL), "Failed to render texture");
}
check(SDL_RenderPresent(appdata->renderer), "Failed to present");
static uint64_t prev = 0;
if (prev != 0) {
uint64_t curr = nanos_since_unspecified_epoch();
double diff = (curr - prev) / (double) NANOS_PER_SEC;
prev = curr;
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "[%s] Time between draw: %f s / %f FPS", THREAD_NAME, diff, 1 / diff);
} else {
prev = nanos_since_unspecified_epoch();
}
return G_SOURCE_CONTINUE;
}
#undef check
static gboolean handle_input_and_draw(gpointer data) {
AppData *appdata = data;
g_mutex_lock(&appdata->mutex);
gboolean result = handle_input_and_draw_unlocked(appdata);
g_mutex_unlock(&appdata->mutex);
return result;
}
GstFlowReturn new_sample_cb(GstElement *object, gpointer data) {
AppData *appdata = (AppData *) data;
GstSample *sample = gst_app_sink_pull_sample(GST_APP_SINK(object));
if (sample == NULL) {
return GST_FLOW_EOS;
}
GstBuffer *buffer = gst_sample_get_buffer(sample);
gchar *caps = gst_caps_to_string(gst_sample_get_caps(sample));
g_free(caps);
// TODO: get GstVulkanImageMemory->device and maybe initialize window here
// or how else would it choose the proper GPU?
// Or better yet, make GStreamer use the SDL VkInstance... There is no such API yet though.
g_mutex_lock(&appdata->mutex);
assert(gst_buffer_n_memory(buffer) == 1);
GstMemory *memory = gst_buffer_peek_memory(buffer, 0);
assert(gst_is_vulkan_image_memory(memory));
g_queue_push_tail(&appdata->queue, gst_memory_ref(memory));
// handle_input_and_draw_unlocked(appdata); // must not call this here as it needs to be in the main thread
g_mutex_unlock(&appdata->mutex);
gst_sample_unref(sample);
return GST_FLOW_OK;
}
#define check(what, msg) \
if (!(what)) { \
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, msg ": %s", SDL_GetError()); \
return 1; \
}
int main(int argc, char **argv) {
check(SDL_Init(SDL_INIT_VIDEO), "Failed to initialize SDL");
SDL_Window *window = SDL_CreateWindow("SDL GStreamer Vulkan Demo", 1280, 800, SDL_WINDOW_RESIZABLE | SDL_WINDOW_VULKAN | SDL_WINDOW_HIDDEN);
check(window != NULL, "Failed to create SDL Vulkan window");
gst_init (&argc, &argv);
GMainLoop *loop = g_main_loop_new(NULL, FALSE);
GError *error = NULL;
GstPipeline *pipeline = GST_PIPELINE(gst_parse_launch("videotestsrc ! video/x-raw,format=NV12,width=1280,height=800,framerate=60/1 ! vulkanupload ! vulkancolorconvert ! video/x-raw(memory:VulkanImage),format=RGBA ! appsink name=vksink emit-signals=true", &error));
assert(pipeline != NULL && error == NULL);
AppData appdata = {
.loop = loop,
.pipeline = pipeline,
.window = window,
.queue = G_QUEUE_INIT,
};
g_mutex_init(&appdata.mutex);
GstAppSink *vksink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(pipeline), "vksink"));
g_signal_connect(vksink, "new-sample", G_CALLBACK(new_sample_cb), &appdata);
GstBus *bus = gst_pipeline_get_bus(pipeline);
gst_bus_add_signal_watch(bus);
g_signal_connect(bus, "message::eos", G_CALLBACK(end_stream_cb), &appdata);
gst_bus_enable_sync_message_emission(bus);
g_signal_connect(bus, "sync-message", G_CALLBACK(sync_bus_call), &appdata);
assert(pipeline != NULL);
gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PAUSED);
gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_PLAYING);
g_timeout_add(10, handle_input_and_draw, &appdata);
g_main_loop_run(loop);
gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL);
gst_object_unref(pipeline);
SDL_DestroyRenderer(appdata.renderer);
SDL_DestroyWindow(window);
SDL_Quit();
}
Makefile:
CC?=gcc
AR?=ar
CXX?=g++
INCLUDES=
DEFINES=
LIBS=
PKGCONF=gstreamer-1.0 \
gstreamer-app-1.0 \
gstreamer-vulkan-1.0 \
vulkan \
sdl3
CFLAGS=-ggdb -fPIC -O3 $(addprefix -I,$(INCLUDES)) $(addprefix -D,$(DEFINES)) $(shell pkg-config --cflags $(PKGCONF))
LDFLAGS=$(addprefix -l,$(LIBS)) $(shell pkg-config --libs $(PKGCONF))
all: build/main
MAIN_EXE_OBJS=$(addprefix build/,main.o)
build/main: $(MAIN_EXE_OBJS) | build
$(CC) $(LDFLAGS) -o $@ $^
build/%.o: %.c | build
$(CC) $(CFLAGS) $^ -c -o $@
build:
mkdir -pv $@
Any ideas what could be wrong?