/**************************************************************************/
/* render_raytracing.h */
/**************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/**************************************************************************/
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/**************************************************************************/
#pragma once
#include "core/math/transform_3d.h"
#include "core/string/string_name.h"
#include "core/templates/hash_map.h"
#include "core/templates/local_vector.h"
#include "core/templates/vector.h"
#include "servers/rendering/renderer_rd/bindless_block.h"
#include "servers/rendering/rendering_device.h"
#include "servers/rendering/shader_language.h"
#define RB_TEX_RAYTRACING SNAME("raytracing")
#define RB_TEX_RT_DEPTH SNAME("rt_depth")
#define RB_SCOPE_DLSS_RR SNAME("dlss_rr")
#define RB_TEX_DLSS_RR_DIFFUSE_ALBEDO SNAME("diffuse_albedo")
#define RB_TEX_DLSS_RR_SPECULAR_ALBEDO SNAME("specular_albedo")
#define RB_TEX_DLSS_RR_NORMAL_ROUGHNESS SNAME("normal_roughness")
#define RB_TEX_DLSS_RR_SPECULAR_HIT_DIST SNAME("specular_hit_dist")
class RenderDataRD;
class RenderSceneBuffersRD;
namespace RendererSceneRenderImplementation {
class RenderForwardClustered;
class SceneShaderRaytracing;
// Must match GLSL GeometryData (std430, 128 bytes).
struct alignas(16) RT_GeometryData {
uint64_t vertex_buffer_address;
uint64_t attribute_buffer_address;
uint64_t index_buffer_address;
uint32_t vertex_count;
uint32_t position_stride;
uint32_t normal_byte_offset;
uint32_t normal_stride;
uint32_t tangent_byte_offset;
uint32_t tangent_stride;
uint32_t attribute_stride;
uint32_t uv_byte_offset;
uint32_t uv_scale_packed;
uint32_t index_format;
uint32_t primitive_count;
uint32_t flags;
float aabb_size_x;
float aabb_size_y;
float aabb_size_z;
uint32_t color_byte_offset;
float aabb_pos_x;
float aabb_pos_y;
float aabb_pos_z;
// For deformed geometry: previous-frame position buffer used for motion vectors.
uint32_t prev_vertex_buffer_address_lo;
uint32_t prev_vertex_buffer_address_hi;
// CUSTOM0-3 attribute location, one packed word each:
// bits 0-23 byte offset inside attribute_stride
// bits 24-31 ARRAY_CUSTOM_* format
// RT_OFFSET_NONE (all bits set) when the surface lacks that custom array --
// unambiguous because format 0xFF is not a valid ARRAY_CUSTOM_* value.
uint32_t custom_packed[4];
uint32_t _pad[1];
};
static_assert(sizeof(RT_GeometryData) == 128, "RT_GeometryData must be 128 bytes for std430");
/// Per-instance motion data for velocity computation (matches GLSL InstanceMotionData, 48 bytes).
struct RT_InstanceMotionData {
float prev_object_to_world[12]; // Previous object-to-world (mat3x4, transposed 3x4).
};
static_assert(sizeof(RT_InstanceMotionData) == 48, "RT_InstanceMotionData must be 48 bytes");
// Must match GLSL MaterialData (std430, 96 bytes).
struct alignas(16) RT_MaterialData {
uint32_t albedo_texture_idx;
uint32_t normal_texture_idx;
uint32_t orm_texture_idx;
uint32_t emission_texture_idx;
float albedo_color[4];
float emission_color[3];
float emission_strength;
float metallic;
float roughness;
float ao_strength;
uint32_t flags;
float uv1_scale[2];
float uv1_offset[2];
float normal_map_depth; // Strength [0..N], default 1.0 (not Z-depth).
float specular; // Dielectric specular [0..1], default 0.5 -> F0 = 0.04.
uint64_t uniform_address; // BDA for custom shader uniform buffer (0 = none).
};
static_assert(sizeof(RT_MaterialData) == 96, "RT_MaterialData must be 96 bytes for std430");
// Light types for raytracing (matches GLSL RT_LIGHT_TYPE_* defines).
enum RTLightType : uint32_t {
RT_LIGHT_TYPE_OMNI = 0,
RT_LIGHT_TYPE_DIRECTIONAL = 1,
RT_LIGHT_TYPE_SPOT = 3,
};
// Must match GLSL RTLightData (std430, 80 bytes).
struct alignas(16) RT_LightData {
float position[3]; // World position (omni/spot) or direction (directional, normalized).
uint32_t type;
float emission[3];
float radius;
float attenuation;
float inv_max_range; // 1.0/range, or -1.0 for infinite.
float max_range_squared; // range*range, or 0.0 for infinite.
float specular_amount;
float indirect_energy;
float inv_spot_attenuation;
float cos_spot_angle;
float _pad0;
float spot_direction[3];
float _pad1;
};
static_assert(sizeof(RT_LightData) == 80, "RT_LightData must be 80 bytes for std430");
enum {
RT_LIGHTS_MAX = 64,
RT_LIGHTS_FRUSTUM_BUDGET = 48,
RT_LIGHTS_INDIRECT_BUDGET = RT_LIGHTS_MAX - RT_LIGHTS_FRUSTUM_BUDGET,
};
enum {
RT_OFFSET_NONE = 0xFFFFFFFFu,
RT_CACHE_CHUNK_SIZE = 256,
RT_CACHE_CHUNK_SHIFT = 8,
RT_CACHE_CHUNK_MASK = 255,
};
// Material flags for RT (matches GLSL mat_flags bit layout).
enum {
RT_MAT_FLAG_HAS_NORMAL_MAP = 1u,
RT_MAT_FLAG_HAS_EMISSION_TEX = 2u,
RT_MAT_FLAG_POINT_FILTER = 4u,
};
// Index format for RT geometry (matches GLSL fetch_indices).
enum {
RT_INDEX_FORMAT_UINT16 = 0,
RT_INDEX_FORMAT_UINT32 = 1,
RT_INDEX_FORMAT_NONE = 2,
};
enum {
RT_GEOM_FLAG_COMPRESSED = 1u,
RT_GEOM_FLAG_PROCEDURAL = 2u,
// Set when the BLAS uses a per-frame-deformed vertex buffer.
RT_GEOM_FLAG_DEFORMED = 4u,
};
/// Per-instance state for procedural RT geometry. Heap-allocated, only exists for procedural instances.
struct RTProceduralState {
AABB culling_aabb;
PackedFloat32Array aabb_data; // N * 6 floats (min/max per AABB). Empty = single AABB.
bool expose_bounds = false;
bool dirty = true;
RID blas;
RID gpu_buffer;
uint32_t gpu_buffer_capacity = 0; // Bytes, grow-only.
uint64_t gpu_buffer_address = 0; // BDA (0 = not exposed).
uint32_t aabb_count = 0;
};
struct RTSurfaceData {
RID blas;
RT_GeometryData geometry = {};
Transform3D aabb_transform;
bool is_compressed = false;
uint64_t blas_size = 0;
};
/// Inputs for a surface backed by a per-frame-deformed vertex buffer.
struct RTDeformedGeometrySource {
RID current_vb; ///< Vertex buffer (object-space positions) the BLAS is built / refit against.
RID prev_vb; ///< Previous-frame positions for motion vectors. Optional.
uint64_t change_stamp = 0; ///< Changes when `current_vb` contents change.
uint64_t cache_key = 0; ///< Caller-defined cache identity.
uint32_t cache_version = 0; ///< Changes when the resource behind `cache_key` is recycled.
uint32_t surface_counter = 0; ///< Changes when the underlying mesh surface changes.
};
struct RTMaterialData {
alignas(16) RT_MaterialData data = {};
uint32_t global_buffer_index = UINT32_MAX;
uint32_t rt_sbt_offset = 0;
bool is_custom_shader = false;
RID uniform_buffer; // Buffer pointer for mats > 512 bytes.
uint32_t uniform_pool_slot = UINT32_MAX; // Index into the material UBO pool, or UINT32_MAX (unused)
RID albedo_texture_rd;
RID normal_texture_rd;
RID orm_texture_rd;
RID emission_texture_rd;
};
struct RTCacheEntry {
RTSurfaceData *ptr = nullptr;
uint32_t last_used_frame = 0;
uint32_t cached_counter = 0;
uint32_t cached_rid_version = 0;
uint8_t failed_attempts = 0;
uint64_t size_bytes = 0;
};
struct RTMaterialCacheEntry {
RTMaterialData *ptr = nullptr;
uint32_t last_used_frame = 0;
uint16_t cached_counter = 0;
uint32_t cached_rid_version = 0;
};
/// Per-viewport raytracing state.
///
/// Each viewport has its own visibility set (frustum/LOD/visibility ranges), so
/// the TLAS instance composition and per-instance SSBO contents differ across
/// viewports. Sharing them caused the wrong `gl_InstanceCustomIndexEXT` to
/// resolve to the wrong `geometries[]` / `materials[]` entries, dereferencing
/// stale BDAs and faulting the GPU.
///
/// Lifetime is tied to a `RenderSceneBuffersRD`: created lazily on first
/// `build_tlas` for that viewport, freed via `RenderRaytracing::free_viewport_state`
/// from `RenderBufferDataForwardClustered::free_data()`.
struct RTViewportState {
RID tlas;
uint32_t tlas_max_instances = 0;
RID geometry_buffer;
uint32_t geometry_buffer_capacity = 0;
RID material_buffer;
uint32_t material_buffer_capacity = 0;
RID motion_index_buffer;
uint32_t motion_index_buffer_capacity = 0;
RID motion_transform_buffer;
uint32_t motion_transform_buffer_capacity = 0;
RID light_buffer;
RID params_buffer;
RID uniform_set;
// RIDs the live uniform_set was built from. The set is only rebuilt when one
// of them actually changes; buffers whose CONTENTS change (params, lights)
// keep the same RID and so need no new descriptor set.
LocalVector<RID> uniform_set_rids;
uint32_t frame_counter = 0;
};
class RenderRaytracing {
friend class RenderForwardClustered;
RenderForwardClustered *owner = nullptr;
SceneShaderRaytracing *shader = nullptr;
// One bindless table per texture type, all sharing descriptor set 1.
BindlessBlock *bindless_blocks[BindlessBlock::TEXTURE_KIND_MAX] = {};
RID bindless_uniform_set;
// Routes a shader uniform's sampler type to the table that can serve it.
static BindlessBlock::TextureKind _texture_kind_for_type(ShaderLanguage::DataType p_type);
_FORCE_INLINE_ BindlessBlock *_bindless(BindlessBlock::TextureKind p_kind) const {
return bindless_blocks[p_kind];
}
// Caching (chunked sparse caches indexed by RID low bits / 256).
Vector<RTCacheEntry *> surface_chunks;
Vector<RTMaterialCacheEntry *> material_chunks;
// BLAS cache for surfaces driven by per-frame-deformed vertex buffers.
struct RTDeformedCacheEntry {
RTSurfaceData *ptr = nullptr;
uint32_t last_used_frame = 0;
uint64_t cached_change_stamp = 0;
uint32_t cached_key_version = 0;
uint32_t cached_surface_counter = 0;
uint64_t cached_buffer_id = 0; // RID id of the deformed vertex buffer at the time of build.
bool blas_built_once = false; // True once the BLAS has been fully built; subsequent ticks can refit.
};
HashMap<uint64_t, RTDeformedCacheEntry> deformed_surface_cache;
LocalVector<uint32_t> material_free_slots;
uint32_t next_material_slot = 0;
uint64_t vram_used = 0;
uint32_t cache_hits = 0;
uint32_t cache_misses = 0;
// Per-frame scratch arrays. Refilled per viewport's build_tlas; immediately
// consumed by build_acceleration_structures + finalize_buffers, so they
// don't need to be per-viewport.
LocalVector<RT_GeometryData> geometry_data;
LocalVector<RT_MaterialData> material_data;
LocalVector<int32_t> motion_indices; ///< Per-instance: index into motion_transforms[], or -1.
LocalVector<RT_InstanceMotionData> motion_transforms; ///< Compact: only moving instances.
LocalVector<RID> blass;
LocalVector<Transform3D> blas_transforms;
LocalVector<uint32_t> instance_flags;
LocalVector<uint32_t> sbt_offsets; // 0 = default material hit group
HashMap<RenderSceneBuffersRD *, RTViewportState *> viewport_states;
RTViewportState *_get_or_create_viewport_state(const RenderDataRD *p_render_data);
RTViewportState *_get_viewport_state(const RenderDataRD *p_render_data) const;
void _free_viewport_state_internal(RTViewportState *p_state);
// Material UBO sub-allocation pool. One large device-address buffer divided
// into fixed-size slots for performance reasons, and easier to debug.
RID mat_ubo_pool_buffer;
uint64_t mat_ubo_pool_bda = 0;
LocalVector<uint32_t> mat_ubo_pool_free_slots;
uint32_t mat_ubo_pool_free_count = 0;
uint32_t mat_ubo_pool_next_slot = 0;
void mat_ubo_pool_ensure_initialized();
uint32_t mat_ubo_pool_allocate(); // Returns slot index or UINT32_MAX if pool is exhausted.
void mat_ubo_pool_release(uint32_t p_slot);
void mat_ubo_pool_update(uint32_t p_slot, const void *p_data, uint32_t p_size);
uint64_t mat_ubo_pool_get_address(uint32_t p_slot) const;
// Cache helpers.
static uint32_t get_rid_index(RID p_rid);
static uint32_t get_rid_version(RID p_rid);
RTCacheEntry *get_surface_cache_entry(uint32_t p_index);
RTMaterialCacheEntry *get_material_cache_entry(uint32_t p_index);
uint32_t allocate_material_slot();
// Internal methods.
RTSurfaceData *process_surface(
const void *p_surf,
void *p_mesh_surface,
uint32_t p_surface_invalidation_counter,
const Transform3D &p_transform,
LocalVector<RID> &r_dirty_blas_list);
RTSurfaceData *process_deformed_surface(
const void *p_surf,
void *p_mesh_surface,
const struct RTDeformedGeometrySource &p_source,
LocalVector<RID> &r_dirty_blas_list,
LocalVector<RID> &r_dirty_blas_update_list);
void _populate_surface_blas(
void *p_mesh_surface,
RID p_vertex_buffer_override,
bool p_force_uncompressed,
bool p_prefer_fast_build,
bool p_allow_update,
uint32_t p_cache_key,
RTSurfaceData *r_surf_data,
LocalVector<RID> &r_dirty_blas_list);
RTMaterialData *process_material(RID p_material_rid, uint16_t p_material_invalidation_counter);
void update_procedural_blas(RTProceduralState *p_state, LocalVector<RID> &r_dirty_blas_list);
void build_acceleration_structures(RTViewportState *p_state, const LocalVector<RID> &p_dirty_blas_list, const LocalVector<RID> &p_dirty_blas_update_list);
void finalize_buffers(RTViewportState *p_state);
void prepare_frame();
public:
void initialize(RenderForwardClustered *p_owner);
void cleanup_caches();
void _update_bindless_uniform_set(RID p_shader_rd);
RTViewportState *build_tlas(const RenderDataRD *p_render_data, uint32_t p_rt_flags);
uint32_t gather_lights(const RenderDataRD *p_render_data, RT_LightData *r_light_data, uint32_t p_max_lights);
RID update_uniform_set(RTViewportState *p_state, const RenderDataRD *p_render_data, uint32_t p_rt_flags);
void copy_output_texture(const RenderDataRD *p_render_data);
void free_viewport_state(RenderSceneBuffersRD *p_render_buffers);
SceneShaderRaytracing *get_shader() const { return shader; }
RID get_bindless_uniform_set() const { return bindless_uniform_set; }
RID get_mat_ubo_pool_buffer() const { return mat_ubo_pool_buffer; }
~RenderRaytracing();
};
} // namespace RendererSceneRenderImplementation