Stencil shadows (cg_shadows = 2).

src/engine/renderer/tr_shadows.c

@@ -44,8 +44,10 @@ typedef struct {
 static	edgeDef_t	edgeDefs[SHADER_MAX_VERTEXES][MAX_EDGE_DEFS];
 static	int			numEdgeDefs[SHADER_MAX_VERTEXES];
 static	int			facing[SHADER_MAX_INDEXES/3];
+static vec4_t		extrudedEdges[SHADER_MAX_VERTEXES * 4];
+static int			numExtrudedEdges;
 
-void R_AddEdgeDef( int i1, int i2, int facing ) {
+static void R_AddEdgeDef( int i1, int i2, int facing ) {
 	int		c;
 
 	c = numEdgeDefs[ i1 ];
@@ -58,51 +60,18 @@ void R_AddEdgeDef( int i1, int i2, int facing ) {
 	numEdgeDefs[ i1 ]++;
 }
 
-void R_RenderShadowEdges( void ) {
+static void R_ExtrudeShadowEdges( void ) {
 	int		i;
-
-#if 0
-	int		numTris;
-
-	// dumb way -- render every triangle's edges
-	numTris = tess.numIndexes / 3;
-
-	for ( i = 0 ; i < numTris ; i++ ) {
-		int		i1, i2, i3;
-
-		if ( !facing[i] ) {
-			continue;
-		}
-
-		i1 = tess.indexes[ i*3 + 0 ];
-		i2 = tess.indexes[ i*3 + 1 ];
-		i3 = tess.indexes[ i*3 + 2 ];
-
-		qglBegin( GL_TRIANGLE_STRIP );
-		qglVertex3fv( tess.xyz[ i1 ] );
-		qglVertex3fv( tess.xyz[ i1 + tess.numVertexes ] );
-		qglVertex3fv( tess.xyz[ i2 ] );
-		qglVertex3fv( tess.xyz[ i2 + tess.numVertexes ] );
-		qglVertex3fv( tess.xyz[ i3 ] );
-		qglVertex3fv( tess.xyz[ i3 + tess.numVertexes ] );
-		qglVertex3fv( tess.xyz[ i1 ] );
-		qglVertex3fv( tess.xyz[ i1 + tess.numVertexes ] );
-		qglEnd();
-	}
-#else
 	int		c, c2;
 	int		j, k;
 	int		i2;
-	int		c_edges, c_rejected;
+
-	int		hit[2];
+	numExtrudedEdges = 0;
 
 	// an edge is NOT a silhouette edge if its face doesn't face the light,
 	// or if it has a reverse paired edge that also faces the light.
 	// A well behaved polyhedron would have exactly two faces for each edge,
 	// but lots of models have dangling edges or overfanned edges
-	c_edges = 0;
-	c_rejected = 0;
-
 	for ( i = 0 ; i < tess.numVertexes ; i++ ) {
 		c = numEdgeDefs[ i ];
 		for ( j = 0 ; j < c ; j++ ) {
@@ -110,33 +79,78 @@ void R_RenderShadowEdges( void ) {
 				continue;
 			}
 
-			hit[0] = 0;
+			bool sil_edge = true;
-			hit[1] = 0;
-
 			i2 = edgeDefs[ i ][ j ].i2;
 			c2 = numEdgeDefs[ i2 ];
 			for ( k = 0 ; k < c2 ; k++ ) {
-				if ( edgeDefs[ i2 ][ k ].i2 == i ) {
+				if ( edgeDefs[ i2 ][ k ].i2 == i && edgeDefs[ i2 ][ k ].facing) {
-					hit[ edgeDefs[ i2 ][ k ].facing ]++;
+					sil_edge = false;
+					break;
 				}
 			}
 
 			// if it doesn't share the edge with another front facing
 			// triangle, it is a sil edge
-			if ( hit[ 1 ] == 0 ) {
+			if ( sil_edge ) {
-				qglBegin( GL_TRIANGLE_STRIP );
+				VectorCopy(tess.xyz[ i ],						extrudedEdges[numExtrudedEdges * 4 + 0]);
-				qglVertex3fv( tess.xyz[ i ] );
+				VectorCopy(tess.xyz[ i + tess.numVertexes ],	extrudedEdges[numExtrudedEdges * 4 + 1]);
-				qglVertex3fv( tess.xyz[ i + tess.numVertexes ] );
+				VectorCopy(tess.xyz[ i2 ],						extrudedEdges[numExtrudedEdges * 4 + 2]);
-				qglVertex3fv( tess.xyz[ i2 ] );
+				VectorCopy(tess.xyz[ i2 + tess.numVertexes ],	extrudedEdges[numExtrudedEdges * 4 + 3]);
-				qglVertex3fv( tess.xyz[ i2 + tess.numVertexes ] );
+				numExtrudedEdges++;
+			}
+		}
+	}
+}
+
+static void R_GL_RenderShadowEdges() {
+	qglBegin( GL_QUADS);
+	for (int i = 0; i < numExtrudedEdges; i++) {
+		qglVertex3fv(extrudedEdges[i*4 + 0]);
+		qglVertex3fv(extrudedEdges[i*4 + 1]);
+		qglVertex3fv(extrudedEdges[i*4 + 3]);
+		qglVertex3fv(extrudedEdges[i*4 + 2]);
+	}
 	qglEnd();
-				c_edges++;
+}
-			} else {
+
-				c_rejected++;
+// VULKAN
+static void R_Vk_RenderShadowEdges(VkPipeline pipeline) {
+	if (!vk.active)
+		return;
+
+	int i = 0;
+	while (i < numExtrudedEdges) {
+		int count = numExtrudedEdges - i;
+		if (count > (SHADER_MAX_VERTEXES - 1) / 4)
+			count = (SHADER_MAX_VERTEXES - 1) / 4;
+
+		Com_Memcpy(tess.xyz, extrudedEdges[i*4], 4 * count * sizeof(vec4_t));
+		tess.numVertexes = count * 4;
+
+		for (int k = 0; k < count; k++) {
+			tess.indexes[k * 6 + 0] = k * 4 + 0;
+			tess.indexes[k * 6 + 1] = k * 4 + 2;
+			tess.indexes[k * 6 + 2] = k * 4 + 1;
+
+			tess.indexes[k * 6 + 3] = k * 4 + 2;
+			tess.indexes[k * 6 + 4] = k * 4 + 3;
+			tess.indexes[k * 6 + 5] = k * 4 + 1;
 		}
+		tess.numIndexes = count * 6;
+
+		for (int k = 0; k < tess.numVertexes; k++) {
+			VectorSet(tess.svars.colors[k], 50, 50, 50);
+			tess.svars.colors[k][3] = 255;
 		}
+
+		vk_bind_resources_shared_between_stages();
+		vk_bind_stage_specific_resources(pipeline, false, false);
+		vkCmdDrawIndexed(vk.command_buffer, tess.numIndexes, 1, 0, 0, 0);
+		vk_resources.dirty_attachments = true;
+		vk.xyz_elements += tess.numVertexes;
+
+		i += count;
 	}
-#endif
 }
 
 /*
@@ -220,27 +234,33 @@ void RB_ShadowTessEnd( void ) {
 	qglEnable( GL_STENCIL_TEST );
 	qglStencilFunc( GL_ALWAYS, 1, 255 );
 
+	R_ExtrudeShadowEdges();
+
 	// mirrors have the culling order reversed
 	if ( backEnd.viewParms.isMirror ) {
 		qglCullFace( GL_FRONT );
 		qglStencilOp( GL_KEEP, GL_KEEP, GL_INCR );
-
+		R_GL_RenderShadowEdges();
-		R_RenderShadowEdges();
 
 		qglCullFace( GL_BACK );
 		qglStencilOp( GL_KEEP, GL_KEEP, GL_DECR );
+		R_GL_RenderShadowEdges();
 
-		R_RenderShadowEdges();
+		// VULKAN
+		R_Vk_RenderShadowEdges(vk.shadow_volume_pipelines[0][1]);
+		R_Vk_RenderShadowEdges(vk.shadow_volume_pipelines[1][1]);
 	} else {
 		qglCullFace( GL_BACK );
 		qglStencilOp( GL_KEEP, GL_KEEP, GL_INCR );
-
+		R_GL_RenderShadowEdges();
-		R_RenderShadowEdges();
 
 		qglCullFace( GL_FRONT );
 		qglStencilOp( GL_KEEP, GL_KEEP, GL_DECR );
+		R_GL_RenderShadowEdges();
 
-		R_RenderShadowEdges();
+		// VULKAN
+		R_Vk_RenderShadowEdges(vk.shadow_volume_pipelines[0][0]);
+		R_Vk_RenderShadowEdges(vk.shadow_volume_pipelines[1][0]);
 	}
 
 
@@ -288,6 +308,48 @@ void RB_ShadowFinish( void ) {
 
 	qglColor3f(1,1,1);
 	qglDisable( GL_STENCIL_TEST );
+
+	// VULKAN
+	if (vk.active) {
+		tess.indexes[0] = 0;
+		tess.indexes[1] = 1;
+		tess.indexes[2] = 2;
+		tess.indexes[3] = 0;
+		tess.indexes[4] = 2;
+		tess.indexes[5] = 3;
+		tess.numIndexes = 6;
+
+		VectorSet(tess.xyz[0], -100,  100, -10);
+		VectorSet(tess.xyz[1],  100,  100, -10);
+		VectorSet(tess.xyz[2],  100, -100, -10);
+		VectorSet(tess.xyz[3], -100, -100, -10);
+		for (int i = 0; i < 4; i++) {
+			VectorSet(tess.svars.colors[i], 153, 153, 153);
+			tess.svars.colors[i][3] = 255;
+		}
+		tess.numVertexes = 4;
+
+		// set backEnd.or.modelMatrix to identity matrix
+		float tmp[16];
+		Com_Memcpy(tmp, backEnd.or.modelMatrix, 64);
+		Com_Memset(backEnd.or.modelMatrix, 0, 64);
+		backEnd.or.modelMatrix[0] = 1.0f;
+		backEnd.or.modelMatrix[5] = 1.0f;
+		backEnd.or.modelMatrix[10] = 1.0f;
+		backEnd.or.modelMatrix[15] = 1.0f;
+
+		vk_bind_resources_shared_between_stages();
+
+		Com_Memcpy(backEnd.or.modelMatrix, tmp, 64);
+
+		vk_bind_stage_specific_resources(vk.shadow_finish_pipeline, false, false);
+		vkCmdDrawIndexed(vk.command_buffer, tess.numIndexes, 1, 0, 0, 0);
+		vk_resources.dirty_attachments = true;
+		vk.xyz_elements += tess.numVertexes;
+
+		tess.numIndexes = 0;
+		tess.numVertexes = 0;
+	}
 }
 
 

src/engine/renderer/vk.cpp

@@ -1089,10 +1089,44 @@ void vk_initialize() {
             def.polygon_offset = false;
             def.clipping_plane = false;
             def.mirror = false;
-
             vk.skybox_pipeline = create_pipeline(def);
         }
 
+		// Q3 stencil shadows
+		{
+			{
+				Vk_Pipeline_Def def;
+				def.polygon_offset = false;
+				def.state_bits = 0;
+				def.shader_type = Vk_Shader_Type::single_texture;
+				def.clipping_plane = false;
+				def.shadow_phase = Vk_Shadow_Phase::shadow_edges_rendering;
+
+				cullType_t cull_types[2] = {CT_FRONT_SIDED, CT_BACK_SIDED};
+				bool mirror_flags[2] = {false, true};
+
+				for (int i = 0; i < 2; i++) {
+					def.face_culling = cull_types[i];
+					for (int j = 0; j < 2; j++) {
+						def.mirror = mirror_flags[j];
+						vk.shadow_volume_pipelines[i][j] = create_pipeline(def);
+					}
+				}
+			}
+
+			{
+				Vk_Pipeline_Def def;
+				def.face_culling = CT_FRONT_SIDED;
+				def.polygon_offset = false;
+				def.state_bits = GLS_DEPTHMASK_TRUE | GLS_SRCBLEND_DST_COLOR | GLS_DSTBLEND_ZERO;
+				def.shader_type = Vk_Shader_Type::single_texture;
+				def.clipping_plane = false;
+				def.mirror = false;
+				def.shadow_phase = Vk_Shadow_Phase::fullscreen_quad_rendering;
+				vk.shadow_finish_pipeline = create_pipeline(def);
+			}
+		}
+
         // fog and dlights
         {
             Vk_Pipeline_Def def;
@@ -1170,6 +1204,13 @@ void vk_shutdown() {
     vkDestroyShaderModule(vk.device, vk.multi_texture_add_fs, nullptr);
 
     vkDestroyPipeline(vk.device, vk.skybox_pipeline, nullptr);
+
+	for (int i = 0; i < 2; i++)
+		for (int j = 0; j < 2; j++) {
+			vkDestroyPipeline(vk.device, vk.shadow_volume_pipelines[i][j], nullptr);
+		}
+	vkDestroyPipeline(vk.device, vk.shadow_finish_pipeline, nullptr);
+
     for (int i = 0; i < 2; i++)
         for (int j = 0; j < 3; j++)
             for (int k = 0; k < 2; k++) {
@@ -1581,14 +1622,44 @@ static VkPipeline create_pipeline(const Vk_Pipeline_Def& def) {
     depth_stencil_state.depthWriteEnable = (def.state_bits & GLS_DEPTHMASK_TRUE) ? VK_TRUE : VK_FALSE;
     depth_stencil_state.depthCompareOp = (def.state_bits & GLS_DEPTHFUNC_EQUAL) ? VK_COMPARE_OP_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
     depth_stencil_state.depthBoundsTestEnable = VK_FALSE;
-    depth_stencil_state.stencilTestEnable = VK_FALSE;
+    depth_stencil_state.stencilTestEnable = (def.shadow_phase != Vk_Shadow_Phase::disabled) ? VK_TRUE : VK_FALSE;
+
+	if (def.shadow_phase == Vk_Shadow_Phase::shadow_edges_rendering) {
+		depth_stencil_state.front.failOp = VK_STENCIL_OP_KEEP;
+		depth_stencil_state.front.passOp = (def.face_culling == CT_FRONT_SIDED) ? VK_STENCIL_OP_INCREMENT_AND_CLAMP : VK_STENCIL_OP_DECREMENT_AND_CLAMP;
+		depth_stencil_state.front.depthFailOp = VK_STENCIL_OP_KEEP;
+		depth_stencil_state.front.compareOp = VK_COMPARE_OP_ALWAYS;
+		depth_stencil_state.front.compareMask = 255;
+		depth_stencil_state.front.writeMask = 255;
+		depth_stencil_state.front.reference = 0;
+
+		depth_stencil_state.back = depth_stencil_state.front;
+
+	}  else if (def.shadow_phase == Vk_Shadow_Phase::fullscreen_quad_rendering) {
+		depth_stencil_state.front.failOp = VK_STENCIL_OP_KEEP;
+		depth_stencil_state.front.passOp = VK_STENCIL_OP_KEEP;
+		depth_stencil_state.front.depthFailOp = VK_STENCIL_OP_KEEP;
+		depth_stencil_state.front.compareOp = VK_COMPARE_OP_NOT_EQUAL;
+		depth_stencil_state.front.compareMask = 255;
+		depth_stencil_state.front.writeMask = 255;
+		depth_stencil_state.front.reference = 0;
+
+		depth_stencil_state.back = depth_stencil_state.front;
+
+	} else {
 		depth_stencil_state.front = {};
 		depth_stencil_state.back = {};
+	}
+
     depth_stencil_state.minDepthBounds = 0.0;
     depth_stencil_state.maxDepthBounds = 0.0;
 
     VkPipelineColorBlendAttachmentState attachment_blend_state = {};
     attachment_blend_state.blendEnable = (def.state_bits & (GLS_SRCBLEND_BITS | GLS_DSTBLEND_BITS)) ? VK_TRUE : VK_FALSE;
+
+	if (def.shadow_phase == Vk_Shadow_Phase::shadow_edges_rendering)
+		attachment_blend_state.colorWriteMask = 0;
+	else
 		attachment_blend_state.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
     
     if (attachment_blend_state.blendEnable) {
@@ -1813,7 +1884,8 @@ VkPipeline vk_find_pipeline(const Vk_Pipeline_Def& def) {
             cur_def.face_culling == def.face_culling &&
             cur_def.polygon_offset == def.polygon_offset &&
             cur_def.clipping_plane == def.clipping_plane &&
-            cur_def.mirror == def.mirror)
+            cur_def.mirror == def.mirror &&
+			cur_def.shadow_phase == def.shadow_phase)
         {
             return vk_resources.pipelines[i];
         }

src/engine/renderer/vk.h

@@ -28,6 +28,13 @@ enum class Vk_Shader_Type {
     multi_texture_add
 };
 
+// used with cg_shadows == 2
+enum class Vk_Shadow_Phase {
+	disabled,
+	shadow_edges_rendering,
+	fullscreen_quad_rendering
+};
+
 struct Vk_Sampler_Def {
     bool repeat_texture = false; // clamp/repeat texture addressing mode
     int gl_mag_filter = 0; // GL_XXX mag filter
@@ -41,6 +48,7 @@ struct Vk_Pipeline_Def {
     bool polygon_offset = false;
     bool clipping_plane = false;
     bool mirror = false;
+	Vk_Shadow_Phase shadow_phase = Vk_Shadow_Phase::disabled;
 };
 
 struct Vk_Image {
@@ -147,6 +155,11 @@ struct Vk_Instance {
 
     VkPipeline skybox_pipeline = VK_NULL_HANDLE;
 
+	// dim 0: 0 - front side, 1 - back size
+	// dim 1: 0 - normal view, 1 - mirror view
+	VkPipeline shadow_volume_pipelines[2][2];
+	VkPipeline shadow_finish_pipeline;
+
     // dim 0 is based on fogPass_t: 0 - corresponds to FP_EQUAL, 1 - corresponds to FP_LE.
     // dim 1 is directly a cullType_t enum value.
     // dim 2 is a polygon offset value (0 - off, 1 - on).