force the first stage of subsampling on luxels that are hit through an alphashadow surface

tools/quake3/q3map2/light_trace.c

@@ -1512,6 +1512,9 @@ qboolean TraceTriangle( traceInfo_t *ti, traceTriangle_t *tt, trace_t *trace )
 		return qtrue;
 	}
 
+	/* force subsampling because the lighting is texture dependent */
+	trace->forceSubsampling = qtrue;
+	
 	/* try to avoid double shadows near triangle seams */
 	if( u < -ASLF_EPSILON || u > (1.0f + ASLF_EPSILON) ||
 		v < -ASLF_EPSILON || (u + v) > (1.0f + ASLF_EPSILON) )
@@ -1778,6 +1781,7 @@ sets up certain trace values
 
 float SetupTrace( trace_t *trace )
 {
+	trace->forceSubsampling = qfalse;
 	VectorSubtract( trace->end, trace->origin, trace->displacement );
 	trace->distance = VectorNormalize( trace->displacement, trace->direction );
 	VectorCopy( trace->origin, trace->hit );

tools/quake3/q3map2/light_ydnar.c

@@ -1884,6 +1884,7 @@ void IlluminateRawLightmap( int rawLightmapNum )
 	qboolean			filterColor, filterDir;
 	float				brightness;
 	float				*origin, *normal, *dirt, *luxel, *luxel2, *deluxel, *deluxel2;
+	unsigned char			*flag;
 	float				*lightLuxels, *lightLuxel, samples, filterRadius, weight;
 	vec3_t				color, averageColor, averageDir, total, temp, temp2;
 	float				tests[ 4 ][ 2 ] = { { 0.0f, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } };
@@ -2078,6 +2079,27 @@ void IlluminateRawLightmap( int rawLightmapNum )
 			memset( lightLuxels, 0, llSize );
 			totalLighted = 0;
 			
+			/* determine filter radius */
+			filterRadius = lm->filterRadius > trace.light->filterRadius
+				? lm->filterRadius
+				: trace.light->filterRadius;
+			if( filterRadius < 0.0f )
+				filterRadius = 0.0f;
+			
+			/* set luxel filter radius */
+			luxelFilterRadius = superSample * filterRadius / lm->sampleSize;
+			if( luxelFilterRadius == 0 && (filterRadius > 0.0f || filter) )
+				luxelFilterRadius = 1;
+
+			/* allocate sampling flags storage */
+			if(lightSamples > 1 && luxelFilterRadius == 0)
+			{
+				size = lm->sw * lm->sh * SUPER_LUXEL_SIZE * sizeof( unsigned char );
+				if(lm->superFlags == NULL)
+					lm->superFlags = safe_malloc( size );
+				memset( (void *) lm->superFlags, 0, size );
+			}
+
 			/* initial pass, one sample per luxel */
 			for( y = 0; y < lm->sh; y++ )
 			{
@@ -2093,6 +2115,7 @@ void IlluminateRawLightmap( int rawLightmapNum )
 					deluxel = SUPER_DELUXEL( x, y );
 					origin = SUPER_ORIGIN( x, y );
 					normal = SUPER_NORMAL( x, y );
+					flag = SUPER_FLAG( x, y );
 
 #if 0
 					////////// 27's temp hack for testing edge clipping ////
@@ -2121,8 +2144,14 @@ void IlluminateRawLightmap( int rawLightmapNum )
 						if( deluxemap )
 							VectorAdd( deluxel, trace.directionContribution, deluxel );
 
+						/* check for evilness */
+						if(trace.forceSubsampling && lightSamples > 1 && luxelFilterRadius == 0)
+						{
+							totalLighted++;
+							*flag |= FLAG_FORCE_SUBSAMPLING; /* force */
+						}
 						/* add to count */
-						if( trace.color[ 0 ] || trace.color[ 1 ] || trace.color[ 2 ] )
+						else if( trace.color[ 0 ] || trace.color[ 1 ] || trace.color[ 2 ] )
 							totalLighted++;
 					}
 				}
@@ -2132,18 +2161,6 @@ void IlluminateRawLightmap( int rawLightmapNum )
 			if( totalLighted == 0 )
 				continue;
 			
-			/* determine filter radius */
-			filterRadius = lm->filterRadius > trace.light->filterRadius
-				? lm->filterRadius
-				: trace.light->filterRadius;
-			if( filterRadius < 0.0f )
-				filterRadius = 0.0f;
-			
-			/* set luxel filter radius */
-			luxelFilterRadius = superSample * filterRadius / lm->sampleSize;
-			if( luxelFilterRadius == 0 && (filterRadius > 0.0f || filter) )
-				luxelFilterRadius = 1;
-			
 			/* secondary pass, adaptive supersampling (fixme: use a contrast function to determine if subsampling is necessary) */
 			/* 2003-09-27: changed it so filtering disamples supersampling, as it would waste time */
 			if( lightSamples > 1 && luxelFilterRadius == 0 )
@@ -2172,6 +2189,14 @@ void IlluminateRawLightmap( int rawLightmapNum )
 							mapped++;
 							
 							/* get luxel */
+							flag = SUPER_FLAG( sx, sy );
+							if(*flag & FLAG_FORCE_SUBSAMPLING)
+							{
+								/* force a lighted/mapped discrepancy so we subsample */
+								++lighted;
+								++mapped;
+								++mapped;
+							}
 							lightLuxel = LIGHT_LUXEL( sx, sy );
 							VectorAdd( total, lightLuxel, total );
 							if( (lightLuxel[ 0 ] + lightLuxel[ 1 ] + lightLuxel[ 2 ]) > 0.0f )
@@ -2195,6 +2220,9 @@ void IlluminateRawLightmap( int rawLightmapNum )
 								cluster = SUPER_CLUSTER( sx, sy );
 								if( *cluster < 0 )
 									continue;
+								flag = SUPER_FLAG( sx, sy );
+								if(*flag & FLAG_ALREADY_SUBSAMPLED) // already subsampled
+									continue;
 								lightLuxel = LIGHT_LUXEL( sx, sy );
 								origin = SUPER_ORIGIN( sx, sy );
 								
@@ -2205,6 +2233,8 @@ void IlluminateRawLightmap( int rawLightmapNum )
 								/* subsample it */
 								SubsampleRawLuxel_r( lm, &trace, origin, sx, sy, 0.25f * lightSamplesSearchBoxSize, lightLuxel );
 
+								*flag |= FLAG_ALREADY_SUBSAMPLED;
+								
 								/* debug code to colorize subsampled areas to yellow */
 								//%	luxel = SUPER_LUXEL( lightmapNum, sx, sy );
 								//%	VectorSet( luxel, 255, 204, 0 );

tools/quake3/q3map2/q3map2.h

@@ -268,6 +268,9 @@ constants
 #define BSP_LUXEL_SIZE			3
 #define RAD_LUXEL_SIZE			3
 #define SUPER_LUXEL_SIZE		4
+#define SUPER_FLAG_SIZE			4
+#define FLAG_FORCE_SUBSAMPLING 1
+#define FLAG_ALREADY_SUBSAMPLED 2
 #define SUPER_ORIGIN_SIZE		3
 #define SUPER_NORMAL_SIZE		4
 #define SUPER_DELUXEL_SIZE		3
@@ -279,6 +282,7 @@ constants
 #define BSP_LUXEL( s, x, y )	(lm->bspLuxels[ s ] + ((((y) * lm->w) + (x)) * BSP_LUXEL_SIZE))
 #define RAD_LUXEL( s, x, y )	(lm->radLuxels[ s ] + ((((y) * lm->w) + (x)) * RAD_LUXEL_SIZE))
 #define SUPER_LUXEL( s, x, y )	(lm->superLuxels[ s ] + ((((y) * lm->sw) + (x)) * SUPER_LUXEL_SIZE))
+#define SUPER_FLAG( x, y )	(lm->superFlags + ((((y) * lm->sw) + (x)) * SUPER_FLAG_SIZE))
 #define SUPER_DELUXEL( x, y )	(lm->superDeluxels + ((((y) * lm->sw) + (x)) * SUPER_DELUXEL_SIZE))
 #define BSP_DELUXEL( x, y )		(lm->bspDeluxels + ((((y) * lm->w) + (x)) * BSP_DELUXEL_SIZE))
 #define SUPER_CLUSTER( x, y )	(lm->superClusters + (((y) * lm->sw) + (x)))
@@ -1359,6 +1363,7 @@ typedef struct
 	int					compileFlags;	/* for determining surface compile flags traced through */
 	qboolean			passSolid;
 	qboolean			opaque;
+	qboolean			forceSubsampling; /* needs subsampling (alphashadow) */
 	
 	/* working data */
 	int					numTestNodes;
@@ -1450,6 +1455,7 @@ typedef struct rawLightmap_s
 	float					*bspLuxels[ MAX_LIGHTMAPS ];
 	float					*radLuxels[ MAX_LIGHTMAPS ];
 	float					*superLuxels[ MAX_LIGHTMAPS ];
+	unsigned char				*superFlags;
 	float					*superOrigins;
 	float					*superNormals;
 	int						*superClusters;