Fix #31168: cycles mask layer should only affect objects for camera rays.

Fix: texture coordinate normal output was not correct, still changed under
object transform.

intern/cycles/blender/blender_mesh.cpp

@@ -198,11 +198,11 @@ static void create_subd_mesh(Mesh *mesh, BL::Mesh b_mesh, PointerRNA *cmesh, con
 
 /* Sync */
 
-Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool holdout, bool object_updated)
+Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool object_updated)
 {
 	/* test if we can instance or if the object is modified */
 	BL::ID b_ob_data = b_ob.data();
-	BL::ID key = (object_is_modified(b_ob) || holdout)? b_ob: b_ob_data;
+	BL::ID key = (object_is_modified(b_ob))? b_ob: b_ob_data;
 	BL::Material material_override = render_layer.material_override;
 
 	/* find shader indices */
@@ -212,18 +212,14 @@ Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool holdout, bool object_updated)
 	for(b_ob.material_slots.begin(slot); slot != b_ob.material_slots.end(); ++slot) {
 		BL::Material material_override = render_layer.material_override;
 
-		if(holdout)
+		if(material_override)
-			find_shader(PointerRNA_NULL, used_shaders, scene->default_holdout);
-		else if(material_override)
 			find_shader(material_override, used_shaders, scene->default_surface);
 		else
 			find_shader(slot->material(), used_shaders, scene->default_surface);
 	}
 
 	if(used_shaders.size() == 0) {
-		if(holdout)
+		if(material_override)
-			used_shaders.push_back(scene->default_holdout);
-		else if(material_override)
 			find_shader(material_override, used_shaders, scene->default_surface);
 		else
 			used_shaders.push_back(scene->default_surface);

intern/cycles/blender/blender_object.cpp

@@ -232,11 +232,15 @@ void BlenderSync::sync_object(BL::Object b_parent, int b_index, BL::Object b_ob,
 	if(object_map.sync(&object, b_ob, b_parent, key))
 		object_updated = true;
 	
-	/* holdout? */
+	bool use_holdout = (layer_flag & render_layer.holdout_layer) != 0;
-	bool holdout = (layer_flag & render_layer.holdout_layer) != 0;
+	
-
 	/* mesh sync */
-	object->mesh = sync_mesh(b_ob, holdout, object_updated);
+	object->mesh = sync_mesh(b_ob, object_updated);
+
+	if(use_holdout != object->use_holdout) {
+		object->use_holdout = use_holdout;
+		scene->object_manager->tag_update(scene);
+	}
 
 	/* object sync */
 	if(object_updated || (object->mesh && object->mesh->need_update)) {

intern/cycles/blender/blender_sync.h

@@ -79,7 +79,7 @@ private:
 	void sync_shaders();
 
 	void sync_nodes(Shader *shader, BL::ShaderNodeTree b_ntree);
-	Mesh *sync_mesh(BL::Object b_ob, bool holdout, bool object_updated);
+	Mesh *sync_mesh(BL::Object b_ob, bool object_updated);
 	void sync_object(BL::Object b_parent, int b_index, BL::Object b_object, Transform& tfm, uint layer_flag, int motion);
 	void sync_light(BL::Object b_parent, int b_index, BL::Object b_ob, Transform& tfm);
 	void sync_background_light();

intern/cycles/kernel/kernel_camera.h

@@ -64,7 +64,7 @@ __device void camera_sample_perspective(KernelGlobals *kg, float raster_x, float
 	Transform cameratoworld = kernel_data.cam.cameratoworld;
 
 #ifdef __MOTION__
-	if(ray->time != TIME_INVALID)
+	if(kernel_data.cam.have_motion)
 		transform_motion_interpolate(&cameratoworld, &kernel_data.cam.motion, ray->time);
 #endif
 
@@ -107,7 +107,7 @@ __device void camera_sample_orthographic(KernelGlobals *kg, float raster_x, floa
 	Transform cameratoworld = kernel_data.cam.cameratoworld;
 
 #ifdef __MOTION__
-	if(ray->time != TIME_INVALID)
+	if(kernel_data.cam.have_motion)
 		transform_motion_interpolate(&cameratoworld, &kernel_data.cam.motion, ray->time);
 #endif
 
@@ -147,7 +147,7 @@ __device void camera_sample_environment(KernelGlobals *kg, float raster_x, float
 	Transform cameratoworld = kernel_data.cam.cameratoworld;
 
 #ifdef __MOTION__
-	if(ray->time != TIME_INVALID)
+	if(kernel_data.cam.have_motion)
 		transform_motion_interpolate(&cameratoworld, &kernel_data.cam.motion, ray->time);
 #endif
 

intern/cycles/kernel/kernel_object.h

@@ -32,34 +32,28 @@ __device_inline Transform object_fetch_transform(KernelGlobals *kg, int object,
 
 #ifdef __MOTION__
 	/* if we do motion blur */
-	if(time != TIME_INVALID) {
+	if(sd->flag & SD_OBJECT_MOTION) {
-		int offset = object*OBJECT_SIZE + (int)OBJECT_TRANSFORM_MOTION_PRE;
+		/* fetch motion transforms */
-		float4 have_motion = kernel_tex_fetch(__objects, offset + 0);
+		MotionTransform motion;
 
-		/* if this object have motion */
+		motion.pre.x = have_motion;
-		if(have_motion.x != FLT_MAX) {
+		motion.pre.y = kernel_tex_fetch(__objects, offset + 1);
-			/* fetch motion transforms */
+		motion.pre.z = kernel_tex_fetch(__objects, offset + 2);
-			MotionTransform motion;
+		motion.pre.w = kernel_tex_fetch(__objects, offset + 3);
 
-			motion.pre.x = have_motion;
+		motion.post.x = kernel_tex_fetch(__objects, offset + 4);
-			motion.pre.y = kernel_tex_fetch(__objects, offset + 1);
+		motion.post.y = kernel_tex_fetch(__objects, offset + 5);
-			motion.pre.z = kernel_tex_fetch(__objects, offset + 2);
+		motion.post.z = kernel_tex_fetch(__objects, offset + 6);
-			motion.pre.w = kernel_tex_fetch(__objects, offset + 3);
+		motion.post.w = kernel_tex_fetch(__objects, offset + 7);
 
-			motion.post.x = kernel_tex_fetch(__objects, offset + 4);
+		/* interpolate (todo: do only once per object) */
-			motion.post.y = kernel_tex_fetch(__objects, offset + 5);
+		transform_motion_interpolate(&tfm, &motion, time);
-			motion.post.z = kernel_tex_fetch(__objects, offset + 6);
-			motion.post.w = kernel_tex_fetch(__objects, offset + 7);
 
-			/* interpolate (todo: do only once per object) */
+		/* invert */
-			transform_motion_interpolate(&tfm, &motion, time);
+		if(type == OBJECT_INVERSE_TRANSFORM)
+			tfm = transform_quick_inverse(tfm);
 
-			/* invert */
+		return tfm;
-			if(type == OBJECT_INVERSE_TRANSFORM)
-				tfm = transform_quick_inverse(tfm);
-
-			return tfm;
-		}
 	}
 #endif
 
@@ -83,6 +77,16 @@ __device_inline void object_position_transform(KernelGlobals *kg, ShaderData *sd
 #endif
 }
 
+__device_inline void object_inverse_normal_transform(KernelGlobals *kg, ShaderData *sd, float3 *N)
+{
+#ifdef __MOTION__
+	*N = normalize(transform_direction_transposed(&sd->ob_tfm, *N));
+#else
+	Transform tfm = object_fetch_transform(kg, sd->object, TIME_INVALID, OBJECT_TRANSFORM);
+	*N = normalize(transform_direction_transposed(&tfm, *N));
+#endif
+}
+
 __device_inline void object_normal_transform(KernelGlobals *kg, ShaderData *sd, float3 *N)
 {
 #ifdef __MOTION__

intern/cycles/kernel/kernel_path.h

@@ -277,12 +277,21 @@ __device float4 kernel_path_integrate(KernelGlobals *kg, RNG *rng, int sample, R
 
 		/* holdout */
 #ifdef __HOLDOUT__
-		if((sd.flag & SD_HOLDOUT) && (state.flag & PATH_RAY_CAMERA)) {
+		if((sd.flag & (SD_HOLDOUT|SD_HOLDOUT_MASK)) && (state.flag & PATH_RAY_CAMERA)) {
-			float3 holdout_weight = shader_holdout_eval(kg, &sd);
+			if(kernel_data.background.transparent) {
+				float3 holdout_weight;
+				
+				if(sd.flag & SD_HOLDOUT_MASK)
+					holdout_weight = make_float3(1.0f, 1.0f, 1.0f);
+				else
+					shader_holdout_eval(kg, &sd);
 
-			if(kernel_data.background.transparent)
 				/* any throughput is ok, should all be identical here */
 				L_transparent += average(holdout_weight*throughput);
+			}
+
+			if(sd.flag & SD_HOLDOUT_MASK)
+				break;
 		}
 #endif
 

intern/cycles/kernel/kernel_shader.h

@@ -83,6 +83,7 @@ __device_inline void shader_setup_from_ray(KernelGlobals *kg, ShaderData *sd,
 		sd->N = triangle_smooth_normal(kg, sd->prim, sd->u, sd->v);
 
 	sd->flag = kernel_tex_fetch(__shader_flag, (sd->shader & SHADER_MASK)*2);
+	sd->flag |= kernel_tex_fetch(__object_flag, sd->object);
 
 #ifdef __DPDU__
 	/* dPdu/dPdv */
@@ -177,6 +178,7 @@ __device void shader_setup_from_sample(KernelGlobals *kg, ShaderData *sd,
 	}
 
 	sd->flag = kernel_tex_fetch(__shader_flag, (sd->shader & SHADER_MASK)*2);
+	sd->flag |= kernel_tex_fetch(__object_flag, sd->object);
 
 #ifdef __DPDU__
 	/* dPdu/dPdv */

intern/cycles/kernel/kernel_textures.h

@@ -39,6 +39,7 @@ KERNEL_TEX(float2, texture_float2, __light_background_conditional_cdf)
 /* shaders */
 KERNEL_TEX(uint4, texture_uint4, __svm_nodes)
 KERNEL_TEX(uint, texture_uint, __shader_flag)
+KERNEL_TEX(uint, texture_uint, __object_flag)
 
 /* camera/film */
 KERNEL_TEX(float, texture_float, __filter_table)

intern/cycles/kernel/kernel_types.h

@@ -38,7 +38,7 @@ CCL_NAMESPACE_BEGIN
 /* device capabilities */
 #ifdef __KERNEL_CPU__
 #define __KERNEL_SHADING__
-#define __KERNEL_ADV_SHADING__
+//#define __KERNEL_ADV_SHADING__
 #endif
 
 #ifdef __KERNEL_CUDA__
@@ -370,7 +370,11 @@ enum ShaderDataFlag {
 	SD_SAMPLE_AS_LIGHT = 128,			/* direct light sample */
 	SD_HAS_SURFACE_TRANSPARENT = 256,	/* has surface transparency */
 	SD_HAS_VOLUME = 512,				/* has volume shader */
-	SD_HOMOGENEOUS_VOLUME = 1024		/* has homogeneous volume */
+	SD_HOMOGENEOUS_VOLUME = 1024,		/* has homogeneous volume */
+
+	/* object flags */
+	SD_HOLDOUT_MASK = 2048,				/* holdout for camera rays */
+	SD_OBJECT_MOTION = 4096				/* has object motion blur */
 };
 
 typedef struct ShaderData {
@@ -463,7 +467,7 @@ typedef struct KernelCamera {
 
 	/* motion blur */
 	float shuttertime;
-	float pad;
+	int have_motion;
 
 	/* clipping */
 	float nearclip;

intern/cycles/kernel/svm/svm_tex_coord.h

@@ -43,7 +43,7 @@ __device void svm_node_tex_coord(KernelGlobals *kg, ShaderData *sd, float *stack
 		case NODE_TEXCO_NORMAL: {
 			if(sd->object != ~0) {
 				data = sd->N;
-				object_normal_transform(kg, sd, &data);
+				object_inverse_normal_transform(kg, sd, &data);
 			}
 			else
 				data = sd->N;
@@ -97,7 +97,7 @@ __device void svm_node_tex_coord_bump_dx(KernelGlobals *kg, ShaderData *sd, floa
 		case NODE_TEXCO_NORMAL: {
 			if(sd->object != ~0) {
 				data = sd->N;
-				object_normal_transform(kg, sd, &data);
+				object_inverse_normal_transform(kg, sd, &data);
 			}
 			else
 				data = sd->N;
@@ -154,7 +154,7 @@ __device void svm_node_tex_coord_bump_dy(KernelGlobals *kg, ShaderData *sd, floa
 		case NODE_TEXCO_NORMAL: {
 			if(sd->object != ~0) {
 				data = sd->N;
-				object_normal_transform(kg, sd, &data);
+				object_inverse_normal_transform(kg, sd, &data);
 			}
 			else
 				data = sd->N;

intern/cycles/render/camera.cpp

@@ -149,6 +149,7 @@ void Camera::device_update(Device *device, DeviceScene *dscene, Scene *scene)
 
 	/* camera motion */
 	Scene::MotionType need_motion = scene->need_motion();
+	kcam->have_motion = 0;
 
 	if(need_motion == Scene::MOTION_PASS) {
 		if(use_motion) {
@@ -162,7 +163,10 @@ void Camera::device_update(Device *device, DeviceScene *dscene, Scene *scene)
 	}
 	else if(need_motion == Scene::MOTION_BLUR) {
 		/* todo: exact camera position will not be hit this way */
-		transform_motion_decompose(&kcam->motion, &motion);
+		if(use_motion) {
+			transform_motion_decompose(&kcam->motion, &motion);
+			kcam->have_motion = 1;
+		}
 	}
 
 	/* depth of field */

intern/cycles/render/object.cpp

@@ -41,6 +41,7 @@ Object::Object()
 	motion.pre = transform_identity();
 	motion.post = transform_identity();
 	use_motion = false;
+	use_holdout = false;
 }
 
 Object::~Object()
@@ -143,12 +144,14 @@ ObjectManager::~ObjectManager()
 void ObjectManager::device_update_transforms(Device *device, DeviceScene *dscene, Scene *scene, Progress& progress)
 {
 	float4 *objects = dscene->objects.resize(OBJECT_SIZE*scene->objects.size());
+	uint *object_flag = dscene->object_flag.resize(OBJECT_SIZE*scene->objects.size());
 	int i = 0;
 	map<Mesh*, float> surface_area_map;
 	Scene::MotionType need_motion = scene->need_motion();
 
 	foreach(Object *ob, scene->objects) {
 		Mesh *mesh = ob->mesh;
+		uint flag = 0;
 
 		/* compute transformations */
 		Transform tfm = ob->tfm;
@@ -219,6 +222,7 @@ void ObjectManager::device_update_transforms(Device *device, DeviceScene *dscene
 
 				transform_motion_decompose(&decomp, &ob->motion);
 				memcpy(&objects[offset+8], &decomp, sizeof(float4)*8);
+				flag |= SD_OBJECT_MOTION;
 			}
 			else {
 				float4 no_motion = make_float4(FLT_MAX);
@@ -226,12 +230,18 @@ void ObjectManager::device_update_transforms(Device *device, DeviceScene *dscene
 			}
 		}
 
+		/* object flag */
+		if(ob->use_holdout)
+			flag |= SD_HOLDOUT_MASK;
+		object_flag[i] = flag;
+
 		i++;
 
 		if(progress.get_cancel()) return;
 	}
 
 	device->tex_alloc("__objects", dscene->objects);
+	device->tex_alloc("__object_flag", dscene->object_flag);
 }
 
 void ObjectManager::device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress& progress)
@@ -266,6 +276,9 @@ void ObjectManager::device_free(Device *device, DeviceScene *dscene)
 {
 	device->tex_free(dscene->objects);
 	dscene->objects.clear();
+
+	device->tex_free(dscene->object_flag);
+	dscene->object_flag.clear();
 }
 
 void ObjectManager::apply_static_transforms(Scene *scene, Progress& progress)

intern/cycles/render/object.h

@@ -46,6 +46,7 @@ public:
 	uint visibility;
 	MotionTransform motion;
 	bool use_motion;
+	bool use_holdout;
 
 	Object();
 	~Object();

intern/cycles/render/scene.h

@@ -85,6 +85,7 @@ public:
 	/* shaders */
 	device_vector<uint4> svm_nodes;
 	device_vector<uint> shader_flag;
+	device_vector<uint> object_flag;
 
 	/* filter */
 	device_vector<float> filter_table;

intern/cycles/util/util_math.h

@@ -436,6 +436,11 @@ __device_inline float len(const float3 a)
 	return sqrtf(dot(a, a));
 }
 
+__device_inline float len_squared(const float3 a)
+{
+	return dot(a, a);
+}
+
 #ifndef __KERNEL_OPENCL__
 
 __device_inline float3 normalize(const float3 a)

intern/cycles/util/util_transform.h

@@ -255,12 +255,12 @@ __device_inline bool transform_uniform_scale(const Transform& tfm, float& scale)
 	Transform ttfm = transform_transpose(tfm);
 	float eps = 1e-7f; 
 	
-	float sx = len(float4_to_float3(tfm.x));
+	float sx = len_squared(float4_to_float3(tfm.x));
-	float sy = len(float4_to_float3(tfm.y));
+	float sy = len_squared(float4_to_float3(tfm.y));
-	float sz = len(float4_to_float3(tfm.z));
+	float sz = len_squared(float4_to_float3(tfm.z));
-	float stx = len(float4_to_float3(ttfm.x));
+	float stx = len_squared(float4_to_float3(ttfm.x));
-	float sty = len(float4_to_float3(ttfm.y));
+	float sty = len_squared(float4_to_float3(ttfm.y));
-	float stz = len(float4_to_float3(ttfm.z));
+	float stz = len_squared(float4_to_float3(ttfm.z));
 	
 	if(fabsf(sx - sy) < eps && fabsf(sx - sz) < eps &&
 	   fabsf(sx - stx) < eps && fabsf(sx - sty) < eps &&