Cycles: support for mask layers in render layer, this has the same effect as

assigning holdout shaders to every object in the specified layers.

http://wiki.blender.org/index.php/Doc:2.6/Manual/Render/Cycles/Passes#Layers

intern/cycles/blender/addon/ui.py

@@ -177,11 +177,13 @@ class CyclesRender_PT_layers(CyclesButtonsPanel, Panel):
 
         col = split.column()
         col.prop(scene, "layers", text="Scene")
+        col.label(text="Material:")
+        col.prop(rl, "material_override", text="")
 
         col = split.column()
         col.prop(rl, "layers", text="Layer")
-
+        col.label(text="Mask Layers:")
-        layout.separator()
+        col.prop(rl, "layers_zmask", text="")
 
         split = layout.split()
 
@@ -213,11 +215,6 @@ class CyclesRender_PT_layers(CyclesButtonsPanel, Panel):
         row.prop(rl, "use_pass_transmission_indirect", text="Indirect", toggle=True)
         row.prop(rl, "use_pass_transmission_color", text="Color", toggle=True)
 
-        layout.separator()
-
-        rl = rd.layers[0]
-        layout.prop(rl, "material_override", text="Material")
-
 
 class Cycles_PT_post_processing(CyclesButtonsPanel, Panel):
     bl_label = "Post Processing"

intern/cycles/blender/blender_mesh.cpp

@@ -222,20 +222,22 @@ static void create_subd_mesh(Mesh *mesh, BL::Mesh b_mesh, PointerRNA *cmesh, con
 
 /* Sync */
 
-Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool object_updated)
+Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool holdout, bool object_updated)
 {
 	/* test if we can instance or if the object is modified */
 	BL::ID b_ob_data = b_ob.data();
-	BL::ID key = (object_is_modified(b_ob))? b_ob: b_ob_data;
+	BL::ID key = (object_is_modified(b_ob) || holdout)? b_ob: b_ob_data;
 
 	/* find shader indices */
 	vector<uint> used_shaders;
 
 	BL::Object::material_slots_iterator slot;
 	for(b_ob.material_slots.begin(slot); slot != b_ob.material_slots.end(); ++slot) {
-		BL::Material material_override = render_layers.front().material_override;
+		BL::Material material_override = render_layer.material_override;
 
-		if(material_override)
+		if(holdout)
+			find_shader(PointerRNA_NULL, used_shaders, scene->default_holdout);
+		else if(material_override)
 			find_shader(material_override, used_shaders, scene->default_surface);
 		else
 			find_shader(slot->material(), used_shaders, scene->default_surface);

intern/cycles/blender/blender_object.cpp

@@ -208,8 +208,11 @@ void BlenderSync::sync_object(BL::Object b_parent, int b_index, BL::Object b_ob,
 	if(object_map.sync(&object, b_ob, b_parent, key))
 		object_updated = true;
 	
+	/* holdout? */
+	bool holdout = (layer_flag & render_layer.holdout_layer) != 0;
+
 	/* mesh sync */
-	object->mesh = sync_mesh(b_ob, object_updated);
+	object->mesh = sync_mesh(b_ob, holdout, object_updated);
 
 	/* object sync */
 	if(object_updated || (object->mesh && object->mesh->need_update)) {
@@ -238,7 +241,7 @@ void BlenderSync::sync_object(BL::Object b_parent, int b_index, BL::Object b_ob,
 void BlenderSync::sync_objects(BL::SpaceView3D b_v3d)
 {
 	/* layer data */
-	uint scene_layer = render_layers.front().scene_layer;
+	uint scene_layer = render_layer.scene_layer;
 	
 	/* prepare for sync */
 	light_map.pre_sync();

intern/cycles/blender/blender_sync.cpp

@@ -48,8 +48,7 @@ BlenderSync::BlenderSync(BL::BlendData b_data_, BL::Scene b_scene_, Scene *scene
   light_map(&scene_->lights),
   world_map(NULL),
   world_recalc(false),
-  experimental(false),
+  experimental(false)
-  active_layer(0)
 {
 	scene = scene_;
 	preview = preview_;
@@ -124,8 +123,8 @@ bool BlenderSync::sync_recalc()
 
 void BlenderSync::sync_data(BL::SpaceView3D b_v3d, const char *layer)
 {
-	sync_render_layers(b_v3d);
+	sync_render_layers(b_v3d, layer);
-	sync_integrator(layer);
+	sync_integrator();
 	sync_film();
 	sync_shaders();
 	sync_objects(b_v3d);
@@ -133,7 +132,7 @@ void BlenderSync::sync_data(BL::SpaceView3D b_v3d, const char *layer)
 
 /* Integrator */
 
-void BlenderSync::sync_integrator(const char *layer)
+void BlenderSync::sync_integrator()
 {
 	PointerRNA cscene = RNA_pointer_get(&b_scene.ptr, "cycles");
 
@@ -156,19 +155,7 @@ void BlenderSync::sync_integrator(const char *layer)
 	integrator->no_caustics = get_boolean(cscene, "no_caustics");
 	integrator->seed = get_int(cscene, "seed");
 
-	/* render layer */
+	integrator->layer_flag = render_layer.layer;
-	int active_layer = 0;
-
-	if(layer) {
-		for(int i = 0; i < render_layers.size(); i++) {
-			if(render_layers[i].name == layer) {
-				active_layer = i;
-				break;
-			}
-		}
-	}
-
-	integrator->layer_flag = render_layers[active_layer].layer;
 
 	if(integrator->modified(previntegrator))
 		integrator->tag_update(scene);
@@ -200,33 +187,28 @@ void BlenderSync::sync_film()
 
 /* Render Layer */
 
-void BlenderSync::sync_render_layers(BL::SpaceView3D b_v3d)
+void BlenderSync::sync_render_layers(BL::SpaceView3D b_v3d, const char *layer)
 {
-	render_layers.clear();
-
 	if(b_v3d) {
-		RenderLayerInfo rlay;
+		render_layer.scene_layer = get_layer(b_v3d.layers());
-
+		render_layer.layer = render_layer.scene_layer;
-		rlay.scene_layer = get_layer(b_v3d.layers());
+		render_layer.material_override = PointerRNA_NULL;
-		rlay.layer = rlay.scene_layer;
-		rlay.material_override = PointerRNA_NULL;
-
-		render_layers.push_back(rlay);
 	}
 	else {
 		BL::RenderSettings r = b_scene.render();
 		BL::RenderSettings::layers_iterator b_rlay;
+		bool first_layer = true;
 
 		for(r.layers.begin(b_rlay); b_rlay != r.layers.end(); ++b_rlay) {
-			/* single layer for now */
+			if((!layer && first_layer) || (layer && b_rlay->name() == layer)) {
-			RenderLayerInfo rlay;
+				render_layer.name = b_rlay->name();
+				render_layer.scene_layer = get_layer(b_scene.layers());
+				render_layer.layer = get_layer(b_rlay->layers());
+				render_layer.holdout_layer = get_layer(b_rlay->layers_zmask());
+				render_layer.material_override = b_rlay->material_override();
+			}
 
-			rlay.name = b_rlay->name();
+			first_layer = false;
-			rlay.scene_layer = get_layer(b_scene.layers());
-			rlay.layer = get_layer(b_rlay->layers());
-			rlay.material_override = b_rlay->material_override();
-
-			render_layers.push_back(rlay);
 		}
 	}
 }

intern/cycles/blender/blender_sync.h

@@ -70,14 +70,14 @@ private:
 	void sync_materials();
 	void sync_objects(BL::SpaceView3D b_v3d);
 	void sync_film();
-	void sync_integrator(const char *layer);
+	void sync_integrator();
 	void sync_view();
 	void sync_world();
-	void sync_render_layers(BL::SpaceView3D b_v3d);
+	void sync_render_layers(BL::SpaceView3D b_v3d, const char *layer);
 	void sync_shaders();
 
 	void sync_nodes(Shader *shader, BL::ShaderNodeTree b_ntree);
-	Mesh *sync_mesh(BL::Object b_ob, bool object_updated);
+	Mesh *sync_mesh(BL::Object b_ob, bool holdout, bool object_updated);
 	void sync_object(BL::Object b_parent, int b_index, BL::Object b_object, Transform& tfm, uint layer_flag);
 	void sync_light(BL::Object b_parent, int b_index, BL::Object b_ob, Transform& tfm);
 	void sync_background_light();
@@ -113,11 +113,9 @@ private:
 		string name;
 		uint scene_layer;
 		uint layer;
+		uint holdout_layer;
 		BL::Material material_override;
-	};
+	} render_layer;
-
-	vector<RenderLayerInfo> render_layers;
-	int active_layer;
 };
 
 CCL_NAMESPACE_END

intern/cycles/render/scene.h

@@ -156,6 +156,7 @@ public:
 	int default_surface;
 	int default_light;
 	int default_background;
+	int default_holdout;
 
 	/* device */
 	Device *device;

intern/cycles/render/shader.cpp

@@ -256,6 +256,22 @@ void ShaderManager::add_default(Scene *scene)
 		scene->shaders.push_back(shader);
 		scene->default_background = scene->shaders.size() - 1;
 	}
+
+	/* default holdout */
+	{
+		graph = new ShaderGraph();
+
+		closure = graph->add(new HoldoutNode());
+		out = graph->output();
+
+		graph->connect(closure->output("Holdout"), out->input("Surface"));
+
+		shader = new Shader();
+		shader->name = "default_holdout";
+		shader->graph = graph;
+		scene->shaders.push_back(shader);
+		scene->default_holdout = scene->shaders.size() - 1;
+	}
 }
 
 CCL_NAMESPACE_END