WM linking code rework.

New code loops much less, does re-open & close .blend files for each data type,
and is much much more flexible - it is also ready for id-remap & co work being done in branches.

Main idea is to store libraries & datablocks to link in a dedicated struct, in a way
that avoids too much looping, and also allows to search for a single datablock in several libraries.

Here again, no change is expected in current behavior of link/append tool, please report
if anything goes different!

source/blender/windowmanager/intern/wm_operators.c

@@ -59,10 +59,13 @@
 #include "PIL_time.h"
 
 #include "BLI_blenlib.h"
+#include "BLI_bitmap.h"
 #include "BLI_dial.h"
 #include "BLI_dynstr.h" /*for WM_operator_pystring */
+#include "BLI_linklist.h"
 #include "BLI_linklist_stack.h"
 #include "BLI_math.h"
+#include "BLI_memarena.h"
 #include "BLI_utildefines.h"
 #include "BLI_ghash.h"
 
@@ -2604,106 +2607,150 @@ static int wm_link_append_invoke(bContext *C, wmOperator *op, const wmEvent *UNU
 
 static short wm_link_append_flag(wmOperator *op)
 {
+	PropertyRNA *prop;
 	short flag = 0;
 
-	if (RNA_boolean_get(op->ptr, "autoselect")) flag |= FILE_AUTOSELECT;
+	if (RNA_boolean_get(op->ptr, "autoselect"))
-	if (RNA_boolean_get(op->ptr, "active_layer")) flag |= FILE_ACTIVELAY;
+		flag |= FILE_AUTOSELECT;
-	if (RNA_struct_find_property(op->ptr, "relative_path") && RNA_boolean_get(op->ptr, "relative_path")) flag |= FILE_RELPATH;
+	if (RNA_boolean_get(op->ptr, "active_layer"))
-	if (RNA_boolean_get(op->ptr, "link")) flag |= FILE_LINK;
+		flag |= FILE_ACTIVELAY;
-	if (RNA_boolean_get(op->ptr, "instance_groups")) flag |= FILE_GROUP_INSTANCE;
+	if ((prop = RNA_struct_find_property(op->ptr, "relative_path")) && RNA_property_boolean_get(op->ptr, prop))
+		flag |= FILE_RELPATH;
+	if (RNA_boolean_get(op->ptr, "link"))
+		flag |= FILE_LINK;
+	if (RNA_boolean_get(op->ptr, "instance_groups"))
+		flag |= FILE_GROUP_INSTANCE;
 
 	return flag;
 }
 
-/* Helper.
+typedef struct WMLinkAppendDataItem {
- *     if `name` is non-NULL, we assume a single-item link/append.
+	char *name;
- *     else if `*todo_libraries` is NULL we assume first-run.
+	BLI_bitmap *libraries;  /* All libs (from WMLinkAppendData.libraries) to try to load this ID from. */
- */
+	int idcode;
-static void wm_link_append_do_libgroup(
+
-        bContext *C, wmOperator *op, const char *root, const char *libname, char *group, char *name,
+	ID *new_id;
-        const short flag, GSet **todo_libraries)
+	void *customdata;
+} WMLinkAppendDataItem;
+
+typedef struct WMLinkAppendData {
+	LinkNodePair libraries;
+	LinkNodePair items;
+	int num_libraries;
+	int num_items;
+	short flag;
+
+	/* Internal 'private' data */
+	MemArena *memarena;
+} WMLinkAppendData;
+
+static WMLinkAppendData *wm_link_append_data_new(const int flag)
+{
+	MemArena *ma = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
+	WMLinkAppendData *lapp_data = BLI_memarena_calloc(ma, sizeof(*lapp_data));
+
+	lapp_data->flag = flag;
+	lapp_data->memarena = ma;
+
+	return lapp_data;
+}
+
+static void wm_link_append_data_free(WMLinkAppendData *lapp_data)
+{
+	BLI_memarena_free(lapp_data->memarena);
+}
+
+/* WARNING! *Never* call wm_link_append_data_library_add() after having added some items! */
+
+static void wm_link_append_data_library_add(WMLinkAppendData *lapp_data, const char *libname)
+{
+	size_t len = strlen(libname) + 1;
+	char *libpath = BLI_memarena_alloc(lapp_data->memarena, len);
+
+	BLI_strncpy(libpath, libname, len);
+	BLI_linklist_append_arena(&lapp_data->libraries, libpath, lapp_data->memarena);
+	lapp_data->num_libraries++;
+}
+
+static WMLinkAppendDataItem *wm_link_append_data_item_add(
+        WMLinkAppendData *lapp_data, const char *idname, const int idcode, void *customdata)
+{
+	WMLinkAppendDataItem *item = BLI_memarena_alloc(lapp_data->memarena, sizeof(*item));
+	size_t len = strlen(idname) + 1;
+
+	item->name = BLI_memarena_alloc(lapp_data->memarena, len);
+	BLI_strncpy(item->name, idname, len);
+	item->idcode = idcode;
+	item->libraries = BLI_BITMAP_NEW_MEMARENA(lapp_data->memarena, lapp_data->num_libraries);
+
+	item->new_id = NULL;
+	item->customdata = customdata;
+
+	BLI_linklist_append_arena(&lapp_data->items, item, lapp_data->memarena);
+	lapp_data->num_items++;
+
+	return item;
+}
+
+static void wm_link_do(
+        WMLinkAppendData *lapp_data, ReportList *reports, Main *bmain, Scene *scene, View3D *v3d)
 {
-	Main *bmain = CTX_data_main(C);
 	Main *mainl;
 	BlendHandle *bh;
 	Library *lib;
 
-	char path[FILE_MAX_LIBEXTRA], relname[FILE_MAX];
+	const int flag = lapp_data->flag;
-	int idcode;
-	const bool is_first_run = (*todo_libraries == NULL);
 
-	BLI_assert(group);
+	LinkNode *liblink, *itemlink;
-	idcode = BKE_idcode_from_name(group);
+	int lib_idx, item_idx;
 
-	bh = BLO_blendhandle_from_file(libname, op->reports);
+	BLI_assert(lapp_data->num_items && lapp_data->num_libraries);
+
+	for (lib_idx = 0, liblink = lapp_data->libraries.list; liblink; lib_idx++, liblink = liblink->next) {
+		char *libname = liblink->link;
+
+		bh = BLO_blendhandle_from_file(libname, reports);
 
 		if (bh == NULL) {
-		/* unlikely since we just browsed it, but possible
+			/* Unlikely since we just browsed it, but possible
-		 * error reports will have been made by BLO_blendhandle_from_file() */
+			 * Error reports will have been made by BLO_blendhandle_from_file() */
-		return;
+			continue;
 		}
 
 		/* here appending/linking starts */
 		mainl = BLO_library_link_begin(bmain, &bh, libname);
 		lib = mainl->curlib;
 		BLI_assert(lib);
+		UNUSED_VARS_NDEBUG(lib);
 
 		if (mainl->versionfile < 250) {
-		BKE_reportf(op->reports, RPT_WARNING,
+			BKE_reportf(reports, RPT_WARNING,
 			            "Linking or appending from a very old .blend file format (%d.%d), no animation conversion will "
 			            "be done! You may want to re-save your lib file with current Blender",
 			            mainl->versionfile, mainl->subversionfile);
 		}
 
-	if (name) {
+		/* For each lib file, we try to link all items belonging to that lib,
-		BLO_library_link_named_part_ex(mainl, &bh, name, idcode, flag, CTX_data_scene(C), CTX_wm_view3d(C));
+		 * and tag those successful to not try to load them again with the other libs. */
-	}
+		for (item_idx = 0, itemlink = lapp_data->items.list; itemlink; item_idx++, itemlink = itemlink->next) {
-	else {
+			WMLinkAppendDataItem *item = itemlink->link;
-		if (is_first_run) {
+			ID *new_id;
-			*todo_libraries = BLI_gset_new(BLI_ghashutil_strhash_p, BLI_ghashutil_strcmp, __func__);
-		}
 
-		RNA_BEGIN (op->ptr, itemptr, "files")
+			if (!BLI_BITMAP_TEST(item->libraries, lib_idx)) {
-		{
-			char curr_libname[FILE_MAX];
-			int curr_idcode;
-
-			RNA_string_get(&itemptr, "name", relname);
-
-			BLI_join_dirfile(path, sizeof(path), root, relname);
-
-			if (BLO_library_path_explode(path, curr_libname, &group, &name)) {
-				if (!group || !name) {
 				continue;
 			}
 
-				curr_idcode = BKE_idcode_from_name(group);
+			new_id = BLO_library_link_named_part_ex(mainl, &bh, item->name, item->idcode, flag, scene, v3d);
-
+			if (new_id) {
-				if ((idcode == curr_idcode) && (BLI_path_cmp(curr_libname, libname) == 0)) {
+				/* If the link is sucessful, clear item's libs 'todo' flags.
-					BLO_library_link_named_part_ex(mainl, &bh, name, idcode, flag, CTX_data_scene(C), CTX_wm_view3d(C));
+				 * This avoids trying to link same item with other libraries to come. */
-				}
+				BLI_BITMAP_SET_ALL(item->libraries, false, lapp_data->num_libraries);
-				else if (is_first_run) {
+				item->new_id = new_id;
-					BLI_join_dirfile(path, sizeof(path), curr_libname, group);
+			}
-					if (!BLI_gset_haskey(*todo_libraries, path)) {
+		}
-						BLI_gset_insert(*todo_libraries, BLI_strdup(path));
-					}
-				}
-			}
-		}
-		RNA_END;
-	}
-	BLO_library_link_end(mainl, &bh, flag, CTX_data_scene(C), CTX_wm_view3d(C));
 
+		BLO_library_link_end(mainl, &bh, flag, scene, v3d);
 		BLO_blendhandle_close(bh);
-
-	/* mark all library linked objects to be updated */
-	BKE_main_lib_objects_recalc_all(bmain);
-	IMB_colormanagement_check_file_config(bmain);
-
-	/* append, rather than linking */
-	if ((flag & FILE_LINK) == 0) {
-		BLI_assert(BLI_findindex(&bmain->library, lib) != -1);
-		BKE_library_make_local(bmain, lib, true);
 	}
 }
 
@@ -2712,13 +2759,12 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 	Main *bmain = CTX_data_main(C);
 	Scene *scene = CTX_data_scene(C);
 	PropertyRNA *prop;
+	WMLinkAppendData *lapp_data;
 	char path[FILE_MAX_LIBEXTRA], root[FILE_MAXDIR], libname[FILE_MAX], relname[FILE_MAX];
 	char *group, *name;
 	int totfiles = 0;
 	short flag;
 
-	GSet *todo_libraries = NULL;
-
 	RNA_string_get(op->ptr, "filename", relname);
 	RNA_string_get(op->ptr, "directory", root);
 
@@ -2726,15 +2772,15 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 
 	/* test if we have a valid data */
 	if (!BLO_library_path_explode(path, libname, &group, &name)) {
-		BKE_report(op->reports, RPT_ERROR, "Not a library");
+		BKE_reportf(op->reports, RPT_ERROR, "'%s': not a library", path);
 		return OPERATOR_CANCELLED;
 	}
 	else if (!group) {
-		BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
+		BKE_reportf(op->reports, RPT_ERROR, "'%s': nothing indicated", path);
 		return OPERATOR_CANCELLED;
 	}
 	else if (BLI_path_cmp(bmain->name, libname) == 0) {
-		BKE_report(op->reports, RPT_ERROR, "Cannot use current file as library");
+		BKE_reportf(op->reports, RPT_ERROR, "'%s': cannot use current file as library", path);
 		return OPERATOR_CANCELLED;
 	}
 
@@ -2744,55 +2790,112 @@ static int wm_link_append_exec(bContext *C, wmOperator *op)
 		totfiles = RNA_property_collection_length(op->ptr, prop);
 		if (totfiles == 0) {
 			if (!name) {
-				BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
+				BKE_reportf(op->reports, RPT_ERROR, "'%s': nothing indicated", path);
 				return OPERATOR_CANCELLED;
 			}
 		}
 	}
 	else if (!name) {
-		BKE_report(op->reports, RPT_ERROR, "Nothing indicated");
+		BKE_reportf(op->reports, RPT_ERROR, "'%s': nothing indicated", path);
 		return OPERATOR_CANCELLED;
 	}
 
 	flag = wm_link_append_flag(op);
 
 	/* sanity checks for flag */
-	if (scene->id.lib && (flag & FILE_GROUP_INSTANCE)) {
+	if (scene && scene->id.lib) {
-		/* TODO, user never gets this message */
+		BKE_reportf(op->reports, RPT_WARNING,
-		BKE_reportf(op->reports, RPT_WARNING, "Scene '%s' is linked, group instance disabled", scene->id.name + 2);
+		            "Scene '%s' is linked, instanciation of objects & groups is disabled", scene->id.name + 2);
 		flag &= ~FILE_GROUP_INSTANCE;
+		scene = NULL;
 	}
 
 	/* from here down, no error returns */
 
-	/* now we have or selected, or an indicated file */
+	if (scene && RNA_boolean_get(op->ptr, "autoselect")) {
-	if (RNA_boolean_get(op->ptr, "autoselect"))
 		BKE_scene_base_deselect_all(scene);
+	}
 	
 	/* tag everything, all untagged data can be made local
 	 * its also generally useful to know what is new
 	 *
-	 * take extra care BKE_main_id_flag_all(LIB_LINK_TAG, false) is called after! */
+	 * take extra care BKE_main_id_flag_all(bmain, LIB_PRE_EXISTING, false) is called after! */
-	BKE_main_id_flag_all(bmain, LIB_PRE_EXISTING, 1);
+	BKE_main_id_flag_all(bmain, LIB_PRE_EXISTING, true);
 
+	/* We define our working data...
+	 * Note that here, each item 'uses' one library, and only one. */
+	lapp_data = wm_link_append_data_new(flag);
 	if (totfiles != 0) {
-		name = NULL;
+		GHash *libraries = BLI_ghash_new(BLI_ghashutil_strhash_p, BLI_ghashutil_strcmp, __func__);
+		int lib_idx = 0;
+
+		RNA_BEGIN (op->ptr, itemptr, "files")
+		{
+			RNA_string_get(&itemptr, "name", relname);
+
+			BLI_join_dirfile(path, sizeof(path), root, relname);
+
+			if (BLO_library_path_explode(path, libname, &group, &name)) {
+				if (!group || !name) {
+					continue;
 				}
 
-	wm_link_append_do_libgroup(C, op, root, libname, group, name, flag, &todo_libraries);
+				if (!BLI_ghash_haskey(libraries, libname)) {
+					BLI_ghash_insert(libraries, BLI_strdup(libname), SET_INT_IN_POINTER(lib_idx));
+					lib_idx++;
+					wm_link_append_data_library_add(lapp_data, libname);
+				}
+			}
+		}
+		RNA_END;
 
-	if (todo_libraries) {
+		RNA_BEGIN (op->ptr, itemptr, "files")
-		GSetIterator libs_it;
+		{
+			RNA_string_get(&itemptr, "name", relname);
 
-		GSET_ITER(libs_it, todo_libraries) {
+			BLI_join_dirfile(path, sizeof(path), root, relname);
-			char *libpath = (char *)BLI_gsetIterator_getKey(&libs_it);
 
-			BLO_library_path_explode(libpath, libname, &group, NULL);
+			if (BLO_library_path_explode(path, libname, &group, &name)) {
-
+				WMLinkAppendDataItem *item;
-			wm_link_append_do_libgroup(C, op, root, libname, group, NULL, flag, &todo_libraries);
+				if (!group || !name) {
+					printf("skipping %s\n", path);
+					continue;
 				}
 
-		BLI_gset_free(todo_libraries, MEM_freeN);
+				lib_idx = GET_INT_FROM_POINTER(BLI_ghash_lookup(libraries, libname));
+
+				item = wm_link_append_data_item_add(lapp_data, name, BKE_idcode_from_name(group), NULL);
+				BLI_BITMAP_ENABLE(item->libraries, lib_idx);
+			}
+		}
+		RNA_END;
+
+		BLI_ghash_free(libraries, MEM_freeN, NULL);
+	}
+	else {
+		WMLinkAppendDataItem *item;
+
+		wm_link_append_data_library_add(lapp_data, libname);
+		item = wm_link_append_data_item_add(lapp_data, name, BKE_idcode_from_name(group), NULL);
+		BLI_BITMAP_ENABLE(item->libraries, 0);
+	}
+
+	/* XXX We'd need re-entrant locking on Main for this to work... */
+	/* BKE_main_lock(bmain); */
+
+	wm_link_do(lapp_data, op->reports, bmain, scene, CTX_wm_view3d(C));
+
+	/* BKE_main_unlock(bmain); */
+
+	wm_link_append_data_free(lapp_data);
+
+	/* mark all library linked objects to be updated */
+	BKE_main_lib_objects_recalc_all(bmain);
+	IMB_colormanagement_check_file_config(bmain);
+
+	/* append, rather than linking */
+	if ((flag & FILE_LINK) == 0) {
+		BKE_library_make_local(bmain, NULL, true);
 	}
 
 	/* important we unset, otherwise these object wont