Add generic IPC mechanism between Lua envs

doc/lua_api.md

@@ -6855,17 +6855,6 @@ This allows you easy interoperability for delegating work to jobs.
     * Register a path to a Lua file to be imported when an async environment
       is initialized. You can use this to preload code which you can then call
       later using `minetest.handle_async()`.
-* `minetest.register_portable_metatable(name, mt)`:
-    * Register a metatable that should be preserved when data is transferred
-    between the main thread and the async environment.
-    * `name` is a string that identifies the metatable. It is recommended to
-      follow the `modname:name` convention for this identifier.
-    * `mt` is the metatable to register.
-    * Note that it is allowed to register the same metatable under multiple
-      names, but it is not allowed to register multiple metatables under the
-      same name.
-    * You must register the metatable in both the main environment
-      and the async environment for this mechanism to work.
 
 
 ### List of APIs available in an async environment
@@ -6895,7 +6884,8 @@ Functions:
 
 * Standalone helpers such as logging, filesystem, encoding,
   hashing or compression APIs
-* `minetest.register_portable_metatable` (see above)
+* `minetest.register_portable_metatable`
+* IPC
 
 Variables:
 
@@ -6973,6 +6963,7 @@ Functions:
 * `minetest.get_node`, `set_node`, `find_node_near`, `find_nodes_in_area`,
   `spawn_tree` and similar
     * these only operate on the current chunk (if inside a callback)
+* IPC
 
 Variables:
 
@@ -7050,6 +7041,31 @@ Server
       this can make transfer of bigger files painless (if set up). Nevertheless
       it is advised not to use dynamic media for big media files.
 
+IPC
+---
+
+The engine provides a generalized mechanism to enable sharing data between the
+different Lua environments (main, mapgen and async).
+It is essentially a shared in-memory key-value store.
+
+* `minetest.ipc_get(key)`:
+  * Read a value from the shared data area.
+  * `key`: string, should use the `"modname:thing"` convention to avoid conflicts.
+  * returns an arbitrary Lua value, or `nil` if this key does not exist
+* `minetest.ipc_set(key, value)`:
+  * Write a value to the shared data area.
+  * `key`: as above
+  * `value`: an arbitrary Lua value, cannot be or contain userdata.
+
+Interacting with the shared data will perform an operation comparable to
+(de)serialization on each access.
+For that reason modifying references will not have any effect, as in this example:
+```lua
+minetest.ipc_set("test:foo", {})
+minetest.ipc_get("test:foo").subkey = "value" -- WRONG!
+minetest.ipc_get("test:foo") -- returns an empty table
+```
+
 Bans
 ----
 
@@ -7449,6 +7465,17 @@ Misc.
 * `minetest.global_exists(name)`
     * Checks if a global variable has been set, without triggering a warning.
 
+* `minetest.register_portable_metatable(name, mt)`:
+    * Register a metatable that should be preserved when Lua data is transferred
+      between environments (via IPC or `handle_async`).
+    * `name` is a string that identifies the metatable. It is recommended to
+      follow the `modname:name` convention for this identifier.
+    * `mt` is the metatable to register.
+    * Note that the same metatable can be registered under multiple names,
+      but multiple metatables must not be registered under the same name.
+    * You must register the metatable in both the main environment
+      and the async environment for this mechanism to work.
+
 Global objects
 --------------
 

games/devtest/mods/unittests/inside_mapgen_env.lua

@@ -22,9 +22,8 @@ local function do_tests()
 	assert(core.registered_items["unittests:description_test"].on_place == true)
 end
 
--- there's no (usable) communcation path between mapgen and the regular env
+-- this is checked from the main env
--- so we just run the test unconditionally
+core.ipc_set("unittests:mg", { pcall(do_tests) })
-do_tests()
 
 core.register_on_generated(function(vm, pos1, pos2, blockseed)
 	local n = tonumber(core.get_mapgen_setting("chunksize")) * 16 - 1

games/devtest/mods/unittests/misc.lua

@@ -254,3 +254,28 @@ local function test_gennotify_api()
 	assert(#custom == 0, "custom ids not empty")
 end
 unittests.register("test_gennotify_api", test_gennotify_api)
+
+-- <=> inside_mapgen_env.lua
+local function test_mapgen_env(cb)
+	-- emerge threads start delayed so this can take a second
+	local res = core.ipc_get("unittests:mg")
+	if res == nil then
+		return core.after(0, test_mapgen_env, cb)
+	end
+	-- handle error status
+	if res[1] then
+		cb()
+	else
+		cb(res[2])
+	end
+end
+unittests.register("test_mapgen_env", test_mapgen_env, {async=true})
+
+local function test_ipc_vector_preserve(cb)
+	-- the IPC also uses register_portable_metatable
+	core.ipc_set("unittests:v", vector.new(4, 0, 4))
+	local v = core.ipc_get("unittests:v")
+	assert(type(v) == "table")
+	assert(vector.check(v))
+end
+unittests.register("test_ipc_vector_preserve", test_ipc_vector_preserve)

src/gamedef.h

@@ -34,19 +34,19 @@ class Camera;
 class ModChannel;
 class ModStorage;
 class ModStorageDatabase;
+struct SubgameSpec;
+struct ModSpec;
+struct ModIPCStore;
 
 namespace irr::scene {
 	class IAnimatedMesh;
 	class ISceneManager;
 }
 
-struct SubgameSpec;
-struct ModSpec;
 /*
 	An interface for fetching game-global definitions like tool and
 	mapnode properties
 */
-
 class IGameDef
 {
 public:
@@ -63,6 +63,9 @@ public:
 	// environment thread.
 	virtual IRollbackManager* getRollbackManager() { return NULL; }
 
+	// Only usable on server.
+	virtual ModIPCStore *getModIPCStore() { return nullptr; }
+
 	// Shorthands
 	// TODO: these should be made const-safe so that a const IGameDef* is
 	//       actually usable

src/script/cpp_api/s_async.cpp

@@ -50,11 +50,12 @@ AsyncEngine::~AsyncEngine()
 	}
 
 	// Wait for threads to finish
+	infostream << "AsyncEngine: Waiting for " << workerThreads.size()
+		<< " threads" << std::endl;
 	for (AsyncWorkerThread *workerThread : workerThreads) {
 		workerThread->wait();
 	}
 
-	// Force kill all threads
 	for (AsyncWorkerThread *workerThread : workerThreads) {
 		delete workerThread;
 	}

src/script/lua_api/CMakeLists.txt

@@ -6,6 +6,7 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_env.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_http.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_inventory.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/l_ipc.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_item.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_itemstackmeta.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_mapgen.cpp

src/script/lua_api/l_ipc.cpp

@@ -0,0 +1,68 @@
+// Minetest
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "lua_api/l_ipc.h"
+#include "lua_api/l_internal.h"
+#include "common/c_packer.h"
+#include "server.h"
+#include "debug.h"
+
+typedef std::shared_lock<std::shared_mutex> SharedReadLock;
+typedef std::unique_lock<std::shared_mutex> SharedWriteLock;
+
+int ModApiIPC::l_ipc_get(lua_State *L)
+{
+	auto *store = getGameDef(L)->getModIPCStore();
+
+	auto key = readParam<std::string>(L, 1);
+
+	{
+		SharedReadLock autolock(store->mutex);
+		auto it = store->map.find(key);
+		if (it == store->map.end())
+			lua_pushnil(L);
+		else
+			script_unpack(L, it->second.get());
+	}
+	return 1;
+}
+
+int ModApiIPC::l_ipc_set(lua_State *L)
+{
+	auto *store = getGameDef(L)->getModIPCStore();
+
+	auto key = readParam<std::string>(L, 1);
+
+	luaL_checkany(L, 2);
+	std::unique_ptr<PackedValue> pv;
+	if (!lua_isnil(L, 2)) {
+		pv.reset(script_pack(L, 2));
+		if (pv->contains_userdata)
+			throw LuaError("Userdata not allowed");
+	}
+
+	{
+		SharedWriteLock autolock(store->mutex);
+		if (pv)
+			store->map[key] = std::move(pv);
+		else
+			store->map.erase(key); // delete the map value for nil
+	}
+	return 0;
+}
+
+/*
+ * Implementation note:
+ * Iterating over the IPC table is intentionally not supported.
+ * Mods should know what they have set.
+ * This has the nice side effect that mods are able to use a randomly generated key
+ * if they really *really* want to avoid other code touching their data.
+ */
+
+void ModApiIPC::Initialize(lua_State *L, int top)
+{
+	FATAL_ERROR_IF(!getGameDef(L)->getModIPCStore(), "ModIPCStore missing from gamedef");
+
+	API_FCT(ipc_get);
+	API_FCT(ipc_set);
+}

src/script/lua_api/l_ipc.h

@@ -0,0 +1,15 @@
+// Minetest
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#pragma once
+
+#include "lua_api/l_base.h"
+
+class ModApiIPC : public ModApiBase {
+private:
+	static int l_ipc_get(lua_State *L);
+	static int l_ipc_set(lua_State *L);
+
+public:
+	static void Initialize(lua_State *L, int top);
+};

src/script/scripting_emerge.cpp

@@ -35,6 +35,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_util.h"
 #include "lua_api/l_vmanip.h"
 #include "lua_api/l_settings.h"
+#include "lua_api/l_ipc.h"
 
 extern "C" {
 #include <lualib.h>
@@ -89,5 +90,6 @@ void EmergeScripting::InitializeModApi(lua_State *L, int top)
 	ModApiMapgen::InitializeEmerge(L, top);
 	ModApiServer::InitializeAsync(L, top);
 	ModApiUtil::InitializeAsync(L, top);
+	ModApiIPC::Initialize(L, top);
 	// TODO ^ these should also be renamed to InitializeRO or such
 }

src/script/scripting_server.cpp

@@ -46,6 +46,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_settings.h"
 #include "lua_api/l_http.h"
 #include "lua_api/l_storage.h"
+#include "lua_api/l_ipc.h"
 
 extern "C" {
 #include <lualib.h>
@@ -121,6 +122,7 @@ void ServerScripting::initAsync()
 	asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync);
 	asyncEngine.registerStateInitializer(ModApiItem::InitializeAsync);
 	asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiIPC::Initialize);
 	// not added: ModApiMapgen is a minefield for thread safety
 	// not added: ModApiHttp async api can't really work together with our jobs
 	// not added: ModApiStorage is probably not thread safe(?)
@@ -176,6 +178,7 @@ void ServerScripting::InitializeModApi(lua_State *L, int top)
 	ModApiHttp::Initialize(L, top);
 	ModApiStorage::Initialize(L, top);
 	ModApiChannels::Initialize(L, top);
+	ModApiIPC::Initialize(L, top);
 }
 
 void ServerScripting::InitializeAsync(lua_State *L, int top)

src/server.cpp

@@ -86,6 +86,15 @@ public:
 	{}
 };
 
+ModIPCStore::~ModIPCStore()
+{
+	// we don't have to do this, it's pure debugging aid
+	if (!std::unique_lock(mutex, std::try_to_lock).owns_lock()) {
+		errorstream << FUNCTION_NAME << ": lock is still in use!" << std::endl;
+		assert(0);
+	}
+}
+
 class ServerThread : public Thread
 {
 public:

src/server.h

@@ -47,6 +47,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <unordered_set>
 #include <optional>
 #include <string_view>
+#include <shared_mutex>
 
 class ChatEvent;
 struct ChatEventChat;
@@ -142,6 +143,20 @@ struct ClientInfo {
 	std::string vers_string, lang_code;
 };
 
+struct ModIPCStore {
+	ModIPCStore() = default;
+	~ModIPCStore();
+
+	/// RW lock for this entire structure
+	std::shared_mutex mutex;
+	/**
+	 * Map storing the data
+	 *
+	 * @note Do not store `nil` data in this map, instead remove the whole key.
+	 */
+	std::unordered_map<std::string, std::unique_ptr<PackedValue>> map;
+};
+
 class Server : public con::PeerHandler, public MapEventReceiver,
 		public IGameDef
 {
@@ -301,12 +316,14 @@ public:
 	NodeDefManager* getWritableNodeDefManager();
 	IWritableCraftDefManager* getWritableCraftDefManager();
 
+	// Not under envlock
 	virtual const std::vector<ModSpec> &getMods() const;
 	virtual const ModSpec* getModSpec(const std::string &modname) const;
 	virtual const SubgameSpec* getGameSpec() const { return &m_gamespec; }
 	static std::string getBuiltinLuaPath();
 	virtual std::string getWorldPath() const { return m_path_world; }
 	virtual std::string getModDataPath() const { return m_path_mod_data; }
+	virtual ModIPCStore *getModIPCStore() { return &m_ipcstore; }
 
 	inline bool isSingleplayer() const
 			{ return m_simple_singleplayer_mode; }
@@ -666,6 +683,8 @@ private:
 
 	std::unordered_map<std::string, Translations> server_translations;
 
+	ModIPCStore m_ipcstore;
+
 	/*
 		Threads
 	*/