diff --git a/doc/lua_api.md b/doc/lua_api.md index a78afc847..eb3c111b9 100644 --- a/doc/lua_api.md +++ b/doc/lua_api.md @@ -6855,17 +6855,6 @@ This allows you easy interoperability for delegating work to jobs. * Register a path to a Lua file to be imported when an async environment is initialized. You can use this to preload code which you can then call later using `minetest.handle_async()`. -* `minetest.register_portable_metatable(name, mt)`: - * Register a metatable that should be preserved when data is transferred - between the main thread and the async environment. - * `name` is a string that identifies the metatable. It is recommended to - follow the `modname:name` convention for this identifier. - * `mt` is the metatable to register. - * Note that it is allowed to register the same metatable under multiple - names, but it is not allowed to register multiple metatables under the - same name. - * You must register the metatable in both the main environment - and the async environment for this mechanism to work. ### List of APIs available in an async environment @@ -6895,7 +6884,8 @@ Functions: * Standalone helpers such as logging, filesystem, encoding, hashing or compression APIs -* `minetest.register_portable_metatable` (see above) +* `minetest.register_portable_metatable` +* IPC Variables: @@ -6973,6 +6963,7 @@ Functions: * `minetest.get_node`, `set_node`, `find_node_near`, `find_nodes_in_area`, `spawn_tree` and similar * these only operate on the current chunk (if inside a callback) +* IPC Variables: @@ -7050,6 +7041,31 @@ Server this can make transfer of bigger files painless (if set up). Nevertheless it is advised not to use dynamic media for big media files. +IPC +--- + +The engine provides a generalized mechanism to enable sharing data between the +different Lua environments (main, mapgen and async). +It is essentially a shared in-memory key-value store. + +* `minetest.ipc_get(key)`: + * Read a value from the shared data area. + * `key`: string, should use the `"modname:thing"` convention to avoid conflicts. + * returns an arbitrary Lua value, or `nil` if this key does not exist +* `minetest.ipc_set(key, value)`: + * Write a value to the shared data area. + * `key`: as above + * `value`: an arbitrary Lua value, cannot be or contain userdata. + +Interacting with the shared data will perform an operation comparable to +(de)serialization on each access. +For that reason modifying references will not have any effect, as in this example: +```lua +minetest.ipc_set("test:foo", {}) +minetest.ipc_get("test:foo").subkey = "value" -- WRONG! +minetest.ipc_get("test:foo") -- returns an empty table +``` + Bans ---- @@ -7449,6 +7465,17 @@ Misc. * `minetest.global_exists(name)` * Checks if a global variable has been set, without triggering a warning. +* `minetest.register_portable_metatable(name, mt)`: + * Register a metatable that should be preserved when Lua data is transferred + between environments (via IPC or `handle_async`). + * `name` is a string that identifies the metatable. It is recommended to + follow the `modname:name` convention for this identifier. + * `mt` is the metatable to register. + * Note that the same metatable can be registered under multiple names, + but multiple metatables must not be registered under the same name. + * You must register the metatable in both the main environment + and the async environment for this mechanism to work. + Global objects -------------- diff --git a/games/devtest/mods/unittests/inside_mapgen_env.lua b/games/devtest/mods/unittests/inside_mapgen_env.lua index a8df004de..021a4a44c 100644 --- a/games/devtest/mods/unittests/inside_mapgen_env.lua +++ b/games/devtest/mods/unittests/inside_mapgen_env.lua @@ -22,9 +22,8 @@ local function do_tests() assert(core.registered_items["unittests:description_test"].on_place == true) end --- there's no (usable) communcation path between mapgen and the regular env --- so we just run the test unconditionally -do_tests() +-- this is checked from the main env +core.ipc_set("unittests:mg", { pcall(do_tests) }) core.register_on_generated(function(vm, pos1, pos2, blockseed) local n = tonumber(core.get_mapgen_setting("chunksize")) * 16 - 1 diff --git a/games/devtest/mods/unittests/misc.lua b/games/devtest/mods/unittests/misc.lua index 6ff5c7e84..1b39708d9 100644 --- a/games/devtest/mods/unittests/misc.lua +++ b/games/devtest/mods/unittests/misc.lua @@ -254,3 +254,28 @@ local function test_gennotify_api() assert(#custom == 0, "custom ids not empty") end unittests.register("test_gennotify_api", test_gennotify_api) + +-- <=> inside_mapgen_env.lua +local function test_mapgen_env(cb) + -- emerge threads start delayed so this can take a second + local res = core.ipc_get("unittests:mg") + if res == nil then + return core.after(0, test_mapgen_env, cb) + end + -- handle error status + if res[1] then + cb() + else + cb(res[2]) + end +end +unittests.register("test_mapgen_env", test_mapgen_env, {async=true}) + +local function test_ipc_vector_preserve(cb) + -- the IPC also uses register_portable_metatable + core.ipc_set("unittests:v", vector.new(4, 0, 4)) + local v = core.ipc_get("unittests:v") + assert(type(v) == "table") + assert(vector.check(v)) +end +unittests.register("test_ipc_vector_preserve", test_ipc_vector_preserve) diff --git a/src/gamedef.h b/src/gamedef.h index 9a6c55ab1..f8d6d79e7 100644 --- a/src/gamedef.h +++ b/src/gamedef.h @@ -34,19 +34,19 @@ class Camera; class ModChannel; class ModStorage; class ModStorageDatabase; +struct SubgameSpec; +struct ModSpec; +struct ModIPCStore; namespace irr::scene { class IAnimatedMesh; class ISceneManager; } -struct SubgameSpec; -struct ModSpec; /* An interface for fetching game-global definitions like tool and mapnode properties */ - class IGameDef { public: @@ -63,6 +63,9 @@ public: // environment thread. virtual IRollbackManager* getRollbackManager() { return NULL; } + // Only usable on server. + virtual ModIPCStore *getModIPCStore() { return nullptr; } + // Shorthands // TODO: these should be made const-safe so that a const IGameDef* is // actually usable diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index 75b1a8205..bfcfb4f7d 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -50,11 +50,12 @@ AsyncEngine::~AsyncEngine() } // Wait for threads to finish + infostream << "AsyncEngine: Waiting for " << workerThreads.size() + << " threads" << std::endl; for (AsyncWorkerThread *workerThread : workerThreads) { workerThread->wait(); } - // Force kill all threads for (AsyncWorkerThread *workerThread : workerThreads) { delete workerThread; } diff --git a/src/script/lua_api/CMakeLists.txt b/src/script/lua_api/CMakeLists.txt index d9405e4fe..2e12f8c56 100644 --- a/src/script/lua_api/CMakeLists.txt +++ b/src/script/lua_api/CMakeLists.txt @@ -6,6 +6,7 @@ set(common_SCRIPT_LUA_API_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/l_env.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_http.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_inventory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/l_ipc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_item.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_itemstackmeta.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_mapgen.cpp diff --git a/src/script/lua_api/l_ipc.cpp b/src/script/lua_api/l_ipc.cpp new file mode 100644 index 000000000..eb1eaedd7 --- /dev/null +++ b/src/script/lua_api/l_ipc.cpp @@ -0,0 +1,68 @@ +// Minetest +// SPDX-License-Identifier: LGPL-2.1-or-later + +#include "lua_api/l_ipc.h" +#include "lua_api/l_internal.h" +#include "common/c_packer.h" +#include "server.h" +#include "debug.h" + +typedef std::shared_lock SharedReadLock; +typedef std::unique_lock SharedWriteLock; + +int ModApiIPC::l_ipc_get(lua_State *L) +{ + auto *store = getGameDef(L)->getModIPCStore(); + + auto key = readParam(L, 1); + + { + SharedReadLock autolock(store->mutex); + auto it = store->map.find(key); + if (it == store->map.end()) + lua_pushnil(L); + else + script_unpack(L, it->second.get()); + } + return 1; +} + +int ModApiIPC::l_ipc_set(lua_State *L) +{ + auto *store = getGameDef(L)->getModIPCStore(); + + auto key = readParam(L, 1); + + luaL_checkany(L, 2); + std::unique_ptr pv; + if (!lua_isnil(L, 2)) { + pv.reset(script_pack(L, 2)); + if (pv->contains_userdata) + throw LuaError("Userdata not allowed"); + } + + { + SharedWriteLock autolock(store->mutex); + if (pv) + store->map[key] = std::move(pv); + else + store->map.erase(key); // delete the map value for nil + } + return 0; +} + +/* + * Implementation note: + * Iterating over the IPC table is intentionally not supported. + * Mods should know what they have set. + * This has the nice side effect that mods are able to use a randomly generated key + * if they really *really* want to avoid other code touching their data. + */ + +void ModApiIPC::Initialize(lua_State *L, int top) +{ + FATAL_ERROR_IF(!getGameDef(L)->getModIPCStore(), "ModIPCStore missing from gamedef"); + + API_FCT(ipc_get); + API_FCT(ipc_set); +} diff --git a/src/script/lua_api/l_ipc.h b/src/script/lua_api/l_ipc.h new file mode 100644 index 000000000..ca2cde22f --- /dev/null +++ b/src/script/lua_api/l_ipc.h @@ -0,0 +1,15 @@ +// Minetest +// SPDX-License-Identifier: LGPL-2.1-or-later + +#pragma once + +#include "lua_api/l_base.h" + +class ModApiIPC : public ModApiBase { +private: + static int l_ipc_get(lua_State *L); + static int l_ipc_set(lua_State *L); + +public: + static void Initialize(lua_State *L, int top); +}; diff --git a/src/script/scripting_emerge.cpp b/src/script/scripting_emerge.cpp index 3467b1495..f96a6c294 100644 --- a/src/script/scripting_emerge.cpp +++ b/src/script/scripting_emerge.cpp @@ -35,6 +35,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_util.h" #include "lua_api/l_vmanip.h" #include "lua_api/l_settings.h" +#include "lua_api/l_ipc.h" extern "C" { #include @@ -89,5 +90,6 @@ void EmergeScripting::InitializeModApi(lua_State *L, int top) ModApiMapgen::InitializeEmerge(L, top); ModApiServer::InitializeAsync(L, top); ModApiUtil::InitializeAsync(L, top); + ModApiIPC::Initialize(L, top); // TODO ^ these should also be renamed to InitializeRO or such } diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 324850011..d7d2513bb 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -46,6 +46,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_settings.h" #include "lua_api/l_http.h" #include "lua_api/l_storage.h" +#include "lua_api/l_ipc.h" extern "C" { #include @@ -121,6 +122,7 @@ void ServerScripting::initAsync() asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync); asyncEngine.registerStateInitializer(ModApiItem::InitializeAsync); asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiIPC::Initialize); // not added: ModApiMapgen is a minefield for thread safety // not added: ModApiHttp async api can't really work together with our jobs // not added: ModApiStorage is probably not thread safe(?) @@ -176,6 +178,7 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModApiHttp::Initialize(L, top); ModApiStorage::Initialize(L, top); ModApiChannels::Initialize(L, top); + ModApiIPC::Initialize(L, top); } void ServerScripting::InitializeAsync(lua_State *L, int top) diff --git a/src/server.cpp b/src/server.cpp index 405af63ef..df2d14a1d 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -86,6 +86,15 @@ public: {} }; +ModIPCStore::~ModIPCStore() +{ + // we don't have to do this, it's pure debugging aid + if (!std::unique_lock(mutex, std::try_to_lock).owns_lock()) { + errorstream << FUNCTION_NAME << ": lock is still in use!" << std::endl; + assert(0); + } +} + class ServerThread : public Thread { public: diff --git a/src/server.h b/src/server.h index 0318ec6f6..e8fa6b0da 100644 --- a/src/server.h +++ b/src/server.h @@ -47,6 +47,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include #include +#include class ChatEvent; struct ChatEventChat; @@ -142,6 +143,20 @@ struct ClientInfo { std::string vers_string, lang_code; }; +struct ModIPCStore { + ModIPCStore() = default; + ~ModIPCStore(); + + /// RW lock for this entire structure + std::shared_mutex mutex; + /** + * Map storing the data + * + * @note Do not store `nil` data in this map, instead remove the whole key. + */ + std::unordered_map> map; +}; + class Server : public con::PeerHandler, public MapEventReceiver, public IGameDef { @@ -301,12 +316,14 @@ public: NodeDefManager* getWritableNodeDefManager(); IWritableCraftDefManager* getWritableCraftDefManager(); + // Not under envlock virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; virtual const SubgameSpec* getGameSpec() const { return &m_gamespec; } static std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } virtual std::string getModDataPath() const { return m_path_mod_data; } + virtual ModIPCStore *getModIPCStore() { return &m_ipcstore; } inline bool isSingleplayer() const { return m_simple_singleplayer_mode; } @@ -666,6 +683,8 @@ private: std::unordered_map server_translations; + ModIPCStore m_ipcstore; + /* Threads */