[MP] Improve network profiler.

Fix RPC profiler and add average RPC size.

Improve bandwidth debugger to account for all multiplayer traffic
(excluding the lower level peer transformations).

10 files changed, 72 insertions, 61 deletions

diff --git a/modules/multiplayer/editor/editor_network_profiler.cpp b/modules/multiplayer/editor/editor_network_profiler.cpp
index a7e5b80b66..c78b32ed49 100644
--- a/modules/multiplayer/editor/editor_network_profiler.cpp
+++ b/modules/multiplayer/editor/editor_network_profiler.cpp
@@ -67,8 +67,8 @@ void EditorNetworkProfiler::_update_frame() {
 		}
 
 		node->set_text(0, E.value.node_path);
-		node->set_text(1, E.value.incoming_rpc == 0 ? "-" : itos(E.value.incoming_rpc));
-		node->set_text(2, E.value.outgoing_rpc == 0 ? "-" : itos(E.value.outgoing_rpc));
+		node->set_text(1, E.value.incoming_rpc == 0 ? "-" : vformat(TTR("%d (%s)"), E.value.incoming_rpc, String::humanize_size(E.value.incoming_size)));
+		node->set_text(2, E.value.outgoing_rpc == 0 ? "-" : vformat(TTR("%d (%s)"), E.value.outgoing_rpc, String::humanize_size(E.value.outgoing_size)));
 	}
 }
 
@@ -99,6 +99,12 @@ void EditorNetworkProfiler::add_node_frame_data(const RPCNodeInfo p_frame) {
 		nodes_data[p_frame.node].incoming_rpc += p_frame.incoming_rpc;
 		nodes_data[p_frame.node].outgoing_rpc += p_frame.outgoing_rpc;
 	}
+	if (p_frame.incoming_rpc) {
+		nodes_data[p_frame.node].incoming_size = p_frame.incoming_size / p_frame.incoming_rpc;
+	}
+	if (p_frame.outgoing_rpc) {
+		nodes_data[p_frame.node].outgoing_size = p_frame.outgoing_size / p_frame.outgoing_rpc;
+	}
 
 	if (frame_delay->is_stopped()) {
 		frame_delay->set_wait_time(0.1);
diff --git a/modules/multiplayer/editor/editor_network_profiler.h b/modules/multiplayer/editor/editor_network_profiler.h
index 98d12e3c0a..65d08dcd56 100644
--- a/modules/multiplayer/editor/editor_network_profiler.h
+++ b/modules/multiplayer/editor/editor_network_profiler.h
@@ -66,7 +66,7 @@ protected:
 	static void _bind_methods();
 
 public:
-	void add_node_frame_data(const RPCNodeInfo p_frame);
+	void add_node_frame_data(RPCNodeInfo p_frame);
 	void set_bandwidth(int p_incoming, int p_outgoing);
 	bool is_profiling();
 
diff --git a/modules/multiplayer/multiplayer_debugger.cpp b/modules/multiplayer/multiplayer_debugger.cpp
index 3d22af04dc..f6d22b1637 100644
--- a/modules/multiplayer/multiplayer_debugger.cpp
+++ b/modules/multiplayer/multiplayer_debugger.cpp
@@ -126,12 +126,14 @@ void MultiplayerDebugger::BandwidthProfiler::tick(double p_frame_time, double p_
 
 Array MultiplayerDebugger::RPCFrame::serialize() {
 	Array arr;
-	arr.push_back(infos.size() * 4);
+	arr.push_back(infos.size() * 6);
 	for (int i = 0; i < infos.size(); ++i) {
 		arr.push_back(uint64_t(infos[i].node));
 		arr.push_back(infos[i].node_path);
 		arr.push_back(infos[i].incoming_rpc);
+		arr.push_back(infos[i].incoming_size);
 		arr.push_back(infos[i].outgoing_rpc);
+		arr.push_back(infos[i].outgoing_size);
 	}
 	return arr;
 }
@@ -139,15 +141,18 @@ Array MultiplayerDebugger::RPCFrame::serialize() {
 bool MultiplayerDebugger::RPCFrame::deserialize(const Array &p_arr) {
 	ERR_FAIL_COND_V(p_arr.size() < 1, false);
 	uint32_t size = p_arr[0];
-	ERR_FAIL_COND_V(size % 4, false);
+	ERR_FAIL_COND_V(size % 6, false);
 	ERR_FAIL_COND_V((uint32_t)p_arr.size() != size + 1, false);
-	infos.resize(size / 4);
+	infos.resize(size / 6);
 	int idx = 1;
-	for (uint32_t i = 0; i < size / 4; ++i) {
+	for (uint32_t i = 0; i < size / 6; i++) {
 		infos.write[i].node = uint64_t(p_arr[idx]);
 		infos.write[i].node_path = p_arr[idx + 1];
 		infos.write[i].incoming_rpc = p_arr[idx + 2];
-		infos.write[i].outgoing_rpc = p_arr[idx + 3];
+		infos.write[i].incoming_size = p_arr[idx + 3];
+		infos.write[i].outgoing_rpc = p_arr[idx + 4];
+		infos.write[i].outgoing_size = p_arr[idx + 5];
+		idx += 6;
 	}
 	return true;
 }
@@ -159,8 +164,6 @@ void MultiplayerDebugger::RPCProfiler::init_node(const ObjectID p_node) {
 	rpc_node_data.insert(p_node, RPCNodeInfo());
 	rpc_node_data[p_node].node = p_node;
 	rpc_node_data[p_node].node_path = Object::cast_to<Node>(ObjectDB::get_instance(p_node))->get_path();
-	rpc_node_data[p_node].incoming_rpc = 0;
-	rpc_node_data[p_node].outgoing_rpc = 0;
 }
 
 void MultiplayerDebugger::RPCProfiler::toggle(bool p_enable, const Array &p_opts) {
@@ -168,15 +171,18 @@ void MultiplayerDebugger::RPCProfiler::toggle(bool p_enable, const Array &p_opts
 }
 
 void MultiplayerDebugger::RPCProfiler::add(const Array &p_data) {
-	ERR_FAIL_COND(p_data.size() < 2);
-	const ObjectID id = p_data[0];
-	const String what = p_data[1];
+	ERR_FAIL_COND(p_data.size() != 3);
+	const String what = p_data[0];
+	const ObjectID id = p_data[1];
+	const int size = p_data[2];
 	init_node(id);
 	RPCNodeInfo &info = rpc_node_data[id];
 	if (what == "rpc_in") {
 		info.incoming_rpc++;
+		info.incoming_size += size;
 	} else if (what == "rpc_out") {
 		info.outgoing_rpc++;
+		info.outgoing_size += size;
 	}
 }
 
diff --git a/modules/multiplayer/multiplayer_debugger.h b/modules/multiplayer/multiplayer_debugger.h
index 4efd1da016..c8acd2eeb4 100644
--- a/modules/multiplayer/multiplayer_debugger.h
+++ b/modules/multiplayer/multiplayer_debugger.h
@@ -41,7 +41,9 @@ public:
 		ObjectID node;
 		String node_path;
 		int incoming_rpc = 0;
+		int incoming_size = 0;
 		int outgoing_rpc = 0;
+		int outgoing_size = 0;
 	};
 
 	struct RPCFrame {
diff --git a/modules/multiplayer/scene_cache_interface.cpp b/modules/multiplayer/scene_cache_interface.cpp
index 7df9b95b30..f0da4f9dfc 100644
--- a/modules/multiplayer/scene_cache_interface.cpp
+++ b/modules/multiplayer/scene_cache_interface.cpp
@@ -99,10 +99,6 @@ void SceneCacheInterface::process_simplify_path(int p_from, const uint8_t *p_pac
 	Ref<MultiplayerPeer> multiplayer_peer = multiplayer->get_multiplayer_peer();
 	ERR_FAIL_COND(multiplayer_peer.is_null());
 
-#ifdef DEBUG_ENABLED
-	multiplayer->profile_bandwidth("out", packet.size());
-#endif
-
 	multiplayer_peer->set_transfer_channel(0);
 	multiplayer_peer->set_transfer_mode(MultiplayerPeer::TRANSFER_MODE_RELIABLE);
 	multiplayer->send_command(p_from, packet.ptr(), packet.size());
@@ -155,10 +151,6 @@ Error SceneCacheInterface::_send_confirm_path(Node *p_node, NodePath p_path, Pat
 	Ref<MultiplayerPeer> multiplayer_peer = multiplayer->get_multiplayer_peer();
 	ERR_FAIL_COND_V(multiplayer_peer.is_null(), ERR_BUG);
 
-#ifdef DEBUG_ENABLED
-	multiplayer->profile_bandwidth("out", packet.size() * p_peers.size());
-#endif
-
 	Error err = OK;
 	for (int peer_id : p_peers) {
 		multiplayer_peer->set_transfer_channel(0);
diff --git a/modules/multiplayer/scene_multiplayer.cpp b/modules/multiplayer/scene_multiplayer.cpp
index db7c5037cd..93089cd50a 100644
--- a/modules/multiplayer/scene_multiplayer.cpp
+++ b/modules/multiplayer/scene_multiplayer.cpp
@@ -40,12 +40,12 @@
 #endif
 
 #ifdef DEBUG_ENABLED
-void SceneMultiplayer::profile_bandwidth(const String &p_inout, int p_size) {
+_FORCE_INLINE_ void SceneMultiplayer::_profile_bandwidth(const String &p_what, int p_value) {
 	if (EngineDebugger::is_profiling("multiplayer")) {
 		Array values;
-		values.push_back(p_inout);
+		values.push_back(p_what);
 		values.push_back(OS::get_singleton()->get_ticks_msec());
-		values.push_back(p_size);
+		values.push_back(p_value);
 		EngineDebugger::profiler_add_frame_data("multiplayer", values);
 	}
 }
@@ -91,6 +91,10 @@ Error SceneMultiplayer::poll() {
 		Error err = multiplayer_peer->get_packet(&packet, len);
 		ERR_FAIL_COND_V_MSG(err != OK, err, vformat("Error getting packet! %d", err));
 
+#ifdef DEBUG_ENABLED
+		_profile_bandwidth("in", len);
+#endif
+
 		if (pending_peers.has(sender)) {
 			if (pending_peers[sender].local) {
 				// If the auth is over, admit the peer at the first packet.
@@ -220,10 +224,6 @@ void SceneMultiplayer::_process_packet(int p_from, const uint8_t *p_packet, int
 	ERR_FAIL_COND_MSG(root_path.is_empty(), "Multiplayer root was not initialized. If you are using custom multiplayer, remember to set the root path via SceneMultiplayer.set_root_path before using it.");
 	ERR_FAIL_COND_MSG(p_packet_len < 1, "Invalid packet received. Size too small.");
 
-#ifdef DEBUG_ENABLED
-	profile_bandwidth("in", p_packet_len);
-#endif
-
 	// Extract the `packet_type` from the LSB three bits:
 	uint8_t packet_type = p_packet[0] & CMD_MASK;
 
@@ -258,6 +258,13 @@ void SceneMultiplayer::_process_packet(int p_from, const uint8_t *p_packet, int
 	}
 }
 
+#ifdef DEBUG_ENABLED
+_FORCE_INLINE_ Error SceneMultiplayer::_send(const uint8_t *p_packet, int p_packet_len) {
+	_profile_bandwidth("out", p_packet_len);
+	return multiplayer_peer->put_packet(p_packet, p_packet_len);
+}
+#endif
+
 Error SceneMultiplayer::send_command(int p_to, const uint8_t *p_packet, int p_packet_len) {
 	if (server_relay && get_unique_id() != 1 && p_to != 1 && multiplayer_peer->is_server_relay_supported()) {
 		// Send relay packet.
@@ -268,19 +275,19 @@ Error SceneMultiplayer::send_command(int p_to, const uint8_t *p_packet, int p_pa
 		relay_buffer->put_data(p_packet, p_packet_len);
 		multiplayer_peer->set_target_peer(1);
 		const Vector<uint8_t> data = relay_buffer->get_data_array();
-		return multiplayer_peer->put_packet(data.ptr(), relay_buffer->get_position());
+		return _send(data.ptr(), relay_buffer->get_position());
 	}
 	if (p_to > 0) {
 		ERR_FAIL_COND_V(!connected_peers.has(p_to), ERR_BUG);
 		multiplayer_peer->set_target_peer(p_to);
-		return multiplayer_peer->put_packet(p_packet, p_packet_len);
+		return _send(p_packet, p_packet_len);
 	} else {
 		for (const int &pid : connected_peers) {
 			if (p_to && pid == -p_to) {
 				continue;
 			}
 			multiplayer_peer->set_target_peer(pid);
-			multiplayer_peer->put_packet(p_packet, p_packet_len);
+			_send(p_packet, p_packet_len);
 		}
 		return OK;
 	}
@@ -319,7 +326,7 @@ void SceneMultiplayer::_process_sys(int p_from, const uint8_t *p_packet, int p_p
 				multiplayer_peer->set_transfer_channel(p_channel);
 				if (peer > 0) {
 					multiplayer_peer->set_target_peer(peer);
-					multiplayer_peer->put_packet(data.ptr(), relay_buffer->get_position());
+					_send(data.ptr(), relay_buffer->get_position());
 				} else {
 					for (const int &P : connected_peers) {
 						// Not to sender, nor excluded.
@@ -327,7 +334,7 @@ void SceneMultiplayer::_process_sys(int p_from, const uint8_t *p_packet, int p_p
 							continue;
 						}
 						multiplayer_peer->set_target_peer(P);
-						multiplayer_peer->put_packet(data.ptr(), relay_buffer->get_position());
+						_send(data.ptr(), relay_buffer->get_position());
 					}
 				}
 				if (peer == 0 || peer == -1) {
@@ -373,11 +380,11 @@ void SceneMultiplayer::_admit_peer(int p_id) {
 			// Send new peer to already connected.
 			encode_uint32(p_id, &buf[2]);
 			multiplayer_peer->set_target_peer(P);
-			multiplayer_peer->put_packet(buf, sizeof(buf));
+			_send(buf, sizeof(buf));
 			// Send already connected to new peer.
 			encode_uint32(P, &buf[2]);
 			multiplayer_peer->set_target_peer(p_id);
-			multiplayer_peer->put_packet(buf, sizeof(buf));
+			_send(buf, sizeof(buf));
 		}
 	}
 
@@ -412,7 +419,7 @@ void SceneMultiplayer::_del_peer(int p_id) {
 				continue;
 			}
 			multiplayer_peer->set_target_peer(P);
-			multiplayer_peer->put_packet(buf, sizeof(buf));
+			_send(buf, sizeof(buf));
 		}
 	}
 
@@ -468,7 +475,7 @@ Error SceneMultiplayer::send_auth(int p_to, Vector<uint8_t> p_data) {
 	multiplayer_peer->set_target_peer(p_to);
 	multiplayer_peer->set_transfer_channel(0);
 	multiplayer_peer->set_transfer_mode(MultiplayerPeer::TRANSFER_MODE_RELIABLE);
-	return multiplayer_peer->put_packet(packet_cache.ptr(), p_data.size() + 2);
+	return _send(packet_cache.ptr(), p_data.size() + 2);
 }
 
 Error SceneMultiplayer::complete_auth(int p_peer) {
@@ -478,7 +485,7 @@ Error SceneMultiplayer::complete_auth(int p_peer) {
 	pending_peers[p_peer].local = true;
 	// Notify the remote peer that the authentication has completed.
 	uint8_t buf[2] = { NETWORK_COMMAND_SYS, SYS_COMMAND_AUTH };
-	Error err = multiplayer_peer->put_packet(buf, 2);
+	Error err = _send(buf, 2);
 	// The remote peer already reported the authentication as completed, so admit the peer.
 	// May generate new packets, so it must happen after sending confirmation.
 	if (pending_peers[p_peer].remote) {
diff --git a/modules/multiplayer/scene_multiplayer.h b/modules/multiplayer/scene_multiplayer.h
index b0ecc48f8c..da8c762134 100644
--- a/modules/multiplayer/scene_multiplayer.h
+++ b/modules/multiplayer/scene_multiplayer.h
@@ -103,6 +103,15 @@ private:
 	Ref<SceneReplicationInterface> replicator;
 	Ref<SceneRPCInterface> rpc;
 
+#ifdef DEBUG_ENABLED
+	_FORCE_INLINE_ void _profile_bandwidth(const String &p_what, int p_value);
+	_FORCE_INLINE_ Error _send(const uint8_t *p_packet, int p_packet_len); // Also profiles.
+#else
+	_FORCE_INLINE_ Error _send(const uint8_t *p_packet, int p_packet_len) {
+		return multiplayer_peer->put_packet(p_packet, p_packet_len);
+	}
+#endif
+
 protected:
 	static void _bind_methods();
 
@@ -163,10 +172,6 @@ public:
 
 	Ref<SceneCacheInterface> get_path_cache() { return cache; }
 
-#ifdef DEBUG_ENABLED
-	void profile_bandwidth(const String &p_inout, int p_size);
-#endif
-
 	SceneMultiplayer();
 	~SceneMultiplayer();
 };
diff --git a/modules/multiplayer/scene_replication_interface.cpp b/modules/multiplayer/scene_replication_interface.cpp
index f1bab7327a..bc7eaba6d4 100644
--- a/modules/multiplayer/scene_replication_interface.cpp
+++ b/modules/multiplayer/scene_replication_interface.cpp
@@ -362,13 +362,8 @@ Error SceneReplicationInterface::_update_spawn_visibility(int p_peer, const Obje
 
 Error SceneReplicationInterface::_send_raw(const uint8_t *p_buffer, int p_size, int p_peer, bool p_reliable) {
 	ERR_FAIL_COND_V(!p_buffer || p_size < 1, ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(!multiplayer, ERR_UNCONFIGURED);
 	ERR_FAIL_COND_V(!multiplayer->has_multiplayer_peer(), ERR_UNCONFIGURED);
 
-#ifdef DEBUG_ENABLED
-	multiplayer->profile_bandwidth("out", p_size);
-#endif
-
 	Ref<MultiplayerPeer> peer = multiplayer->get_multiplayer_peer();
 	peer->set_transfer_channel(0);
 	peer->set_transfer_mode(p_reliable ? MultiplayerPeer::TRANSFER_MODE_RELIABLE : MultiplayerPeer::TRANSFER_MODE_UNRELIABLE);
diff --git a/modules/multiplayer/scene_rpc_interface.cpp b/modules/multiplayer/scene_rpc_interface.cpp
index acc113c901..bfb1623077 100644
--- a/modules/multiplayer/scene_rpc_interface.cpp
+++ b/modules/multiplayer/scene_rpc_interface.cpp
@@ -52,16 +52,15 @@
 #define BYTE_ONLY_OR_NO_ARGS_FLAG (1 << BYTE_ONLY_OR_NO_ARGS_SHIFT)
 
 #ifdef DEBUG_ENABLED
-_FORCE_INLINE_ void SceneRPCInterface::_profile_node_data(const String &p_what, ObjectID p_id) {
+_FORCE_INLINE_ void SceneRPCInterface::_profile_node_data(const String &p_what, ObjectID p_id, int p_size) {
 	if (EngineDebugger::is_profiling("rpc")) {
 		Array values;
-		values.push_back(p_id);
 		values.push_back(p_what);
+		values.push_back(p_id);
+		values.push_back(p_size);
 		EngineDebugger::profiler_add_frame_data("rpc", values);
 	}
 }
-#else
-_FORCE_INLINE_ void SceneRPCInterface::_profile_node_data(const String &p_what, ObjectID p_id) {}
 #endif
 
 // Returns the packet size stripping the node path added when the node is not yet cached.
@@ -277,7 +276,7 @@ void SceneRPCInterface::_process_rpc(Node *p_node, const uint16_t p_rpc_method_i
 	argp.resize(argc);
 
 #ifdef DEBUG_ENABLED
-	_profile_node_data("rpc_in", p_node->get_instance_id());
+	_profile_node_data("rpc_in", p_node->get_instance_id(), p_packet_len);
 #endif
 
 	int out;
@@ -399,13 +398,13 @@ void SceneRPCInterface::_send_rpc(Node *p_from, int p_to, uint16_t p_rpc_id, con
 	ERR_FAIL_COND(node_id_compression > 3);
 	ERR_FAIL_COND(name_id_compression > 1);
 
-	// We can now set the meta
-	packet_cache.write[0] = command_type + (node_id_compression << NODE_ID_COMPRESSION_SHIFT) + (name_id_compression << NAME_ID_COMPRESSION_SHIFT) + (byte_only_or_no_args ? BYTE_ONLY_OR_NO_ARGS_FLAG : 0);
-
 #ifdef DEBUG_ENABLED
-	multiplayer->profile_bandwidth("out", ofs);
+	_profile_node_data("rpc_out", p_from->get_instance_id(), ofs);
 #endif
 
+	// We can now set the meta
+	packet_cache.write[0] = command_type + (node_id_compression << NODE_ID_COMPRESSION_SHIFT) + (name_id_compression << NAME_ID_COMPRESSION_SHIFT) + (byte_only_or_no_args ? BYTE_ONLY_OR_NO_ARGS_FLAG : 0);
+
 	// Take chance and set transfer mode, since all send methods will use it.
 	peer->set_transfer_channel(p_config.channel);
 	peer->set_transfer_mode(p_config.transfer_mode);
@@ -477,10 +476,6 @@ Error SceneRPCInterface::rpcp(Object *p_obj, int p_peer_id, const StringName &p_
 	}
 
 	if (p_peer_id != caller_id) {
-#ifdef DEBUG_ENABLED
-		_profile_node_data("rpc_out", node->get_instance_id());
-#endif
-
 		_send_rpc(node, p_peer_id, rpc_id, config, p_method, p_arg, p_argcount);
 	}
 
diff --git a/modules/multiplayer/scene_rpc_interface.h b/modules/multiplayer/scene_rpc_interface.h
index aa9be525a2..800293714c 100644
--- a/modules/multiplayer/scene_rpc_interface.h
+++ b/modules/multiplayer/scene_rpc_interface.h
@@ -81,8 +81,11 @@ private:
 
 	HashMap<ObjectID, RPCConfigCache> rpc_cache;
 
+#ifdef DEBUG_ENABLED
+	_FORCE_INLINE_ void _profile_node_data(const String &p_what, ObjectID p_id, int p_size);
+#endif
+
 protected:
-	_FORCE_INLINE_ void _profile_node_data(const String &p_what, ObjectID p_id);
 	void _process_rpc(Node *p_node, const uint16_t p_rpc_method_id, int p_from, const uint8_t *p_packet, int p_packet_len, int p_offset);
 
 	void _send_rpc(Node *p_from, int p_to, uint16_t p_rpc_id, const RPCConfig &p_config, const StringName &p_name, const Variant **p_arg, int p_argcount);