Fixed message ordering issues
This commit is contained in:
parent
61e0b2ff24
commit
cda773915e
8 changed files with 48 additions and 29 deletions
|
|
@ -61,7 +61,7 @@ void NetClient::parse_received_messages() {
|
||||||
lastMessageTime = std::chrono::steady_clock::now();
|
lastMessageTime = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
size_t messageOrderDisplacement = 0;
|
size_t messageOrderDisplacement = 0;
|
||||||
if(receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL || receivedMessage.channel == RELIABLE_COMMAND_CHANNEL) {
|
if(NetLibrary::is_ordered_channel(receivedMessage.channel)) {
|
||||||
MessageOrder messageOrder = NetLibrary::get_message_order(receivedMessage.data);
|
MessageOrder messageOrder = NetLibrary::get_message_order(receivedMessage.data);
|
||||||
if(nextMessageOrderToExpect > messageOrder && receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL) {
|
if(nextMessageOrderToExpect > messageOrder && receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL) {
|
||||||
receiveQueue->messages.pop();
|
receiveQueue->messages.pop();
|
||||||
|
|
@ -95,16 +95,16 @@ void NetClient::parse_received_messages() {
|
||||||
|
|
||||||
void NetClient::send_queued_messages() {
|
void NetClient::send_queued_messages() {
|
||||||
for(auto& [channelName, messageQueue] : messageQueues) {
|
for(auto& [channelName, messageQueue] : messageQueues) {
|
||||||
bool addMessageOrder = channelName == UNRELIABLE_COMMAND_CHANNEL || channelName == RELIABLE_COMMAND_CHANNEL;
|
bool addMessageOrder = NetLibrary::is_ordered_channel(channelName);
|
||||||
if(directConnectServer) {
|
if(directConnectServer) {
|
||||||
auto& c = directConnectServer->directConnectClientData;
|
auto& c = directConnectServer->directConnectClientData;
|
||||||
std::scoped_lock rMessLock(c->receivedMessagesMutex);
|
std::scoped_lock rMessLock(c->receivedMessagesMutex);
|
||||||
while(!messageQueue.empty()) {
|
while(!messageQueue.empty()) {
|
||||||
std::string toInsert;
|
std::string toInsert;
|
||||||
if(addMessageOrder)
|
if(addMessageOrder)
|
||||||
toInsert = NetLibrary::attach_message_order_and_update(nextMessageOrderToSend, messageQueue.front()->str());
|
toInsert = NetLibrary::attach_message_order(messageQueue.front().order, messageQueue.front().ss->str());
|
||||||
else
|
else
|
||||||
toInsert = messageQueue.front()->str();
|
toInsert = messageQueue.front().ss->str();
|
||||||
rtc::binary b((std::byte*)toInsert.c_str(), (std::byte*)(toInsert.c_str() + toInsert.length()));
|
rtc::binary b((std::byte*)toInsert.c_str(), (std::byte*)(toInsert.c_str() + toInsert.length()));
|
||||||
c->receivedMessages.emplace(channelName, b);
|
c->receivedMessages.emplace(channelName, b);
|
||||||
messageQueue.pop();
|
messageQueue.pop();
|
||||||
|
|
@ -120,14 +120,14 @@ void NetClient::send_queued_messages() {
|
||||||
break;
|
break;
|
||||||
#endif
|
#endif
|
||||||
if(addMessageOrder)
|
if(addMessageOrder)
|
||||||
send_str_as_bytes(channel, NetLibrary::attach_message_order_and_update(nextMessageOrderToSend, messageQueue.front()->str()));
|
send_str_as_bytes(channel, NetLibrary::attach_message_order(messageQueue.front().order, messageQueue.front().ss->str()));
|
||||||
else
|
else
|
||||||
send_str_as_bytes(channel, messageQueue.front()->str());
|
send_str_as_bytes(channel, messageQueue.front().ss->str());
|
||||||
messageQueue.pop();
|
messageQueue.pop();
|
||||||
}
|
}
|
||||||
#ifdef __EMSCRIPTEN__
|
#ifdef __EMSCRIPTEN__
|
||||||
if(channel->bufferedAmount() >= NetLibrary::MAX_BUFFERED_DATA_PER_CHANNEL && channelName == UNRELIABLE_COMMAND_CHANNEL)
|
if(channel->bufferedAmount() >= NetLibrary::MAX_BUFFERED_DATA_PER_CHANNEL && channelName == UNRELIABLE_COMMAND_CHANNEL)
|
||||||
messageQueue = std::queue<std::shared_ptr<std::stringstream>>();
|
messageQueue = std::queue<OutgoingMessage>();
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,10 +28,10 @@ class NetClient : public std::enable_shared_from_this<NetClient> {
|
||||||
std::vector<std::shared_ptr<std::stringstream>> fragmentedMessage = fragment_message(ss->view(), NetLibrary::FRAGMENT_MESSAGE_STRIDE);
|
std::vector<std::shared_ptr<std::stringstream>> fragmentedMessage = fragment_message(ss->view(), NetLibrary::FRAGMENT_MESSAGE_STRIDE);
|
||||||
auto& messageQueue = messageQueues[channel];
|
auto& messageQueue = messageQueues[channel];
|
||||||
if(fragmentedMessage.empty())
|
if(fragmentedMessage.empty())
|
||||||
messageQueue.emplace(ss);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, nextMessageOrderToSend), ss);
|
||||||
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
||||||
for(auto& ss2 : fragmentedMessage)
|
for(auto& ss2 : fragmentedMessage)
|
||||||
messageQueue.emplace(ss2);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, nextMessageOrderToSend), ss2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
void update();
|
void update();
|
||||||
|
|
@ -45,7 +45,11 @@ class NetClient : public std::enable_shared_from_this<NetClient> {
|
||||||
void send_queued_messages();
|
void send_queued_messages();
|
||||||
void send_str_as_bytes(std::shared_ptr<rtc::DataChannel> channel, const std::string& str);
|
void send_str_as_bytes(std::shared_ptr<rtc::DataChannel> channel, const std::string& str);
|
||||||
|
|
||||||
std::unordered_map<std::string, std::queue<std::shared_ptr<std::stringstream>>> messageQueues;
|
struct OutgoingMessage {
|
||||||
|
MessageOrder order;
|
||||||
|
std::shared_ptr<std::stringstream> ss;
|
||||||
|
};
|
||||||
|
std::unordered_map<std::string, std::queue<OutgoingMessage>> messageQueues;
|
||||||
|
|
||||||
struct ReceiveQueue {
|
struct ReceiveQueue {
|
||||||
std::mutex mut;
|
std::mutex mut;
|
||||||
|
|
|
||||||
|
|
@ -341,13 +341,20 @@ void NetLibrary::channel_created_in_client(std::shared_ptr<rtc::DataChannel> cha
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string NetLibrary::attach_message_order_and_update(MessageOrder& order, const std::string& message) {
|
bool NetLibrary::is_ordered_channel(std::string_view channelName) {
|
||||||
|
return channelName == RELIABLE_COMMAND_CHANNEL || channelName == UNRELIABLE_COMMAND_CHANNEL;
|
||||||
|
}
|
||||||
|
|
||||||
|
MessageOrder NetLibrary::calc_order_for_queued_message(std::string_view channelName, MessageOrder& mOrder) {
|
||||||
|
return is_ordered_channel(channelName) ? mOrder++ : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string NetLibrary::attach_message_order(MessageOrder order, const std::string& message) {
|
||||||
union {
|
union {
|
||||||
MessageOrder o;
|
MessageOrder o;
|
||||||
char b[sizeof(MessageOrder)];
|
char b[sizeof(MessageOrder)];
|
||||||
} u;
|
} u;
|
||||||
u.o = (std::endian::native == std::endian::little) ? std::byteswap(order) : order;
|
u.o = (std::endian::native == std::endian::little) ? std::byteswap(order) : order;
|
||||||
order++;
|
|
||||||
return std::string(u.b, sizeof(MessageOrder)) + message;
|
return std::string(u.b, sizeof(MessageOrder)) + message;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,15 +33,18 @@ class NetLibrary {
|
||||||
static constexpr size_t GLOBALID_LEN = 20;
|
static constexpr size_t GLOBALID_LEN = 20;
|
||||||
private:
|
private:
|
||||||
static constexpr size_t FRAGMENT_MESSAGE_STRIDE = 4096;
|
static constexpr size_t FRAGMENT_MESSAGE_STRIDE = 4096;
|
||||||
static constexpr size_t MAX_BUFFERED_DATA_PER_CHANNEL = 65500;
|
static constexpr size_t MAX_BUFFERED_DATA_PER_CHANNEL = 64000;
|
||||||
static constexpr std::chrono::seconds TIMEOUT_DURATION = std::chrono::seconds(30);
|
static constexpr std::chrono::seconds TIMEOUT_DURATION = std::chrono::seconds(30);
|
||||||
|
|
||||||
|
static bool is_ordered_channel(std::string_view channelName);
|
||||||
|
static MessageOrder calc_order_for_queued_message(std::string_view channelName, MessageOrder& mOrder);
|
||||||
|
|
||||||
static void finish_client_registration(std::shared_ptr<NetClient> client);
|
static void finish_client_registration(std::shared_ptr<NetClient> client);
|
||||||
|
|
||||||
static void assign_client_connection_to_server(const std::string& serverLocalID, const std::string& clientLocalID, std::shared_ptr<rtc::PeerConnection> connection);
|
static void assign_client_connection_to_server(const std::string& serverLocalID, const std::string& clientLocalID, std::shared_ptr<rtc::PeerConnection> connection);
|
||||||
static void channel_created_in_client(std::shared_ptr<rtc::DataChannel> channel);
|
static void channel_created_in_client(std::shared_ptr<rtc::DataChannel> channel);
|
||||||
|
|
||||||
static std::string attach_message_order_and_update(MessageOrder& order, const std::string& message);
|
static std::string attach_message_order(MessageOrder order, const std::string& message);
|
||||||
static MessageOrder get_message_order(rtc::binary& message);
|
static MessageOrder get_message_order(rtc::binary& message);
|
||||||
|
|
||||||
struct PeerData {
|
struct PeerData {
|
||||||
|
|
|
||||||
|
|
@ -95,16 +95,16 @@ void NetServer::client_connected(std::shared_ptr<rtc::PeerConnection> connection
|
||||||
|
|
||||||
void NetServer::ClientData::send_queued_messages(NetServer& server) {
|
void NetServer::ClientData::send_queued_messages(NetServer& server) {
|
||||||
for(auto& [channelName, messageQueue] : messageQueues) {
|
for(auto& [channelName, messageQueue] : messageQueues) {
|
||||||
bool addMessageOrder = channelName == UNRELIABLE_COMMAND_CHANNEL || channelName == RELIABLE_COMMAND_CHANNEL;
|
bool addMessageOrder = NetLibrary::is_ordered_channel(channelName);
|
||||||
if(this == server.directConnectClientData.get()) {
|
if(this == server.directConnectClientData.get()) {
|
||||||
NetClient* c = server.directConnectClient;
|
NetClient* c = server.directConnectClient;
|
||||||
std::scoped_lock recvQueueLock(c->receiveQueue->mut);
|
std::scoped_lock recvQueueLock(c->receiveQueue->mut);
|
||||||
while(!messageQueue.empty()) {
|
while(!messageQueue.empty()) {
|
||||||
std::string toInsert;
|
std::string toInsert;
|
||||||
if(addMessageOrder)
|
if(addMessageOrder)
|
||||||
toInsert = NetLibrary::attach_message_order_and_update(nextMessageOrderToSend, messageQueue.front()->str());
|
toInsert = NetLibrary::attach_message_order(messageQueue.front().order, messageQueue.front().ss->str());
|
||||||
else
|
else
|
||||||
toInsert = messageQueue.front()->str();
|
toInsert = messageQueue.front().ss->str();
|
||||||
rtc::binary b((std::byte*)toInsert.c_str(), (std::byte*)(toInsert.c_str() + toInsert.length()));
|
rtc::binary b((std::byte*)toInsert.c_str(), (std::byte*)(toInsert.c_str() + toInsert.length()));
|
||||||
c->receiveQueue->messages.emplace(channelName, b);
|
c->receiveQueue->messages.emplace(channelName, b);
|
||||||
messageQueue.pop();
|
messageQueue.pop();
|
||||||
|
|
@ -120,14 +120,14 @@ void NetServer::ClientData::send_queued_messages(NetServer& server) {
|
||||||
break;
|
break;
|
||||||
#endif
|
#endif
|
||||||
if(addMessageOrder)
|
if(addMessageOrder)
|
||||||
send_str_as_bytes(channel, NetLibrary::attach_message_order_and_update(nextMessageOrderToSend, messageQueue.front()->str()));
|
send_str_as_bytes(channel, NetLibrary::attach_message_order(messageQueue.front().order, messageQueue.front().ss->str()));
|
||||||
else
|
else
|
||||||
send_str_as_bytes(channel, messageQueue.front()->str());
|
send_str_as_bytes(channel, messageQueue.front().ss->str());
|
||||||
messageQueue.pop();
|
messageQueue.pop();
|
||||||
}
|
}
|
||||||
#ifdef __EMSCRIPTEN__
|
#ifdef __EMSCRIPTEN__
|
||||||
if(channel->bufferedAmount() >= NetLibrary::MAX_BUFFERED_DATA_PER_CHANNEL && channelName == UNRELIABLE_COMMAND_CHANNEL)
|
if(channel->bufferedAmount() >= NetLibrary::MAX_BUFFERED_DATA_PER_CHANNEL && channelName == UNRELIABLE_COMMAND_CHANNEL)
|
||||||
messageQueue = std::queue<std::shared_ptr<std::stringstream>>();
|
messageQueue = std::queue<OutgoingMessage>();
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -146,7 +146,7 @@ void NetServer::ClientData::parse_received_messages(NetServer& server) {
|
||||||
lastMessageTime = std::chrono::steady_clock::now();
|
lastMessageTime = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
size_t messageOrderDisplacement = 0;
|
size_t messageOrderDisplacement = 0;
|
||||||
if(receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL || receivedMessage.channel == RELIABLE_COMMAND_CHANNEL) {
|
if(NetLibrary::is_ordered_channel(receivedMessage.channel)) {
|
||||||
MessageOrder messageOrder = NetLibrary::get_message_order(receivedMessage.data);
|
MessageOrder messageOrder = NetLibrary::get_message_order(receivedMessage.data);
|
||||||
if(nextMessageOrderToExpect > messageOrder && receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL) {
|
if(nextMessageOrderToExpect > messageOrder && receivedMessage.channel == UNRELIABLE_COMMAND_CHANNEL) {
|
||||||
receivedMessages.pop();
|
receivedMessages.pop();
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,11 @@ class NetServer : public std::enable_shared_from_this<NetServer> {
|
||||||
};
|
};
|
||||||
std::queue<ReceivedMessage> receivedMessages;
|
std::queue<ReceivedMessage> receivedMessages;
|
||||||
|
|
||||||
std::unordered_map<std::string, std::queue<std::shared_ptr<std::stringstream>>> messageQueues;
|
struct OutgoingMessage {
|
||||||
|
MessageOrder order;
|
||||||
|
std::shared_ptr<std::stringstream> ss;
|
||||||
|
};
|
||||||
|
std::unordered_map<std::string, std::queue<OutgoingMessage>> messageQueues;
|
||||||
|
|
||||||
uint64_t customID;
|
uint64_t customID;
|
||||||
|
|
||||||
|
|
@ -62,10 +66,10 @@ class NetServer : public std::enable_shared_from_this<NetServer> {
|
||||||
std::vector<std::shared_ptr<std::stringstream>> fragmentedMessage = fragment_message(ss->view(), NetLibrary::FRAGMENT_MESSAGE_STRIDE);
|
std::vector<std::shared_ptr<std::stringstream>> fragmentedMessage = fragment_message(ss->view(), NetLibrary::FRAGMENT_MESSAGE_STRIDE);
|
||||||
auto& messageQueue = client->messageQueues[channel];
|
auto& messageQueue = client->messageQueues[channel];
|
||||||
if(fragmentedMessage.empty())
|
if(fragmentedMessage.empty())
|
||||||
messageQueue.emplace(ss);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, client->nextMessageOrderToSend), ss);
|
||||||
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
||||||
for(auto& ss2 : fragmentedMessage)
|
for(auto& ss2 : fragmentedMessage)
|
||||||
messageQueue.emplace(ss2);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, client->nextMessageOrderToSend), ss2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
template <typename... Args> void send_items_to_all_clients(const std::string& channel, Args&&... items) {
|
template <typename... Args> void send_items_to_all_clients(const std::string& channel, Args&&... items) {
|
||||||
|
|
@ -91,10 +95,10 @@ class NetServer : public std::enable_shared_from_this<NetServer> {
|
||||||
if(client && clientChecker(client)) {
|
if(client && clientChecker(client)) {
|
||||||
auto& messageQueue = client->messageQueues[channel];
|
auto& messageQueue = client->messageQueues[channel];
|
||||||
if(fragmentedMessage.empty())
|
if(fragmentedMessage.empty())
|
||||||
messageQueue.emplace(ss);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, client->nextMessageOrderToSend), ss);
|
||||||
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
else if(channel != UNRELIABLE_COMMAND_CHANNEL) { // Just drop unreliable messages that are fragmented
|
||||||
for(auto& ss2 : fragmentedMessage)
|
for(auto& ss2 : fragmentedMessage)
|
||||||
messageQueue.emplace(ss2);
|
messageQueue.emplace(NetLibrary::calc_order_for_queued_message(channel, client->nextMessageOrderToSend), ss2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -74,9 +74,9 @@ void DrawComponent::server_send_erase_set(MainServer& server, const std::unorder
|
||||||
|
|
||||||
void DrawComponent::server_send_update(MainServer& server, bool final) {
|
void DrawComponent::server_send_update(MainServer& server, bool final) {
|
||||||
if(final)
|
if(final)
|
||||||
server.netServer->send_items_to_all_clients(RELIABLE_COMMAND_CHANNEL, CLIENT_UPDATE_COMPONENT, id, *this);
|
server.netServer->send_items_to_all_clients(RELIABLE_COMMAND_CHANNEL, CLIENT_UPDATE_COMPONENT, final, id, *this);
|
||||||
else
|
else
|
||||||
server.netServer->send_items_to_all_clients(UNRELIABLE_COMMAND_CHANNEL, CLIENT_UPDATE_COMPONENT, id, *this);
|
server.netServer->send_items_to_all_clients(UNRELIABLE_COMMAND_CHANNEL, CLIENT_UPDATE_COMPONENT, final, id, *this);
|
||||||
}
|
}
|
||||||
|
|
||||||
void DrawComponent::server_send_transform_many(MainServer& server, const std::vector<std::pair<ServerClientID, CoordSpaceHelper>>& transforms) {
|
void DrawComponent::server_send_transform_many(MainServer& server, const std::vector<std::pair<ServerClientID, CoordSpaceHelper>>& transforms) {
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,8 @@ DrawingProgram::DrawingProgram(World& initWorld):
|
||||||
void DrawingProgram::init_client_callbacks() {
|
void DrawingProgram::init_client_callbacks() {
|
||||||
world.con.client_add_recv_callback(CLIENT_UPDATE_COMPONENT, [&](cereal::PortableBinaryInputArchive& message) {
|
world.con.client_add_recv_callback(CLIENT_UPDATE_COMPONENT, [&](cereal::PortableBinaryInputArchive& message) {
|
||||||
ServerClientID id;
|
ServerClientID id;
|
||||||
message(id);
|
bool isFinal; // Not actually used, but sent for debugging purposes
|
||||||
|
message(isFinal, id);
|
||||||
auto objPtr = components.get_item_by_id(id);
|
auto objPtr = components.get_item_by_id(id);
|
||||||
if(!objPtr)
|
if(!objPtr)
|
||||||
return;
|
return;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue