Compress large messages with zstd
This commit is contained in:
parent
eb9db31d31
commit
1300be0227
10 changed files with 81 additions and 60 deletions
|
|
@ -134,6 +134,8 @@ if((NOT EMSCRIPTEN) AND (NOT APPLE))
|
||||||
find_package(TBB REQUIRED)
|
find_package(TBB REQUIRED)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
find_package(zstd REQUIRED)
|
||||||
|
|
||||||
message("Skia root: " ${SKIA_INCLUDE})
|
message("Skia root: " ${SKIA_INCLUDE})
|
||||||
message("Build type: " ${CMAKE_BUILD_TYPE})
|
message("Build type: " ${CMAKE_BUILD_TYPE})
|
||||||
|
|
||||||
|
|
@ -236,7 +238,7 @@ if((NOT EMSCRIPTEN) AND (NOT APPLE))
|
||||||
endif()
|
endif()
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
target_link_libraries(infinipaint SDL3::SDL3)
|
target_link_libraries(infinipaint SDL3::SDL3 zstd::libzstd_static)
|
||||||
|
|
||||||
if(EMSCRIPTEN)
|
if(EMSCRIPTEN)
|
||||||
add_subdirectory(deps/datachannel-wasm EXCLUDE_FROM_ALL)
|
add_subdirectory(deps/datachannel-wasm EXCLUDE_FROM_ALL)
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,8 @@ class CompressorRecipe(ConanFile):
|
||||||
if self.settings.os != "Emscripten":
|
if self.settings.os != "Emscripten":
|
||||||
self.requires("libdatachannel/0.23.1")
|
self.requires("libdatachannel/0.23.1")
|
||||||
|
|
||||||
|
self.requires("zstd/1.5.7")
|
||||||
|
|
||||||
def build_requirements(self):
|
def build_requirements(self):
|
||||||
self.tool_requires("cmake/3.27.0")
|
self.tool_requires("cmake/3.27.0")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
#include <cereal/archives/portable_binary.hpp>
|
#include <cereal/archives/portable_binary.hpp>
|
||||||
#include <cereal/types/string.hpp>
|
#include <cereal/types/string.hpp>
|
||||||
#include "NetLibrary.hpp"
|
#include "NetLibrary.hpp"
|
||||||
|
#include <zstd.h>
|
||||||
|
|
||||||
ByteMemBuf::ByteMemBuf(char* p, size_t l) {
|
ByteMemBuf::ByteMemBuf(char* p, size_t l) {
|
||||||
setg(p, p, p + l);
|
setg(p, p, p + l);
|
||||||
|
|
@ -15,16 +16,30 @@ ByteMemStream::ByteMemStream(char* p, size_t l):
|
||||||
rdbuf(&buffer);
|
rdbuf(&buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view v, uint64_t stride) {
|
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view uncompressedView, uint64_t stride) {
|
||||||
if(v.length() <= stride)
|
if(uncompressedView.length() <= stride)
|
||||||
return {};
|
return {};
|
||||||
|
|
||||||
|
std::vector<char> compressedData(ZSTD_compressBound(uncompressedView.size()));
|
||||||
|
size_t trueCompressedSize = ZSTD_compress(compressedData.data(), compressedData.size(), uncompressedView.data(), uncompressedView.size(), ZSTD_CLEVEL_DEFAULT);
|
||||||
|
|
||||||
|
std::string_view v(compressedData.data(), trueCompressedSize);
|
||||||
|
|
||||||
uint64_t nextByte = 0;
|
uint64_t nextByte = 0;
|
||||||
std::vector<std::shared_ptr<std::stringstream>> toRet;
|
std::vector<std::shared_ptr<std::stringstream>> toRet;
|
||||||
|
|
||||||
toRet.emplace_back(std::make_shared<std::stringstream>());
|
toRet.emplace_back(std::make_shared<std::stringstream>());
|
||||||
|
|
||||||
|
size_t messFirstStride = std::min(stride, v.size());
|
||||||
|
|
||||||
{
|
{
|
||||||
cereal::PortableBinaryOutputArchive m(*toRet.back());
|
cereal::PortableBinaryOutputArchive m(*toRet.back());
|
||||||
m((MessageCommandType)0, (uint64_t)v.length(), cereal::binary_data(v.data() + nextByte, stride));
|
m((MessageCommandType)0, (uint64_t)v.length(), cereal::binary_data(v.data() + nextByte, messFirstStride));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(messFirstStride == v.size())
|
||||||
|
return toRet;
|
||||||
|
|
||||||
nextByte += stride;
|
nextByte += stride;
|
||||||
for(;;) {
|
for(;;) {
|
||||||
if((v.length() - nextByte) <= stride) {
|
if((v.length() - nextByte) <= stride) {
|
||||||
|
|
@ -44,3 +59,39 @@ std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_vie
|
||||||
}
|
}
|
||||||
return toRet;
|
return toRet;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void uncompress_and_run_func(PartialFragmentMessage& spfm, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded) {
|
||||||
|
std::vector<char> uncompressedData(ZSTD_getFrameContentSize(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size()));
|
||||||
|
size_t trueUncompressedSize = ZSTD_decompress(uncompressedData.data(), uncompressedData.size(), spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
|
||||||
|
|
||||||
|
ByteMemStream completeStrm(uncompressedData.data(), trueUncompressedSize);
|
||||||
|
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
|
||||||
|
runAfterDecoded(completeArchive);
|
||||||
|
spfm.partialFragmentMessage.clear();
|
||||||
|
spfm.partialFragmentMessageLoc = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void decode_fragmented_message(cereal::PortableBinaryInputArchive& inArchive, PartialFragmentMessage& spfm, size_t fragmentMessageStride, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded) {
|
||||||
|
if(spfm.partialFragmentMessage.size() == 0) {
|
||||||
|
spfm.partialFragmentMessageLoc = 0;
|
||||||
|
uint64_t messageSize;
|
||||||
|
inArchive(messageSize);
|
||||||
|
spfm.partialFragmentMessage.resize(messageSize);
|
||||||
|
|
||||||
|
size_t messStride = std::min(fragmentMessageStride, spfm.partialFragmentMessage.size());
|
||||||
|
|
||||||
|
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, messStride));
|
||||||
|
spfm.partialFragmentMessageLoc += messStride;
|
||||||
|
|
||||||
|
if(messStride == spfm.partialFragmentMessage.size())
|
||||||
|
uncompress_and_run_func(spfm, runAfterDecoded);
|
||||||
|
}
|
||||||
|
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= fragmentMessageStride) {
|
||||||
|
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
|
||||||
|
uncompress_and_run_func(spfm, runAfterDecoded);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, fragmentMessageStride));
|
||||||
|
spfm.partialFragmentMessageLoc += fragmentMessageStride;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
#include <cereal/archives/portable_binary.hpp>
|
||||||
|
|
||||||
// https://tuttlem.github.io/2014/08/18/getting-istream-to-work-off-a-byte-array.html
|
// https://tuttlem.github.io/2014/08/18/getting-istream-to-work-off-a-byte-array.html
|
||||||
class ByteMemBuf : public std::basic_streambuf<char> {
|
class ByteMemBuf : public std::basic_streambuf<char> {
|
||||||
|
|
@ -16,4 +17,10 @@ class ByteMemStream : public std::istream {
|
||||||
ByteMemBuf buffer;
|
ByteMemBuf buffer;
|
||||||
};
|
};
|
||||||
|
|
||||||
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view v, uint64_t stride);
|
struct PartialFragmentMessage {
|
||||||
|
std::string partialFragmentMessage;
|
||||||
|
uint64_t partialFragmentMessageLoc = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view uncompressedView, uint64_t stride);
|
||||||
|
void decode_fragmented_message(cereal::PortableBinaryInputArchive& inArchive, PartialFragmentMessage& spfm, size_t fragmentMessageStride, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded);
|
||||||
|
|
|
||||||
|
|
@ -80,28 +80,11 @@ void NetClient::parse_received_messages() {
|
||||||
inArchive(commandID);
|
inArchive(commandID);
|
||||||
|
|
||||||
if(commandID == 0) {
|
if(commandID == 0) {
|
||||||
if(spfm.partialFragmentMessage.size() == 0) {
|
decode_fragmented_message(inArchive, spfm, NetLibrary::FRAGMENT_MESSAGE_STRIDE, [&](cereal::PortableBinaryInputArchive& completeArchive) {
|
||||||
spfm.partialFragmentMessageLoc = 0;
|
|
||||||
uint64_t messageSize;
|
|
||||||
inArchive(messageSize);
|
|
||||||
spfm.partialFragmentMessage.resize(messageSize);
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
|
|
||||||
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
|
|
||||||
}
|
|
||||||
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= NetLibrary::FRAGMENT_MESSAGE_STRIDE) {
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
|
|
||||||
ByteMemStream completeStrm(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
|
|
||||||
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
|
|
||||||
MessageCommandType completeCommandID;
|
MessageCommandType completeCommandID;
|
||||||
completeArchive(completeCommandID);
|
completeArchive(completeCommandID);
|
||||||
recvCallbacks[completeCommandID](completeArchive);
|
recvCallbacks[completeCommandID](completeArchive);
|
||||||
spfm.partialFragmentMessage.clear();
|
});
|
||||||
spfm.partialFragmentMessageLoc = 0;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
|
|
||||||
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
recvCallbacks[commandID](inArchive);
|
recvCallbacks[commandID](inArchive);
|
||||||
|
|
|
||||||
|
|
@ -59,10 +59,6 @@ class NetClient : public std::enable_shared_from_this<NetClient> {
|
||||||
|
|
||||||
std::unordered_map<std::string, std::shared_ptr<rtc::DataChannel>> channels;
|
std::unordered_map<std::string, std::shared_ptr<rtc::DataChannel>> channels;
|
||||||
|
|
||||||
struct PartialFragmentMessage {
|
|
||||||
std::string partialFragmentMessage;
|
|
||||||
uint64_t partialFragmentMessageLoc = 0;
|
|
||||||
};
|
|
||||||
std::unordered_map<std::string, PartialFragmentMessage> pfm;
|
std::unordered_map<std::string, PartialFragmentMessage> pfm;
|
||||||
|
|
||||||
std::unordered_map<MessageCommandType, NetClientRecvCallback> recvCallbacks;
|
std::unordered_map<MessageCommandType, NetClientRecvCallback> recvCallbacks;
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
#include "NetServer.hpp"
|
#include "NetServer.hpp"
|
||||||
|
#include "Helpers/Networking/ByteStream.hpp"
|
||||||
#include "NetLibrary.hpp"
|
#include "NetLibrary.hpp"
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <rtc/peerconnection.hpp>
|
#include <rtc/peerconnection.hpp>
|
||||||
#include <Helpers/Random.hpp>
|
#include <Helpers/Random.hpp>
|
||||||
#include <Helpers/Logger.hpp>
|
#include <Helpers/Logger.hpp>
|
||||||
#include "NetClient.hpp"
|
#include "NetClient.hpp"
|
||||||
|
#include "cereal/archives/portable_binary.hpp"
|
||||||
|
|
||||||
NetServer::NetServer(const std::string& serverLocalID) {
|
NetServer::NetServer(const std::string& serverLocalID) {
|
||||||
localID = serverLocalID;
|
localID = serverLocalID;
|
||||||
|
|
@ -163,28 +165,11 @@ void NetServer::ClientData::parse_received_messages(NetServer& server) {
|
||||||
inArchive(commandID);
|
inArchive(commandID);
|
||||||
|
|
||||||
if(commandID == 0) {
|
if(commandID == 0) {
|
||||||
if(spfm.partialFragmentMessage.size() == 0) {
|
decode_fragmented_message(inArchive, spfm, NetLibrary::FRAGMENT_MESSAGE_STRIDE, [&](cereal::PortableBinaryInputArchive& completeArchive) {
|
||||||
spfm.partialFragmentMessageLoc = 0;
|
|
||||||
uint64_t messageSize;
|
|
||||||
inArchive(messageSize);
|
|
||||||
spfm.partialFragmentMessage.resize(messageSize);
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
|
|
||||||
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
|
|
||||||
}
|
|
||||||
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= NetLibrary::FRAGMENT_MESSAGE_STRIDE) {
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
|
|
||||||
ByteMemStream completeStrm(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
|
|
||||||
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
|
|
||||||
MessageCommandType completeCommandID;
|
MessageCommandType completeCommandID;
|
||||||
completeArchive(completeCommandID);
|
completeArchive(completeCommandID);
|
||||||
server.recvCallbacks[completeCommandID](shared_from_this(), completeArchive);
|
server.recvCallbacks[completeCommandID](shared_from_this(), completeArchive);
|
||||||
spfm.partialFragmentMessage.clear();
|
});
|
||||||
spfm.partialFragmentMessageLoc = 0;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
|
|
||||||
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
server.recvCallbacks[commandID](shared_from_this(), inArchive);
|
server.recvCallbacks[commandID](shared_from_this(), inArchive);
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,6 @@ class NetServer : public std::enable_shared_from_this<NetServer> {
|
||||||
typedef std::function<void(std::shared_ptr<ClientData>)> NetServerDisconnectCallback;
|
typedef std::function<void(std::shared_ptr<ClientData>)> NetServerDisconnectCallback;
|
||||||
struct ClientData : public std::enable_shared_from_this<ClientData> {
|
struct ClientData : public std::enable_shared_from_this<ClientData> {
|
||||||
std::unordered_map<std::string, std::queue<std::shared_ptr<std::stringstream>>> messages;
|
std::unordered_map<std::string, std::queue<std::shared_ptr<std::stringstream>>> messages;
|
||||||
struct PartialFragmentMessage {
|
|
||||||
std::string partialFragmentMessage;
|
|
||||||
uint64_t partialFragmentMessageLoc = 0;
|
|
||||||
};
|
|
||||||
std::unordered_map<std::string, PartialFragmentMessage> pfm;
|
std::unordered_map<std::string, PartialFragmentMessage> pfm;
|
||||||
|
|
||||||
MessageOrder nextMessageOrderToSend = 0;
|
MessageOrder nextMessageOrderToSend = 0;
|
||||||
|
|
|
||||||
|
|
@ -344,15 +344,14 @@ void DrawBrushStroke::create_collider() {
|
||||||
return false;
|
return false;
|
||||||
}, points, 0);
|
}, points, 0);
|
||||||
strokeObjects.recalculate_bounds();
|
strokeObjects.recalculate_bounds();
|
||||||
collisionTree.clear();
|
|
||||||
|
SCollision::BVHContainer<float> collisionTree;
|
||||||
|
|
||||||
collisionTree.calculate_bvh_recursive(strokeObjects);
|
collisionTree.calculate_bvh_recursive(strokeObjects);
|
||||||
precheckAABBLevels.clear();
|
precheckAABBLevels.clear();
|
||||||
add_precheck_aabb_level(0, collisionTree.children);
|
add_precheck_aabb_level(0, collisionTree.children);
|
||||||
|
|
||||||
collisionTree.objects.aabb.clear();
|
bounds = collisionTree.objects.bounds;
|
||||||
collisionTree.objects.circle.clear();
|
|
||||||
collisionTree.objects.triangle.clear();
|
|
||||||
collisionTree.children.clear();
|
|
||||||
|
|
||||||
calculate_world_bounds();
|
calculate_world_bounds();
|
||||||
}
|
}
|
||||||
|
|
@ -377,7 +376,7 @@ void DrawBrushStroke::update(DrawingProgram& drawP) {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DrawBrushStroke::collides_within_coords(const SCollision::ColliderCollection<float>& checkAgainst) {
|
bool DrawBrushStroke::collides_within_coords(const SCollision::ColliderCollection<float>& checkAgainst) {
|
||||||
if(!SCollision::collide(checkAgainst, collisionTree.objects.bounds))
|
if(!SCollision::collide(checkAgainst, bounds))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
bool deepPrecheck = false;
|
bool deepPrecheck = false;
|
||||||
|
|
@ -400,7 +399,7 @@ bool DrawBrushStroke::collides_within_coords(const SCollision::ColliderCollectio
|
||||||
}
|
}
|
||||||
|
|
||||||
SCollision::AABB<float> DrawBrushStroke::get_obj_coord_bounds() const {
|
SCollision::AABB<float> DrawBrushStroke::get_obj_coord_bounds() const {
|
||||||
return collisionTree.objects.bounds;
|
return bounds;
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,7 @@ class DrawBrushStroke : public DrawComponent {
|
||||||
|
|
||||||
sk_sp<SkVertices> vertices;
|
sk_sp<SkVertices> vertices;
|
||||||
std::array<sk_sp<SkVertices>, 3> verticesMipMap;
|
std::array<sk_sp<SkVertices>, 3> verticesMipMap;
|
||||||
SCollision::BVHContainer<float> collisionTree;
|
SCollision::AABB<float> bounds;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void add_precheck_aabb_level(size_t level, const std::vector<SCollision::BVHContainer<float>>& levelArray);
|
void add_precheck_aabb_level(size_t level, const std::vector<SCollision::BVHContainer<float>>& levelArray);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue