copilot/infinipaint
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions 1

Compress large messages with zstd

Browse source
This commit is contained in:
Yousef Khadadeh 2025-07-28 17:07:39 +03:00
parent eb9db31d31
commit 1300be0227
10 changed files with 81 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CMakeLists.txt
View file

@ -134,6 +134,8 @@ if((NOT EMSCRIPTEN) AND (NOT APPLE))
find_package(TBB REQUIRED)
endif()
find_package(zstd REQUIRED)
message("Skia root: " ${SKIA_INCLUDE})
message("Build type: " ${CMAKE_BUILD_TYPE})
@ -236,7 +238,7 @@ if((NOT EMSCRIPTEN) AND (NOT APPLE))
endif()
endif()
target_link_libraries(infinipaint SDL3::SDL3)
target_link_libraries(infinipaint SDL3::SDL3 zstd::libzstd_static)
if(EMSCRIPTEN)
add_subdirectory(deps/datachannel-wasm EXCLUDE_FROM_ALL)

2
conanfile.py
View file

@ -53,6 +53,8 @@ class CompressorRecipe(ConanFile):
if self.settings.os != "Emscripten":
self.requires("libdatachannel/0.23.1")
self.requires("zstd/1.5.7")
def build_requirements(self):
self.tool_requires("cmake/3.27.0")

57
include/Helpers/Networking/ByteStream.cpp
View file

@ -2,6 +2,7 @@
#include <cereal/archives/portable_binary.hpp>
#include <cereal/types/string.hpp>
#include "NetLibrary.hpp"
#include <zstd.h>
ByteMemBuf::ByteMemBuf(char* p, size_t l) {
setg(p, p, p + l);
@ -15,16 +16,30 @@ ByteMemStream::ByteMemStream(char* p, size_t l):
rdbuf(&buffer);
}
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view v, uint64_t stride) {
if(v.length() <= stride)
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view uncompressedView, uint64_t stride) {
if(uncompressedView.length() <= stride)
return {};
std::vector<char> compressedData(ZSTD_compressBound(uncompressedView.size()));
size_t trueCompressedSize = ZSTD_compress(compressedData.data(), compressedData.size(), uncompressedView.data(), uncompressedView.size(), ZSTD_CLEVEL_DEFAULT);
std::string_view v(compressedData.data(), trueCompressedSize);
uint64_t nextByte = 0;
std::vector<std::shared_ptr<std::stringstream>> toRet;
toRet.emplace_back(std::make_shared<std::stringstream>());
size_t messFirstStride = std::min(stride, v.size());
{
cereal::PortableBinaryOutputArchive m(*toRet.back());
m((MessageCommandType)0, (uint64_t)v.length(), cereal::binary_data(v.data() + nextByte, stride));
m((MessageCommandType)0, (uint64_t)v.length(), cereal::binary_data(v.data() + nextByte, messFirstStride));
}
if(messFirstStride == v.size())
return toRet;
nextByte += stride;
for(;;) {
if((v.length() - nextByte) <= stride) {
@ -44,3 +59,39 @@ std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_vie
}
return toRet;
}
void uncompress_and_run_func(PartialFragmentMessage& spfm, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded) {
std::vector<char> uncompressedData(ZSTD_getFrameContentSize(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size()));
size_t trueUncompressedSize = ZSTD_decompress(uncompressedData.data(), uncompressedData.size(), spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
ByteMemStream completeStrm(uncompressedData.data(), trueUncompressedSize);
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
runAfterDecoded(completeArchive);
spfm.partialFragmentMessage.clear();
spfm.partialFragmentMessageLoc = 0;
}
void decode_fragmented_message(cereal::PortableBinaryInputArchive& inArchive, PartialFragmentMessage& spfm, size_t fragmentMessageStride, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded) {
if(spfm.partialFragmentMessage.size() == 0) {
spfm.partialFragmentMessageLoc = 0;
uint64_t messageSize;
inArchive(messageSize);
spfm.partialFragmentMessage.resize(messageSize);
size_t messStride = std::min(fragmentMessageStride, spfm.partialFragmentMessage.size());
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, messStride));
spfm.partialFragmentMessageLoc += messStride;
if(messStride == spfm.partialFragmentMessage.size())
uncompress_and_run_func(spfm, runAfterDecoded);
}
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= fragmentMessageStride) {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
uncompress_and_run_func(spfm, runAfterDecoded);
}
else {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, fragmentMessageStride));
spfm.partialFragmentMessageLoc += fragmentMessageStride;
}
}

9
include/Helpers/Networking/ByteStream.hpp
View file

@ -3,6 +3,7 @@
#include <sstream>
#include <memory>
#include <vector>
#include <cereal/archives/portable_binary.hpp>
// https://tuttlem.github.io/2014/08/18/getting-istream-to-work-off-a-byte-array.html
class ByteMemBuf : public std::basic_streambuf<char> {
@ -16,4 +17,10 @@ class ByteMemStream : public std::istream {
ByteMemBuf buffer;
};
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view v, uint64_t stride);
struct PartialFragmentMessage {
std::string partialFragmentMessage;
uint64_t partialFragmentMessageLoc = 0;
};
std::vector<std::shared_ptr<std::stringstream>> fragment_message(std::string_view uncompressedView, uint64_t stride);
void decode_fragmented_message(cereal::PortableBinaryInputArchive& inArchive, PartialFragmentMessage& spfm, size_t fragmentMessageStride, std::function<void(cereal::PortableBinaryInputArchive& completeArchive)> runAfterDecoded);

21
include/Helpers/Networking/NetClient.cpp
View file

@ -80,28 +80,11 @@ void NetClient::parse_received_messages() {
inArchive(commandID);
if(commandID == 0) {
if(spfm.partialFragmentMessage.size() == 0) {
spfm.partialFragmentMessageLoc = 0;
uint64_t messageSize;
inArchive(messageSize);
spfm.partialFragmentMessage.resize(messageSize);
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
}
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= NetLibrary::FRAGMENT_MESSAGE_STRIDE) {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
ByteMemStream completeStrm(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
decode_fragmented_message(inArchive, spfm, NetLibrary::FRAGMENT_MESSAGE_STRIDE, [&](cereal::PortableBinaryInputArchive& completeArchive) {
MessageCommandType completeCommandID;
completeArchive(completeCommandID);
recvCallbacks[completeCommandID](completeArchive);
spfm.partialFragmentMessage.clear();
spfm.partialFragmentMessageLoc = 0;
}
else {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
}
});
}
else
recvCallbacks[commandID](inArchive);

4
include/Helpers/Networking/NetClient.hpp
View file

@ -59,10 +59,6 @@ class NetClient : public std::enable_shared_from_this<NetClient> {
std::unordered_map<std::string, std::shared_ptr<rtc::DataChannel>> channels;
struct PartialFragmentMessage {
std::string partialFragmentMessage;
uint64_t partialFragmentMessageLoc = 0;
};
std::unordered_map<std::string, PartialFragmentMessage> pfm;
std::unordered_map<MessageCommandType, NetClientRecvCallback> recvCallbacks;

23
include/Helpers/Networking/NetServer.cpp
View file

@ -1,10 +1,12 @@
#include "NetServer.hpp"
#include "Helpers/Networking/ByteStream.hpp"
#include "NetLibrary.hpp"
#include <chrono>
#include <rtc/peerconnection.hpp>
#include <Helpers/Random.hpp>
#include <Helpers/Logger.hpp>
#include "NetClient.hpp"
#include "cereal/archives/portable_binary.hpp"
NetServer::NetServer(const std::string& serverLocalID) {
localID = serverLocalID;
@ -163,28 +165,11 @@ void NetServer::ClientData::parse_received_messages(NetServer& server) {
inArchive(commandID);
if(commandID == 0) {
if(spfm.partialFragmentMessage.size() == 0) {
spfm.partialFragmentMessageLoc = 0;
uint64_t messageSize;
inArchive(messageSize);
spfm.partialFragmentMessage.resize(messageSize);
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
}
else if(spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc <= NetLibrary::FRAGMENT_MESSAGE_STRIDE) {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, spfm.partialFragmentMessage.size() - spfm.partialFragmentMessageLoc));
ByteMemStream completeStrm(spfm.partialFragmentMessage.data(), spfm.partialFragmentMessage.size());
cereal::PortableBinaryInputArchive completeArchive(completeStrm);
decode_fragmented_message(inArchive, spfm, NetLibrary::FRAGMENT_MESSAGE_STRIDE, [&](cereal::PortableBinaryInputArchive& completeArchive) {
MessageCommandType completeCommandID;
completeArchive(completeCommandID);
server.recvCallbacks[completeCommandID](shared_from_this(), completeArchive);
spfm.partialFragmentMessage.clear();
spfm.partialFragmentMessageLoc = 0;
}
else {
inArchive(cereal::binary_data(spfm.partialFragmentMessage.data() + spfm.partialFragmentMessageLoc, NetLibrary::FRAGMENT_MESSAGE_STRIDE));
spfm.partialFragmentMessageLoc += NetLibrary::FRAGMENT_MESSAGE_STRIDE;
}
});
}
else
server.recvCallbacks[commandID](shared_from_this(), inArchive);

4
include/Helpers/Networking/NetServer.hpp
View file

@ -17,10 +17,6 @@ class NetServer : public std::enable_shared_from_this<NetServer> {
typedef std::function<void(std::shared_ptr<ClientData>)> NetServerDisconnectCallback;
struct ClientData : public std::enable_shared_from_this<ClientData> {
std::unordered_map<std::string, std::queue<std::shared_ptr<std::stringstream>>> messages;
struct PartialFragmentMessage {
std::string partialFragmentMessage;
uint64_t partialFragmentMessageLoc = 0;
};
std::unordered_map<std::string, PartialFragmentMessage> pfm;
MessageOrder nextMessageOrderToSend = 0;

13
src/DrawComponents/DrawBrushStroke.cpp
View file

@ -344,15 +344,14 @@ void DrawBrushStroke::create_collider() {
return false;
}, points, 0);
strokeObjects.recalculate_bounds();
collisionTree.clear();
SCollision::BVHContainer<float> collisionTree;
collisionTree.calculate_bvh_recursive(strokeObjects);
precheckAABBLevels.clear();
add_precheck_aabb_level(0, collisionTree.children);
collisionTree.objects.aabb.clear();
collisionTree.objects.circle.clear();
collisionTree.objects.triangle.clear();
collisionTree.children.clear();
bounds = collisionTree.objects.bounds;
calculate_world_bounds();
}
@ -377,7 +376,7 @@ void DrawBrushStroke::update(DrawingProgram& drawP) {
}
bool DrawBrushStroke::collides_within_coords(const SCollision::ColliderCollection<float>& checkAgainst) {
if(!SCollision::collide(checkAgainst, collisionTree.objects.bounds))
if(!SCollision::collide(checkAgainst, bounds))
return false;
bool deepPrecheck = false;
@ -400,7 +399,7 @@ bool DrawBrushStroke::collides_within_coords(const SCollision::ColliderCollectio
}
SCollision::AABB<float> DrawBrushStroke::get_obj_coord_bounds() const {
return collisionTree.objects.bounds;
return bounds;
}
#endif

2
src/DrawComponents/DrawBrushStroke.hpp
View file

@ -45,7 +45,7 @@ class DrawBrushStroke : public DrawComponent {
sk_sp<SkVertices> vertices;
std::array<sk_sp<SkVertices>, 3> verticesMipMap;
SCollision::BVHContainer<float> collisionTree;
SCollision::AABB<float> bounds;
private:
void add_precheck_aabb_level(size_t level, const std::vector<SCollision::BVHContainer<float>>& levelArray);