Compress savefiles using zstd
This commit is contained in:
parent
43f7027651
commit
c499f597d9
4 changed files with 55 additions and 56 deletions
|
|
@ -39,6 +39,8 @@ enum ClientCommands : MessageCommandType {
|
||||||
CLIENT_REMOVE_BOOKMARK
|
CLIENT_REMOVE_BOOKMARK
|
||||||
};
|
};
|
||||||
|
|
||||||
// NOTE: Change these values whenever savefile is incompatible with previous version
|
// NOTE: Do not change savefile header length
|
||||||
#define SAVEFILE_HEADER_LEN 12
|
#define SAVEFILE_HEADER_LEN 12
|
||||||
#define SAVEFILE_HEADER "INFPNT000001"
|
// NOTE: Change these values whenever savefile is incompatible with previous version
|
||||||
|
#define SAVEFILE_HEADER "INFPNT000002"
|
||||||
|
#define SAVEFILE_HEADER_V1 "INFPNT000001"
|
||||||
|
|
|
||||||
|
|
@ -342,13 +342,8 @@ void Toolbar::update() {
|
||||||
void Toolbar::save_func() {
|
void Toolbar::save_func() {
|
||||||
if(main.world->filePath == std::filesystem::path())
|
if(main.world->filePath == std::filesystem::path())
|
||||||
save_as_func();
|
save_as_func();
|
||||||
else {
|
else
|
||||||
#ifdef __EMSCRIPTEN__
|
|
||||||
main.world->download_file(main.world->filePath.string());
|
|
||||||
#else
|
|
||||||
main.world->save_to_file(main.world->filePath.string());
|
main.world->save_to_file(main.world->filePath.string());
|
||||||
#endif
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void Toolbar::save_as_func() {
|
void Toolbar::save_as_func() {
|
||||||
|
|
@ -1390,7 +1385,7 @@ void Toolbar::options_menu() {
|
||||||
gui.input_text_field("file name", "File Name", &downloadNameSet);
|
gui.input_text_field("file name", "File Name", &downloadNameSet);
|
||||||
gui.left_to_right_line_layout([&]() {
|
gui.left_to_right_line_layout([&]() {
|
||||||
if(gui.text_button_wide("download save button", "Save")) {
|
if(gui.text_button_wide("download save button", "Save")) {
|
||||||
main.world->download_file(downloadNameSet);
|
main.world->save_to_file(downloadNameSet);
|
||||||
optionsMenuOpen = false;
|
optionsMenuOpen = false;
|
||||||
}
|
}
|
||||||
if(gui.text_button_wide("cancel", "Cancel"))
|
if(gui.text_button_wide("cancel", "Cancel"))
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@
|
||||||
#include <Helpers/Logger.hpp>
|
#include <Helpers/Logger.hpp>
|
||||||
#include <cereal/types/vector.hpp>
|
#include <cereal/types/vector.hpp>
|
||||||
#include <Helpers/Networking/NetLibrary.hpp>
|
#include <Helpers/Networking/NetLibrary.hpp>
|
||||||
|
#include <zstd.h>
|
||||||
|
|
||||||
#ifdef __EMSCRIPTEN__
|
#ifdef __EMSCRIPTEN__
|
||||||
#include <EmscriptenHelpers/emscripten_browser_file.h>
|
#include <EmscriptenHelpers/emscripten_browser_file.h>
|
||||||
|
|
@ -223,16 +224,27 @@ void World::start_hosting(const std::string& initNetSource, const std::string& s
|
||||||
clientStillConnecting = true;
|
clientStillConnecting = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void World::download_file(const std::filesystem::path& fileName) {
|
void World::save_to_file(const std::filesystem::path& fileName) {
|
||||||
try {
|
try {
|
||||||
filePath = force_extension_on_path(fileName, FILE_EXTENSION);
|
filePath = force_extension_on_path(fileName, FILE_EXTENSION);
|
||||||
|
|
||||||
std::stringstream f;
|
std::stringstream f;
|
||||||
f.write(SAVEFILE_HEADER, SAVEFILE_HEADER_LEN);
|
f.write(SAVEFILE_HEADER, SAVEFILE_HEADER_LEN);
|
||||||
cereal::PortableBinaryOutputArchive a(f);
|
|
||||||
|
{
|
||||||
|
std::stringstream fWorldDataToCompress;
|
||||||
|
{
|
||||||
|
cereal::PortableBinaryOutputArchive a(fWorldDataToCompress);
|
||||||
SaveLoadFileOp op{.world = this};
|
SaveLoadFileOp op{.world = this};
|
||||||
a(op);
|
a(op);
|
||||||
Logger::get().log("USERINFO", "File saved");
|
}
|
||||||
|
|
||||||
|
std::vector<char> compressedData(ZSTD_compressBound(fWorldDataToCompress.view().size()));
|
||||||
|
size_t trueCompressedSize = ZSTD_compress(compressedData.data(), compressedData.size(), fWorldDataToCompress.view().data(), fWorldDataToCompress.view().size(), ZSTD_CLEVEL_DEFAULT);
|
||||||
|
std::string_view compressedF(compressedData.data(), trueCompressedSize);
|
||||||
|
|
||||||
|
f << compressedF;
|
||||||
|
}
|
||||||
|
|
||||||
set_name(filePath.stem().string());
|
set_name(filePath.stem().string());
|
||||||
|
|
||||||
|
|
@ -242,26 +254,13 @@ void World::download_file(const std::filesystem::path& fileName) {
|
||||||
"application/octet-stream",
|
"application/octet-stream",
|
||||||
f.view()
|
f.view()
|
||||||
);
|
);
|
||||||
|
#else
|
||||||
|
std::ofstream fi(filePath, std::ios::out | std::ios::binary);
|
||||||
|
fi << f.view();
|
||||||
|
fi.close();
|
||||||
#endif
|
#endif
|
||||||
}
|
|
||||||
catch(const std::exception& e) {
|
|
||||||
Logger::get().log("WORLDFATAL", std::string("Save error: ") + e.what());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void World::save_to_file(const std::filesystem::path& fileName) {
|
|
||||||
try {
|
|
||||||
filePath = force_extension_on_path(fileName, FILE_EXTENSION);
|
|
||||||
|
|
||||||
std::ofstream f(filePath, std::ios::out | std::ios::binary);
|
|
||||||
f.write(SAVEFILE_HEADER, SAVEFILE_HEADER_LEN);
|
|
||||||
cereal::PortableBinaryOutputArchive a(f);
|
|
||||||
SaveLoadFileOp op{.world = this};
|
|
||||||
a(op);
|
|
||||||
f.close();
|
|
||||||
Logger::get().log("USERINFO", "File saved");
|
Logger::get().log("USERINFO", "File saved");
|
||||||
|
|
||||||
set_name(filePath.stem().string());
|
|
||||||
}
|
}
|
||||||
catch(const std::exception& e) {
|
catch(const std::exception& e) {
|
||||||
Logger::get().log("WORLDFATAL", std::string("Save error: ") + e.what());
|
Logger::get().log("WORLDFATAL", std::string("Save error: ") + e.what());
|
||||||
|
|
@ -271,34 +270,38 @@ void World::save_to_file(const std::filesystem::path& fileName) {
|
||||||
void World::load_from_file(const std::filesystem::path& fileName, std::string_view buffer) {
|
void World::load_from_file(const std::filesystem::path& fileName, std::string_view buffer) {
|
||||||
filePath = force_extension_on_path(fileName, FILE_EXTENSION);
|
filePath = force_extension_on_path(fileName, FILE_EXTENSION);
|
||||||
|
|
||||||
|
std::string byteDataFromFile;
|
||||||
|
|
||||||
if(buffer.empty()) {
|
if(buffer.empty()) {
|
||||||
std::ifstream f(filePath, std::ios::in | std::ios::binary);
|
byteDataFromFile = read_file_to_string(fileName);
|
||||||
std::string fileHeader;
|
buffer = byteDataFromFile;
|
||||||
fileHeader.resize(SAVEFILE_HEADER_LEN);
|
}
|
||||||
f.read(fileHeader.data(), SAVEFILE_HEADER_LEN);
|
|
||||||
if(fileHeader != std::string(SAVEFILE_HEADER))
|
if(buffer.size() < SAVEFILE_HEADER_LEN)
|
||||||
|
throw std::runtime_error("[World::load_from_file] File too small");
|
||||||
|
|
||||||
|
std::string_view fileHeader = buffer.substr(0, SAVEFILE_HEADER_LEN);
|
||||||
|
std::string_view uncompressedDataView;
|
||||||
|
std::vector<char> uncompressedDataVector;
|
||||||
|
|
||||||
|
if(fileHeader == SAVEFILE_HEADER) {
|
||||||
|
uncompressedDataVector.resize(ZSTD_getFrameContentSize(buffer.data() + SAVEFILE_HEADER_LEN, buffer.size() - SAVEFILE_HEADER_LEN));
|
||||||
|
size_t trueUncompressedSize = ZSTD_decompress(uncompressedDataVector.data(), uncompressedDataVector.size(), buffer.data() + SAVEFILE_HEADER_LEN, buffer.size() - SAVEFILE_HEADER_LEN);
|
||||||
|
uncompressedDataView = std::string_view(uncompressedDataVector.data(), trueUncompressedSize);
|
||||||
|
}
|
||||||
|
else if(fileHeader == SAVEFILE_HEADER_V1)
|
||||||
|
uncompressedDataView = std::string_view(buffer.data() + SAVEFILE_HEADER_LEN, buffer.size() - SAVEFILE_HEADER_LEN);
|
||||||
|
else
|
||||||
throw std::runtime_error("[World::load_from_file] File does not have correct header");
|
throw std::runtime_error("[World::load_from_file] File does not have correct header");
|
||||||
|
|
||||||
cereal::PortableBinaryInputArchive a(f);
|
|
||||||
SaveLoadFileOp op{.world = this};
|
|
||||||
a(op);
|
ByteMemStream f((char*)uncompressedDataView.data(), uncompressedDataView.size());
|
||||||
f.close();
|
|
||||||
nextClientID = std::max(rMan.get_max_id(ownID), drawProg.get_max_id(ownID));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
ByteMemStream f((char*)buffer.data(), buffer.size());
|
|
||||||
std::string fileHeader;
|
|
||||||
fileHeader.resize(SAVEFILE_HEADER_LEN);
|
|
||||||
f.read(fileHeader.data(), SAVEFILE_HEADER_LEN);
|
|
||||||
if(fileHeader != std::string(SAVEFILE_HEADER))
|
|
||||||
throw std::runtime_error("[World::load_from_file] File does not have correct header");
|
|
||||||
|
|
||||||
cereal::PortableBinaryInputArchive a(f);
|
cereal::PortableBinaryInputArchive a(f);
|
||||||
SaveLoadFileOp op{.world = this};
|
SaveLoadFileOp op{.world = this};
|
||||||
a(op);
|
a(op);
|
||||||
nextClientID = std::max(rMan.get_max_id(ownID), drawProg.get_max_id(ownID));
|
nextClientID = std::max(rMan.get_max_id(ownID), drawProg.get_max_id(ownID));
|
||||||
}
|
|
||||||
|
|
||||||
Logger::get().log("USERINFO", "File loaded");
|
Logger::get().log("USERINFO", "File loaded");
|
||||||
set_name(filePath.stem().string());
|
set_name(filePath.stem().string());
|
||||||
|
|
|
||||||
|
|
@ -74,7 +74,6 @@ class World {
|
||||||
ConnectionType conType;
|
ConnectionType conType;
|
||||||
bool clientStillConnecting = false;
|
bool clientStillConnecting = false;
|
||||||
|
|
||||||
void download_file(const std::filesystem::path& fileName);
|
|
||||||
void save_to_file(const std::filesystem::path& fileName);
|
void save_to_file(const std::filesystem::path& fileName);
|
||||||
void load_from_file(const std::filesystem::path& fileName, std::string_view buffer);
|
void load_from_file(const std::filesystem::path& fileName, std::string_view buffer);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue