Added FileDownloader utility based on libcurl to download files from URLs on native platforms
This commit is contained in:
parent
220eb34ab9
commit
08680c6e94
8 changed files with 232 additions and 3 deletions
|
|
@ -104,6 +104,7 @@ set(sources "src/main.cpp"
|
||||||
"include/Helpers/Logger.cpp"
|
"include/Helpers/Logger.cpp"
|
||||||
"include/Helpers/StringHelpers.cpp"
|
"include/Helpers/StringHelpers.cpp"
|
||||||
"include/Helpers/Serializers.cpp"
|
"include/Helpers/Serializers.cpp"
|
||||||
|
"include/Helpers/FileDownloader.cpp"
|
||||||
)
|
)
|
||||||
|
|
||||||
if((NOT EMSCRIPTEN) AND (GRAPHICS_BACKEND STREQUAL "Vulkan"))
|
if((NOT EMSCRIPTEN) AND (GRAPHICS_BACKEND STREQUAL "Vulkan"))
|
||||||
|
|
@ -144,6 +145,7 @@ endif()
|
||||||
if(NOT EMSCRIPTEN)
|
if(NOT EMSCRIPTEN)
|
||||||
find_package(SDL3 REQUIRED CONFIG REQUIRED COMPONENTS SDL3-shared)
|
find_package(SDL3 REQUIRED CONFIG REQUIRED COMPONENTS SDL3-shared)
|
||||||
find_package(LibDataChannel REQUIRED)
|
find_package(LibDataChannel REQUIRED)
|
||||||
|
find_package(CURL REQUIRED)
|
||||||
else()
|
else()
|
||||||
find_package(SDL3 REQUIRED CONFIG REQUIRED COMPONENTS SDL3-static)
|
find_package(SDL3 REQUIRED CONFIG REQUIRED COMPONENTS SDL3-static)
|
||||||
endif()
|
endif()
|
||||||
|
|
@ -266,7 +268,7 @@ if(EMSCRIPTEN)
|
||||||
add_subdirectory(deps/datachannel-wasm EXCLUDE_FROM_ALL)
|
add_subdirectory(deps/datachannel-wasm EXCLUDE_FROM_ALL)
|
||||||
target_link_libraries(infinipaint datachannel-wasm)
|
target_link_libraries(infinipaint datachannel-wasm)
|
||||||
else()
|
else()
|
||||||
target_link_libraries(infinipaint LibDataChannel::LibDataChannel)
|
target_link_libraries(infinipaint LibDataChannel::LibDataChannel CURL::libcurl)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if(LINK_SKIA_MANUALLY)
|
if(LINK_SKIA_MANUALLY)
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||||
|
|
||||||
conan export "$(dirname "$0")/libjuice/all/conanfile.py" --version=1.6.1
|
conan export "$(dirname "$0")/libjuice/all/conanfile.py" --version=1.6.1
|
||||||
conan export "$(dirname "$0")/libdatachannel/all/conanfile.py" --version=0.23.1
|
conan export "$(dirname "$0")/libdatachannel/all/conanfile.py" --version=0.23.2
|
||||||
conan export "$(dirname "$0")/mpg123/all/conanfile.py" --version=1.32.10
|
conan export "$(dirname "$0")/mpg123/all/conanfile.py" --version=1.32.10
|
||||||
conan export "$(dirname "$0")/conan-skia/recipes/skia/all/conanfile.py" --version=134.20250327.0
|
conan export "$(dirname "$0")/conan-skia/recipes/skia/all/conanfile.py" --version=134.20250327.0
|
||||||
#conan create . -o use_system_expat=False -o use_system_freetype=False -o use_system_harfbuzz=False -o use_system_icu=False -o use_system_libjpeg_turbo=False -o use_system_libpng=False -o use_system_libwebp=False -o use_system_zlib=False --build=missing --version=134.20250327.0
|
#conan create . -o use_system_expat=False -o use_system_freetype=False -o use_system_harfbuzz=False -o use_system_icu=False -o use_system_libjpeg_turbo=False -o use_system_libpng=False -o use_system_libwebp=False -o use_system_zlib=False --build=missing --version=134.20250327.0
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,7 @@
|
||||||
sources:
|
sources:
|
||||||
|
"0.23.2":
|
||||||
|
url: "https://github.com/paullouisageneau/libdatachannel/archive/refs/tags/v0.23.2.tar.gz"
|
||||||
|
sha256: "b9606efc5b2b173f2d22d0be3f6ba4f12af78c00ca02cde5932f3ff902980eb9"
|
||||||
"0.23.1":
|
"0.23.1":
|
||||||
url: "https://github.com/paullouisageneau/libdatachannel/archive/refs/tags/v0.23.1.tar.gz"
|
url: "https://github.com/paullouisageneau/libdatachannel/archive/refs/tags/v0.23.1.tar.gz"
|
||||||
sha256: "63e14d619ac4d9cc310a0c7620b80e6da88abf878f27ccc78cd099f95d47b121"
|
sha256: "63e14d619ac4d9cc310a0c7620b80e6da88abf878f27ccc78cd099f95d47b121"
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,8 @@ class CompressorRecipe(ConanFile):
|
||||||
self.requires("onetbb/2022.0.0")
|
self.requires("onetbb/2022.0.0")
|
||||||
|
|
||||||
if self.settings.os != "Emscripten":
|
if self.settings.os != "Emscripten":
|
||||||
self.requires("libdatachannel/0.23.1")
|
self.requires("libdatachannel/0.23.2")
|
||||||
|
self.requires("libcurl/8.15.0")
|
||||||
|
|
||||||
self.requires("zstd/1.5.7")
|
self.requires("zstd/1.5.7")
|
||||||
|
|
||||||
|
|
|
||||||
22
data/third_party_licenses/libcurl
Normal file
22
data/third_party_licenses/libcurl
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
COPYRIGHT AND PERMISSION NOTICE
|
||||||
|
|
||||||
|
Copyright (c) 1996 - 2025, Daniel Stenberg, <daniel@haxx.se>, and many
|
||||||
|
contributors, see the THANKS file.
|
||||||
|
|
||||||
|
All rights reserved.
|
||||||
|
|
||||||
|
Permission to use, copy, modify, and distribute this software for any purpose
|
||||||
|
with or without fee is hereby granted, provided that the above copyright
|
||||||
|
notice and this permission notice appear in all copies.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
|
||||||
|
NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||||
|
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
||||||
|
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
|
||||||
|
OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
Except as contained in this notice, the name of a copyright holder shall not
|
||||||
|
be used in advertising or otherwise to promote the sale, use or other dealings
|
||||||
|
in this Software without prior written authorization of the copyright holder.
|
||||||
135
include/Helpers/FileDownloader.cpp
Normal file
135
include/Helpers/FileDownloader.cpp
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
#include "FileDownloader.hpp"
|
||||||
|
#ifndef __EMSCRIPTEN__
|
||||||
|
|
||||||
|
#include <curl/multi.h>
|
||||||
|
#include <iostream>
|
||||||
|
#include <curl/curl.h>
|
||||||
|
|
||||||
|
std::mutex FileDownloader::newDownloadsMutex;
|
||||||
|
std::vector<FileDownloader::DownloadHandler> FileDownloader::newDownloads;
|
||||||
|
std::atomic<bool> FileDownloader::shutdownThread;
|
||||||
|
FileDownloader::MultiHandle FileDownloader::mHandle;
|
||||||
|
std::vector<FileDownloader::DownloadHandler> FileDownloader::currentDownloads;
|
||||||
|
std::unique_ptr<std::thread> FileDownloader::downloadThread;
|
||||||
|
|
||||||
|
void FileDownloader::init() {
|
||||||
|
curl_global_init(CURL_GLOBAL_DEFAULT);
|
||||||
|
shutdownThread = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<FileDownloader::DownloadData> FileDownloader::download_data_from_url(std::string_view url) {
|
||||||
|
DownloadHandler h;
|
||||||
|
auto downloadData = std::make_shared<DownloadData>();
|
||||||
|
h.data = downloadData;
|
||||||
|
h.data->status = DownloadData::Status::IN_PROGRESS;
|
||||||
|
h.eHandle = create_easy_handle();
|
||||||
|
curl_easy_setopt(h.eHandle.get(), CURLOPT_URL, url.data());
|
||||||
|
disable_ssl_verification(h.eHandle.get());
|
||||||
|
save_to_string(h.eHandle.get(), &h.data->str);
|
||||||
|
|
||||||
|
{
|
||||||
|
std::scoped_lock newDownloadsLock(newDownloadsMutex);
|
||||||
|
newDownloads.emplace_back(std::move(h));
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!downloadThread)
|
||||||
|
downloadThread = std::make_unique<std::thread>(download_thread_update);
|
||||||
|
|
||||||
|
return downloadData;
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileDownloader::add_new_handlers_to_multi() {
|
||||||
|
std::scoped_lock newDownloadsLock(newDownloadsMutex);
|
||||||
|
for(auto& handler : newDownloads) {
|
||||||
|
curl_multi_add_handle(mHandle.get(), handler.eHandle.get());
|
||||||
|
currentDownloads.emplace_back(std::move(handler));
|
||||||
|
}
|
||||||
|
newDownloads.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileDownloader::download_thread_update() {
|
||||||
|
int stillRunning = 0;
|
||||||
|
|
||||||
|
mHandle = create_multi_handle();
|
||||||
|
add_new_handlers_to_multi();
|
||||||
|
|
||||||
|
do {
|
||||||
|
if(shutdownThread)
|
||||||
|
break;
|
||||||
|
|
||||||
|
CURLMcode mc;
|
||||||
|
int numfds;
|
||||||
|
|
||||||
|
mc = curl_multi_perform(mHandle.get(), &stillRunning);
|
||||||
|
|
||||||
|
for(;;) {
|
||||||
|
int msgCount;
|
||||||
|
CURLMsg* eHandleMsg = curl_multi_info_read(mHandle.get(), &msgCount);
|
||||||
|
if(!eHandleMsg)
|
||||||
|
break;
|
||||||
|
if(eHandleMsg->msg == CURLMSG_DONE) {
|
||||||
|
auto it = std::find_if(currentDownloads.begin(), currentDownloads.end(), [eHandleMsg](auto& d) {
|
||||||
|
return eHandleMsg->easy_handle == d.eHandle.get();
|
||||||
|
});
|
||||||
|
if(it != currentDownloads.end()) {
|
||||||
|
DownloadHandler& download = *it;
|
||||||
|
download.data->status = eHandleMsg->data.result == CURLE_OK ? DownloadData::Status::SUCCESS : DownloadData::Status::FAILURE;
|
||||||
|
currentDownloads.erase(it);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(mc == CURLM_OK)
|
||||||
|
curl_multi_poll(mHandle.get(), nullptr, 0, 1000, &numfds);
|
||||||
|
else {
|
||||||
|
std::cout << "curl_multi failed" << std::endl;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} while(stillRunning);
|
||||||
|
|
||||||
|
for(auto& download : currentDownloads)
|
||||||
|
download.data->status = DownloadData::Status::FAILURE;
|
||||||
|
currentDownloads.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileDownloader::save_to_string(CURL* curl, std::string* str) {
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_to_string);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, reinterpret_cast<void*>(str));
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileDownloader::disable_ssl_verification(CURL* curl) {
|
||||||
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
FileDownloader::EasyHandle FileDownloader::create_easy_handle() {
|
||||||
|
auto easyHandle = EasyHandle(curl_easy_init(), curl_easy_cleanup);
|
||||||
|
if(!easyHandle)
|
||||||
|
throw std::runtime_error("[FileDownloader::create_easy_handle] Failed to create CURL easy handle");
|
||||||
|
return easyHandle;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileDownloader::MultiHandle FileDownloader::create_multi_handle() {
|
||||||
|
auto multiHandle = EasyHandle(curl_multi_init(), curl_multi_cleanup);
|
||||||
|
if(!multiHandle)
|
||||||
|
throw std::runtime_error("[FileDownloader::create_multi_handle] Failed to create CURL multi handle");
|
||||||
|
return multiHandle;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t FileDownloader::write_to_string(void* contents, size_t size, size_t nmemb, void* userp) {
|
||||||
|
size_t realsize = size * nmemb;
|
||||||
|
auto str = reinterpret_cast<std::string*>(userp);
|
||||||
|
str->append(reinterpret_cast<const char*>(contents), realsize);
|
||||||
|
return realsize;
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileDownloader::cleanup() {
|
||||||
|
if(downloadThread && downloadThread->joinable()) {
|
||||||
|
shutdownThread = true;
|
||||||
|
downloadThread->join();
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_global_cleanup();
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
57
include/Helpers/FileDownloader.hpp
Normal file
57
include/Helpers/FileDownloader.hpp
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
#pragma once
|
||||||
|
#ifndef __EMSCRIPTEN__
|
||||||
|
|
||||||
|
#include <curl/curl.h>
|
||||||
|
#include <memory>
|
||||||
|
#include <functional>
|
||||||
|
#include <atomic>
|
||||||
|
#include <thread>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
|
class FileDownloader {
|
||||||
|
public:
|
||||||
|
struct DownloadData {
|
||||||
|
enum class Status {
|
||||||
|
IN_PROGRESS = 0,
|
||||||
|
SUCCESS,
|
||||||
|
FAILURE
|
||||||
|
};
|
||||||
|
std::string str;
|
||||||
|
std::atomic<Status> status;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void init();
|
||||||
|
static std::shared_ptr<DownloadData> download_data_from_url(std::string_view url);
|
||||||
|
static void cleanup();
|
||||||
|
private:
|
||||||
|
typedef std::unique_ptr<CURL, std::function<void(CURL*)>> EasyHandle;
|
||||||
|
typedef std::unique_ptr<CURLM, std::function<void(CURLM*)>> MultiHandle;
|
||||||
|
|
||||||
|
static void add_new_handlers_to_multi();
|
||||||
|
static void download_thread_update();
|
||||||
|
|
||||||
|
struct DownloadHandler {
|
||||||
|
EasyHandle eHandle;
|
||||||
|
std::shared_ptr<DownloadData> data;
|
||||||
|
};
|
||||||
|
|
||||||
|
static std::mutex newDownloadsMutex;
|
||||||
|
static std::vector<DownloadHandler> newDownloads;
|
||||||
|
static std::atomic<bool> shutdownThread;
|
||||||
|
static MultiHandle mHandle;
|
||||||
|
static std::vector<DownloadHandler> currentDownloads;
|
||||||
|
static std::unique_ptr<std::thread> downloadThread;
|
||||||
|
|
||||||
|
static void save_to_string(CURL* curl, std::string* str);
|
||||||
|
static void disable_ssl_verification(CURL* curl);
|
||||||
|
|
||||||
|
static timeval get_timeout(CURLM* mCurl);
|
||||||
|
static int wait_if_needed(CURLM* mCurl, timeval& timeout);
|
||||||
|
|
||||||
|
static EasyHandle create_easy_handle();
|
||||||
|
static MultiHandle create_multi_handle();
|
||||||
|
|
||||||
|
static size_t write_to_string(void* contents, size_t size, size_t nmemb, void* userp);
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
#include "Helpers/SCollision.hpp"
|
#include "Helpers/SCollision.hpp"
|
||||||
#include "Helpers/StringHelpers.hpp"
|
#include "Helpers/StringHelpers.hpp"
|
||||||
|
#include "Helpers/FileDownloader.hpp"
|
||||||
#include "include/gpu/GpuTypes.h"
|
#include "include/gpu/GpuTypes.h"
|
||||||
#include "sago/platform_folders.h"
|
#include "sago/platform_folders.h"
|
||||||
#include <SDL3/SDL_hints.h>
|
#include <SDL3/SDL_hints.h>
|
||||||
|
|
@ -387,6 +388,10 @@ SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) {
|
||||||
|
|
||||||
init_logs(mS);
|
init_logs(mS);
|
||||||
|
|
||||||
|
#ifndef __EMSCRIPTEN__
|
||||||
|
FileDownloader::init();
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef NDEBUG
|
#ifdef NDEBUG
|
||||||
try {
|
try {
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -770,5 +775,9 @@ void SDL_AppQuit(void *appstate, SDL_AppResult result) {
|
||||||
|
|
||||||
delete (&mS);
|
delete (&mS);
|
||||||
|
|
||||||
|
#ifndef __EMSCRIPTEN__
|
||||||
|
FileDownloader::cleanup();
|
||||||
|
#endif
|
||||||
|
|
||||||
SDL_Quit();
|
SDL_Quit();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue