From 35561b7931fe9da7380f4e354cd326bc31c8052f Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Wed, 11 Jun 2025 03:46:58 +0000 Subject: [PATCH 1/7] forked and branched to start work on Meala suuport --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bfcf132..42f9fce 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ mavlink_proto.msgid == 117 || mavlink_proto.msgid == 118 || mavlink_proto.msgid Watch your beautiful logs arrive ### Future developments -- Multiple backends: e.g. RobotoAI, DroneLogbook, Auterion Suite, Aloft etc +- Multiple backends: e.g. RobotoAI, DroneLogbook, Auterion Suite, Aloft, Meala etc From a512afc14b3daaadbf833b46b78aa07885d80759 Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Tue, 19 Aug 2025 01:06:25 -0400 Subject: [PATCH 2/7] first pass at Meala remote upload support --- .gitmodules | 3 + CMakeLists.txt | 2 + config.toml | 2 + src/LogLoader.cpp | 13 ++- src/LogLoader.hpp | 7 ++ src/MealaServerInterface.cpp | 124 ++++++++++++++++++++++++++ src/ServerInterface.cpp | 3 +- src/ServerInterface.hpp | 21 +++++ src/main.cpp | 4 +- third_party/nlohmann_json | 1 + third_party/third_party/nlohmann_json | 1 + 11 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 src/MealaServerInterface.cpp create mode 160000 third_party/nlohmann_json create mode 160000 third_party/third_party/nlohmann_json diff --git a/.gitmodules b/.gitmodules index 5b8b83f..e7c62b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "third_party/tomlplusplus"] path = third_party/tomlplusplus url = https://github.com/marzer/tomlplusplus.git +[submodule "third_party/nlohmann_json"] + path = third_party/nlohmann_json + url = https://github.com/nlohmann/json.git \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 6529639..c3d2d31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,11 +28,13 @@ find_package(MAVSDK REQUIRED) include_directories(third_party/cpp-httplib/) include_directories(third_party/tomlplusplus/) +include_directories(third_party/nlohmann_json/) include_directories(${SQLite3_INCLUDE_DIRS}) add_executable(${PROJECT_NAME} src/main.cpp src/ServerInterface.cpp + src/MealaServerInterface.cpp src/LogLoader.cpp) target_link_libraries(${PROJECT_NAME} diff --git a/config.toml b/config.toml index ff208c2..58b87fd 100644 --- a/config.toml +++ b/config.toml @@ -1,6 +1,8 @@ connection_url = "udp://:14551" local_server = "http://127.0.0.1:5006" +upload_service = 0 remote_server = "https://review.px4.io" +credentials_file = "meala_creds.cert" email = "" upload_enabled = false public_logs = false diff --git a/src/LogLoader.cpp b/src/LogLoader.cpp index 357b0fe..8d145bd 100644 --- a/src/LogLoader.cpp +++ b/src/LogLoader.cpp @@ -37,10 +37,21 @@ LogLoader::LogLoader(const LogLoader::Settings& settings) .db_path = _settings.application_directory + "remote_server.db", .upload_enabled = settings.upload_enabled, .public_logs = settings.public_logs, + .credentials_file = settings.credentials_file, + .upload_service = settings.upload_service }; _local_server = std::make_shared(local_server_settings); - _remote_server = std::make_shared(remote_server_settings); + + // _remote_server = std::make_shared(remote_server_settings); + if (_settings.upload_service == UploadService::Meala) { + MealaCredentials creds; + creds.credentials_file = _settings.credentials_file; + _remote_server = std::make_shared(remote_server_settings, creds); + + } else { + _remote_server = std::make_shared(remote_server_settings); + } std::cout << std::fixed << std::setprecision(8); diff --git a/src/LogLoader.hpp b/src/LogLoader.hpp index 0f2df34..62ca2db 100644 --- a/src/LogLoader.hpp +++ b/src/LogLoader.hpp @@ -8,6 +8,11 @@ #include "ServerInterface.hpp" +enum class UploadService { + FlightReview = 0, + Meala = 1 +}; + class LogLoader { public: @@ -19,6 +24,8 @@ class LogLoader std::string application_directory; bool upload_enabled; bool public_logs; + std::string credentials_file; + UploadService upload_service; }; LogLoader(const Settings& settings); diff --git a/src/MealaServerInterface.cpp b/src/MealaServerInterface.cpp new file mode 100644 index 0000000..d816120 --- /dev/null +++ b/src/MealaServerInterface.cpp @@ -0,0 +1,124 @@ +#include "ServerInterface.hpp" +#include +#include +#include +#include +#define CPPHTTPLIB_OPENSSL_SUPPORT +#include +#include + +namespace fs = std::filesystem; +using json = nlohmann::json; + +MealaServerInterface::MealaServerInterface(const Settings& settings, const MealaCredentials& creds) + : ServerInterface(settings), _creds(creds) +{} + +bool MealaServerInterface::login() +{ + std::string username, password, token; + + if (!_creds.credentials_file.empty() && fs::exists(_creds.credentials_file)) { + std::ifstream f(_creds.credentials_file); + json creds_json; + f >> creds_json; + username = creds_json.value("username", ""); + token = creds_json.value("token", ""); + } else { + username = _creds.username; + password = _creds.password; + } + + httplib::Params params; + params.emplace("username", username); + if (!token.empty()) { + params.emplace("token", token); + } else { + params.emplace("password", password); + } + + std::string url_path = "/login"; + httplib::Result res; + if (_protocol == Protocol::Https) { + httplib::SSLClient cli(_settings.server_url); + res = cli.Post(url_path.c_str(), params); + } else { + httplib::Client cli(_settings.server_url); + res = cli.Post(url_path.c_str(), params); + } + + if (res && res->status == 200) { + auto it = res->headers.find("Set-Cookie"); + if (it != res->headers.end()) { + _session_cookie = it->second; + _logged_in = true; + return true; + } + } + _logged_in = false; + return false; +} + +ServerInterface::UploadResult MealaServerInterface::upload_log(const std::string& file_path) +{ + if (!_logged_in && !login()) { + return {false, 401, "Login to Meala failed"}; + } + + constexpr size_t chunk_size = 5 * 1024 * 1024; // 5 MB + + if (!fs::exists(file_path)) { + return {false, 404, "Log file does not exist: " + file_path}; + } + size_t file_size = fs::file_size(file_path); + size_t total_chunks = (file_size / chunk_size) + (file_size % chunk_size ? 1 : 0); + std::string filename = fs::path(file_path).filename().string(); + + std::ifstream file(file_path, std::ios::binary); + if (!file) { + return {false, 0, "Failed to open file"}; + } + + for (size_t chunk_index = 0; chunk_index < total_chunks; ++chunk_index) { + size_t offset = chunk_index * chunk_size; + size_t this_chunk_size = std::min(chunk_size, file_size - offset); + std::vector buffer(this_chunk_size); + file.read(buffer.data(), this_chunk_size); + + httplib::MultipartFormDataItems items = { + {"comments", "Log uploaded from C++ API", "", ""}, + {"battery", "", "", ""}, + {"pic", "", "", ""}, + {"gso", "", "", ""}, + {"vehicle_id", "", "", ""}, + {"dzchunkbyteoffset", std::to_string(offset), "", ""}, + {"dzchunkindex", std::to_string(chunk_index), "", ""}, + {"dztotalchunkcount", std::to_string(total_chunks), "", ""}, + {"files", std::string(buffer.begin(), buffer.end()), filename, "application/octet-stream"} + }; + + std::string url_path = "/upload/api"; + httplib::Headers headers; + if (!_session_cookie.empty()) { + headers.emplace("Cookie", _session_cookie); + } + + httplib::Result res; + if (_protocol == Protocol::Https) { + httplib::SSLClient cli(_settings.server_url); + res = cli.Post(url_path.c_str(), headers, items); + } else { + httplib::Client cli(_settings.server_url); + res = cli.Post(url_path.c_str(), headers, items); + } + + if (!res || res->status != 200) { + std::ostringstream oss; + oss << "Upload failed on chunk " << (chunk_index + 1) << ": " + << (res ? std::to_string(res->status) : "No response"); + return {false, res ? res->status : 0, oss.str()}; + } + } + + return {true, 200, "Upload completed successfully."}; +} \ No newline at end of file diff --git a/src/ServerInterface.cpp b/src/ServerInterface.cpp index 554893b..493ae21 100644 --- a/src/ServerInterface.cpp +++ b/src/ServerInterface.cpp @@ -11,6 +11,7 @@ #include #define CPPHTTPLIB_OPENSSL_SUPPORT #include +#include namespace fs = std::filesystem; @@ -597,4 +598,4 @@ ServerInterface::DatabaseEntry ServerInterface::row_to_db_entry(sqlite3_stmt* st entry.downloaded = sqlite3_column_int(stmt, 4) != 0; return entry; -} +} \ No newline at end of file diff --git a/src/ServerInterface.hpp b/src/ServerInterface.hpp index ea47559..63c6f27 100644 --- a/src/ServerInterface.hpp +++ b/src/ServerInterface.hpp @@ -4,6 +4,7 @@ #include #include #include +#include class ServerInterface { @@ -79,3 +80,23 @@ class ServerInterface bool _should_exit = false; sqlite3* _db = nullptr; }; + +struct MealaCredentials { + std::string username; + std::string password; + std::string token; + std::string credentials_file; +}; + +class MealaServerInterface : public ServerInterface { +public: + MealaServerInterface(const Settings& settings, const MealaCredentials& creds); + + bool login(); + UploadResult upload_log(const std::string& filepath) override; + +private: + MealaCredentials _creds; + std::string _session_cookie; + bool _logged_in = false; +}; \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 6000d2b..850a6c9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -37,7 +37,9 @@ int main() .mavsdk_connection_url = config["connection_url"].value_or("0.0.0"), .application_directory = std::string(getenv("HOME")) + "/.local/share/logloader/", .upload_enabled = config["upload_enabled"].value_or(false), - .public_logs = config["public_logs"].value_or(false) + .public_logs = config["public_logs"].value_or(false), + .credentials_file = config["credentials_file"].value_or(std::string(getenv("HOME")) + "/.local/share/logloader/meala_creds.cert"), + .upload_service = static_cast(config["upload_service"].value_or(0)) // 0 for FlightReview, 1 for Meala }; _log_loader = std::make_shared(settings); diff --git a/third_party/nlohmann_json b/third_party/nlohmann_json new file mode 160000 index 0000000..52d3f6e --- /dev/null +++ b/third_party/nlohmann_json @@ -0,0 +1 @@ +Subproject commit 52d3f6e034504b4097741b2da5b644a38ee273ff diff --git a/third_party/third_party/nlohmann_json b/third_party/third_party/nlohmann_json new file mode 160000 index 0000000..52d3f6e --- /dev/null +++ b/third_party/third_party/nlohmann_json @@ -0,0 +1 @@ +Subproject commit 52d3f6e034504b4097741b2da5b644a38ee273ff From 34b09ca8369db6eea27660319643596e36811826 Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Tue, 19 Aug 2025 16:03:23 -0400 Subject: [PATCH 3/7] added support for Meala remote log upload. Tested and working --- CMakeLists.txt | 2 +- config.toml | 2 +- src/LogLoader.cpp | 8 +- src/LogLoader.hpp | 8 +- src/MealaServerInterface.cpp | 220 ++++++++++++++------------ src/ServerInterface.hpp | 39 +++-- src/main.cpp | 4 +- third_party/third_party/nlohmann_json | 1 - 8 files changed, 153 insertions(+), 131 deletions(-) delete mode 160000 third_party/third_party/nlohmann_json diff --git a/CMakeLists.txt b/CMakeLists.txt index c3d2d31..4aad746 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ find_package(MAVSDK REQUIRED) include_directories(third_party/cpp-httplib/) include_directories(third_party/tomlplusplus/) -include_directories(third_party/nlohmann_json/) +include_directories(third_party/nlohmann_json/single_include) include_directories(${SQLite3_INCLUDE_DIRS}) add_executable(${PROJECT_NAME} diff --git a/config.toml b/config.toml index 58b87fd..2e4cdee 100644 --- a/config.toml +++ b/config.toml @@ -2,7 +2,7 @@ connection_url = "udp://:14551" local_server = "http://127.0.0.1:5006" upload_service = 0 remote_server = "https://review.px4.io" -credentials_file = "meala_creds.cert" +credentials_file = "" email = "" upload_enabled = false public_logs = false diff --git a/src/LogLoader.cpp b/src/LogLoader.cpp index 8d145bd..b062dcc 100644 --- a/src/LogLoader.cpp +++ b/src/LogLoader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include "ServerInterface.hpp" namespace fs = std::filesystem; @@ -27,6 +28,9 @@ LogLoader::LogLoader(const LogLoader::Settings& settings) .db_path = _settings.application_directory + "local_server.db", .upload_enabled = true, // Always upload to local server .public_logs = true, // Public required true for searching using Web UI + .upload_service = UploadService::FlightReview, // Local server is always FlightReview, + .credentials_file = "" // FlightReview does not need credentials + }; // Setup remote server interface @@ -37,8 +41,8 @@ LogLoader::LogLoader(const LogLoader::Settings& settings) .db_path = _settings.application_directory + "remote_server.db", .upload_enabled = settings.upload_enabled, .public_logs = settings.public_logs, - .credentials_file = settings.credentials_file, - .upload_service = settings.upload_service + .upload_service = settings.upload_service, + .credentials_file = settings.credentials_file }; _local_server = std::make_shared(local_server_settings); diff --git a/src/LogLoader.hpp b/src/LogLoader.hpp index 62ca2db..08f9ff3 100644 --- a/src/LogLoader.hpp +++ b/src/LogLoader.hpp @@ -8,14 +8,10 @@ #include "ServerInterface.hpp" -enum class UploadService { - FlightReview = 0, - Meala = 1 -}; - class LogLoader { public: + struct Settings { std::string email; std::string local_server; @@ -24,8 +20,8 @@ class LogLoader std::string application_directory; bool upload_enabled; bool public_logs; - std::string credentials_file; UploadService upload_service; + std::string credentials_file; }; LogLoader(const Settings& settings); diff --git a/src/MealaServerInterface.cpp b/src/MealaServerInterface.cpp index d816120..572f416 100644 --- a/src/MealaServerInterface.cpp +++ b/src/MealaServerInterface.cpp @@ -6,119 +6,133 @@ #define CPPHTTPLIB_OPENSSL_SUPPORT #include #include +#include "Log.hpp" namespace fs = std::filesystem; using json = nlohmann::json; MealaServerInterface::MealaServerInterface(const Settings& settings, const MealaCredentials& creds) - : ServerInterface(settings), _creds(creds) + : ServerInterface(settings), _creds(creds) {} bool MealaServerInterface::login() { - std::string username, password, token; - - if (!_creds.credentials_file.empty() && fs::exists(_creds.credentials_file)) { - std::ifstream f(_creds.credentials_file); - json creds_json; - f >> creds_json; - username = creds_json.value("username", ""); - token = creds_json.value("token", ""); - } else { - username = _creds.username; - password = _creds.password; - } - - httplib::Params params; - params.emplace("username", username); - if (!token.empty()) { - params.emplace("token", token); - } else { - params.emplace("password", password); - } - - std::string url_path = "/login"; - httplib::Result res; - if (_protocol == Protocol::Https) { - httplib::SSLClient cli(_settings.server_url); - res = cli.Post(url_path.c_str(), params); - } else { - httplib::Client cli(_settings.server_url); - res = cli.Post(url_path.c_str(), params); - } - - if (res && res->status == 200) { - auto it = res->headers.find("Set-Cookie"); - if (it != res->headers.end()) { - _session_cookie = it->second; - _logged_in = true; - return true; - } - } - _logged_in = false; - return false; + std::string username, password, token; + + if (!_creds.credentials_file.empty() && fs::exists(_creds.credentials_file)) { + std::ifstream f(_creds.credentials_file); + json creds_json; + f >> creds_json; + username = creds_json.value("username", ""); + token = creds_json.value("token", ""); + + } else { + username = _creds.username; + password = _creds.password; + } + + httplib::Params params; + params.emplace("username", username); + + if (!token.empty()) { + params.emplace("token", token); + + } else { + params.emplace("password", password); + } + + std::string url_path = "/login"; + httplib::Result res; + + if (_protocol == Protocol::Https) { + httplib::SSLClient cli(_settings.server_url); + res = cli.Post(url_path.c_str(), params); + + } else { + httplib::Client cli(_settings.server_url); + res = cli.Post(url_path.c_str(), params); + } + + if (res && res->status == 200) { + auto it = res->headers.find("Set-Cookie"); + + if (it != res->headers.end()) { + _session_cookie = it->second; + _logged_in = true; + return true; + } + } + + _logged_in = false; + return false; } -ServerInterface::UploadResult MealaServerInterface::upload_log(const std::string& file_path) +ServerInterface::UploadResult MealaServerInterface::upload(const std::string& filepath) { - if (!_logged_in && !login()) { - return {false, 401, "Login to Meala failed"}; - } - - constexpr size_t chunk_size = 5 * 1024 * 1024; // 5 MB - - if (!fs::exists(file_path)) { - return {false, 404, "Log file does not exist: " + file_path}; - } - size_t file_size = fs::file_size(file_path); - size_t total_chunks = (file_size / chunk_size) + (file_size % chunk_size ? 1 : 0); - std::string filename = fs::path(file_path).filename().string(); - - std::ifstream file(file_path, std::ios::binary); - if (!file) { - return {false, 0, "Failed to open file"}; - } - - for (size_t chunk_index = 0; chunk_index < total_chunks; ++chunk_index) { - size_t offset = chunk_index * chunk_size; - size_t this_chunk_size = std::min(chunk_size, file_size - offset); - std::vector buffer(this_chunk_size); - file.read(buffer.data(), this_chunk_size); - - httplib::MultipartFormDataItems items = { - {"comments", "Log uploaded from C++ API", "", ""}, - {"battery", "", "", ""}, - {"pic", "", "", ""}, - {"gso", "", "", ""}, - {"vehicle_id", "", "", ""}, - {"dzchunkbyteoffset", std::to_string(offset), "", ""}, - {"dzchunkindex", std::to_string(chunk_index), "", ""}, - {"dztotalchunkcount", std::to_string(total_chunks), "", ""}, - {"files", std::string(buffer.begin(), buffer.end()), filename, "application/octet-stream"} - }; - - std::string url_path = "/upload/api"; - httplib::Headers headers; - if (!_session_cookie.empty()) { - headers.emplace("Cookie", _session_cookie); - } - - httplib::Result res; - if (_protocol == Protocol::Https) { - httplib::SSLClient cli(_settings.server_url); - res = cli.Post(url_path.c_str(), headers, items); - } else { - httplib::Client cli(_settings.server_url); - res = cli.Post(url_path.c_str(), headers, items); - } - - if (!res || res->status != 200) { - std::ostringstream oss; - oss << "Upload failed on chunk " << (chunk_index + 1) << ": " - << (res ? std::to_string(res->status) : "No response"); - return {false, res ? res->status : 0, oss.str()}; - } - } - - return {true, 200, "Upload completed successfully."}; + if (!_logged_in && !login()) { + return {false, 401, "Login to Meala failed"}; + } + + constexpr size_t chunk_size = 5 * 1024 * 1024; // 5 MB + + if (!fs::exists(filepath)) { + return {false, 404, "Log file does not exist: " + filepath}; + } + + size_t file_size = fs::file_size(filepath); + size_t total_chunks = (file_size / chunk_size) + (file_size % chunk_size ? 1 : 0); + std::string filename = fs::path(filepath).filename().string(); + + std::ifstream file(filepath, std::ios::binary); + + if (!file) { + return {false, 0, "Failed to open file"}; + } + + for (size_t chunk_index = 0; chunk_index < total_chunks; ++chunk_index) { + size_t offset = chunk_index * chunk_size; + size_t this_chunk_size = std::min(chunk_size, file_size - offset); + std::vector buffer(this_chunk_size); + file.read(buffer.data(), this_chunk_size); + + httplib::MultipartFormDataItems items = { + {"comments", "Log uploaded from C++ API", "", ""}, + {"battery", "", "", ""}, + {"pic", "", "", ""}, + {"gso", "", "", ""}, + {"vehicle_id", "", "", ""}, + {"dzchunkbyteoffset", std::to_string(offset), "", ""}, + {"dzchunkindex", std::to_string(chunk_index), "", ""}, + {"dztotalchunkcount", std::to_string(total_chunks), "", ""}, + {"files", std::string(buffer.begin(), buffer.end()), filename, "application/octet-stream"} + }; + + std::string url_path = "/upload/api"; + httplib::Headers headers; + + if (!_session_cookie.empty()) { + headers.emplace("Cookie", _session_cookie); + } + + LOG("Uploading " << filename << " to " << _settings.server_url); + + httplib::Result res; + + if (_protocol == Protocol::Https) { + httplib::SSLClient cli(_settings.server_url); + res = cli.Post(url_path.c_str(), headers, items); + + } else { + httplib::Client cli(_settings.server_url); + res = cli.Post(url_path.c_str(), headers, items); + } + + if (!res || res->status != 200) { + LOG("Upload failed on chunk " << (chunk_index + 1) << ": " + << (res ? std::to_string(res->status) : "No response")); + return {false, res ? res->status : 0, "No response from server"}; + } + } + + return {true, 200, "Upload completed successfully."}; } \ No newline at end of file diff --git a/src/ServerInterface.hpp b/src/ServerInterface.hpp index 63c6f27..c9b3ba2 100644 --- a/src/ServerInterface.hpp +++ b/src/ServerInterface.hpp @@ -6,6 +6,11 @@ #include #include +enum class UploadService { + FlightReview = 0, + Meala = 1 +}; + class ServerInterface { public: @@ -16,6 +21,8 @@ class ServerInterface std::string db_path; // Path to this server's database bool upload_enabled {}; bool public_logs {}; + UploadService upload_service; + std::string credentials_file; // For Meala credentials }; struct UploadResult { @@ -60,14 +67,17 @@ class ServerInterface void start(); void stop(); -private: +protected: enum class Protocol { Http, Https }; + virtual UploadResult upload(const std::string& filepath); + Protocol _protocol {Protocol::Https}; + Settings _settings; +private: void sanitize_url_and_determine_protocol(); - UploadResult upload(const std::string& filepath); bool server_reachable(); // Database operations @@ -75,28 +85,27 @@ class ServerInterface bool add_to_blacklist(const std::string& uuid, const std::string& reason); DatabaseEntry row_to_db_entry(sqlite3_stmt* stmt); - Settings _settings; - Protocol _protocol {Protocol::Https}; bool _should_exit = false; sqlite3* _db = nullptr; }; struct MealaCredentials { - std::string username; - std::string password; - std::string token; - std::string credentials_file; + std::string username; + std::string password; + std::string token; + std::string credentials_file; }; -class MealaServerInterface : public ServerInterface { +class MealaServerInterface : public ServerInterface +{ public: - MealaServerInterface(const Settings& settings, const MealaCredentials& creds); + MealaServerInterface(const Settings& settings, const MealaCredentials& creds); - bool login(); - UploadResult upload_log(const std::string& filepath) override; + bool login(); + UploadResult upload(const std::string& filepath) override; private: - MealaCredentials _creds; - std::string _session_cookie; - bool _logged_in = false; + MealaCredentials _creds; + std::string _session_cookie; + bool _logged_in = false; }; \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 850a6c9..774e4ef 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -38,8 +38,8 @@ int main() .application_directory = std::string(getenv("HOME")) + "/.local/share/logloader/", .upload_enabled = config["upload_enabled"].value_or(false), .public_logs = config["public_logs"].value_or(false), - .credentials_file = config["credentials_file"].value_or(std::string(getenv("HOME")) + "/.local/share/logloader/meala_creds.cert"), - .upload_service = static_cast(config["upload_service"].value_or(0)) // 0 for FlightReview, 1 for Meala + .upload_service = static_cast(config["upload_service"].value_or(UploadService::FlightReview)), // 0 for FlightReview, 1 for Meala + .credentials_file = config["credentials_file"].value_or(std::string(getenv("HOME")) + "/.local/share/logloader/meala_creds.cert") }; _log_loader = std::make_shared(settings); diff --git a/third_party/third_party/nlohmann_json b/third_party/third_party/nlohmann_json deleted file mode 160000 index 52d3f6e..0000000 --- a/third_party/third_party/nlohmann_json +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 52d3f6e034504b4097741b2da5b644a38ee273ff From 314455a256453d0b94221ce2815b534f965e0f49 Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Sat, 12 Sep 2026 14:19:15 +0000 Subject: [PATCH 4/7] Add Meala as a selectable upload backend Reimplements the Meala integration on top of the current UploadTarget architecture and addresses the review feedback from PR #23. A target now carries a backend ("flight_review" by default, or "meala") and an optional credentials_file. UploadTarget grows a virtual upload() plus protected helpers for the parts every backend shares -- the reachability probe, the 401/403 cooldown, and the local-file checks -- and make_upload_target() builds the one the config names. Backend is per target, so the local Flight Review and the remote server are independent, and since a target is keyed in the database by its name rather than its url, switching backends does not re-upload what is already up. MealaUploadTarget logs in for a session cookie and posts the log in 5 MB Dropzone-style chunks, mapping Meala's replies onto the existing Outcome set so the upload loop's retry, rejection and cooldown handling applies unchanged. The credentials file is read on first login rather than at construction, so one that appears after startup is picked up. A meala target with no credentials_file is disabled at startup with a warning instead of failing a login every upload pass. Review comments addressed: - MealaCredentials and the Meala class live in their own header, not in the base class's. - The unused includes ( and in the old ServerInterface, the hpp-duplicating include in LogLoader.cpp) are gone; nothing here includes what it does not use. - The commented-out _remote_server line is gone. - README documents the supported backends in a table, with a link to Meala and what each backend needs. Verified against a stub Meala server: login, cookie reuse, and a 12 MB log in three chunks reassembling to a matching sha256, plus the refused-login, no-cookie, rejected-chunk, missing/empty-file, missing/malformed-credentials and unreachable-server paths. Full build with -Werror, ctest and make check-format all pass. --- CMakeLists.txt | 1 + README.md | 29 +++++- config.toml | 19 ++++ src/Config.cpp | 75 ++++++++++++++ src/Config.hpp | 19 ++++ src/LogLoader.cpp | 2 +- src/MealaUploadTarget.cpp | 200 ++++++++++++++++++++++++++++++++++++++ src/MealaUploadTarget.hpp | 47 +++++++++ src/UploadTarget.cpp | 70 +++++++++---- src/UploadTarget.hpp | 34 ++++++- 10 files changed, 470 insertions(+), 26 deletions(-) create mode 100644 src/MealaUploadTarget.cpp create mode 100644 src/MealaUploadTarget.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c36215..f72073b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(${PROJECT_NAME} src/Log.cpp src/LogDatabase.cpp src/LogLoader.cpp + src/MealaUploadTarget.cpp src/Sqlite.cpp src/StatusBoard.cpp src/UploadTarget.cpp) diff --git a/README.md b/README.md index f18ca59..a09dcff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![image](logloader_logo.png) -Downloads flight logs from the vehicle over MAVLink FTP and uploads them to a local Flight Review and, optionally, a remote one. Works with PX4 (`.ulg`) and ArduPilot (`.BIN`). +Downloads flight logs from the vehicle over MAVLink FTP and uploads them to a local Flight Review and, optionally, a remote server. Works with PX4 (`.ulg`) and ArduPilot (`.BIN`). ### What it fetches, and when @@ -59,6 +59,29 @@ Downloads are staged in a temporary directory and only moved next to the finishe Older versions kept a database per server and identified logs by the timestamp `LOG_ENTRY` reported, which MAVLink FTP cannot reproduce. On first start `local_server.db` and `remote_server.db` are imported into a single `logloader.db` and their rows matched against the FTP listing by size, so logs already downloaded and uploaded are not fetched or uploaded a second time. The old files are left untouched. +### Upload backends + +A target's `backend` says which upload API it speaks. It is per target, so the local Flight Review on the companion and the remote server do not have to be the same kind of thing. + +| `backend` | Server | Auth | Notes | +| --- | --- | --- | --- | +| `flight_review` (default) | [Flight Review](https://github.com/PX4/flight_review), e.g. [review.px4.io](https://review.px4.io) | None, or `api_key` on an authenticated instance | Records the `/plot_app?log=` path it redirects to, which is what lets the UI link to the plot. `.BIN` is not accepted by review.px4.io | +| `meala` | [Meala](https://www.apisdynamics.ca) | `credentials_file`, required | Logs in for a session cookie, then uploads in 5 MB chunks. Returns no per-log url, so nothing is recorded to link to | + +Switching a target's backend does not re-upload what is already up: a target is identified in the database by its name (`local`, `remote`), not by where it points. + +To upload to Meala, sign up, download the account's JSON credentials, and point a target at it: + +```toml +[upload_remote] +enabled = true +backend = "meala" +url = "https://www.apisdynamics.ca" +credentials_file = "/home/pilot/.config/ark/logloader/meala_creds.json" +``` + +The credentials file is the one Meala hands out, `{"username": "...", "token": "..."}`. Give an absolute path: like every other path in the config, it is taken as written and `~` is not expanded. Meala has no anonymous upload, so a `meala` target with no `credentials_file` is disabled at startup with a warning rather than failing one login per upload pass. `email`, `public` and `api_key` are Flight Review's and are ignored. + ### Configuration `config.toml`, in full, is documented inline in the shipped file. The keys: @@ -78,6 +101,8 @@ Older versions kept a database per server and identified logs by the timestamp ` | `upload.interval` | `10` | Seconds between upload passes | | `upload_local.*` | enabled, `http://127.0.0.1:5006` | Flight Review on the companion | | `upload_remote.*` | disabled, `https://review.px4.io` | `url`, `email`, `public`, `api_key` | +| `*.backend` | `flight_review` | Upload API this target speaks; see above | +| `*.credentials_file` | `""` | Meala account credentials; required by that backend | Tables are one level deep on purpose: ARK-OS's config editor renders exactly that, and a setting an operator cannot reach from the web UI may as well not exist. @@ -152,4 +177,4 @@ make install ### Future developments - Resume interrupted transfers rather than restarting them. -- Multiple backends: e.g. RobotoAI, DroneLogbook, Auterion Suite, Aloft. +- Further upload backends: e.g. RobotoAI, DroneLogbook, Auterion Suite, Aloft. diff --git a/config.toml b/config.toml index 3dada54..cc9156e 100644 --- a/config.toml +++ b/config.toml @@ -61,3 +61,22 @@ public = false # Per-account key for an authenticated Flight Review. When empty no # authorization headers are sent at all, which is what open servers expect. api_key = "" + +# Which upload API this target speaks: "flight_review" (the default) or +# "meala". Only the transport changes -- the target is still "remote" +# everywhere else, so switching backends does not re-upload what is already up. +# +# For Meala, point url at the Meala server and credentials_file at the JSON +# credentials downloaded from your Meala account: +# +# backend = "meala" +# url = "https://www.apisdynamics.ca" +# credentials_file = "/home/pilot/.config/ark/logloader/meala_creds.json" +# +# Paths are taken as written, so give an absolute one -- "~" is not expanded. +# +# Meala has no anonymous upload, so a meala target without credentials_file is +# disabled at startup. email, public and api_key are Flight Review's and are +# ignored by it. +backend = "flight_review" +credentials_file = "" diff --git a/src/Config.cpp b/src/Config.cpp index 27379d0..0b354e6 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,6 +1,7 @@ #include "Config.hpp" #include +#include #include #include #include @@ -63,6 +64,65 @@ T value_or(const Preferred& preferred, const Legacy& legacy, T fallback) } // namespace +bool parse_upload_backend(const std::string& text, UploadBackend& backend) +{ + std::string key; + + // Accept "flight_review", "flight-review" and "flightreview" alike; the + // separator is the kind of thing an operator gets wrong once per install. + for (const char c : text) { + if (c != '_' && c != '-' && c != ' ') { + key += static_cast(std::tolower(static_cast(c))); + } + } + + if (key == "flightreview") { + backend = UploadBackend::FlightReview; + return true; + } + + if (key == "meala") { + backend = UploadBackend::Meala; + return true; + } + + return false; +} + +const char* to_string(UploadBackend backend) +{ + switch (backend) { + case UploadBackend::Meala: + return "meala"; + + case UploadBackend::FlightReview: + default: + return "flight_review"; + } +} + +namespace +{ + +// Reads one target's backend, warning rather than throwing on a typo: an +// unreadable backend name should not stop the daemon from starting with the +// default one. +void load_backend(const toml::node_view& table, UploadTargetConfig& target) +{ + const auto text = table["backend"].value(); + + if (!text.has_value() || text->empty()) { + return; + } + + if (!parse_upload_backend(*text, target.backend)) { + LOG_WARN("Unknown backend \"" << *text << "\" for upload target " << target.name + << ", using " << to_string(target.backend)); + } +} + +} // namespace + std::string resolve_config_path(int argc, char** argv) { for (int i = 1; i < argc; i++) { @@ -147,6 +207,8 @@ Config load_config(const std::string& path) config.local.public_logs = local["public"].value_or(true); config.local.email = local["email"].value_or(""); config.local.api_key = trim(local["api_key"].value_or("")); + load_backend(local, config.local); + config.local.credentials_file = trim(local["credentials_file"].value_or("")); config.remote.name = kTargetRemote; config.remote.url = value_or(remote["url"], file["remote_server"], "https://review.px4.io"); @@ -154,6 +216,8 @@ Config load_config(const std::string& path) config.remote.public_logs = value_or(remote["public"], file["public_logs"], false); config.remote.email = value_or(remote["email"], file["email"], ""); config.remote.api_key = trim(value_or(remote["api_key"], file["remote_api_key"], "")); + load_backend(remote, config.remote); + config.remote.credentials_file = trim(remote["credentials_file"].value_or("")); if (config.local.url.empty()) { config.local.enabled = false; @@ -163,6 +227,17 @@ Config load_config(const std::string& path) config.remote.enabled = false; } + // Meala authenticates every upload, so a target without credentials would + // fail the login once per pass forever. Saying so at startup beats one + // warning per upload interval. + for (UploadTargetConfig* target : {&config.local, &config.remote}) { + if (target->enabled && target->backend == UploadBackend::Meala && target->credentials_file.empty()) { + LOG_WARN("Upload target " << target->name << " uses the meala backend but has no " + "credentials_file; disabling it"); + target->enabled = false; + } + } + if (g_used_legacy_key) { LOG_WARN(path << " uses the pre-overhaul flat key layout. It still works, but the " "shipped config.toml shows the current one and the fallback will be removed."); diff --git a/src/Config.hpp b/src/Config.hpp index b819746..c2165c8 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -10,14 +10,33 @@ constexpr const char* kTargetLocal = "local"; constexpr const char* kTargetRemote = "remote"; +// Which upload API a target speaks. The name is a target's identity in the +// database, so it stays "remote" whatever it is pointed at; this is how the +// bytes get there. +enum class UploadBackend { + FlightReview, + Meala, +}; + +// Parses the config spelling ("flight_review", "meala"). False leaves the +// backend untouched and tells the caller the text was not recognised. +bool parse_upload_backend(const std::string& text, UploadBackend& backend); + +const char* to_string(UploadBackend backend); + struct UploadTargetConfig { std::string name; bool enabled {false}; + UploadBackend backend {UploadBackend::FlightReview}; std::string url; std::string email; // Per-account key for authenticated Flight Review instances. Empty means // no auth headers are sent at all, which is what open servers expect. std::string api_key; + // Meala only: JSON file holding the account's username and token, as + // downloaded from the Meala account page. Meala has no anonymous upload, so + // a target without this cannot be enabled. + std::string credentials_file; bool public_logs {false}; }; diff --git a/src/LogLoader.cpp b/src/LogLoader.cpp index eea052b..1ebf1e2 100644 --- a/src/LogLoader.cpp +++ b/src/LogLoader.cpp @@ -53,7 +53,7 @@ LogLoader::LogLoader(const Config& config) fs::create_directories(_config.logs_directory, ec); for (const auto* target : _config.targets()) { - _targets.push_back(std::make_unique(*target)); + _targets.push_back(make_upload_target(*target)); } } diff --git a/src/MealaUploadTarget.cpp b/src/MealaUploadTarget.cpp new file mode 100644 index 0000000..4496cc7 --- /dev/null +++ b/src/MealaUploadTarget.cpp @@ -0,0 +1,200 @@ +#include "MealaUploadTarget.hpp" +#include "Log.hpp" + +#include +#include +#include +#include + +// CPPHTTPLIB_OPENSSL_SUPPORT is set for the whole target in CMakeLists.txt; +// see the note in UploadTarget.cpp for why it is not defined per-file. +#include +#include + +namespace fs = std::filesystem; + +MealaUploadTarget::MealaUploadTarget(const UploadTargetConfig& config) + : UploadTarget(config) +{} + +std::optional MealaUploadTarget::load_credentials() +{ + std::error_code ec; + + if (!fs::exists(_config.credentials_file, ec)) { + return Result{Outcome::Unauthorized, 0, "credentials file is missing: " + _config.credentials_file, ""}; + } + + std::ifstream file(_config.credentials_file); + + if (!file) { + return Result{Outcome::Unauthorized, 0, "cannot open credentials file: " + _config.credentials_file, ""}; + } + + MealaCredentials credentials; + + try { + const nlohmann::json parsed = nlohmann::json::parse(file); + credentials.username = parsed.value("username", ""); + credentials.token = parsed.value("token", ""); + + } catch (const nlohmann::json::exception& error) { + // A truncated or hand-edited file is a stable failure, not a transient + // one: it stays broken until somebody replaces it. + return Result{Outcome::Unauthorized, 0, std::string("cannot parse credentials file: ") + error.what(), ""}; + } + + if (credentials.username.empty() || credentials.token.empty()) { + return Result{Outcome::Unauthorized, 0, "credentials file has no username/token pair: " + _config.credentials_file, ""}; + } + + _credentials = std::move(credentials); + return std::nullopt; +} + +std::optional MealaUploadTarget::login() +{ + if (_logged_in) { + return std::nullopt; + } + + if (const auto problem = load_credentials(); problem.has_value()) { + note_unauthorized(); + return problem; + } + + httplib::Params params; + params.emplace("username", _credentials.username); + params.emplace("token", _credentials.token); + + httplib::Client client(_config.url); + client.set_connection_timeout(30, 0); + client.set_read_timeout(30, 0); + + const httplib::Result response = client.Post("/login", params); + + if (!response) { + return Result{Outcome::Unreachable, 0, "connection failed during login", ""}; + } + + if (response->status != 200) { + note_unauthorized(); + return Result{Outcome::Unauthorized, response->status, "login was refused", ""}; + } + + // The session cookie is the only thing that makes the upload calls + // authenticated; a 200 without one would upload as nobody. + if (!response->has_header("Set-Cookie")) { + note_unauthorized(); + return Result{Outcome::Unauthorized, response->status, "login returned no session cookie", ""}; + } + + _session_cookie = response->get_header_value("Set-Cookie"); + _logged_in = true; + return std::nullopt; +} + +UploadTarget::Result MealaUploadTarget::upload(const std::string& file_path) +{ + // Same reasoning as the base class: a rejected account stays rejected, and + // re-posting a whole log to find that out again helps nobody. + if (in_unauthorized_cooldown()) { + return {Outcome::Unreachable, 0, "waiting out an unauthorized response", ""}; + } + + size_t size = 0; + + if (const auto problem = check_local_file(file_path, size); problem.has_value()) { + return *problem; + } + + if (!reachable()) { + return {Outcome::Unreachable, 0, "server unreachable", ""}; + } + + if (const auto problem = login(); problem.has_value()) { + return *problem; + } + + std::ifstream file(file_path, std::ios::binary); + + if (!file) { + return {Outcome::Missing, 0, "cannot open local file: " + file_path, ""}; + } + + const std::string name = fs::path(file_path).filename().string(); + const size_t total_chunks = (size + kChunkSize - 1) / kChunkSize; + + LOG("Uploading " << name << " to " << _config.name << " (" << _config.url << ") in " + << total_chunks << (total_chunks == 1 ? " chunk" : " chunks")); + + httplib::Client client(_config.url); + client.set_connection_timeout(30, 0); + client.set_read_timeout(300, 0); + client.set_write_timeout(300, 0); + client.set_follow_location(false); + + httplib::Headers headers; + headers.emplace("Cookie", _session_cookie); + + std::vector buffer; + + for (size_t index = 0; index < total_chunks; index++) { + const size_t offset = index * kChunkSize; + const size_t length = std::min(kChunkSize, size - offset); + + buffer.resize(length); + + if (!file.read(buffer.data(), static_cast(length))) { + // The file shrank or went away mid-upload; fetching it again is the + // way out, which is what Missing asks the caller to do. + return {Outcome::Missing, 0, "local file ended early: " + file_path, ""}; + } + + // Meala reads every one of these off the form. The ones an automated + // upload has nothing to say about still have to be present. + const httplib::MultipartFormDataItems fields = { + {"comments", "Uploaded by logloader", "", ""}, + {"battery", "", "", ""}, + {"pic", "", "", ""}, + {"gso", "", "", ""}, + {"vehicle_id", "", "", ""}, + {"dzchunkbyteoffset", std::to_string(offset), "", ""}, + {"dzchunkindex", std::to_string(index), "", ""}, + {"dztotalchunkcount", std::to_string(total_chunks), "", ""}, + {"files", std::string(buffer.begin(), buffer.end()), name, "application/octet-stream"}, + }; + + const httplib::Result response = client.Post("/upload/api", headers, fields); + + if (!response) { + return {Outcome::Retry, 0, "connection failed during upload", ""}; + } + + const int status = response->status; + + if (status == 200 || status == 201) { + continue; + } + + if (status == 401 || status == 403) { + // The session expired mid-upload, or the account lost its access. + // Either way the next attempt logs in again from the top. + _logged_in = false; + _session_cookie.clear(); + note_unauthorized(); + return {Outcome::Unauthorized, status, "not authorized", ""}; + } + + if (status == 400) { + return {Outcome::Rejected, status, "rejected by the server", ""}; + } + + return {Outcome::Retry, status, "server error on chunk " + std::to_string(index + 1) + + "/" + std::to_string(total_chunks), ""}; + } + + // Meala's upload API answers per chunk and hands back no per-log url, so + // there is no location to record the way Flight Review's redirect gives one. + return {Outcome::Success, 200, "uploaded", ""}; +} diff --git a/src/MealaUploadTarget.hpp b/src/MealaUploadTarget.hpp new file mode 100644 index 0000000..2cc22be --- /dev/null +++ b/src/MealaUploadTarget.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include "UploadTarget.hpp" + +// One Meala account, as read from the JSON credentials file the Meala account +// page hands out: +// +// {"username": "...", "token": "..."} +// +// Meala has no anonymous upload, so a target without these cannot upload at +// all; Config rejects one that is enabled without a credentials_file. +struct MealaCredentials { + std::string username; + std::string token; +}; + +// Uploads to Meala instead of Flight Review. Two things differ from the base +// class: every request carries a session cookie obtained from /login, and the +// log goes up in fixed-size chunks rather than as one streamed body, because +// /upload/api takes the Dropzone-style chunk fields. +class MealaUploadTarget : public UploadTarget +{ +public: + explicit MealaUploadTarget(const UploadTargetConfig& config); + + Result upload(const std::string& file_path) override; + +private: + // POSTs /login and keeps the session cookie. Returns an unset optional once + // there is a usable session, and the failure to report otherwise. + std::optional login(); + + // Read on the first login rather than at construction, so a credentials file + // that appears after logloader started is picked up on the next attempt. + std::optional load_credentials(); + + MealaCredentials _credentials; + std::string _session_cookie; + bool _logged_in {false}; + + // What /upload/api expects per part. The whole log is never held in memory; + // one chunk at a time is. + static constexpr size_t kChunkSize = 5 * 1024 * 1024; +}; diff --git a/src/UploadTarget.cpp b/src/UploadTarget.cpp index cc3e545..9e4ab7e 100644 --- a/src/UploadTarget.cpp +++ b/src/UploadTarget.cpp @@ -1,5 +1,6 @@ #include "UploadTarget.hpp" #include "Log.hpp" +#include "MealaUploadTarget.hpp" #include #include @@ -58,6 +59,51 @@ UploadTarget::UploadTarget(const UploadTargetConfig& config) } } +void UploadTarget::note_unauthorized() +{ + _unauthorized_until = std::chrono::steady_clock::now() + kUnauthorizedCooldown; +} + +bool UploadTarget::in_unauthorized_cooldown() const +{ + return std::chrono::steady_clock::now() < _unauthorized_until; +} + +std::optional UploadTarget::check_local_file(const std::string& file_path, size_t& size) const +{ + std::error_code ec; + + if (!fs::exists(file_path, ec)) { + return Result{Outcome::Missing, 0, "local file is missing: " + file_path, ""}; + } + + size = fs::file_size(file_path, ec); + + if (ec) { + return Result{Outcome::Missing, 0, "cannot stat local file: " + file_path, ""}; + } + + // Not Missing: re-fetching a log the vehicle reports as zero bytes would + // download nothing, succeed, and come straight back here forever. + if (size == 0) { + return Result{Outcome::Rejected, 0, "log is empty: " + file_path, ""}; + } + + return std::nullopt; +} + +std::unique_ptr make_upload_target(const UploadTargetConfig& config) +{ + switch (config.backend) { + case UploadBackend::Meala: + return std::make_unique(config); + + case UploadBackend::FlightReview: + default: + return std::make_unique(config); + } +} + bool UploadTarget::reachable() { const auto now = std::chrono::steady_clock::now(); @@ -103,26 +149,14 @@ UploadTarget::Result UploadTarget::upload(const std::string& file_path) // Reported once when the 401/403 arrived; repeating a multi-megabyte POST // every upload interval just to be told no again is the spam this daemon // exists to avoid. Unreachable keeps the caller quiet and the log queued. - if (std::chrono::steady_clock::now() < _unauthorized_until) { + if (in_unauthorized_cooldown()) { return {Outcome::Unreachable, 0, "waiting out an unauthorized response", ""}; } - std::error_code ec; + size_t size = 0; - if (!fs::exists(file_path, ec)) { - return {Outcome::Missing, 0, "local file is missing: " + file_path, ""}; - } - - const auto size = fs::file_size(file_path, ec); - - if (ec) { - return {Outcome::Missing, 0, "cannot stat local file: " + file_path, ""}; - } - - // Not Missing: re-fetching a log the vehicle reports as zero bytes would - // download nothing, succeed, and come straight back here forever. - if (size == 0) { - return {Outcome::Rejected, 0, "log is empty: " + file_path, ""}; + if (const auto problem = check_local_file(file_path, size); problem.has_value()) { + return *problem; } if (!reachable()) { @@ -196,7 +230,7 @@ UploadTarget::Result UploadTarget::upload(const std::string& file_path) const size_t in_file = offset - prologue.size(); if (in_file < size) { - const size_t n = std::min(kChunk, static_cast(size) - in_file); + const size_t n = std::min(kChunk, size - in_file); std::vector buffer(n); file->seekg(static_cast(in_file)); @@ -230,7 +264,7 @@ UploadTarget::Result UploadTarget::upload(const std::string& file_path) const std::string detail = one_line(response->body); if (status == 401 || status == 403) { - _unauthorized_until = std::chrono::steady_clock::now() + kUnauthorizedCooldown; + note_unauthorized(); return {Outcome::Unauthorized, status, detail.empty() ? "not authorized" : detail, ""}; } diff --git a/src/UploadTarget.hpp b/src/UploadTarget.hpp index 2b4fb5d..204c694 100644 --- a/src/UploadTarget.hpp +++ b/src/UploadTarget.hpp @@ -1,13 +1,19 @@ #pragma once #include +#include +#include #include #include "Config.hpp" -// One Flight Review endpoint. Knows how to reach it and how to post a log to -// it; it holds no record of which logs exist or what has been uploaded, which -// is what LogDatabase is for. +// One upload endpoint. Knows how to reach it and how to post a log to it; it +// holds no record of which logs exist or what has been uploaded, which is what +// LogDatabase is for. +// +// This class speaks Flight Review's upload API, which is the default and the +// only one that needs no account. Another backend subclasses it and overrides +// upload(); make_upload_target() builds the one the config asks for. class UploadTarget { public: @@ -37,21 +43,36 @@ class UploadTarget }; explicit UploadTarget(const UploadTargetConfig& config); + virtual ~UploadTarget() = default; + + UploadTarget(const UploadTarget&) = delete; + UploadTarget& operator=(const UploadTarget&) = delete; const std::string& name() const { return _config.name; } const std::string& url() const { return _config.url; } bool enabled() const { return _config.enabled; } - Result upload(const std::string& file_path); + virtual Result upload(const std::string& file_path); -private: +protected: // Probes at most once per cooldown and logs only on the down/up // transitions, so a server that is simply off does not produce one failure // line per pending log. bool reachable(); + // A subclass reports a 401/403 through these so every backend gets the same + // cooldown; see kUnauthorizedCooldown. + void note_unauthorized(); + bool in_unauthorized_cooldown() const; + + // Common to every backend: the local file has to exist and have something in + // it before any of them is worth opening a socket for. Returns an unset + // optional when the file is fit to upload. + std::optional check_local_file(const std::string& file_path, size_t& size) const; + UploadTargetConfig _config; +private: static constexpr auto kUnreachableCooldown = std::chrono::seconds(60); std::chrono::steady_clock::time_point _unreachable_until {}; bool _reported_unreachable {false}; @@ -62,3 +83,6 @@ class UploadTarget static constexpr auto kUnauthorizedCooldown = std::chrono::minutes(5); std::chrono::steady_clock::time_point _unauthorized_until {}; }; + +// Builds the target config.backend asks for. +std::unique_ptr make_upload_target(const UploadTargetConfig& config); From a036168d9c6a7faee5a2c05adb671e2ad31a4ea6 Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Sat, 12 Sep 2026 14:41:05 +0000 Subject: [PATCH 5/7] Align the Meala backend with the published API, verified against the server Checked the implementation against Meala's OpenAPI spec and a real upload. The endpoints and form fields were already right; three things were not. 202 was treated as a server error. The spec defines it as "upload received; database temporarily unavailable -- will be processed later", so the log is up and re-sending it would have pushed a second copy rather than retried a failed one. It now counts as success alongside 200. The session cookie was sent back verbatim, attributes and all. Only the name=value pair belongs in a Cookie header; an Expires attribute contains a comma, so echoing the rest risks a malformed header. It is now trimmed at the first ";", and an empty cookie is treated as a failed login. A refused login is documented as 400, but Meala actually answers it with 200 and a session cookie -- an unauthenticated session. The first thing to notice was the chunk POST, so bad credentials meant pushing a multi-megabyte body every cooldown just to be told 401. login() now confirms the session with one parameterless call that answers in a few dozen bytes. Only an explicit 401/403 stops the upload: if that endpoint ever moves or errs, the upload proceeds as it did before, so the check cannot become a new way to fail. Verified against https://apisdynamics.ca with a 6.6 MB PX4 log: uploaded in two chunks, accepted, parsed server-side as PX4 with a plausible duration and takeoff time, and downloaded back with a sha256 matching the source byte for byte. Bad credentials now return Unauthorized before any chunk is sent. The stub-server cases still pass, including one where the session-check endpoint is absent and the upload goes through regardless. Full build with -Werror, ctest and make check-format all pass. --- README.md | 6 ++--- config.toml | 2 +- src/MealaUploadTarget.cpp | 56 ++++++++++++++++++++++++++++++++++++++- src/MealaUploadTarget.hpp | 9 +++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a09dcff..2bd96fd 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ A target's `backend` says which upload API it speaks. It is per target, so the l | `backend` | Server | Auth | Notes | | --- | --- | --- | --- | | `flight_review` (default) | [Flight Review](https://github.com/PX4/flight_review), e.g. [review.px4.io](https://review.px4.io) | None, or `api_key` on an authenticated instance | Records the `/plot_app?log=` path it redirects to, which is what lets the UI link to the plot. `.BIN` is not accepted by review.px4.io | -| `meala` | [Meala](https://www.apisdynamics.ca) | `credentials_file`, required | Logs in for a session cookie, then uploads in 5 MB chunks. Returns no per-log url, so nothing is recorded to link to | +| `meala` | [Meala](https://apisdynamics.ca) ([API docs](https://apisdynamics.ca/apidocs/)) | `credentials_file`, required | Logs in for a session cookie, then uploads in 5 MB Dropzone-style chunks. Returns no per-log url, so nothing is recorded to link to | Switching a target's backend does not re-upload what is already up: a target is identified in the database by its name (`local`, `remote`), not by where it points. @@ -76,11 +76,11 @@ To upload to Meala, sign up, download the account's JSON credentials, and point [upload_remote] enabled = true backend = "meala" -url = "https://www.apisdynamics.ca" +url = "https://apisdynamics.ca" credentials_file = "/home/pilot/.config/ark/logloader/meala_creds.json" ``` -The credentials file is the one Meala hands out, `{"username": "...", "token": "..."}`. Give an absolute path: like every other path in the config, it is taken as written and `~` is not expanded. Meala has no anonymous upload, so a `meala` target with no `credentials_file` is disabled at startup with a warning rather than failing one login per upload pass. `email`, `public` and `api_key` are Flight Review's and are ignored. +The credentials file is the one Meala hands out, `{"username": "...", "token": "..."}`, where the token is the API token from the account page. Give an absolute path: like every other path in the config, it is taken as written and `~` is not expanded. Meala has no anonymous upload, so a `meala` target with no `credentials_file` is disabled at startup with a warning rather than failing one login per upload pass. `email`, `public` and `api_key` are Flight Review's and are ignored. ### Configuration diff --git a/config.toml b/config.toml index cc9156e..666ddb4 100644 --- a/config.toml +++ b/config.toml @@ -70,7 +70,7 @@ api_key = "" # credentials downloaded from your Meala account: # # backend = "meala" -# url = "https://www.apisdynamics.ca" +# url = "https://apisdynamics.ca" # credentials_file = "/home/pilot/.config/ark/logloader/meala_creds.json" # # Paths are taken as written, so give an absolute one -- "~" is not expanded. diff --git a/src/MealaUploadTarget.cpp b/src/MealaUploadTarget.cpp index 4496cc7..9b2e28b 100644 --- a/src/MealaUploadTarget.cpp +++ b/src/MealaUploadTarget.cpp @@ -77,6 +77,8 @@ std::optional MealaUploadTarget::login() return Result{Outcome::Unreachable, 0, "connection failed during login", ""}; } + // A refused login is documented as 400, but anything that is not a 200 leaves + // us without a session and will keep doing so until the credentials change. if (response->status != 200) { note_unauthorized(); return Result{Outcome::Unauthorized, response->status, "login was refused", ""}; @@ -89,11 +91,57 @@ std::optional MealaUploadTarget::login() return Result{Outcome::Unauthorized, response->status, "login returned no session cookie", ""}; } + // Set-Cookie carries attributes after the first ";" (HttpOnly, Path, and an + // Expires whose value contains a comma). Only the name=value pair belongs in + // a Cookie header; sending the attributes back would be sending junk pairs. _session_cookie = response->get_header_value("Set-Cookie"); + _session_cookie = _session_cookie.substr(0, _session_cookie.find(';')); + + if (_session_cookie.empty()) { + note_unauthorized(); + return Result{Outcome::Unauthorized, response->status, "login returned an empty session cookie", ""}; + } + + if (const auto problem = verify_session(client); problem.has_value()) { + _session_cookie.clear(); + return problem; + } + _logged_in = true; return std::nullopt; } +std::optional MealaUploadTarget::verify_session(httplib::Client& client) +{ + // A refused login is documented as a 400, but in practice Meala answers one + // with 200 and a session cookie all the same -- the session is simply not + // authenticated. Without this check the first thing to notice would be a + // chunk POST, so every upload pass would push a multi-megabyte body just to + // be told 401. This endpoint takes no parameters and answers in a few dozen + // bytes, which is a cheap way to find out before sending the log. + httplib::Headers headers; + headers.emplace("Cookie", _session_cookie); + + const httplib::Result response = client.Post("/api/get-dashboard-shares", headers, "{}", "application/json"); + + // Only an explicit refusal stops the upload. This call is an optimisation, + // not a gate: if it fails for any other reason -- the endpoint moved, the + // server erred, the connection dropped -- the upload goes ahead and reports + // whatever it finds, exactly as it would have without the check. + if (response && (response->status == 401 || response->status == 403)) { + note_unauthorized(); + return Result{Outcome::Unauthorized, response->status, "the credentials were not accepted", ""}; + } + + if (!response || response->status != 200) { + LOG_DEBUG("Could not confirm the Meala session (" + << (response ? std::to_string(response->status) : "no response") + << "); continuing with the upload"); + } + + return std::nullopt; +} + UploadTarget::Result MealaUploadTarget::upload(const std::string& file_path) { // Same reasoning as the base class: a rejected account stays rejected, and @@ -173,10 +221,16 @@ UploadTarget::Result MealaUploadTarget::upload(const std::string& file_path) const int status = response->status; - if (status == 200 || status == 201) { + // 200 is a chunk accepted (and, on the last one, the log processed). 202 + // means the server has the bytes but its database was busy and will get to + // it later -- the log is up either way, so re-sending it would be a second + // copy rather than a retry. + if (status == 200 || status == 202) { continue; } + // 401 is an expired session; 403 is a lapsed subscription. Both are a + // human's problem and stable until they fix it. if (status == 401 || status == 403) { // The session expired mid-upload, or the account lost its access. // Either way the next attempt logs in again from the top. diff --git a/src/MealaUploadTarget.hpp b/src/MealaUploadTarget.hpp index 2cc22be..f91ff8b 100644 --- a/src/MealaUploadTarget.hpp +++ b/src/MealaUploadTarget.hpp @@ -5,6 +5,11 @@ #include "UploadTarget.hpp" +namespace httplib +{ +class Client; +} + // One Meala account, as read from the JSON credentials file the Meala account // page hands out: // @@ -37,6 +42,10 @@ class MealaUploadTarget : public UploadTarget // that appears after logloader started is picked up on the next attempt. std::optional load_credentials(); + // Confirms the session the login handed back is actually authenticated; see + // the comment on the definition for why a 200 from /login is not enough. + std::optional verify_session(httplib::Client& client); + MealaCredentials _credentials; std::string _session_cookie; bool _logged_in {false}; From 7d5e1a19ff92b9f65d4e6ff6fa64664e3dca07fa Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Wed, 23 Sep 2026 23:44:24 -0400 Subject: [PATCH 6/7] Make Meala upload form fields configurable comment, battery, pic, gso and vehicle_id were hardcoded in the upload form. They are now read from the upload target's table in config.toml, defaulting to the previous values, so an operator can fill in the aircraft and crew without rebuilding. --- README.md | 8 ++++++++ config.toml | 10 ++++++++++ src/Config.cpp | 33 +++++++++++++++++++++++++++++++++ src/Config.hpp | 7 +++++++ src/MealaUploadTarget.cpp | 14 +++++++------- 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2bd96fd..dc36ff7 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,13 @@ enabled = true backend = "meala" url = "https://apisdynamics.ca" credentials_file = "/home/pilot/.config/ark/logloader/meala_creds.json" + +# Filled in on every log this target uploads. +comment = "Uploaded by logloader" +battery = "" +pic = "" +gso = "" +vehicle_id = "" ``` The credentials file is the one Meala hands out, `{"username": "...", "token": "..."}`, where the token is the API token from the account page. Give an absolute path: like every other path in the config, it is taken as written and `~` is not expanded. Meala has no anonymous upload, so a `meala` target with no `credentials_file` is disabled at startup with a warning rather than failing one login per upload pass. `email`, `public` and `api_key` are Flight Review's and are ignored. @@ -103,6 +110,7 @@ The credentials file is the one Meala hands out, `{"username": "...", "token": " | `upload_remote.*` | disabled, `https://review.px4.io` | `url`, `email`, `public`, `api_key` | | `*.backend` | `flight_review` | Upload API this target speaks; see above | | `*.credentials_file` | `""` | Meala account credentials; required by that backend | +| `*.comment` / `battery` / `pic` / `gso` / `vehicle_id` | `Uploaded by logloader` / `""` | Meala form fields sent with every upload | Tables are one level deep on purpose: ARK-OS's config editor renders exactly that, and a setting an operator cannot reach from the web UI may as well not exist. diff --git a/config.toml b/config.toml index 666ddb4..27e70bc 100644 --- a/config.toml +++ b/config.toml @@ -80,3 +80,13 @@ api_key = "" # ignored by it. backend = "flight_review" credentials_file = "" + +# Meala only: sent with every log this target uploads, as the flight's details +# on Meala. Edit these to match the aircraft and crew; empty fields are sent +# empty. comment is the log's comment, battery the pack flown, pic the pilot in +# command, gso the ground safety officer, vehicle_id the aircraft. +comment = "Uploaded by logloader" +battery = "" +pic = "" +gso = "" +vehicle_id = "" diff --git a/src/Config.cpp b/src/Config.cpp index 0b354e6..4649594 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -121,6 +121,37 @@ void load_backend(const toml::node_view& table, UploadTargetConfig& } } +// Meala's form fields are free text, but an operator writing vehicle_id = 3 +// means the same as vehicle_id = "3"; refusing the bare number would silently +// send an empty field instead. +std::string text_or(const toml::node_view& node, const std::string& fallback) +{ + if (auto value = node.value(); value.has_value()) { + return trim(*value); + } + + if (auto value = node.value(); value.has_value()) { + return std::to_string(*value); + } + + if (auto value = node.value(); value.has_value()) { + std::ostringstream text; + text << *value; + return text.str(); + } + + return fallback; +} + +void load_meala_fields(const toml::node_view& table, UploadTargetConfig& target) +{ + target.meala_comment = text_or(table["comment"], target.meala_comment); + target.meala_battery = text_or(table["battery"], target.meala_battery); + target.meala_pic = text_or(table["pic"], target.meala_pic); + target.meala_gso = text_or(table["gso"], target.meala_gso); + target.meala_vehicle_id = text_or(table["vehicle_id"], target.meala_vehicle_id); +} + } // namespace std::string resolve_config_path(int argc, char** argv) @@ -209,6 +240,7 @@ Config load_config(const std::string& path) config.local.api_key = trim(local["api_key"].value_or("")); load_backend(local, config.local); config.local.credentials_file = trim(local["credentials_file"].value_or("")); + load_meala_fields(local, config.local); config.remote.name = kTargetRemote; config.remote.url = value_or(remote["url"], file["remote_server"], "https://review.px4.io"); @@ -218,6 +250,7 @@ Config load_config(const std::string& path) config.remote.api_key = trim(value_or(remote["api_key"], file["remote_api_key"], "")); load_backend(remote, config.remote); config.remote.credentials_file = trim(remote["credentials_file"].value_or("")); + load_meala_fields(remote, config.remote); if (config.local.url.empty()) { config.local.enabled = false; diff --git a/src/Config.hpp b/src/Config.hpp index c2165c8..5619418 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -38,6 +38,13 @@ struct UploadTargetConfig { // a target without this cannot be enabled. std::string credentials_file; bool public_logs {false}; + // Meala only: what each upload fills its form fields with. Meala requires + // every field to be present, so empty is sent as an empty field. + std::string meala_comment {"Uploaded by logloader"}; + std::string meala_battery; + std::string meala_pic; + std::string meala_gso; + std::string meala_vehicle_id; }; struct Config { diff --git a/src/MealaUploadTarget.cpp b/src/MealaUploadTarget.cpp index 9b2e28b..54dfee0 100644 --- a/src/MealaUploadTarget.cpp +++ b/src/MealaUploadTarget.cpp @@ -199,14 +199,14 @@ UploadTarget::Result MealaUploadTarget::upload(const std::string& file_path) return {Outcome::Missing, 0, "local file ended early: " + file_path, ""}; } - // Meala reads every one of these off the form. The ones an automated - // upload has nothing to say about still have to be present. + // Meala reads every one of these off the form, so the ones left empty in + // the config still have to be present. const httplib::MultipartFormDataItems fields = { - {"comments", "Uploaded by logloader", "", ""}, - {"battery", "", "", ""}, - {"pic", "", "", ""}, - {"gso", "", "", ""}, - {"vehicle_id", "", "", ""}, + {"comments", _config.meala_comment, "", ""}, + {"battery", _config.meala_battery, "", ""}, + {"pic", _config.meala_pic, "", ""}, + {"gso", _config.meala_gso, "", ""}, + {"vehicle_id", _config.meala_vehicle_id, "", ""}, {"dzchunkbyteoffset", std::to_string(offset), "", ""}, {"dzchunkindex", std::to_string(index), "", ""}, {"dztotalchunkcount", std::to_string(total_chunks), "", ""}, From d102bd95e02bf7a46fd314f46df863da95c621e8 Mon Sep 17 00:00:00 2001 From: Patrick Hinchey Date: Wed, 23 Sep 2026 23:49:04 -0400 Subject: [PATCH 7/7] Correct comments in config.toml Updated comments for clarity in config.toml. --- config.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.toml b/config.toml index 27e70bc..55e4179 100644 --- a/config.toml +++ b/config.toml @@ -83,8 +83,8 @@ credentials_file = "" # Meala only: sent with every log this target uploads, as the flight's details # on Meala. Edit these to match the aircraft and crew; empty fields are sent -# empty. comment is the log's comment, battery the pack flown, pic the pilot in -# command, gso the ground safety officer, vehicle_id the aircraft. +# empty. comment is for observations or descriptions, pic is the Pilot in +# Command, gso the Ground Station Operator or other second crew comment = "Uploaded by logloader" battery = "" pic = ""