Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 37 additions & 28 deletions src/CoreServer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -155,18 +155,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
res.set_content(j.dump(), "text/json");
return;
}
try {
auto taps = _registry->tap_manager()->load_from_str(req.body);
for (auto &mod : taps) {
mod->info_json(j[mod->name()]);
}
res.status = 201;
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
(std::string(e.what()).find("already") != std::string::npos) ? res.status = 409 : res.status = 500;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
}
api::post_taps(*_registry->tap_manager(), req.body, res);
});
_svr.Delete(fmt::format("/api/v1/taps/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
api::delete_tap(*_registry->policy_manager(), req.matches[1].str(), res);
Expand Down Expand Up @@ -204,22 +193,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
res.set_content(j.dump(), "text/json");
return;
}
try {
auto policies = _registry->policy_manager()->load_from_str(req.body);
for (auto &mod : policies) {
mod->info_json(j[mod->name()]);
}
res.status = 201;
res.set_content(j.dump(), "text/json");
} catch (const std::invalid_argument &e) {
res.status = 422;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
(std::string(e.what()).find("already") != std::string::npos) ? res.status = 409 : res.status = 500;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
}
api::post_policies(*_registry->policy_manager(), req.body, res);
});
_svr.Get(fmt::format("/api/v1/policies/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
api::get_policy(*_registry->policy_manager(), req.matches[1].str(), res);
Expand Down Expand Up @@ -495,5 +469,40 @@ void get_policies_prometheus(PolicyManager &policies, const std::vector<std::str
res.set_content(output.str(), "text/plain");
}

// both answer with what the load rendered: a concurrent DELETE can free what it returns as soon as it returns
void post_taps(TapManager &taps, const std::string &body, httplib::Response &res)
{
json j = json::object();
try {
json loaded = json::object();
taps.load_from_str(body, &loaded);
res.status = 201;
res.set_content(loaded.dump(), "text/json");
} catch (const std::exception &e) {
(std::string(e.what()).find("already") != std::string::npos) ? res.status = 409 : res.status = 500;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
}
}

void post_policies(PolicyManager &policies, const std::string &body, httplib::Response &res)
{
json j = json::object();
try {
json loaded = json::object();
policies.load_from_str(body, &loaded);
res.status = 201;
res.set_content(loaded.dump(), "text/json");
} catch (const std::invalid_argument &e) {
res.status = 422;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
(std::string(e.what()).find("already") != std::string::npos) ? res.status = 409 : res.status = 500;
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
}
}

}
}
4 changes: 4 additions & 0 deletions src/CoreServer.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ void get_default_prometheus(PolicyManager &policies, httplib::Response &res);
// with all set, names is every policy, listed beforehand: one removed since, or without the period, is skipped
void get_policies_metrics(PolicyManager &policies, const std::vector<std::string> &names, bool all, const std::string &period, bool merge, httplib::Response &res);
void get_policies_prometheus(PolicyManager &policies, const std::vector<std::string> &names, bool all, httplib::Response &res);
// for a body whose Content-Type was accepted: answer 201 with the module as loaded, even if a concurrent request
// has removed it since
void post_taps(TapManager &taps, const std::string &body, httplib::Response &res);
void post_policies(PolicyManager &policies, const std::string &body, httplib::Response &res);
}

class CoreServer
Expand Down
33 changes: 22 additions & 11 deletions src/Policies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

namespace visor {

std::vector<Policy *> PolicyManager::load_from_str(const std::string &str)
std::vector<Policy *> PolicyManager::load_from_str(const std::string &str, json *info)
{
if (str.empty()) {
throw PolicyException("empty data");
Expand All @@ -32,14 +32,14 @@ std::vector<Policy *> PolicyManager::load_from_str(const std::string &str)
throw PolicyException("unsupported version");
}
if (node["visor"]["policies"] && node["visor"]["policies"].IsMap()) {
return load(node["visor"]["policies"], true);
return load(node["visor"]["policies"], true, info);
} else {
throw PolicyException("no policies found in schema");
}
}

// needs to be thread safe and transactional: any errors mean resources get cleaned up with no side effects
std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool single)
std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool single, json *info)
{
assert(policy_yaml.IsMap());
assert(spdlog::get("visor"));
Expand Down Expand Up @@ -83,11 +83,9 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si

std::vector<std::string> added_inputs;
std::vector<std::string> added_handlers;
// by name, like the resources policies below: once the policy is detached from an input, nothing keeps
// that input alive, and a concurrent removal of its last policy can free it while this load runs
// by name, like the resources policies below: the rollback looks each one up again
std::vector<std::string> attached_inputs;
std::vector<std::unique_ptr<Policy>> resources_policies;
// by name: once registered, a resources policy can be deleted through the API while this load runs
std::vector<std::string> added_resources_policies;
// Undo everything this load did, one step at a time, so that a step that fails neither skips the rest
// nor replaces the exception that started the rollback. stop() undoes a completed start; after a
Expand All @@ -105,8 +103,7 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
step([&] { policy_ptr->stop(); });
for (auto &name : attached_inputs) {
step([&] {
// under the policy map lock, as remove_policy does: no other removal can free the input
// meanwhile. one that is already gone was freed after stop() detached the policy
// under the policy map lock, as remove_policy does
std::unique_lock policies_lock(_map_mutex);
InputStream *input = nullptr;
{
Expand All @@ -118,8 +115,7 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
input = found->second.get();
}
input->remove_policy(policy_ptr);
// a reused input outlives this load. If its last other policy was removed while this one
// still counted on it, that removal left it running, so finish the teardown here
// a reused input is left to its other policies, and removed only if none is left
if (std::find(added_inputs.begin(), added_inputs.end(), name) == added_inputs.end()) {
_remove_input_if_unused(name, input, policy_name);
}
Expand Down Expand Up @@ -152,6 +148,7 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
window_config.config_set<std::string>("_internal_tap_name", tap_name);

std::string input_stream_name = tap->get_input_name(input_config, input_filter);
// removals wait for the load, so an input found here is still there to reuse below
if (!_registry->input_manager()->module_exists(input_stream_name)) {
try {
spdlog::get("visor")->info("policy [{}]: creating input stream: {}", policy_name, input_stream_name);
Expand All @@ -161,6 +158,7 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
throw PolicyException(fmt::format("unable to instantiate tap '{}': {}", tap_name, e.what()));
}
} else {
// a test holds the load on this message, between finding the input and taking it
spdlog::get("visor")->debug("policy [{}]: input stream already exists. reusing: {}", policy_name, input_stream_name);
}

Expand Down Expand Up @@ -218,7 +216,8 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
}
policy_ptr->start();
for (auto &resources_policy : resources_policies) {
// attached before it is registered, and not touched after: from then on the API can delete it
// attached before it is registered, and not touched after: once registered, a removal after this load
// ends can free it
auto resources_name = resources_policy->name();
resources_policy->start();
resources_policy->input_stream().back()->add_policy(resources_policy.get());
Expand All @@ -231,6 +230,10 @@ std::vector<Policy *> PolicyManager::load(const YAML::Node &policy_yaml, bool si
}
added_resources_policies.push_back(resources_name);
}
if (info) {
// before it is registered: from then on a removal can free it as soon as this load ends
policy_ptr->info_json((*info)[policy_name]);
}
module_add(std::move(policy));
} catch (const std::runtime_error &e) {
rollback();
Expand Down Expand Up @@ -303,6 +306,14 @@ std::string PolicyManager::_get_policy_name(YAML::const_iterator it)

void PolicyManager::remove_policy(const std::string &name)
{
// a missing policy is reported at once, as remove_tap does. One that exists is looked up again below, under the
// locks
if (!module_exists(name)) {
throw ModuleException(name, fmt::format("module name '{}' does not exist", name));
}
// after any load in progress: from finding an input to reuse until its handlers consume it, a load relies on no
// removal taking that input away
std::unique_lock load_lock(_load_mutex);
std::unique_lock lock(_map_mutex);
if (_map.count(name) == 0) {
throw ModuleException(name, fmt::format("module name '{}' does not exist", name));
Expand Down
11 changes: 7 additions & 4 deletions src/Policies.h
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,14 @@ class PolicyManager : public AbstractManager<Policy>
{
}

std::vector<Policy *> load_from_str(const std::string &str);
std::vector<Policy *> load(const YAML::Node &tap_yaml, bool single = false);
// info, if given, gets the info_json of each policy loaded, rendered during the load: once it ends, a removal can
// free the policies returned
std::vector<Policy *> load_from_str(const std::string &str, json *info = nullptr);
std::vector<Policy *> load(const YAML::Node &tap_yaml, bool single = false, json *info = nullptr);
std::pair<std::unique_ptr<Policy>, std::string> create_resources_policy(InputStream *input, const Config &window_config);
// throws ModuleException, having changed nothing, when no policy has that name; any later failure, such as
// removing its handlers or inputs, throws another type
// removes the policy, and its inputs if no other policy uses them, after any policy load in progress. Throws
// ModuleException, having changed nothing, when no policy has that name; a policy already missing is reported
// without waiting for a load. Any later failure, such as removing its handlers or inputs, throws another type
void remove_policy(const std::string &name);
// detaches the tap from every policy and removes it, after any policy load in progress. Throws
// ModuleException, having changed nothing, when no tap has that name; a tap already missing is reported without
Expand Down
9 changes: 6 additions & 3 deletions src/Taps.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

namespace visor {

std::vector<Tap *> TapManager::load_from_str(const std::string &str)
std::vector<Tap *> TapManager::load_from_str(const std::string &str, json *info)
{
if (str.empty()) {
throw TapException("empty data");
Expand All @@ -29,14 +29,14 @@ std::vector<Tap *> TapManager::load_from_str(const std::string &str)
throw PolicyException("unsupported version");
}
if (node["visor"]["taps"] && node["visor"]["taps"].IsMap()) {
return load(node["visor"]["taps"], true, true);
return load(node["visor"]["taps"], true, true, info);
} else {
throw TapException("no taps found in schema");
}
}

// needs to be thread safe and transactional: any errors mean resources get cleaned up with no side effects
std::vector<Tap *> TapManager::load(const YAML::Node &tap_yaml, bool strict, bool single)
std::vector<Tap *> TapManager::load(const YAML::Node &tap_yaml, bool strict, bool single, json *info)
{
assert(tap_yaml.IsMap());
assert(spdlog::get("visor"));
Expand Down Expand Up @@ -86,6 +86,9 @@ std::vector<Tap *> TapManager::load(const YAML::Node &tap_yaml, bool strict, boo
tap_module->tags_set_yaml(it->second["tags"]);
}

if (info) {
tap_module->info_json((*info)[tap_name]);
}
// will throw if it already exists. nothing else to clean up
result.push_back(tap_module.get());
module_add(std::move(tap_module));
Expand Down
6 changes: 4 additions & 2 deletions src/Taps.h
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,10 @@ class TapManager : public AbstractManager<Tap>
{
}

std::vector<Tap *> load_from_str(const std::string &str);
std::vector<Tap *> load(const YAML::Node &tap_yaml, bool strict = true, bool single = false);
// info, if given, gets the info_json of each tap loaded, rendered before it is registered: from then on, a removal
// can free the taps returned
std::vector<Tap *> load_from_str(const std::string &str, json *info = nullptr);
std::vector<Tap *> load(const YAML::Node &tap_yaml, bool strict = true, bool single = false, json *info = nullptr);
std::vector<std::string> get_input_taps_name(const YAML::Node &input_node);
};

Expand Down
81 changes: 74 additions & 7 deletions src/tests/test_core_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,24 @@ int status(const httplib::Response &res)
return res.status == -1 ? 200 : res.status;
}

void load_tap(CoreRegistry &registry)
{
registry.tap_manager()->load_from_str(R"(
auto constexpr TAP = R"(
visor:
taps:
anycast:
input_type: mock
config:
iface: eth0
)");
)";

void load_tap(CoreRegistry &registry)
{
registry.tap_manager()->load_from_str(TAP);
}

// on the tap from load_tap, with an input of its own and a net handler
void load_policy(CoreRegistry &registry, const std::string &name)
std::string policy(const std::string &name)
{
registry.policy_manager()->load_from_str(fmt::format(R"(
return fmt::format(R"(
visor:
policies:
{0}:
Expand All @@ -48,7 +50,12 @@ void load_policy(CoreRegistry &registry, const std::string &name)
{0}_net:
type: net
)",
name));
name);
}

void load_policy(CoreRegistry &registry, const std::string &name)
{
registry.policy_manager()->load_from_str(policy(name));
}

}
Expand Down Expand Up @@ -250,3 +257,63 @@ TEST_CASE("Policy metrics skip a policy without the period under __all and answe
CHECK(res.status == 425);
CHECK_THAT(json::parse(res.body).value("error", ""), Catch::Matchers::ContainsSubstring("has not yet accumulated"));
}

TEST_CASE("POST routes answer 201 with what they loaded and refuse a name already taken", "[api][taps][policies]")
{
CoreRegistry registry;
visor::load_builtin_plugins(registry);
registry.start(nullptr);

SECTION("POST /api/v1/taps")
{
httplib::Response res;
api::post_taps(*registry.tap_manager(), TAP, res);
CHECK(res.status == 201);
auto j = json::parse(res.body);
CHECK(j["anycast"]["input_type"] == "mock");
CHECK(j["anycast"]["config"]["iface"] == "eth0");

httplib::Response again;
api::post_taps(*registry.tap_manager(), TAP, again);
CHECK(again.status == 409);
CHECK(json::parse(again.body) == json{{"error", "module name 'anycast' already exists"}});
}

SECTION("POST /api/v1/policies")
{
REQUIRE_NOTHROW(load_tap(registry));
httplib::Response res;
api::post_policies(*registry.policy_manager(), policy("edge"), res);
CHECK(res.status == 201);
auto j = json::parse(res.body);
CHECK(j["edge"]["taps"].contains("anycast"));
CHECK(j["edge"]["input"].size() == 1);
CHECK(j["edge"]["modules"].contains("edge-anycast-edge_net"));

httplib::Response again;
api::post_policies(*registry.policy_manager(), policy("edge"), again);
CHECK(again.status == 409);
CHECK(json::parse(again.body) == json{{"error", "policy with name 'edge' already defined"}});

// a tap selector that matches no tap
httplib::Response unmatched;
api::post_policies(*registry.policy_manager(), R"(
visor:
policies:
core:
kind: collection
input:
tap_selector:
any:
- site: nowhere
input_type: mock
handlers:
modules:
core_net:
type: net
)",
unmatched);
CHECK(unmatched.status == 422);
CHECK_FALSE(registry.policy_manager()->module_exists("core"));
}
}
Loading
Loading