diff --git a/.gitignore b/.gitignore index 7d5352831..af1600a91 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,6 @@ ext_deps /bindings/cs/rl.net.cli/InternalsVisibleToTest.cs /test_tools/log_parser/reinforcement_learning/ _build -.idea \ No newline at end of file +.idea +dist/ +rl_client.egg-info/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8533c935d..b40548d25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,4 +80,4 @@ add_subdirectory(unit_test) add_subdirectory(unit_test/extensions) if(RL_BUILD_BENCHMARKS) add_subdirectory(benchmarks) -endif() \ No newline at end of file +endif() diff --git a/bindings/python/README.md b/bindings/python/README.md index c261e148b..745e6b51a 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -7,11 +7,11 @@ Commands are relative to repo root. python setup.py install # Or, if vcpkg used for deps -python setup.py --cmake-options="-DCMAKE_TOOLCHAIN_FILE=\"/path_to_vcpkg_root/scripts/buildsystems/vcpkg.cmake\"" install +python setup.py --cmake-options="-DCMAKE_TOOLCHAIN_FILE=/path_to_vcpkg_root/scripts/buildsystems/vcpkg.cmake" install ``` - For Ubuntu 20.04, Python 3.8 a recommended vcpkg version is: `Release 2020.06, commit 6185aa7` ## Usage -After successful installation, an example is in [`examples/python/basic_usage.py`](../../examples/python/basic_usage.py). \ No newline at end of file +After successful installation, an example is in [`examples/python/basic_usage.py`](../../examples/python/basic_usage.py). diff --git a/bindings/python/docs/migration_guide.rst b/bindings/python/docs/migration_guide.rst index e778fc5bb..ef5895230 100644 --- a/bindings/python/docs/migration_guide.rst +++ b/bindings/python/docs/migration_guide.rst @@ -40,7 +40,7 @@ Changes to: # ... - client = rl_client.live_model(_, on_error) + client = rl_client.LiveModel(_, on_error) 3. Init @@ -57,12 +57,12 @@ Changes to: .. code-block:: python - client = rl_client.live_model(config) + client = rl_client.LiveModel(config) 4. `choose_rank` return value ----------------------------- -`choose_rank` no longer returns a tuple, but now returns a :meth:`rl_client.ranking_response` object that contains the same information as was contained in the tuple. +`choose_rank` no longer returns a tuple, but now returns a :meth:`rl_client.RankingResponse` object that contains the same information as was contained in the tuple. .. code-block:: python diff --git a/bindings/python/py_api.cc b/bindings/python/py_api.cc index bc96292c8..f0855e83c 100644 --- a/bindings/python/py_api.cc +++ b/bindings/python/py_api.cc @@ -4,6 +4,7 @@ #include "config_utility.h" #include "constants.h" #include "live_model.h" +#include "multistep.h" #include #include @@ -134,8 +135,10 @@ PYBIND11_MODULE(rl_client, m) { Container class for all configuration values. Generally is constructed from client.json file read from disk )pbdoc") .def(py::init<>()) - .def("get", &rl::utility::configuration::get, py::arg("name"), - py::arg("defval"), R"pbdoc( + .def("get", &rl::utility::configuration::get, + py::arg("name"), + py::arg("defval"), + R"pbdoc( Get a config value or default. :param name: Name of configuration value to get @@ -161,7 +164,8 @@ PYBIND11_MODULE(rl_client, m) { THROW_IF_FAIL(live_model->init(&status)); return live_model; }), - py::arg("config"), py::arg("callback")) + py::arg("config"), + py::arg("callback")) .def( "choose_rank", [](rl::live_model &lm, const char *context, const char *event_id, @@ -174,7 +178,9 @@ PYBIND11_MODULE(rl_client, m) { lm.choose_rank(event_id, context, flags, response, &status)); return response; }, - py::arg("context"), py::arg("event_id"), py::arg("deferred") = false, + py::arg("context"), + py::arg("event_id"), + py::arg("deferred") = false, R"pbdoc( Request prediction for given context and use the given event_id @@ -184,17 +190,31 @@ PYBIND11_MODULE(rl_client, m) { "choose_rank", [](rl::live_model &lm, const char *context, bool deferred) { rl::ranking_response response; + rl::api_status status; unsigned int flags = deferred ? rl::action_flags::DEFERRED : rl::action_flags::DEFAULT; - rl::api_status status; THROW_IF_FAIL(lm.choose_rank(context, flags, response, &status)); return response; }, - py::arg("context"), py::arg("deferred") = false, R"pbdoc( + py::arg("context"), + py::arg("deferred") = false, + R"pbdoc( Request prediction for given context and let an event id be generated :rtype: :class:`rl_client.RankingResponse` )pbdoc") + .def( + "request_episodic_decision", + [](rl::live_model &lm, const char* event_id, const char* previous_id, const char* context, rl::episode_state& episode) { + rl::ranking_response response; + rl::api_status status; + THROW_IF_FAIL(lm.request_episodic_decision(event_id, previous_id, context, response, episode, &status)); + return response; + }, + py::arg("event_id"), + py::arg("previous_id"), + py::arg("context"), + py::arg("episode")) .def( "report_action_taken", [](rl::live_model &lm, const char *event_id) { @@ -208,14 +228,25 @@ PYBIND11_MODULE(rl_client, m) { rl::api_status status; THROW_IF_FAIL(lm.report_outcome(event_id, outcome, &status)); }, - py::arg("event_id"), py::arg("outcome")) + py::arg("event_id"), + py::arg("outcome")) .def( "report_outcome", [](rl::live_model &lm, const char *event_id, float outcome) { rl::api_status status; THROW_IF_FAIL(lm.report_outcome(event_id, outcome, &status)); }, - py::arg("event_id"), py::arg("outcome")) + py::arg("event_id"), + py::arg("outcome")) + .def( + "report_outcome", + [](rl::live_model &lm, const char *episode_id, const char *event_id, float outcome) { + rl::api_status status; + THROW_IF_FAIL(lm.report_outcome(episode_id, event_id, outcome, &status)); + }, + py::arg("episode_id"), + py::arg("event_id"), + py::arg("outcome")) .def("refresh_model", [](rl::live_model &lm) { rl::api_status status; THROW_IF_FAIL(lm.refresh_model(&status)); @@ -267,6 +298,15 @@ PYBIND11_MODULE(rl_client, m) { :rtype: list[(int,float)] )pbdoc"); + // TODO: Expose episode history API. + py::class_(m, "EpisodeState") + .def(py::init()) + .def_property_readonly( + "episode_id", + [](const rl::episode_state &episode) { + return episode.get_episode_id(); + }); + m.def( "create_config_from_json", [](const std::string &config_json) { diff --git a/examples/basic_usage_cpp/basic_usage_cpp.cc b/examples/basic_usage_cpp/basic_usage_cpp.cc index 13a863893..62d108d9a 100644 --- a/examples/basic_usage_cpp/basic_usage_cpp.cc +++ b/examples/basic_usage_cpp/basic_usage_cpp.cc @@ -14,10 +14,11 @@ */ int main() { - return basic_usage_cb(); -// return basic_usage_ca(); -// return basic_usage_ccb(); -// return basic_usage_slates(); + // return basic_usage_cb(); + // return basic_usage_ca(); + // return basic_usage_ccb(); + // return basic_usage_slates(); + return basic_usage_multistep(); } int basic_usage_cb() { @@ -29,7 +30,7 @@ int basic_usage_cb() { u::configuration config; //! Helper method to initialize config from a json file - if( load_config_from_json("client.json", config) != err::success ) { + if (load_config_from_json("client.json", config) != err::success) { std::cout << "Unable to Load file: client.json" << std::endl; return -1; } @@ -44,7 +45,7 @@ int basic_usage_cb() { //! [(1) Instantiate Inference API using config] //! [(2) Initialize the API] - if( rl.init(&status) != err::success ) { + if (rl.init(&status) != err::success) { std::cout << status.get_error_msg() << std::endl; return -1; } @@ -54,7 +55,7 @@ int basic_usage_cb() { // Response class r::ranking_response response; - if( rl.choose_rank(event_id, context, response, &status) != err::success ) { + if (rl.choose_rank(event_id, context, response, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; return -1; } @@ -62,7 +63,7 @@ int basic_usage_cb() { //! [(4) Use the response] size_t chosen_action; - if( response.get_chosen_action_id(chosen_action, &status) != err::success ) { + if (response.get_chosen_action_id(chosen_action, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; return -1; } @@ -72,7 +73,7 @@ int basic_usage_cb() { //! [(5) Report outcome] // Report received outcome (Optional: if this call is not made, default missing outcome is applied) // Missing outcome can be thought of as negative reinforcement - if( rl.report_outcome(event_id, outcome, &status) != err::success ) { + if (rl.report_outcome(event_id, outcome, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; return -1; } @@ -241,6 +242,81 @@ int basic_usage_slates() { return 0; } +int basic_usage_multistep() { + //! name, value based config object used to initialise the API + u::configuration config; + + //! Helper method to initialize config from a json file + if (load_config_from_json("client.json", config) != err::success) { + std::cout << "Unable to Load file: client.json" << std::endl; + return -1; + } + + r::api_status status; + + r::live_model rl(config); + + if (rl.init(&status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + + r::episode_state episode1("my_episode_id_1"); + r::episode_state episode2("my_episode_id_2"); + + { + const std::string context1 = R"({"shared":{"F1": 1.0}, "_multi": [{"AF1": 2.0}, {"AF1": 3.0}]})"; + r::ranking_response response1; + if (rl.request_episodic_decision("event1", nullptr, context1.c_str(), response1, episode1, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + { + const std::string context1 = R"({"shared":{"F2": 1.0}, "_multi": [{"AF2": 2.0}, {"AF2": 3.0}]})"; + r::ranking_response response1; + if (rl.request_episodic_decision("event1", nullptr, context1.c_str(), response1, episode2, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + { + const std::string context2 = R"({"shared":{"F1": 4.0}, "_multi": [{"AF1": 2.0}, {"AF1": 3.0}]})"; + r::ranking_response response2; + if (rl.request_episodic_decision("event2", "event1", context2.c_str(), response2, episode1, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + { + const std::string context2 = R"({"shared":{"F2": 4.0}, "_multi": [{"AF2": 2.0}, {"AF2": 3.0}]})"; + r::ranking_response response2; + if (rl.request_episodic_decision("event2", "event1", context2.c_str(), response2, episode2, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + { + if (rl.report_outcome(episode1.get_episode_id(), "event1", 1.0f, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + { + if (rl.report_outcome(episode2.get_episode_id(), "event2", 1.0f, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + } + + return 0; +} + // Helper methods //! Load config from json file diff --git a/examples/basic_usage_cpp/basic_usage_cpp.h b/examples/basic_usage_cpp/basic_usage_cpp.h index 60143f8a6..c3a3b30c7 100644 --- a/examples/basic_usage_cpp/basic_usage_cpp.h +++ b/examples/basic_usage_cpp/basic_usage_cpp.h @@ -22,6 +22,7 @@ int basic_usage_cb(); int basic_usage_ca(); int basic_usage_ccb(); int basic_usage_slates(); +int basic_usage_multistep(); int load_file(const std::string& file_name, std::string& file_data); int load_config_from_json(const std::string& file_name, u::configuration& cc); diff --git a/examples/python/basic_usage.py b/examples/python/basic_usage.py index 562cf23e3..b96c9b091 100644 --- a/examples/python/basic_usage.py +++ b/examples/python/basic_usage.py @@ -7,11 +7,13 @@ def on_error(error_code, error_message): print(error_code) print(error_message) + def load_config_from_json(file_name): with open(file_name, 'r') as config_file: return rl_client.create_config_from_json(config_file.read()) -def main(): + +def basic_usage_cb(): config = load_config_from_json("client.json") model = rl_client.LiveModel(config, on_error) @@ -41,5 +43,51 @@ def main(): outcome = 1.0 model.report_outcome(event_id, outcome) + +def basic_usage_multistep(): + config = load_config_from_json("client.json") + + model = rl_client.LiveModel(config) + + episode1 = rl_client.EpisodeState("episode1") + episode2 = rl_client.EpisodeState("episode2") + + # episode1, event1 + context1 = '{"shared":{"F1": 1.0}, "_multi": [{"AF1": 2.0}, {"AF1": 3.0}]}' + response1 = model.request_episodic_decision( + "event1", None, context1, episode1) + print("episode id:", episode1.episode_id) + print("event id:", response1.event_id) + print("chosen action:", response1.chosen_action_id) + + # episode2, event1 + context1 = '{"shared":{"F2": 1.0}, "_multi": [{"AF2": 2.0}, {"AF2": 3.0}]}' + response1 = model.request_episodic_decision( + "event1", None, context1, episode2) + print("episode id:", episode2.episode_id) + print("event id:", response1.event_id) + print("chosen action:", response1.chosen_action_id) + + # episode1, event2 + context2 = '{"shared":{"F1": 4.0}, "_multi": [{"AF1": 2.0}, {"AF1": 3.0}]}' + response2 = model.request_episodic_decision( + "event2", "event1", context2, episode1) + print("episode id:", episode1.episode_id) + print("event id:", response2.event_id) + print("chosen action:", response2.chosen_action_id) + + # episode2, event2 + context2 = '{"shared":{"F2": 4.0}, "_multi": [{"AF2": 2.0}, {"AF2": 3.0}]}' + response2 = model.request_episodic_decision( + "event2", "event1", context2, episode2) + print("episode id:", episode2.episode_id) + print("event id:", response2.event_id) + print("chosen action:", response2.chosen_action_id) + + model.report_outcome(episode1.episode_id, "event1", 1.0) + model.report_outcome(episode2.episode_id, "event2", 1.0) + + if __name__ == "__main__": - main() + # basic_usage_cb() + basic_usage_multistep() diff --git a/examples/rl_sim_cpp/main.cc b/examples/rl_sim_cpp/main.cc index 823d94188..6fe5eb88b 100644 --- a/examples/rl_sim_cpp/main.cc +++ b/examples/rl_sim_cpp/main.cc @@ -37,7 +37,10 @@ po::variables_map process_cmd_line(const int argc, char** argv) { ("slates", po::value()-> default_value(false), "Run in slates mode") ("ca", po::value()-> - default_value(false), "Run in continuous actions mode"); + default_value(false), "Run in continuous actions mode") + ("multistep", po::value()-> + default_value(false), "Run in multistep mode") + ; po::variables_map vm; store(parse_command_line(argc, argv, desc), vm); diff --git a/examples/rl_sim_cpp/rl_sim.cc b/examples/rl_sim_cpp/rl_sim.cc index d07ed3d3d..2ef459960 100644 --- a/examples/rl_sim_cpp/rl_sim.cc +++ b/examples/rl_sim_cpp/rl_sim.cc @@ -7,6 +7,7 @@ #include "person.h" #include "simulation_stats.h" #include "constants.h" +#include "multistep.h" using namespace std; @@ -22,6 +23,7 @@ int rl_sim::loop() { case CA: return ca_loop(); case CCB: return ccb_loop(); case Slates: return slates_loop(); + case Multistep : return multistep_loop(); default: std::cout << "Invalid loop kind:" << _loop_kind << std::endl; return -1; @@ -73,6 +75,68 @@ int rl_sim::cb_loop() { return 0; } +std::string create_episode_id(size_t episode_id) { + std::ostringstream oss; + oss << "episode" << "-" << episode_id; + return oss.str(); +} + +int rl_sim::multistep_loop() { + r::ranking_response response; + simulation_stats stats; + + const size_t episode_length = 2; + size_t episode_indx = 0; + while (_run_loop) { + const std::string episode_id = create_episode_id(episode_indx++); + r::api_status status; + r::episode_state episode(episode_id.c_str()); + + std::string previous_id; + float episodic_outcome = 0; + + for (size_t i = 0; i < episode_length; ++i) { + auto& p = pick_a_random_person(); + const auto context_features = p.get_features(); + const auto action_features = get_action_features(); + const auto context_json = create_context_json(context_features, action_features); + const auto req_id = create_event_id(); + + r::ranking_response response1; + if (_rl->request_episodic_decision(req_id.c_str(), i == 0 ? nullptr : previous_id.c_str(), + context_json.c_str(), response, episode, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + return -1; + } + + size_t chosen_action; + if (response.get_chosen_action_id(chosen_action) != err::success) { + std::cout << status.get_error_msg() << std::endl; + continue; + } + + const auto outcome_per_step = p.get_outcome(_topics[chosen_action]); + stats.record(p.id(), chosen_action, outcome_per_step); + + std::cout << " " << stats.count() << ", ctxt, " << p.id() << ", action, " << chosen_action << ", outcome, " << outcome_per_step + << ", dist, " << get_dist_str(response) << ", " << stats.get_stats(p.id(), chosen_action) << std::endl; + + episodic_outcome += outcome_per_step; + previous_id = req_id; + } + + if (_rl->report_outcome(episode.get_episode_id(), episodic_outcome, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + continue; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + } + + return 0; +} + + int rl_sim::ca_loop(){ r::continuous_action_response response; simulation_stats stats; @@ -332,15 +396,15 @@ bool rl_sim::init_sim_world() { bool rl_sim::init_continuous_sim_world() { // initialize continuous actions robot joints - + _friction = {25.4, 41.2, 66.5, 81.9, 104.4}; - + // temperature (C) range: 20.f to 45.f // angular velocity range: 0.f to 200.f // load range: -60.f to 60.f // first joint j1 - joint::friction_prob fb = + joint::friction_prob fb = { { _friction[0], 0.08f }, { _friction[1], 0.03f }, @@ -352,7 +416,7 @@ bool rl_sim::init_continuous_sim_world() { _robot_joints.emplace_back("j1", 20.3, 102.4, -10.2, fb); // second joint j2 - fb = + fb = { { _friction[0],0.08f }, { _friction[1],0.30f }, @@ -441,12 +505,14 @@ std::string rl_sim::create_event_id() { rl_sim::rl_sim(boost::program_options::variables_map vm) : _options(std::move(vm)), _loop_kind(CB) { - if(_options["ccb"].as()) + if (_options["ccb"].as()) _loop_kind = CCB; - else if(_options["slates"].as()) + else if (_options["slates"].as()) _loop_kind = Slates; - else if(_options["ca"].as()) + else if (_options["ca"].as()) _loop_kind = CA; + else if (_options["multistep"].as()) + _loop_kind = Multistep; } std::string get_dist_str(const reinforcement_learning::ranking_response& response) { diff --git a/examples/rl_sim_cpp/rl_sim.h b/examples/rl_sim_cpp/rl_sim.h index 14fa75caa..201e85b05 100644 --- a/examples/rl_sim_cpp/rl_sim.h +++ b/examples/rl_sim_cpp/rl_sim.h @@ -131,6 +131,7 @@ class rl_sim { int ca_loop(); int ccb_loop(); int slates_loop(); + int multistep_loop(); /** * @brief Get the action features as a json string @@ -153,7 +154,8 @@ class rl_sim { CB, CCB, Slates, - CA + CA, + Multistep }; boost::program_options::variables_map _options; diff --git a/examples/test_cpp/CMakeLists.txt b/examples/test_cpp/CMakeLists.txt index 33987f6e8..ce5bec8e3 100644 --- a/examples/test_cpp/CMakeLists.txt +++ b/examples/test_cpp/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(rl_test.out main.cc + options.cc experiment_controller.cc test_data_provider.cc test_loop.cc diff --git a/examples/test_cpp/experiment_controller.cc b/examples/test_cpp/experiment_controller.cc index bbf73c9db..4fd19d0ba 100644 --- a/examples/test_cpp/experiment_controller.cc +++ b/examples/test_cpp/experiment_controller.cc @@ -1,14 +1,9 @@ #include "experiment_controller.h" + #include -void throw_if_conflicting(const boost::program_options::variables_map& vm, const std::string first, const std::string& second) { - if (vm.count(first) && !vm[first].defaulted() && - vm.count(second) && !vm[second].defaulted()) - { - throw std::logic_error(std::string("Conflicting options '") + - first + "' and '" + second + "'."); - } -} +#include "options.h" + void experiment_controller::restart() { _iteration = 0; _is_running = true; @@ -32,7 +27,7 @@ void experiment_controller::stop() { _is_running = false; } -void experiment_controller::progress_bar() { +void experiment_controller::show_progress_bar() const { if (_iteration % 100 == 0) std::cout << "\r" << _iteration << " sent"; } @@ -68,4 +63,4 @@ experiment_controller* experiment_controller_factory::create(const boost::progra return new duration_experiment_controller(vm["duration"].as()); } return new iterations_experiment_controller(vm["examples"].as()); -} \ No newline at end of file +} diff --git a/examples/test_cpp/experiment_controller.h b/examples/test_cpp/experiment_controller.h index 5b9a4b600..6024d230f 100644 --- a/examples/test_cpp/experiment_controller.h +++ b/examples/test_cpp/experiment_controller.h @@ -11,7 +11,7 @@ class experiment_controller { void restart(); void iterate(); - void progress_bar(); + void show_progress_bar() const; size_t get_iteration() const; bool is_running() const; diff --git a/examples/test_cpp/main.cc b/examples/test_cpp/main.cc index 8d7b8f76c..f69705502 100644 --- a/examples/test_cpp/main.cc +++ b/examples/test_cpp/main.cc @@ -1,50 +1,20 @@ -#include "test_loop.h" - +#include #include -namespace po = boost::program_options; - -bool is_help(const po::variables_map& vm) { - return vm.count("help") > 0; -} - -po::variables_map process_cmd_line(const int argc, char** argv) { - po::options_description desc("Options"); - desc.add_options() - ("help", "produce help message") - ("json_config,j", po::value()-> - default_value("client.json"), "JSON file with config information for hosted RL loop") - ("threads,t", po::value()->default_value(1), "Number of threads per instance") - ("examples,n", po::value()->default_value(10), "Number of examples per thread") - ("features,x", po::value()->default_value(10), "Features count") - ("actions,a", po::value()->default_value(2), "Number of actions") - ("experiment_name,e", po::value()->required(), "experiment name") - ("float_outcome,f", "if outcome is float (otherwise - json)") - ("sleep,s", po::value()->default_value(0), "Milliseconds to sleep between loop iterations") - ("duration,d", po::value(), "Duration of experiment (in ms). Alternative to n") - ("instances,i", po::value()->default_value(1), "Number of test loop instances") - ("reward_period,r", po::value()->default_value(0), "Ratio period (0 - no reward, otherwise - every $reward_period interaction is receiving reward)") - ("slots,q", po::value()->default_value(0), "Number of slots (ccb simulation is running if > 0)") - ; - - po::variables_map vm; - store(parse_command_line(argc, argv, desc), vm); - - if (is_help(vm)) - std::cout << desc << std::endl; - - return vm; -} +#include "options.h" +#include "test_loop.h" -int run_test_instance(size_t index, const po::variables_map& vm) { +int run_test_instance(size_t index, + const boost::program_options::variables_map& vm) { test_loop loop(index, vm); + if (!loop.init()) { std::cerr << "Test loop haven't initialized properly." << std::endl; return -1; } - const auto is_ccb = vm["slots"].as() > 0; - loop.run(is_ccb); + loop.run(); + return 0; } @@ -52,13 +22,14 @@ int main(int argc, char** argv) { try { const auto vm = process_cmd_line(argc, argv); if (is_help(vm)) return 0; - const size_t instances = vm["instances"].as(); - std::vector _threads; - for (size_t i = 0; i < instances; ++i) { - _threads.push_back(std::thread(&run_test_instance, i, vm)); + + const size_t num_instances = vm["instances"].as(); + std::vector instances; + for (size_t i = 0; i < num_instances; ++i) { + instances.push_back(std::thread(&run_test_instance, i, vm)); } - for (size_t i = 0; i < instances; ++i) { - _threads[i].join(); + for (size_t i = 0; i < num_instances; ++i) { + instances[i].join(); } } catch (const std::exception& e) { diff --git a/examples/test_cpp/options.cc b/examples/test_cpp/options.cc new file mode 100644 index 000000000..952e08861 --- /dev/null +++ b/examples/test_cpp/options.cc @@ -0,0 +1,48 @@ +#include "options.h" + +#include +#include + +namespace po = boost::program_options; + +bool is_help(const po::variables_map& vm) { + return vm.count("help") > 0; +} + +po::variables_map process_cmd_line(const int argc, char** argv) { + po::options_description desc("Options"); + desc.add_options() + ("help,h", "produce help message") + ("json_config,j", po::value()-> + default_value("client.json"), "JSON file with config information for hosted RL loop") + ("threads,t", po::value()->default_value(1), "Number of threads per instance") + ("examples,n", po::value()->default_value(10), "Number of examples per thread") + ("features,x", po::value()->default_value(10), "Features count") + ("actions,a", po::value()->default_value(2), "Number of actions") + ("experiment_name,e", po::value()->required(), "(REQUIRED) experiment name") + ("float_outcome,f", "if outcome is float (otherwise - json)") + ("sleep,s", po::value()->default_value(0), "Milliseconds to sleep between loop iterations") + ("duration,d", po::value(), "Duration of experiment (in ms). Alternative to n") + ("instances,i", po::value()->default_value(1), "Number of test loop instances") + ("reward_period,r", po::value()->default_value(0), "Ratio period (0 - no reward, otherwise - every $reward_period interaction is receiving reward)") + ("slots,q", po::value()->default_value(0), "Number of slots (ccb simulation is running if > 0)") + ("episode_length,m", po::value()->default_value(0), "Length of an episode (running multistep if > 0)") + ; + + po::variables_map vm; + store(parse_command_line(argc, argv, desc), vm); + + if (is_help(vm)) + std::cout << desc << std::endl; + + return vm; +} + +void throw_if_conflicting(const po::variables_map& vm, const std::string& first, const std::string& second) { + if (vm.count(first) && !vm[first].defaulted() && + vm.count(second) && !vm[second].defaulted()) + { + throw std::logic_error(std::string("Conflicting options '") + + first + "' and '" + second + "'."); + } +} diff --git a/examples/test_cpp/options.h b/examples/test_cpp/options.h new file mode 100644 index 000000000..2e5eabbb8 --- /dev/null +++ b/examples/test_cpp/options.h @@ -0,0 +1,8 @@ +#pragma once + +#include +#include + +boost::program_options::variables_map process_cmd_line(const int argc, char** argv); +bool is_help(const boost::program_options::variables_map& vm); +void throw_if_conflicting(const boost::program_options::variables_map& vm, const std::string& first, const std::string& second); diff --git a/examples/test_cpp/test_cpp.vcxproj b/examples/test_cpp/test_cpp.vcxproj index f3289704a..0d9f5ac12 100644 --- a/examples/test_cpp/test_cpp.vcxproj +++ b/examples/test_cpp/test_cpp.vcxproj @@ -156,6 +156,7 @@ + diff --git a/examples/test_cpp/test_data_provider.cc b/examples/test_cpp/test_data_provider.cc index 7c90b9e30..874b53ac8 100644 --- a/examples/test_cpp/test_data_provider.cc +++ b/examples/test_cpp/test_data_provider.cc @@ -1,9 +1,9 @@ #include "test_data_provider.h" +#include + #include "data_buffer.h" #include "ranking_event.h" - -#include #include "serialization/json_serializer.h" test_data_provider::test_data_provider(const std::string& experiment_name, size_t threads, size_t features, size_t actions, size_t _slots, bool _is_float_outcome, size_t _reward_period) @@ -22,6 +22,7 @@ test_data_provider::test_data_provider(const std::string& experiment_name, size_ } } +// TODO: stringstream is the slowest way to concat strings. std::string test_data_provider::create_event_id(size_t thread_id, size_t example_id) const { std::ostringstream oss; oss << _experiment_name << "-" << thread_id << "-" << example_id; diff --git a/examples/test_cpp/test_data_provider.h b/examples/test_cpp/test_data_provider.h index 6a10ace13..98ac1c22c 100644 --- a/examples/test_cpp/test_data_provider.h +++ b/examples/test_cpp/test_data_provider.h @@ -1,11 +1,12 @@ #pragma once -#include "live_model.h" -#include "ranking_response.h" #include #include #include +#include "live_model.h" +#include "ranking_response.h" + class test_data_provider { public: test_data_provider(const std::string& experiment_name, size_t threads, size_t features, size_t actions, size_t slots, bool _is_float_outcome, size_t reward_ratio); diff --git a/examples/test_cpp/test_loop.cc b/examples/test_cpp/test_loop.cc index b9daa12b5..ab8ddc66c 100644 --- a/examples/test_cpp/test_loop.cc +++ b/examples/test_cpp/test_loop.cc @@ -1,11 +1,14 @@ #include "test_loop.h" -#include "ranking_event.h" -#include "data_buffer.h" -#include "config_utility.h" -#include "constants.h" + #include #include #include + +#include "options.h" +#include "config_utility.h" +#include "constants.h" +#include "data_buffer.h" +#include "ranking_event.h" #include "serialization/fb_serializer.h" namespace r = reinforcement_learning; @@ -16,19 +19,21 @@ namespace po = boost::program_options; namespace chrono = std::chrono; test_loop::test_loop(size_t index, const boost::program_options::variables_map& vm) - : threads(vm["threads"].as()) - , controller(experiment_controller_factory::create(vm)) - , experiment_name(generate_experiment_name(vm["experiment_name"].as(), threads, vm["features"].as(), vm["actions"].as(), vm["slots"].as(), index)) + : loop_kind(get_loop_kind(vm)) + , threads(vm["threads"].as()) + , experiment_name(generate_experiment_name(vm["experiment_name"].as(), threads, vm["features"].as(), vm["actions"].as(), vm["slots"].as(), vm["episode_length"].as(), index)) , json_config(vm["json_config"].as()) , test_inputs(experiment_name, threads, vm["features"].as(), vm["actions"].as(), vm["slots"].as(), vm.count("float_outcome") > 0, vm["reward_period"].as()) + , episode_length(vm["episode_length"].as()) , sleep_interval(vm["sleep"].as()) { for (size_t i = 0; i < threads; ++i) { + controllers.push_back(std::unique_ptr(experiment_controller_factory::create(vm))); loggers.push_back(std::make_shared(experiment_name + "." + std::to_string(i), std::ofstream::out)); } } -void _on_error(const reinforcement_learning::api_status& status, void* nothing) { +void _on_error(const r::api_status& status, void* nothing) { std::cerr << status.get_error_msg() << std::endl; } @@ -68,26 +73,47 @@ int test_loop::load_file(const std::string& file_name, std::string& config_str) return err::success; } -std::string test_loop::generate_experiment_name(const std::string& experiment_name_base, size_t threads, size_t features, size_t actions, size_t slots, size_t index) -{ - return experiment_name_base + "-t" + std::to_string(threads) + "-f" + std::to_string(features) + "-a" + std::to_string(actions) + "-s" + std::to_string(slots) + "-i" + std::to_string(index); +std::string test_loop::generate_experiment_name(const std::string& experiment_name_base, size_t threads, size_t features, size_t actions, size_t slots, size_t episode_length, size_t index) const { + return experiment_name_base + "-t" + std::to_string(threads) + "-x" + std::to_string(features) + "-a" + std::to_string(actions) + "-s" + std::to_string(slots) + "-m" + std::to_string(episode_length) + "-i" + std::to_string(index); +} + +LoopKind test_loop::get_loop_kind(const boost::program_options::variables_map& vm) const { + throw_if_conflicting(vm, "episode_length", "slots"); + if (vm["episode_length"].as() > 0) + return LoopKind::MULTISTEP; + if (vm["slots"].as() > 0) + return LoopKind::CCB; + return LoopKind::CB; } -void test_loop::run(bool is_ccb) { +void test_loop::run() const { + void (test_loop::*loop)(size_t) const; + switch (loop_kind) { + case LoopKind::CB: { + loop = &test_loop::cb_loop; + break; + } + case LoopKind::CCB: { + loop = &test_loop::ccb_loop; + break; + } + case LoopKind::MULTISTEP: { + loop = &test_loop::multistep_loop; + break; + } + } + std::vector _threads; for (size_t i = 0; i < threads; ++i) { - _threads.push_back(is_ccb ? std::thread(&test_loop::ccb_loop, this, i) : std::thread(&test_loop::cb_loop, this, i)); + _threads.push_back(std::thread(loop, this, i)); } for (size_t i = 0; i < threads; ++i) { _threads[i].join(); } } -using namespace reinforcement_learning; -using namespace reinforcement_learning::logger; - -void test_loop::cb_loop(size_t thread_id) -{ +// TODO: why do we need this warmup? +void test_loop::cb_loop(size_t thread_id) const { r::ranking_response response; r::api_status status; std::cout << "Warmup..." << std::endl; @@ -101,40 +127,45 @@ void test_loop::cb_loop(size_t thread_id) } r::utility::data_buffer buffer; - fb_collection_serializer serializer(buffer, r::value::CONTENT_ENCODING_IDENTITY); - auto choose_rank_event = r::ranking_event::choose_rank(warmup_id.c_str(), test_inputs.get_context(0, 0), r::action_flags::DEFAULT, response, timestamp{}); + r::logger::fb_collection_serializer serializer(buffer, r::value::CONTENT_ENCODING_IDENTITY); + auto choose_rank_event = r::ranking_event::choose_rank(warmup_id.c_str(), test_inputs.get_context(0, 0), r::action_flags::DEFAULT, response, r::timestamp{}); serializer.add(choose_rank_event); serializer.finalize(nullptr); choose_rank_size = buffer.body_filled_size(); std::cout << "Choose rank size: " << choose_rank_size << std::endl; - } + } + + auto controller = controllers[thread_id].get(); + auto& logger = *loggers[thread_id]; std::cout << "Perf test is started..." << std::endl; std::cout << "Choose_rank..." << std::endl; const auto choose_rank_start = chrono::high_resolution_clock::now(); for (controller->restart(); controller->is_running(); controller->iterate()) { - if (thread_id == 0) controller->progress_bar(); - const auto event_id = test_inputs.create_event_id(thread_id, controller->get_iteration()); + if (thread_id == 0) controller->show_progress_bar(); - if (rl->choose_rank(event_id.c_str(), test_inputs.get_context(thread_id, controller->get_iteration()), response, &status) != err::success) { + const auto example_id = controller->get_iteration(); + const auto event_id = test_inputs.create_event_id(thread_id, example_id); + + if (rl->choose_rank(event_id.c_str(), test_inputs.get_context(thread_id, example_id), response, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; continue; } - if (test_inputs.is_rewarded(thread_id, controller->get_iteration())) { - if (test_inputs.report_outcome(rl.get(), thread_id, controller->get_iteration(), &status) != err::success) { + if (test_inputs.is_rewarded(thread_id, example_id)) { + if (test_inputs.report_outcome(rl.get(), thread_id, example_id, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; continue; } } } - if (thread_id == 0) controller->progress_bar(); + if (thread_id == 0) controller->show_progress_bar(); std::cout << std::endl; const auto choose_rank_end = chrono::high_resolution_clock::now(); const auto choose_rank_perf = (chrono::duration_cast(choose_rank_end - choose_rank_start).count()) / controller->get_iteration(); - (*loggers[thread_id]) << thread_id << ": Choose_rank: " << choose_rank_perf << " microseconds" << std::endl; + logger << thread_id << ": Choose_rank: " << choose_rank_perf << " microseconds" << std::endl; } std::vector to_const_char(const std::vector& ids) { @@ -145,8 +176,7 @@ std::vector to_const_char(const std::vector& ids) { return result; } -void test_loop::ccb_loop(size_t thread_id) -{ +void test_loop::ccb_loop(size_t thread_id) const { r::decision_response response; r::api_status status; std::cout << "Warmup..." << std::endl; @@ -162,42 +192,112 @@ void test_loop::ccb_loop(size_t thread_id) } r::utility::data_buffer buffer; - fb_collection_serializer serializer(buffer, r::value::CONTENT_ENCODING_IDENTITY); + r::logger::fb_collection_serializer serializer(buffer, r::value::CONTENT_ENCODING_IDENTITY); const std::vector> blank_action_ids(response.size()); const std::vector> blank_pdf(response.size()); - auto decision_event = r::decision_ranking_event::request_decision(event_ids_c, context.c_str(), r::action_flags::DEFAULT, blank_action_ids, blank_pdf, "model", timestamp{}); + auto decision_event = r::decision_ranking_event::request_decision(event_ids_c, context.c_str(), r::action_flags::DEFAULT, blank_action_ids, blank_pdf, "model", r::timestamp{}); serializer.add(decision_event); serializer.finalize(nullptr); choose_rank_size = buffer.body_filled_size(); std::cout << "Decision event size: " << choose_rank_size << std::endl; } + auto controller = controllers[thread_id].get(); + auto& logger = *loggers[thread_id]; + std::cout << "Perf test is started..." << std::endl; std::cout << "Choose_rank..." << std::endl; const auto choose_rank_start = chrono::high_resolution_clock::now(); for (controller->restart(); controller->is_running(); controller->iterate()) { - if (thread_id == 0) controller->progress_bar(); - const auto event_ids = test_inputs.create_event_ids(thread_id, controller->get_iteration()); + if (thread_id == 0) controller->show_progress_bar(); + + const auto example_id = controller->get_iteration(); + const auto event_ids = test_inputs.create_event_ids(thread_id, example_id); const auto event_ids_c = to_const_char(event_ids); - const auto context = test_inputs.get_context(thread_id, controller->get_iteration(), event_ids); + const auto context = test_inputs.get_context(thread_id, example_id, event_ids); if (rl->request_decision(context.c_str(), response, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; continue; } - if (test_inputs.is_rewarded(thread_id, controller->get_iteration())) { - + if (test_inputs.is_rewarded(thread_id, example_id)) { + // TODO: why a fixed reward is used here (probably it doesn't matter for perf) if (rl->report_outcome(event_ids[0].c_str(), 1, &status) != err::success) { std::cout << status.get_error_msg() << std::endl; continue; } } } - if (thread_id == 0) controller->progress_bar(); + if (thread_id == 0) controller->show_progress_bar(); + std::cout << std::endl; + + const auto choose_rank_end = chrono::high_resolution_clock::now(); + + const auto choose_rank_perf = (chrono::duration_cast(choose_rank_end - choose_rank_start).count()) / controller->get_iteration(); + logger << thread_id << ": Choose_rank: " << choose_rank_perf << " microseconds" << std::endl; +} + +void test_loop::multistep_loop(size_t thread_id) const { + r::ranking_response response; + r::api_status status; + std::cout << "Warmup..." << std::endl; + size_t choose_rank_size = 0; + { + r::episode_state episode("warmup_episode"); + const auto event_id = test_inputs.create_event_id(0, 0); + const std::string warmup_id = "_warmup_" + std::string(event_id.c_str()); + if (rl->request_episodic_decision(warmup_id.c_str(), nullptr, test_inputs.get_context(0, 0), response, episode, &status) != err::success) { + std::cout << "Warmup has failed. " << status.get_error_msg() << std::endl; + return; + } + + // TODO: Add serializable request_episodic_decision event. + } + + auto controller = controllers[thread_id].get(); + auto& logger = *loggers[thread_id]; + + size_t episode_id = 0; + std::unique_ptr current_episode = nullptr; + std::string previous_event_id(""); + + std::cout << "Perf test is started..." << std::endl; + std::cout << "Choose_rank..." << std::endl; + const auto choose_rank_start = chrono::high_resolution_clock::now(); + for (controller->restart(); controller->is_running(); controller->iterate()) { + if (thread_id == 0) controller->show_progress_bar(); + + const auto example_id = controller->get_iteration(); + const auto event_id = test_inputs.create_event_id(thread_id, example_id); + + if (example_id % episode_length == 0) { + ++episode_id; + const auto episode_id_str = std::string("episode") + std::to_string(episode_id); + current_episode.reset(new r::episode_state(episode_id_str.c_str())); + previous_event_id = ""; + } + + if (rl->request_episodic_decision(event_id.c_str(), previous_event_id.empty() ? nullptr : previous_event_id.c_str(), test_inputs.get_context(thread_id, example_id), response, *current_episode, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + continue; + } + + if (test_inputs.is_rewarded(thread_id, example_id)) { + float reward = test_inputs.get_outcome(thread_id, example_id); + if (rl->report_outcome(current_episode->get_episode_id(), event_id.c_str(), reward, &status) != err::success) { + std::cout << status.get_error_msg() << std::endl; + continue; + } + } + + // TODO: This makes a chain of events. Add other types of event relationships. + previous_event_id = event_id; + } + if (thread_id == 0) controller->show_progress_bar(); std::cout << std::endl; const auto choose_rank_end = chrono::high_resolution_clock::now(); const auto choose_rank_perf = (chrono::duration_cast(choose_rank_end - choose_rank_start).count()) / controller->get_iteration(); - (*loggers[thread_id]) << thread_id << ": Choose_rank: " << choose_rank_perf << " microseconds" << std::endl; + logger << thread_id << ": Choose_rank: " << choose_rank_perf << " microseconds" << std::endl; } diff --git a/examples/test_cpp/test_loop.h b/examples/test_cpp/test_loop.h index b896bafa6..65723b467 100644 --- a/examples/test_cpp/test_loop.h +++ b/examples/test_cpp/test_loop.h @@ -1,16 +1,23 @@ #pragma once + +#include +#include + #include "experiment_controller.h" #include "test_data_provider.h" #include "live_model.h" -#include -#include +enum class LoopKind { + CB, + CCB, + MULTISTEP, +}; class test_loop { public: test_loop(size_t index, const boost::program_options::variables_map& vm); bool init(); - void run(bool is_ccb); + void run() const; private: int load_file(const std::string& file_name, std::string& config_str) const; @@ -18,19 +25,22 @@ class test_loop { reinforcement_learning::utility::configuration& config, reinforcement_learning::api_status* status) const; std::string generate_experiment_name(const std::string& experiment_name_base, - size_t threads, size_t features, size_t actions, size_t slots, size_t index); + size_t threads, size_t features, size_t actions, size_t slots, size_t episode_length, size_t index) const; + LoopKind get_loop_kind(const boost::program_options::variables_map& vm) const; - void cb_loop(size_t thread_id); - void ccb_loop(size_t thread_id); + void cb_loop(size_t thread_id) const; + void ccb_loop(size_t thread_id) const; + void multistep_loop(size_t thread_id) const; private: + const LoopKind loop_kind; const size_t threads; - std::unique_ptr controller; const size_t sleep_interval; + const size_t episode_length; const std::string experiment_name; const std::string json_config; test_data_provider test_inputs; - + std::vector> controllers; std::vector> loggers; std::unique_ptr rl; }; diff --git a/external_parser/CMakeLists.txt b/external_parser/CMakeLists.txt index 3755e1d7c..ec77e0775 100644 --- a/external_parser/CMakeLists.txt +++ b/external_parser/CMakeLists.txt @@ -42,7 +42,7 @@ set(RL_FLAT_BUFFER_FILES "${CMAKE_CURRENT_SOURCE_DIR}/../rlclientlib/schema/v2/Event.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/../rlclientlib/schema/v2/LearningModeType.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/../rlclientlib/schema/v2/ProblemType.fbs" - "${CMAKE_CURRENT_SOURCE_DIR}/../rlclientlib/schema/v2/MultiStepEvent.fbs" + "${CMAKE_CURRENT_SOURCE_DIR}/../rlclientlib/schema/v2/MultiStepEvent.fbs" ) # set vw cmake flags diff --git a/external_parser/event_processors/metadata.h b/external_parser/event_processors/metadata.h index 1de047295..37e39e0d0 100644 --- a/external_parser/event_processors/metadata.h +++ b/external_parser/event_processors/metadata.h @@ -17,4 +17,4 @@ struct event_metadata_info { std::string event_id; v2::LearningModeType learning_mode; }; -} // namespace metadata \ No newline at end of file +} // namespace metadata diff --git a/external_parser/event_processors/typed_events.h b/external_parser/event_processors/typed_events.h index 20de08b30..0150ddd2d 100644 --- a/external_parser/event_processors/typed_events.h +++ b/external_parser/event_processors/typed_events.h @@ -266,4 +266,4 @@ bool process_compression(const uint8_t *data, size_t size, } return true; } -} // namespace typed_event \ No newline at end of file +} // namespace typed_event diff --git a/external_parser/joiners/example_joiner.cc b/external_parser/joiners/example_joiner.cc index 6c8ce6821..f7d7a6f28 100644 --- a/external_parser/joiners/example_joiner.cc +++ b/external_parser/joiners/example_joiner.cc @@ -576,5 +576,5 @@ metrics::joiner_metrics example_joiner::get_metrics() { return _joiner_metrics; } -void example_joiner::apply_cli_overrides(vw *all, - const input_options &parsed_options) {} +void example_joiner::apply_cli_overrides(vw *, + const input_options &) {} diff --git a/external_parser/joiners/multistep_example_joiner.cc b/external_parser/joiners/multistep_example_joiner.cc index 709994730..e5b984948 100644 --- a/external_parser/joiners/multistep_example_joiner.cc +++ b/external_parser/joiners/multistep_example_joiner.cc @@ -148,7 +148,7 @@ void multistep_example_joiner::set_multistep_reward_function(const multistep_rew /* take forest of tuples as input. Edges are defined using optional previous_id parameter. -get return list of ids ordered topologically with respect to edges and +get return list of ids ordered topologically with respect to edges and according to comp_t comparison for vertices that are not connected */ template>> @@ -177,7 +177,7 @@ class topo_sorter { auto& top = *(states.top()); if (!top.empty()) { const auto& cur = top.top(); - const auto& cur_id = std::get<1>(cur); + const auto& cur_id = std::get<1>(cur); result.push_back(cur_id); states.push(&next[cur_id]); top.pop(); @@ -186,7 +186,7 @@ class topo_sorter { states.pop(); } } - } + } }; bool multistep_example_joiner::populate_order() { @@ -211,7 +211,8 @@ reward::outcome_event multistep_example_joiner::process_outcome( metadata.payload_type(), metadata.pass_probability(), metadata.encoding(), - metadata.id()->str()}; + metadata.id()->str(), + v2::LearningModeType::LearningModeType_Online}; if (event.value_type() == v2::OutcomeValue_literal) { o_event.s_value = event.value_as_literal()->c_str(); @@ -280,7 +281,7 @@ bool multistep_example_joiner::process_joined(v_array &examples) { auto joined = process_interaction(interaction, examples); const auto outcomes = _outcomes[id]; - + for (const auto& o: outcomes) { joined.outcome_events.push_back(o); } @@ -331,7 +332,7 @@ void multistep_example_joiner::populate_episodic_rewards() { for (const std::string& id: _order) { std::vector outcomes = _episodic_outcomes; const auto outcomes_per_step = _outcomes[id]; - outcomes.insert(outcomes.end(), std::make_move_iterator(outcomes_per_step.begin()), + outcomes.insert(outcomes.end(), std::make_move_iterator(outcomes_per_step.begin()), std::make_move_iterator(outcomes_per_step.end())); _rewards.push_back(_reward_calculation.value()(outcomes, _loop_info.default_reward)); } @@ -362,4 +363,4 @@ void multistep_example_joiner::apply_cli_overrides(vw *all, const input_options } set_multistep_reward_function(multistep_reward_func, true); } -} \ No newline at end of file +} diff --git a/external_parser/parse_example_binary.h b/external_parser/parse_example_binary.h index f1bfccd9d..f4d8a428c 100644 --- a/external_parser/parse_example_binary.h +++ b/external_parser/parse_example_binary.h @@ -43,4 +43,4 @@ class binary_parser : public parser { uint64_t _total_size_read; }; } // namespace external -} // namespace VW \ No newline at end of file +} // namespace VW diff --git a/external_parser/parse_example_external.cc b/external_parser/parse_example_external.cc index 4bbf16219..b88a2a8df 100644 --- a/external_parser/parse_example_external.cc +++ b/external_parser/parse_example_external.cc @@ -108,7 +108,7 @@ parser::get_external_parser(vw *all, const input_options &parsed_options) { if (all->options->was_supplied("extra_metrics")) { all->example_parser->metrics = VW::make_unique(); } - + return VW::make_unique(std::move(joiner)); } throw std::runtime_error("external parser type not recognised"); diff --git a/external_parser/unit_tests/test_common.cc b/external_parser/unit_tests/test_common.cc index a7ced881f..967f08253 100644 --- a/external_parser/unit_tests/test_common.cc +++ b/external_parser/unit_tests/test_common.cc @@ -139,4 +139,4 @@ std::vector wrap_into_joined_events( } return event_list; -} \ No newline at end of file +} diff --git a/external_parser/unit_tests/test_vw_binary_parser.cc b/external_parser/unit_tests/test_vw_binary_parser.cc index 435b72c4b..d879fce4d 100644 --- a/external_parser/unit_tests/test_vw_binary_parser.cc +++ b/external_parser/unit_tests/test_vw_binary_parser.cc @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(test_log_file_with_bad_version) { vw->example_parser->input, payload_type), true); BOOST_CHECK_EQUAL(payload_type, MSG_TYPE_FILEMAGIC); - + BOOST_CHECK_EQUAL(bp.read_version(vw->example_parser->input), false); VW::finish(*vw); @@ -409,4 +409,4 @@ BOOST_AUTO_TEST_CASE(test_log_file_with_invalid_cb_context) { clear_examples(examples, vw); VW::finish(*vw); -} \ No newline at end of file +} diff --git a/external_parser/unit_tests/test_vw_external_parser.cc b/external_parser/unit_tests/test_vw_external_parser.cc index 959755b62..7f1fe87f6 100644 --- a/external_parser/unit_tests/test_vw_external_parser.cc +++ b/external_parser/unit_tests/test_vw_external_parser.cc @@ -280,7 +280,7 @@ BOOST_AUTO_TEST_CASE(ca_compare_dsjson_with_fb_models_mixed_skip_learn) { model_name, "--cats 4 --min_value 1 --max_value 100 --bandwidth 1 --id N/A ", file_name); - + // read the models and compare auto buffer_fb_model = read_file(model_name + ".fb"); auto buffer_dsjson_model = read_file(model_name + ".json"); @@ -811,7 +811,7 @@ BOOST_AUTO_TEST_CASE(multistep_2_episodes) { BOOST_CHECK_EQUAL(examples[1]->l.cb.weight, 1); BOOST_CHECK_EQUAL(examples[2]->l.cb.costs.size(), 0); BOOST_CHECK_EQUAL(examples[2]->l.cb.weight, 1); - break; + break; } clear_examples(examples, vw); examples.push_back(&VW::get_unused_example(vw)); @@ -928,7 +928,7 @@ BOOST_AUTO_TEST_CASE(multistep_unordered_episodes) { BOOST_CHECK_EQUAL(examples[1]->l.cb.weight, 1); BOOST_CHECK_EQUAL(examples[2]->l.cb.costs.size(), 0); - BOOST_CHECK_EQUAL(examples[2]->l.cb.weight, 1); + BOOST_CHECK_EQUAL(examples[2]->l.cb.weight, 1); switch (counter++) { case 0: @@ -976,7 +976,7 @@ BOOST_AUTO_TEST_CASE(multistep_unordered_episodes) { clear_examples(examples, vw); examples.push_back(&VW::get_unused_example(vw)); } - + clear_examples(examples, vw); VW::finish(*vw); } @@ -1098,4 +1098,4 @@ BOOST_AUTO_TEST_CASE(multistep_2_episodes_suffix_sum) { clear_examples(examples, vw); VW::finish(*vw); -} \ No newline at end of file +} diff --git a/include/constants.h b/include/constants.h index deef68e78..2d515e8e9 100644 --- a/include/constants.h +++ b/include/constants.h @@ -15,6 +15,15 @@ namespace reinforcement_learning { namespace name { const char* const PROTOCOL_VERSION = "protocol.version"; const char* const HTTP_API_KEY = "http.api.key"; + // Episode + const char *const EPISODE_EH_HOST = "episode.eventhub.host"; + const char *const EPISODE_EH_NAME = "episode.eventhub.name"; + const char *const EPISODE_EH_KEY_NAME = "episode.eventhub.keyname"; + const char *const EPISODE_EH_KEY = "episode.eventhub.key"; + const char *const EPISODE_EH_TASKS_LIMIT = "episode.eventhub.tasks_limit"; + const char *const EPISODE_EH_MAX_HTTP_RETRIES = "episode.eventhub.max_http_retries"; + const char *const EPISODE_SENDER_IMPLEMENTATION = "episode.sender.implementation"; + // Interaction const char *const INTERACTION_EH_HOST = "interaction.eventhub.host"; const char *const INTERACTION_EH_NAME = "interaction.eventhub.name"; @@ -63,6 +72,7 @@ namespace reinforcement_learning { namespace name { const char *const EH_TEST = "eventhub.mock"; const char *const TRACE_LOG_IMPLEMENTATION = "trace.logger.implementation"; + const char *const EPISODE_FILE_NAME = "episode.file.name"; const char *const INTERACTION_FILE_NAME = "interaction.file.name"; const char *const OBSERVATION_FILE_NAME = "observation.file.name"; const char *const TIME_PROVIDER_IMPLEMENTATION = "time_provider.implementation"; @@ -80,8 +90,10 @@ namespace reinforcement_learning { namespace value { const char *const FILE_MODEL_DATA = "FILE_MODEL_DATA"; const char *const VW = "VW"; const char *const PASSTHROUGH_PDF_MODEL = "PASSTHROUGH_PDF"; + const char *const EPISODE_EH_SENDER = "EPISODE_EH_SENDER"; const char *const OBSERVATION_EH_SENDER = "OBSERVATION_EH_SENDER"; const char *const INTERACTION_EH_SENDER = "INTERACTION_EH_SENDER"; + const char *const EPISODE_FILE_SENDER = "EPISODE_FILE_SENDER"; const char *const OBSERVATION_FILE_SENDER = "OBSERVATION_FILE_SENDER"; const char *const INTERACTION_FILE_SENDER = "INTERACTION_FILE_SENDER"; const char* const OBSERVATION_HTTP_API_SENDER = "OBSERVATION_HTTP_API_SENDER"; @@ -103,6 +115,7 @@ namespace reinforcement_learning { namespace value { const int DEFAULT_VW_POOL_INIT_SIZE = 4; const int DEFAULT_PROTOCOL_VERSION = 1; + const char *get_default_episode_sender(); const char *get_default_observation_sender(); const char *get_default_interaction_sender(); const char *get_default_data_transport(); diff --git a/include/internal_constants.h b/include/internal_constants.h index a4d4f8192..9c14ce7e6 100644 --- a/include/internal_constants.h +++ b/include/internal_constants.h @@ -1,6 +1,7 @@ #pragma once namespace reinforcement_learning { namespace config_constants { + const char *const EPISODE = "episode"; const char *const INTERACTION = "interaction"; const char *const OBSERVATION = "observation"; const char *const CONFIG_SECTION = "config.section"; diff --git a/include/live_model.h b/include/live_model.h index d47b9fbc2..e21133ffc 100644 --- a/include/live_model.h +++ b/include/live_model.h @@ -17,6 +17,8 @@ #include "sender.h" #include "future_compat.h" +#include "multistep.h" + #include namespace reinforcement_learning { @@ -142,7 +144,7 @@ namespace reinforcement_learning { */ int choose_rank(const char * context_json, unsigned int flags, ranking_response& resp, api_status* status = nullptr); //event_id is auto-generated - /** + /** * @brief (DEPRECATED) Choose an action from a continuous range, given a list of context features * The inference library chooses an action by sampling the probability density function produced per continuous action range. * The corresponding event_id should be used when reporting the outcome for the continuous action. @@ -227,7 +229,7 @@ namespace reinforcement_learning { RL_DEPRECATED("New interface unifying CB with CCB is coming") int request_decision(const char * context_json, decision_response& resp, api_status* status = nullptr); - /** + /** * @brief (DEPRECATED) Choose an action from the given set for each slot, given a list of actions, slots, * action features, slot features and context features. The inference library chooses an action * per slot by sampling the probability distribution produced per slot. The corresponding event_id should be used when reporting the outcome for each slot. @@ -263,6 +265,10 @@ namespace reinforcement_learning { RL_DEPRECATED("New unified example builder interface is coming") int request_multi_slot_decision(const char * event_id, const char * context_json, unsigned int flags, multi_slot_response_detailed& resp, const int* baseline_actions, size_t baseline_actions_size, api_status* status = nullptr); + //multistep + int request_episodic_decision(const char *event_id, const char *previous_id, const char *context_json, ranking_response &resp, episode_state &episode, api_status *status = nullptr); + int request_episodic_decision(const char *event_id, const char *previous_id, const char *context_json, unsigned int flags, ranking_response &resp, episode_state &episode, api_status *status = nullptr); + /** * @brief Report that action was taken. * @@ -273,6 +279,17 @@ namespace reinforcement_learning { */ int report_action_taken(const char* event_id, api_status* status = nullptr); + /** + * @brief Report that action was taken. + * + * @param primary_id The unique primary_id used when choosing an action should be presented here. This is so that + * the action taken can be matched with feedback received. + * @param secondary_id Index of the partial outcome. + * @param status Optional field with detailed string description if there is an error + * @return int Return error code. This will also be returned in the api_status object + */ + int report_action_taken(const char* primary_id, const char* secondary_id, api_status* status = nullptr); + /** * @brief Report the outcome for the top action. * @@ -410,6 +427,7 @@ namespace reinforcement_learning { live_model& operator=(live_model&) = delete; //! Prevent accidental copy, since destructor will deallocate the implementation ~live_model(); + private: std::unique_ptr _pimpl; //! The actual implementation details are forwarded to this object (PIMPL pattern) bool _initialized = false; //! Guard to ensure that live_model is properly initialized. i.e. init() was called and successfully initialized. diff --git a/include/model_mgmt.h b/include/model_mgmt.h index 371450ee5..dccf39371 100644 --- a/include/model_mgmt.h +++ b/include/model_mgmt.h @@ -6,6 +6,8 @@ #include #include +#include "multistep.h" + // Declare const pointer for internal linkage namespace reinforcement_learning { class ranking_response; @@ -77,6 +79,7 @@ namespace reinforcement_learning { namespace model_management { virtual int choose_continuous_action(const char* features, float& action, float& pdf_value, std::string& model_version, api_status* status = nullptr) = 0; virtual int request_decision(const std::vector& event_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) = 0; virtual int request_multi_slot_decision(const char* event_id, const std::vector& slot_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) = 0; + virtual int choose_rank_multistep(uint64_t rnd_seed, const char* features, const episode_history& history, std::vector& action_ids, std::vector& action_pdf, std::string& model_version, api_status* status = nullptr) = 0; virtual model_type_t model_type() const = 0; virtual ~i_model() = default; }; diff --git a/include/multistep.h b/include/multistep.h new file mode 100644 index 000000000..3dbba72f7 --- /dev/null +++ b/include/multistep.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "ranking_response.h" + +namespace reinforcement_learning { + class episode_history { + public: + episode_history() = default; + episode_history(const episode_history* previous); + + episode_history(const episode_history& other) = default; + episode_history& operator=(const episode_history& other) = default; + + episode_history(episode_history&& other) = default; + episode_history& operator=(episode_history&& other) = default; + + void update(const char* event_id, const char* previous_event_id, const char* context, const ranking_response& resp); + std::string get_context(const char* previous_event_id, const char* context) const; + + size_t size() const; + + private: + int get_depth(const char* id) const; + + private: + std::map _depths; + }; + + class episode_state { + public: + explicit episode_state(const char* episode_id); + + episode_state(const episode_state& other) = default; + episode_state& operator=(const episode_state& other) = default; + + episode_state(episode_state&& other) = default; + episode_state& operator=(episode_state&& other) = default; + + const char* get_episode_id() const; + const episode_history& get_history() const; + size_t size() const; + + int update(const char* event_id, const char* previous_event_id, const char* context, const ranking_response& response, api_status* error = nullptr); + + private: + const std::string _episode_id; + + episode_history _history; + }; + +} diff --git a/rlclientlib/CMakeLists.txt b/rlclientlib/CMakeLists.txt index e896ca84e..dc14ebe42 100644 --- a/rlclientlib/CMakeLists.txt +++ b/rlclientlib/CMakeLists.txt @@ -16,6 +16,7 @@ set(RL_FLAT_BUFFER_FILES_V2 "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/CaEvent.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/FileFormat.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/MultiSlotEvent.fbs" + "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/MultiStepEvent.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/Event.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/LearningModeType.fbs" "${CMAKE_CURRENT_SOURCE_DIR}/schema/v2/MultiStepEvent.fbs") @@ -49,6 +50,7 @@ set(PROJECT_SOURCES model_mgmt/model_downloader.cc model_mgmt/model_mgmt.cc model_mgmt/file_model_loader.cc + multistep.cc generic_event.cc ranking_event.cc ranking_response.cc @@ -100,6 +102,7 @@ set(PROJECT_PUBLIC_HEADERS ../include/future_compat.h ../include/live_model.h ../include/model_mgmt.h + ../include/multistep.h ../include/object_factory.h ../include/personalization.h ../include/ranking_response.h diff --git a/rlclientlib/azure_factories.cc b/rlclientlib/azure_factories.cc index ab3f94b04..23d741936 100644 --- a/rlclientlib/azure_factories.cc +++ b/rlclientlib/azure_factories.cc @@ -14,16 +14,18 @@ namespace reinforcement_learning { namespace u = utility; int restapi_data_transport_create(m::i_data_transport** retval, const u::configuration& config, i_trace* trace_logger, api_status* status); + int episode_sender_create(i_sender** retval, const u::configuration&, error_callback_fn*, i_trace* trace_logger, api_status* status); int observation_sender_create(i_sender** retval, const u::configuration&, error_callback_fn*, i_trace* trace_logger, api_status* status); int interaction_sender_create(i_sender** retval, const u::configuration&, error_callback_fn*, i_trace* trace_logger, api_status* status); int decision_sender_create(i_sender** retval, const u::configuration&, error_callback_fn*, i_trace* trace_logger, api_status* status); int observation_api_sender_create(i_sender** retval, const u::configuration& cfg, error_callback_fn* error_cb, i_trace* trace_logger, api_status* status); int interaction_api_sender_create(i_sender** retval, const u::configuration& cfg, error_callback_fn* error_cb, i_trace* trace_logger, api_status* status); - + void register_azure_factories() { data_transport_factory.register_type(value::AZURE_STORAGE_BLOB, restapi_data_transport_create); sender_factory.register_type(value::OBSERVATION_EH_SENDER, observation_sender_create); sender_factory.register_type(value::INTERACTION_EH_SENDER, interaction_sender_create); + sender_factory.register_type(value::EPISODE_EH_SENDER, episode_sender_create); sender_factory.register_type(value::OBSERVATION_HTTP_API_SENDER, observation_api_sender_create); sender_factory.register_type(value::INTERACTION_HTTP_API_SENDER, interaction_api_sender_create); } @@ -46,7 +48,23 @@ namespace reinforcement_learning { return url; } - int create_apim_http_api_sender(i_sender** retval, const u::configuration& cfg, const char* api_host, int tasks_limit, int max_http_retries, error_callback_fn* error_cb, i_trace* trace_logger, api_status* status) { + int episode_sender_create(i_sender** retval, const u::configuration& cfg, error_callback_fn* error_cb, i_trace* trace_logger, api_status* status) { + const auto eh_host = cfg.get(name::EPISODE_EH_HOST, "localhost:8080"); + const auto eh_name = cfg.get(name::EPISODE_EH_NAME, "episode"); + const auto eh_url = build_eh_url(eh_host, eh_name); + i_http_client* client; + RETURN_IF_FAIL(create_http_client(eh_url.c_str(), cfg, &client, status)); + *retval = new http_transport_client( + client, + cfg.get_int(name::EPISODE_EH_TASKS_LIMIT, 16), + cfg.get_int(name::EPISODE_EH_MAX_HTTP_RETRIES, 4), + trace_logger, + error_cb); + return error_code::success; + } + + int create_apim_http_api_sender(i_sender **retval, const u::configuration &cfg, const char *api_host, int tasks_limit, int max_http_retries, error_callback_fn *error_cb, i_trace *trace_logger, api_status *status) + { i_http_client* client; RETURN_IF_FAIL(create_http_client(api_host, cfg, &client, status)); *retval = new http_transport_client( diff --git a/rlclientlib/constants.cc b/rlclientlib/constants.cc index dbac6d2fc..5cacbb2f5 100644 --- a/rlclientlib/constants.cc +++ b/rlclientlib/constants.cc @@ -3,17 +3,23 @@ namespace reinforcement_learning { namespace value { #ifdef USE_AZURE_FACTORIES + const char *const DEFAULT_EPISODE_SENDER = EPISODE_EH_SENDER; const char *const DEFAULT_OBSERVATION_SENDER = OBSERVATION_EH_SENDER; const char *const DEFAULT_INTERACTION_SENDER = INTERACTION_EH_SENDER; const char *const DEFAULT_DATA_TRANSPORT = AZURE_STORAGE_BLOB; const char *const DEFAULT_TIME_PROVIDER = NULL_TIME_PROVIDER; #else + const char *const DEFAULT_EPISODE_SENDER = EPISODE_FILE_SENDER; const char *const DEFAULT_OBSERVATION_SENDER = OBSERVATION_FILE_SENDER; const char *const DEFAULT_INTERACTION_SENDER = INTERACTION_FILE_SENDER; const char *const DEFAULT_DATA_TRANSPORT = NO_MODEL_DATA; const char *const DEFAULT_TIME_PROVIDER = CLOCK_TIME_PROVIDER; #endif + const char *get_default_episode_sender() { + return DEFAULT_EPISODE_SENDER; + } + const char *get_default_observation_sender() { return DEFAULT_OBSERVATION_SENDER; } diff --git a/rlclientlib/extensions/onnx/src/onnx_model.h b/rlclientlib/extensions/onnx/src/onnx_model.h index 1b37340de..5abd46320 100644 --- a/rlclientlib/extensions/onnx/src/onnx_model.h +++ b/rlclientlib/extensions/onnx/src/onnx_model.h @@ -32,8 +32,13 @@ namespace reinforcement_learning { namespace onnx { return error_code::not_supported; } + int choose_rank_multistep(uint64_t rnd_seed, const char* features, const episode_history& history, std::vector& action_ids, std::vector& action_pdf, std::string& model_version, api_status* status = nullptr) override + { + return error_code::not_supported; + } + model_management::model_type_t model_type() const { return model_management::model_type_t::CB; } - + private: i_trace* _trace_logger; std::string _output_name; diff --git a/rlclientlib/factory_resolver.cc b/rlclientlib/factory_resolver.cc index b719cbf11..e174c5f22 100644 --- a/rlclientlib/factory_resolver.cc +++ b/rlclientlib/factory_resolver.cc @@ -141,6 +141,13 @@ namespace reinforcement_learning { time_provider_factory.register_type(value::CLOCK_TIME_PROVIDER, clock_time_provider_create); // Register File loggers + sender_factory.register_type(value::EPISODE_FILE_SENDER, + [](i_sender** retval, const u::configuration& c, error_callback_fn* cb, i_trace* trace_logger, api_status* status){ + const char* file_name = c.get(name::EPISODE_FILE_NAME,"episode.fb.data"); + return file_sender_create(retval, c , + file_name, + cb, trace_logger, status); + }); sender_factory.register_type(value::OBSERVATION_FILE_SENDER, [](i_sender** retval, const u::configuration& c, error_callback_fn* cb, i_trace* trace_logger, api_status* status){ const char* file_name = c.get(name::OBSERVATION_FILE_NAME,"observation.fb.data"); diff --git a/rlclientlib/live_model.cc b/rlclientlib/live_model.cc index 28e321fce..54bc20956 100644 --- a/rlclientlib/live_model.cc +++ b/rlclientlib/live_model.cc @@ -37,6 +37,7 @@ namespace reinforcement_learning _initialized = other._initialized; return *this; } + int live_model::init(api_status* status) { if (_initialized) return error_code::success; @@ -235,4 +236,20 @@ namespace reinforcement_learning INIT_CHECK(); return _pimpl->refresh_model(status); } + + int live_model::request_episodic_decision(const char* event_id, const char* previous_id, const char* context_json, ranking_response& resp, episode_state& episode, api_status* status) { + INIT_CHECK(); + return _pimpl->request_episodic_decision(event_id, previous_id, context_json, action_flags::DEFAULT, resp, episode, status); + } + + int live_model::request_episodic_decision(const char* event_id, const char* previous_id, const char* context_json, unsigned int flags, ranking_response& resp, episode_state& episode, api_status* status) { + INIT_CHECK(); + return _pimpl->request_episodic_decision(event_id, previous_id, context_json, flags, resp, episode, status); + } + + int live_model::report_action_taken(const char* primary_id, const char* secondary_id, api_status* status) { + INIT_CHECK(); + return _pimpl->report_action_taken(primary_id, secondary_id, status); + } + } diff --git a/rlclientlib/live_model_impl.cc b/rlclientlib/live_model_impl.cc index 1f0a1f600..0a7f45f36 100644 --- a/rlclientlib/live_model_impl.cc +++ b/rlclientlib/live_model_impl.cc @@ -54,7 +54,7 @@ namespace reinforcement_learning { RETURN_IF_FAIL(init_loggers(status)); if (_protocol_version == 1) { - if(_configuration.get_bool("interaction", name::USE_COMPRESSION, false) || + if(_configuration.get_bool("interaction", name::USE_COMPRESSION, false) || _configuration.get_bool("interaction", name::USE_DEDUP, false) || _configuration.get_bool("observation", name::USE_COMPRESSION, false)) { RETURN_ERROR_LS(_trace_logger.get(), status, content_encoding_error); @@ -309,6 +309,13 @@ namespace reinforcement_learning { return _outcome_logger->report_action_taken(event_id, status); } + int live_model_impl::report_action_taken(const char* primary_id, const char* secondary_id, api_status* status) { + // Clear previous errors if any + api_status::try_clear(status); + // Send the outcome event to the backend + return _outcome_logger->report_action_taken(primary_id, secondary_id, status); + } + int live_model_impl::report_outcome(const char* event_id, const char* outcome, api_status* status) { // Check arguments RETURN_IF_FAIL(check_null_or_empty(event_id, outcome, _trace_logger.get(), status)); @@ -437,7 +444,7 @@ namespace reinforcement_learning { i_time_provider* logger_extensions_time_provider; RETURN_IF_FAIL(_time_provider_factory->create(&logger_extensions_time_provider, time_provider_impl, _configuration, _trace_logger.get(), status)); - //Create the logger extension + // Create the logger extension _logger_extensions.reset(logger::i_logger_extensions::get_extensions(_configuration, logger_extensions_time_provider)); i_time_provider* ranking_time_provider; @@ -469,6 +476,31 @@ namespace reinforcement_learning { _outcome_logger.reset(new logger::observation_logger_facade(_configuration, outcome_msg_sender, _watchdog, observation_time_provider, &_error_cb)); RETURN_IF_FAIL(_outcome_logger->init(status)); + // TODO: Use a specific episode message type (for now it is the same with the observation logger, using observation_logger_facade). + if (_configuration.get(name::EPISODE_EH_HOST, nullptr) != nullptr) { + // Get the name of raw data (as opposed to message) sender for episodes. + const auto* const episode_sender_impl = _configuration.get(name::EPISODE_SENDER_IMPLEMENTATION, value::get_default_episode_sender()); + i_sender* episode_sender; + + // Use the name to create an instance of raw data sender for episodes + _configuration.set(config_constants::CONFIG_SECTION, config_constants::EPISODE); + RETURN_IF_FAIL(_sender_factory->create(&episode_sender, episode_sender_impl, _configuration, &_error_cb, _trace_logger.get(), status)); + RETURN_IF_FAIL(episode_sender->init(_configuration, status)); + + // Create a message sender that will prepend the message with a preamble and send the raw data using the + // factory created raw data sender + l::i_message_sender* episode_msg_sender = new l::preamble_message_sender(episode_sender); + RETURN_IF_FAIL(episode_msg_sender->init(status)); + + // Get time provider implementation + i_time_provider* episode_time_provider; + RETURN_IF_FAIL(_time_provider_factory->create(&episode_time_provider, time_provider_impl, _configuration, _trace_logger.get(), status)); + + // Create a logger for episodes that will use msg sender to send episode messages + _episode_logger.reset(new logger::observation_logger_facade(_configuration, episode_msg_sender, _watchdog, episode_time_provider, &_error_cb)); + RETURN_IF_FAIL(_episode_logger->init(status)); + } + return error_code::success; } @@ -585,6 +617,38 @@ namespace reinforcement_learning { return refresh_model(status); } + int live_model_impl::request_episodic_decision(const char* event_id, const char* previous_id, const char* context_json, unsigned int flags, ranking_response& resp, episode_state& episode, api_status* status) { + resp.clear(); + //clear previous errors if any + api_status::try_clear(status); + + //check arguments + RETURN_IF_FAIL(check_null_or_empty(event_id, context_json, _trace_logger.get(), status)); + const uint64_t seed = uniform_hash(event_id, strlen(event_id), 0) + _seed_shift; + + std::vector action_ids; + std::vector action_pdf; + std::string model_version; + + const auto history = episode.get_history(); + const std::string context_patched = history.get_context(previous_id, context_json); + + RETURN_IF_FAIL(_model->choose_rank_multistep(seed, context_patched.c_str(), history, action_ids, action_pdf, model_version, status)); + RETURN_IF_FAIL(sample_and_populate_response(seed, action_ids, action_pdf, std::move(model_version), resp, _trace_logger.get(), status)); + + resp.set_event_id(event_id); + + RETURN_IF_FAIL(episode.update(event_id, previous_id, context_json, resp, status)); + + if (episode.size() == 1) { + // Log the episode id when starting a new episode + RETURN_IF_FAIL(_episode_logger->log(episode.get_episode_id(), "", status)); + } + RETURN_IF_FAIL(_interaction_logger->log(episode.get_episode_id(), previous_id, context_patched.c_str(), flags, resp, status)); + + return error_code::success; + } + //helper: check if at least one of the arguments is null or empty int check_null_or_empty(const char* arg1, const char* arg2, i_trace* trace, api_status* status) { if (!arg1 || !arg2 || strlen(arg1) == 0 || strlen(arg2) == 0) { diff --git a/rlclientlib/live_model_impl.h b/rlclientlib/live_model_impl.h index cab7cc40a..5db82efc0 100644 --- a/rlclientlib/live_model_impl.h +++ b/rlclientlib/live_model_impl.h @@ -1,6 +1,7 @@ #pragma once #include "learning_mode.h" #include "logger/logger_facade.h" +#include "multistep.h" #include "model_mgmt.h" #include "model_mgmt/data_callback_fn.h" #include "model_mgmt/model_downloader.h" @@ -37,8 +38,10 @@ namespace reinforcement_learning int request_multi_slot_decision(const char* context_json, unsigned int flags, multi_slot_response& resp, const std::vector& baseline_actions, api_status* status = nullptr); int request_multi_slot_decision(const char* event_id, const char* context_json, unsigned int flags, multi_slot_response_detailed& resp, const std::vector& baseline_actions, api_status* status = nullptr); int request_multi_slot_decision(const char* context_json, unsigned int flags, multi_slot_response_detailed& resp, const std::vector& baseline_actions, api_status* status = nullptr); + int request_episodic_decision(const char* event_id, const char* previous_id, const char* context_json, unsigned int flags, ranking_response& resp, episode_state& episode, api_status* status = nullptr); int report_action_taken(const char* event_id, api_status* status); + int report_action_taken(const char* primary_id, const char *secondary_id, api_status* status); int report_outcome(const char* event_id, const char* outcome_data, api_status* status); int report_outcome(const char* event_id, float reward, api_status* status); @@ -104,6 +107,7 @@ namespace reinforcement_learning std::unique_ptr _logger_extensions{nullptr}; std::unique_ptr _interaction_logger{nullptr}; std::unique_ptr _outcome_logger{nullptr}; + std::unique_ptr _episode_logger{nullptr}; std::unique_ptr _model_download{nullptr}; std::unique_ptr _trace_logger{nullptr}; diff --git a/rlclientlib/logger/async_batcher.h b/rlclientlib/logger/async_batcher.h index 14ac9cd97..767d9dfba 100644 --- a/rlclientlib/logger/async_batcher.h +++ b/rlclientlib/logger/async_batcher.h @@ -51,7 +51,7 @@ namespace reinforcement_learning { namespace logger { private: int fill_buffer(std::shared_ptr& retbuffer, - size_t& remaining, + size_t& remaining, api_status* status); void flush(); //flush all batches @@ -134,8 +134,8 @@ namespace reinforcement_learning { namespace logger { template class TSerializer> int async_batcher::fill_buffer( - std::shared_ptr& buffer, - size_t& remaining, + std::shared_ptr& buffer, + size_t& remaining, api_status* status) { TEvent evt; diff --git a/rlclientlib/logger/event_logger.cc b/rlclientlib/logger/event_logger.cc index bbdbadc01..327ff530d 100644 --- a/rlclientlib/logger/event_logger.cc +++ b/rlclientlib/logger/event_logger.cc @@ -13,6 +13,7 @@ namespace reinforcement_learning { namespace logger { const auto now = _time_provider != nullptr ? _time_provider->gmt_now() : timestamp(); return append(std::move(decision_ranking_event::request_decision(event_ids, context, flags, action_ids, pdfs, model_version, now)), status); } + int multi_slot_logger::log_decision(const std::string &event_id, const char* context, unsigned int flags, const std::vector>& action_ids, const std::vector>& pdfs, const std::string& model_version, api_status* status) { diff --git a/rlclientlib/logger/logger_extensions.cc b/rlclientlib/logger/logger_extensions.cc index 154a40c83..95180c8c5 100644 --- a/rlclientlib/logger/logger_extensions.cc +++ b/rlclientlib/logger/logger_extensions.cc @@ -21,7 +21,7 @@ class default_extensions : public i_logger_extensions } bool is_object_extraction_enabled() const override { return false; } - bool is_serialization_transform_enabled() const override { return false; } + bool is_serialization_transform_enabled() const override { return false; } int transform_payload_and_extract_objects(const char* context, std::string& edited_payload, generic_event::object_list_t& objects, api_status* status) override { return error_code::success; diff --git a/rlclientlib/logger/logger_facade.cc b/rlclientlib/logger/logger_facade.cc index 0cb11c3f3..0ed76e2d7 100644 --- a/rlclientlib/logger/logger_facade.cc +++ b/rlclientlib/logger/logger_facade.cc @@ -95,6 +95,20 @@ namespace reinforcement_learning { } } + int interaction_logger_facade::log(const char* episode_id, const char* previous_id, const char* context, unsigned int flags, const ranking_response& response, api_status* status) { + switch (_version) { + case 2: { + generic_event::object_list_t actions; + generic_event::payload_buffer_t payload; + event_content_type content_type; + + RETURN_IF_FAIL(wrap_log_call(_ext, _multistep_serializer, context, actions, payload, content_type, status, previous_id, flags, response)); + return _v2->log(episode_id, std::move(payload), _multistep_serializer.type, content_type, std::move(actions), status); + } + default: return protocol_not_supported(status); + } + } + int interaction_logger_facade::log_decisions(std::vector& event_ids, const char* context, unsigned int flags, const std::vector>& action_ids, const std::vector>& pdfs, const std::string& model_version, api_status* status) { switch (_version) { @@ -104,7 +118,6 @@ namespace reinforcement_learning { } int multi_slot_model_type_to_payload_type(model_type_t model_type, generic_event::payload_type_t& payload_type, api_status* status) - { //XXX out params must be always initialized. This is an ok default payload_type = generic_event::payload_type_t::PayloadType_Slates; @@ -171,7 +184,7 @@ namespace reinforcement_learning { , _v2(_version == 2 ? new generic_event_logger( time_provider, create_legacy_async_batcher(c, sender, watchdog, perror_cb, OBSERVATION_SECTION, _serializer_shared_state), - c.get(name::APP_ID, "")) : nullptr) { + c.get(name::APP_ID, "")) : nullptr) { } int observation_logger_facade::init(api_status* status) { @@ -198,7 +211,6 @@ namespace reinforcement_learning { } } - int observation_logger_facade::log(const char* primary_id, int secondary_id, float outcome, api_status* status) { switch (_version) { case 2: return _v2->log(primary_id, _serializer.numeric_event(secondary_id, outcome), _serializer.type, event_content_type::IDENTITY, status); @@ -234,5 +246,12 @@ namespace reinforcement_learning { default: return protocol_not_supported(status); } } + + int observation_logger_facade::report_action_taken(const char* primary_id, const char* secondary_id, api_status* status) { + switch (_version) { + case 2: return _v2->log(primary_id, _serializer.report_action_taken(secondary_id), _serializer.type, event_content_type::IDENTITY, status); + default: return protocol_not_supported(status); + } + } } } diff --git a/rlclientlib/logger/logger_facade.h b/rlclientlib/logger/logger_facade.h index caa722a8d..7ce35982a 100644 --- a/rlclientlib/logger/logger_facade.h +++ b/rlclientlib/logger/logger_facade.h @@ -55,7 +55,6 @@ namespace reinforcement_learning //CB v1/v2 int log(const char* context, unsigned int flags, const ranking_response& response, api_status* status, learning_mode learning_mode = ONLINE); - //CCB v1 int log_decisions(std::vector& event_ids, const char* context, unsigned int flags, const std::vector>& action_ids, const std::vector>& pdfs, const std::string& model_version, api_status* status); @@ -66,6 +65,9 @@ namespace reinforcement_learning //Continuous int log_continuous_action(const char* context, unsigned int flags, const continuous_action_response& response, api_status* status); + //Multistep + int log(const char* episode_id, const char* previous_id, const char* context, unsigned int flags, const ranking_response& response, api_status* status); + private: const reinforcement_learning::model_management::model_type_t _model_type; const int _version; @@ -81,7 +83,7 @@ namespace reinforcement_learning const cb_serializer _serializer_cb; const multi_slot_serializer _serializer_multislot; const ca_serializer _serializer_ca; - + const multistep_serializer _multistep_serializer; }; class observation_logger_facade { @@ -107,6 +109,7 @@ namespace reinforcement_learning int log(const char* event_id, const char* index, const char* outcome, api_status* status); int report_action_taken(const char* event_id, api_status* status); + int report_action_taken(const char* event_id, const char* index, api_status* status); private: const int _version; diff --git a/rlclientlib/multistep.cc b/rlclientlib/multistep.cc new file mode 100644 index 000000000..1d33c6501 --- /dev/null +++ b/rlclientlib/multistep.cc @@ -0,0 +1,44 @@ +#include "multistep.h" +#include "err_constants.h" + +namespace reinforcement_learning { + void episode_history::update(const char* event_id, const char* previous_event_id, const char* context, const ranking_response& resp) { + _depths[event_id] = this->get_depth(previous_event_id) + 1; + } + + std::string episode_history::get_context(const char* previous_event_id, const char* context) const { + return R"({"episode":{"depth":")" + std::to_string(this->get_depth(previous_event_id) + 1) + "\"}," + std::string(context + 1); + } + + int episode_history::get_depth(const char* id) const { + if (id == nullptr) { + return 0; + } + auto result = _depths.find(id); + return (result == _depths.end()) ? 0 : result->second; + } + + size_t episode_history::size() const { + return _depths.size(); + } + + episode_state::episode_state(const char* episode_id) + : _episode_id(episode_id) {} + + const char* episode_state::get_episode_id() const { + return _episode_id.c_str(); + } + + const episode_history& episode_state::get_history() const { + return _history; + } + + size_t episode_state::size() const { + return _history.size(); + } + + int episode_state::update(const char* event_id, const char* previous_event_id, const char* context, const ranking_response& response, api_status* status) { + _history.update(event_id, previous_event_id, context, response); + return error_code::success; + } +} diff --git a/rlclientlib/ranking_event.h b/rlclientlib/ranking_event.h index da1cb6ea7..8b83c6921 100644 --- a/rlclientlib/ranking_event.h +++ b/rlclientlib/ranking_event.h @@ -6,6 +6,8 @@ #include "decision_response.h" #include "multi_slot_response.h" +#include + namespace reinforcement_learning { struct timestamp; namespace utility { class data_buffer; } diff --git a/rlclientlib/rlclientlib.vcxproj b/rlclientlib/rlclientlib.vcxproj index 740645d3f..4c135ba11 100644 --- a/rlclientlib/rlclientlib.vcxproj +++ b/rlclientlib/rlclientlib.vcxproj @@ -183,6 +183,7 @@ $(flatcPath) -o "$(SolutionDir)rlclientlib\generated\v2" --cpp "$(SolutionDir)rl + @@ -255,6 +256,7 @@ $(flatcPath) -o "$(SolutionDir)rlclientlib\generated\v2" --cpp "$(SolutionDir)rl + diff --git a/rlclientlib/rlclientlib.vcxproj.filters b/rlclientlib/rlclientlib.vcxproj.filters index 437d9e50e..8f537b605 100644 --- a/rlclientlib/rlclientlib.vcxproj.filters +++ b/rlclientlib/rlclientlib.vcxproj.filters @@ -46,6 +46,7 @@ + @@ -121,6 +122,7 @@ + @@ -143,6 +145,7 @@ + diff --git a/rlclientlib/serialization/payload_serializer.h b/rlclientlib/serialization/payload_serializer.h index 5a2608bbf..3e3cc8ade 100644 --- a/rlclientlib/serialization/payload_serializer.h +++ b/rlclientlib/serialization/payload_serializer.h @@ -21,6 +21,7 @@ #include "generated/v2/CbEvent_generated.h" #include "generated/v2/CaEvent_generated.h" #include "generated/v2/MultiSlotEvent_generated.h" +#include "generated/v2/MultiStepEvent_generated.h" #include "generated/v2/DedupInfo_generated.h" namespace reinforcement_learning { @@ -164,6 +165,34 @@ namespace reinforcement_learning { fbb.Finish(fb); return fbb.Release(); } + + static generic_event::payload_buffer_t report_action_taken(const char* index) { + flatbuffers::FlatBufferBuilder fbb; + const auto idx = fbb.CreateString(index).Union(); + auto fb = v2::CreateOutcomeEvent(fbb, v2::OutcomeValue_NONE, 0, v2::IndexValue_literal, idx, true); + fbb.Finish(fb); + return fbb.Release(); + } + }; + + struct multistep_serializer : payload_serializer { + static generic_event::payload_buffer_t event(const char* context, const char* previous_id, unsigned int flags, const ranking_response& response) { + flatbuffers::FlatBufferBuilder fbb; + std::vector action_ids; + std::vector probabilities; + for (auto const& r : response) { + action_ids.push_back(r.action_id + 1); + probabilities.push_back(r.probability); + } + std::vector _context; + std::string context_str(context); + copy(context_str.begin(), context_str.end(), std::back_inserter(_context)); + + auto fb = v2::CreateMultiStepEventDirect(fbb, response.get_event_id(), previous_id, &action_ids, + &_context, &probabilities, response.get_model_id(), flags & action_flags::DEFERRED); + fbb.Finish(fb); + return fbb.Release(); + } }; } } diff --git a/rlclientlib/utility/config_utility.cc b/rlclientlib/utility/config_utility.cc index 57262960f..661fbabc8 100644 --- a/rlclientlib/utility/config_utility.cc +++ b/rlclientlib/utility/config_utility.cc @@ -114,6 +114,8 @@ namespace reinforcement_learning { namespace utility { namespace config { }; static const std::unordered_map parsed_translation_mapping = { + { "EventHubEpisodeConnectionString" , "episode" }, + { "eventHubEpisodeConnectionString" , "episode" }, { "EventHubInteractionConnectionString" , "interaction" }, { "eventHubInteractionConnectionString" , "interaction" }, { "EventHubObservationConnectionString" , "observation" }, @@ -121,7 +123,7 @@ namespace reinforcement_learning { namespace utility { namespace config { }; static const std::unordered_set deprecated = { - "QueueMaxSize", + "QueueMaxSize", "queueMaxSize" }; @@ -131,7 +133,7 @@ namespace reinforcement_learning { namespace utility { namespace config { prop_name = legacy_it->second; } - if (trace != nullptr && deprecated.find(prop_name) != deprecated.end()) + if (trace != nullptr && deprecated.find(prop_name) != deprecated.end()) { auto message = concat("Field '", prop_name, "' is unresponsive."); TRACE_WARN(trace, message); @@ -148,7 +150,7 @@ namespace reinforcement_learning { namespace utility { namespace config { // Otherwise, just set the value in the config collection. cc.set(prop_name.c_str(), string_value); } - + return error_code::success; } diff --git a/rlclientlib/vw_model/pdf_model.cc b/rlclientlib/vw_model/pdf_model.cc index b14f76b97..7129860fd 100644 --- a/rlclientlib/vw_model/pdf_model.cc +++ b/rlclientlib/vw_model/pdf_model.cc @@ -64,4 +64,15 @@ namespace reinforcement_learning { namespace model_management { { return model_type_t::CB; } + + int pdf_model::choose_rank_multistep( + uint64_t rnd_seed, + const char* features, + const episode_history& history, + std::vector& action_ids, + std::vector& action_pdf, + std::string& model_version, + api_status* status) { + return error_code::not_supported; + } }} diff --git a/rlclientlib/vw_model/pdf_model.h b/rlclientlib/vw_model/pdf_model.h index ed2478424..5300d8612 100644 --- a/rlclientlib/vw_model/pdf_model.h +++ b/rlclientlib/vw_model/pdf_model.h @@ -19,6 +19,7 @@ namespace reinforcement_learning { namespace model_management { int choose_continuous_action(const char* features, float& action, float& pdf_value, std::string& model_version, api_status* status = nullptr) override; int request_decision(const std::vector& event_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) override; int request_multi_slot_decision(const char *event_id, const std::vector& slot_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) override; + int choose_rank_multistep(uint64_t rnd_seed, const char* features, const episode_history& history, std::vector& action_ids, std::vector& action_pdf, std::string& model_version, api_status* status = nullptr) override; model_type_t model_type() const override; private: std::unique_ptr _vw; diff --git a/rlclientlib/vw_model/vw_model.cc b/rlclientlib/vw_model/vw_model.cc index f6a840342..f32d9dacf 100644 --- a/rlclientlib/vw_model/vw_model.cc +++ b/rlclientlib/vw_model/vw_model.cc @@ -22,7 +22,7 @@ namespace reinforcement_learning { namespace model_management { { std::unique_ptr init_vw(new safe_vw(data.data(), data.data_sz())); - std::unique_ptr factory; + std::unique_ptr factory; if (init_vw->is_CB_to_CCB_model_upgrade(_initial_command_line)) { factory.reset(new safe_vw_factory(std::move(data), _upgrade_to_CCB_vw_commandline_options)); @@ -79,6 +79,17 @@ namespace reinforcement_learning { namespace model_management { } } + int vw_model::choose_rank_multistep( + uint64_t rnd_seed, + const char* features, + const episode_history& history, + std::vector& action_ids, + std::vector& action_pdf, + std::string& model_version, + api_status* status) { + return choose_rank(rnd_seed, features, action_ids, action_pdf, model_version, status); + } + int vw_model::choose_continuous_action(const char* features, float& action, float& pdf_value, std::string& model_version, api_status* status) { try diff --git a/rlclientlib/vw_model/vw_model.h b/rlclientlib/vw_model/vw_model.h index f94f5ee03..3aff8deeb 100644 --- a/rlclientlib/vw_model/vw_model.h +++ b/rlclientlib/vw_model/vw_model.h @@ -1,6 +1,7 @@ #pragma once #include "model_mgmt.h" #include "safe_vw.h" +#include "multistep.h" #include "../utility/versioned_object_pool.h" namespace reinforcement_learning { @@ -20,6 +21,7 @@ namespace reinforcement_learning { namespace model_management { int choose_continuous_action(const char* features, float& action, float& pdf_value, std::string& model_version, api_status* status = nullptr) override; int request_decision(const std::vector& event_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) override; int request_multi_slot_decision(const char *event_id, const std::vector& slot_ids, const char* features, std::vector>& actions_ids, std::vector>& action_pdfs, std::string& model_version, api_status* status = nullptr) override; + int choose_rank_multistep(uint64_t rnd_seed, const char *features, const episode_history &history, std::vector &action_ids, std::vector &action_pdf, std::string &model_version, api_status *status = nullptr) override; model_type_t model_type() const override; private: diff --git a/test_tools/example_gen/example_gen.cc b/test_tools/example_gen/example_gen.cc index 111a81f26..91f9b33c5 100644 --- a/test_tools/example_gen/example_gen.cc +++ b/test_tools/example_gen/example_gen.cc @@ -704,4 +704,4 @@ int main(int argc, char *argv[]) { return run_config(action, count, seed, gen_random_reward, enable_apprentice_mode, deferred_action_count, config_file, rng, epsilon); -} \ No newline at end of file +} diff --git a/test_tools/log_parser/joiner.py b/test_tools/log_parser/joiner.py index c06c9f3f5..32f70bba6 100755 --- a/test_tools/log_parser/joiner.py +++ b/test_tools/log_parser/joiner.py @@ -41,7 +41,7 @@ Respect EUD. """ -# +# fb_api_version = 1 if flatbuffers.Builder.EndVector.__code__.co_argcount == 1: fb_api_version = 2 @@ -174,7 +174,7 @@ def mk_offsets_vector(builder, arr, startFun): startFun(builder, len(arr)) for i in reversed(range(len(arr))): builder.PrependUOffsetTRelative(arr[i]) - + return end_vector_shim(builder, len(arr)) def mk_bytes_vector(builder, arr): @@ -462,4 +462,4 @@ def get_event_id(ser_evt): empty_payload_while_writing = False # only mess with the first payload one_invalid_msg_type_while_writing = False # only mess with the first payload - bin_bad_payload.write_eof() \ No newline at end of file + bin_bad_payload.write_eof() diff --git a/unit_test/mock_util.cc b/unit_test/mock_util.cc index 0b996678d..65294041c 100644 --- a/unit_test/mock_util.cc +++ b/unit_test/mock_util.cc @@ -66,32 +66,38 @@ std::unique_ptr> get_mock_failing_data_transpo std::unique_ptr> get_mock_model(m::model_type_t model_type) { auto mock = std::unique_ptr>(new fakeit::Mock()); - const std::function&, std::vector&, std::string&, r::api_status*)> choose_rank_fn = + + const auto choose_rank_fn = [](uint64_t, const char*, std::vector&, std::vector&, std::string& model_version, r::api_status*) { model_version = "model_id"; return r::error_code::success; }; - const std::function choose_continuous_action_fn = + const auto choose_continuous_action_fn = [](const char*, float&, float&, std::string& model_version, r::api_status*) { model_version = "model_id"; return r::error_code::success; }; - const std::function& event_ids, const char*, std::vector>&, std::vector>&, std::string&, r::api_status*)> request_decision_fn = + const auto request_decision_fn = [](const std::vector& event_ids, const char*, std::vector>&, std::vector>&, std::string& model_version, r::api_status*) { model_version = "model_id"; return r::error_code::success; }; - - const std::function&, const char*, std::vector>&, std::vector>&, std::string&, r::api_status*)> request_multi_slot_decision_fn = + + const auto request_multi_slot_decision_fn = [](const char*, const std::vector&, const char*, std::vector>&, std::vector>&, std::string& model_version, r::api_status*) { model_version = "model_id"; return r::error_code::success; }; + const auto choose_rank_multistep_fn = + [](uint64_t, const char*, const r::episode_history&, std::vector&, std::vector&, std::string& model_version, r::api_status*) { + model_version = "model_id"; + return r::error_code::success; + }; - const std::function get_model_type = [model_type]() { + const auto get_model_type = [model_type]() { return model_type; }; @@ -100,6 +106,7 @@ std::unique_ptr> get_mock_model(m::model_type_t model_t When(Method((*mock), choose_continuous_action)).AlwaysDo(choose_continuous_action_fn); When(Method((*mock), request_decision)).AlwaysDo(request_decision_fn); When(Method((*mock), request_multi_slot_decision)).AlwaysDo(request_multi_slot_decision_fn); + When(Method((*mock), choose_rank_multistep)).AlwaysDo(choose_rank_multistep_fn); When(Method((*mock), model_type)).AlwaysDo(get_model_type); Fake(Dtor((*mock))); diff --git a/unit_test/unit_test.vcxproj b/unit_test/unit_test.vcxproj index d033e6e5c..abdf6fb54 100644 --- a/unit_test/unit_test.vcxproj +++ b/unit_test/unit_test.vcxproj @@ -241,4 +241,4 @@ xcopy /Y $(SolutionDir)packages\boost_unit_test_framework-vc141.1.70.0.0\lib\native\*.dll $(OutDir) - \ No newline at end of file +