Skip to content

Commit

Permalink
add more compilers and mergers
Browse files Browse the repository at this point in the history
  • Loading branch information
@hendrikmuhs
hendrikmuhs committed Dec 29, 2024
1 parent 4627796 commit 3e0fe58
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 10 deletions.
71 changes: 67 additions & 4 deletions python-pybind/src/compiler/py_dictionary_compilers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,21 @@

#include <pybind11/functional.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include <string>
#include <vector>

#include "keyvi/dictionary/dictionary_types.h"

namespace py = pybind11;
namespace kd = keyvi::dictionary;

void init_keyvi_dictionary_compilers(const py::module_ &m) {
void init_keyvi_dictionary_compilers(const py::module_ &module) {
#define CREATE_COMPILER(compiler, name) \
py::class_<compiler>(m, name) \
py::class_<compiler>(module, name) \
.def(py::init<>()) \
.def(py::init<const keyvi::util::parameters_t &>()) \
.def("__enter__", [](compiler &c) { return &c; }) \
.def("__exit__", [](compiler &c, void *exc_type, void *exc_value, void *traceback) { c.Compile(); }) \
.def("__setitem__", &compiler::Add) \
Expand All @@ -47,9 +52,67 @@ void init_keyvi_dictionary_compilers(const py::module_ &m) {
}, \
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
.def("set_manifest", &compiler::SetManifest) \
.def("write_to_file", &compiler::WriteToFile);

.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
#define CREATE_SK_COMPILER(compiler, name) \
py::class_<compiler>(module, name) \
.def(py::init<const std::vector<std::string> &>()) \
.def(py::init<const std::vector<std::string> &, const keyvi::util::parameters_t &>()) \
.def("__enter__", [](compiler &c) { return &c; }) \
.def("__exit__", [](compiler &c, void *exc_type, void *exc_value, void *traceback) { c.Compile(); }) \
.def("__setitem__", &compiler::Add) \
.def("add", &compiler::Add) \
.def( \
"compile", \
[](compiler &c, std::function<void(const size_t a, const size_t b)> progress_callback) { \
pybind11::gil_scoped_release release_gil; \
if (progress_callback == nullptr) { \
c.Compile(); \
return; \
} \
auto progress_compiler_callback = [](size_t a, size_t b, void *user_data) { \
auto py_callback = *reinterpret_cast<std::function<void(const size_t, const size_t)> *>(user_data); \
pybind11::gil_scoped_acquire acquire_gil; \
py_callback(a, b); \
}; \
void *user_data = reinterpret_cast<void *>(&progress_callback); \
c.Compile(progress_compiler_callback, user_data); \
}, \
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
.def("set_manifest", &compiler::SetManifest) \
.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
#define CREATE_MERGER(merger, name) \
py::class_<merger>(module, name) \
.def(py::init<>()) \
.def(py::init<const keyvi::util::parameters_t &>()) \
.def("__enter__", [](merger &m) { return &m; }) \
.def("__exit__", [](merger &m, void *exc_type, void *exc_value, void *traceback) { m.Merge(); }) \
.def("add", &merger::Add) \
.def("merge", \
[](merger &m) { \
pybind11::gil_scoped_release release_gil; \
m.Merge(); \
}) \
.def("merge", \
[](merger &m, const std::string &filename) { \
pybind11::gil_scoped_release release_gil; \
m.Merge(filename); \
}) \
.def("set_manifest", &merger::SetManifest) \
.def("write_to_file", &merger::WriteToFile, py::call_guard<py::gil_scoped_release>());
CREATE_COMPILER(kd::CompletionDictionaryCompiler, "CompletionDictionaryCompiler");
CREATE_COMPILER(kd::FloatVectorDictionaryCompiler, "FloatVectorDictionaryCompiler");
CREATE_COMPILER(kd::IntDictionaryCompiler, "IntDictionaryCompiler");
CREATE_COMPILER(kd::JsonDictionaryCompiler, "JsonDictionaryCompiler");
CREATE_COMPILER(kd::KeyOnlyDictionaryCompiler, "KeyOnlyDictionaryCompiler");
CREATE_COMPILER(kd::StringDictionaryCompiler, "StringDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyCompletionDictionaryCompiler, "SecondaryKeyCompletionDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyFloatVectorDictionaryCompiler, "SecondaryKeyFloatVectorDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyIntDictionaryCompiler, "SecondaryKeyIntDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyJsonDictionaryCompiler, "SecondaryKeyJsonDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyKeyOnlyDictionaryCompiler, "SecondaryKeyKeyOnlyDictionaryCompiler");
CREATE_SK_COMPILER(kd::SecondaryKeyStringDictionaryCompiler, "SecondaryKeyStringDictionaryCompiler");
CREATE_MERGER(kd::CompletionDictionaryMerger, "CompletionDictionaryMerger");
CREATE_MERGER(kd::IntDictionaryMerger, "IntDictionaryMerger");

#undef CREATE_COMPILER
}
1 change: 1 addition & 0 deletions python-pybind/src/dictionary/py_dictionary.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ void init_keyvi_dictionary(const py::module_ &m) {
// 'search_tokenized', 'statistics', 'values'
py::class_<kd::Dictionary>(m, "Dictionary")
.def(py::init<const std::string &>())
.def(py::init<const std::string &, kd::loading_strategy_types>())
.def(
"complete_fuzzy_multiword",
[](const kd::Dictionary &d, const std::string &query, const int32_t max_edit_distance,
Expand Down
14 changes: 14 additions & 0 deletions python-pybind/src/py_keyvi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@

#include <pybind11/pybind11.h>

#include "keyvi/dictionary/fsa/internal/memory_map_flags.h"

#define STRINGIFY(x) #x
#define MACRO_STRINGIFY(x) STRINGIFY(x)

namespace py = pybind11;
namespace kd = keyvi::dictionary;

void init_keyvi_dictionary(const py::module_ &);
void init_keyvi_dictionary_compilers(const py::module_ &);
Expand All @@ -38,6 +41,17 @@ PYBIND11_MODULE(keyvi_scikit_core, m) {
)pbdoc";

py::enum_<kd::loading_strategy_types>(m, "loading_strategy_types")
.value("default_os", kd::loading_strategy_types::default_os)
.value("lazy", kd::loading_strategy_types::lazy)
.value("populate", kd::loading_strategy_types::populate)
.value("populate_key_part", kd::loading_strategy_types::populate_key_part)
.value("populate_lazy", kd::loading_strategy_types::populate_lazy)
.value("lazy_no_readahead", kd::loading_strategy_types::lazy_no_readahead)
.value("lazy_no_readahead_value_part", kd::loading_strategy_types::lazy_no_readahead_value_part)
.value("populate_key_part_no_readahead_value_part",
kd::loading_strategy_types::populate_key_part_no_readahead_value_part);

init_keyvi_match(m);
py::module keyvi_dictionary = m.def_submodule("dictionary", "keyvi_scikit_core.dictionary");
init_keyvi_dictionary(keyvi_dictionary);
Expand Down
12 changes: 6 additions & 6 deletions python-pybind/tests/match_object_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
import warnings


#from keyvi.compiler import (
# JsonDictionaryCompiler,
# CompletionDictionaryCompiler,
# KeyOnlyDictionaryCompiler,
# StringDictionaryCompiler,
#)
from keyvi_scikit_core.compiler import (
JsonDictionaryCompiler,
CompletionDictionaryCompiler,
KeyOnlyDictionaryCompiler,
StringDictionaryCompiler,
)


""" def test_serialization():
Expand Down

0 comments on commit 3e0fe58

Please sign in to comment.