diff --git a/keyvi/src/cpp/dictionary/dictionary_compiler.h b/keyvi/src/cpp/dictionary/dictionary_compiler.h index 15e74793..524a665d 100644 --- a/keyvi/src/cpp/dictionary/dictionary_compiler.h +++ b/keyvi/src/cpp/dictionary/dictionary_compiler.h @@ -45,32 +45,31 @@ namespace dictionary { * structure for internal processing * Note: Not using std::pair because it did not compile with Tpie */ -template struct key_value_pair { key_value_pair() : key(), value() { } - key_value_pair(const std::string& k, const value_t v): key(k), value(v) {} + key_value_pair(const std::string& k, const fsa::ValueHandle& v): key(k), value(v) {} bool operator<(const key_value_pair kv) const { return key < kv.key; } std::string key; - value_t value; + fsa::ValueHandle value; }; /** * Tpie serialization and deserialization for sorting. */ -template -void serialize(Dst & d, const keyvi::dictionary::key_value_pair & pt) { +template +void serialize(Dst & d, const keyvi::dictionary::key_value_pair & pt) { using tpie::serialize; serialize(d, pt.key); serialize(d, pt.value); } -template -void unserialize(Src & s, keyvi::dictionary::key_value_pair & pt) { +template +void unserialize(Src & s, keyvi::dictionary::key_value_pair & pt) { using tpie::unserialize; unserialize(s, pt.key); unserialize(s, pt.value); @@ -83,7 +82,7 @@ template class DictionaryCompiler final { typedef const fsa::internal::IValueStoreWriter::vs_param_t compiler_param_t; - typedef key_value_pair key_value_t; + typedef key_value_pair key_value_t; typedef std::function callback_t; public: @@ -104,12 +103,15 @@ class DictionaryCompiler sorter_.set_available_memory(memory_limit); sorter_.begin(); - generator_ = new fsa::GeneratorAdapter(memory_limit, params); + value_store_= new ValueStoreT(params); } ~DictionaryCompiler(){ if (generator_) { delete generator_; + } else { + // if generator was not created we have to delete the value store ourselves + delete value_store_; } } @@ -118,7 +120,8 @@ class DictionaryCompiler void Add(const std::string& input_key, typename ValueStoreT::value_t value = ValueStoreT::no_value) { - sorter_.push(key_value_t(input_key, value)); + + sorter_.push(key_value_t(input_key, RegisterValue(value))); size_of_keys_ += input_key.size(); } @@ -133,9 +136,9 @@ class DictionaryCompiler /** * Do the final compilation - * todo: implement some sort of progress callback */ void Compile(callback_t progress_callback = nullptr, void* user_data = nullptr) { + value_store_->CloseFeeding(); sorter_.end(); sorter_.merge_runs(); CreateGenerator(); @@ -165,99 +168,6 @@ class DictionaryCompiler generator_->CloseFeeding(); } - /** - * Compile a partition - * - * @param maximum_partition_size - * @param stream - * @param maximum_partition_depth - * @return - */ - bool CompileNext(size_t maximum_partition_size, std::ostream& stream, int maximum_partition_depth = 2, - callback_t progress_callback = nullptr, void* user_data = nullptr) - { - if (!sort_finalized_){ - CreateGenerator(); - - sorter_.end(); - sorter_.merge_runs(); - sort_finalized_ = true; - - number_of_items_ = sorter_.item_count(); - callback_trigger_ = 1+(number_of_items_-1)/100; - if (callback_trigger_ > 100000) { - callback_trigger_ = 100000; - } - } - - size_t partition_conservative_size = (maximum_partition_size / 10) * 9; - - TRACE("Compile next partition"); - { - int i = 0; - - //size_t number_of_items = sorter_.item_count(); - - while (sorter_.can_pull()) { - key_value_t key_value = sorter_.pull(); - - generator_->Add(key_value.key, key_value.value); - ++i; - ++added_key_values_; - - if (progress_callback && (added_key_values_ % callback_trigger_ == 0)){ - progress_callback(added_key_values_, number_of_items_, user_data); - } - - if (i % 1000 == 0 && generator_->GetFsaSize() > partition_conservative_size){ - // todo: continue feeding until we found a good point to persist the partition - - TRACE("finish partition: find good partition end"); - std::string last_key (key_value.key); - bool make_new_partition = false; - - while (sorter_.can_pull()) { - key_value = sorter_.pull(); - - if (fsa::get_common_prefix_length(last_key.c_str(), key_value.key.c_str()) < maximum_partition_depth) { - make_new_partition = true; - break; - } - - generator_->Add(key_value.key, key_value.value); - ++added_key_values_; - - if (progress_callback && (added_key_values_ % callback_trigger_ == 0)){ - progress_callback(added_key_values_, number_of_items_, user_data); - } - - last_key = key_value.key; - } - TRACE("finish partition: finalize partition"); - - generator_->CloseFeeding(); - generator_->Write(stream); - - // handle case where we do not have to start a new partition - if (!make_new_partition){ - return false; - } - - generator_->Reset(); - generator_->Add(key_value.key, key_value.value); - - return true; - } - } - } - - // finalize the last partition - generator_->CloseFeeding(); - generator_->Write(stream); - - return false; - } - /** * Set a custom manifest to be embedded into the index file. * @@ -297,7 +207,7 @@ class DictionaryCompiler tpie::serialization_sorter sorter_; size_t memory_limit_; compiler_param_t params_; - + ValueStoreT* value_store_; fsa::GeneratorAdapterInterface* generator_ = nullptr; bool sort_finalized_ = false; size_t added_key_values_ = 0; @@ -309,7 +219,25 @@ class DictionaryCompiler void CreateGenerator(); + /** + * Register a value before inserting the key(for optimization purposes). + * + * @param value The Value + * @return a handle that later needs to be passed to Add() + */ + fsa::ValueHandle RegisterValue(typename ValueStoreT::value_t value = + ValueStoreT::no_value){ + + fsa::ValueHandle handle; + handle.no_minimization = false; + handle.value_idx = value_store_->GetValue(value, handle.no_minimization); + + // if inner weights are used update them + handle.weight = value_store_->GetWeightValue(value); + + return handle; + } }; /** @@ -323,15 +251,15 @@ inline void DictionaryCompiler::CreateGenerator() // todo: find good parameters for auto-guessing this if (size_of_keys_ > UINT32_MAX){ if (memory_limit_ > 0x280000000UL /* 10 GB */) { - generator_ = new fsa::GeneratorAdapter(memory_limit_, params_); + generator_ = new fsa::GeneratorAdapter(memory_limit_, params_, value_store_); } else { - generator_ = new fsa::GeneratorAdapter(memory_limit_, params_); + generator_ = new fsa::GeneratorAdapter(memory_limit_, params_, value_store_); } } else { if (memory_limit_ > 0x140000000UL) /* 5GB */ { - generator_ = new fsa::GeneratorAdapter(memory_limit_, params_); + generator_ = new fsa::GeneratorAdapter(memory_limit_, params_, value_store_); } else { - generator_ = new fsa::GeneratorAdapter(memory_limit_, params_); + generator_ = new fsa::GeneratorAdapter(memory_limit_, params_, value_store_); } } diff --git a/keyvi/src/cpp/dictionary/fsa/generator.h b/keyvi/src/cpp/dictionary/fsa/generator.h index 3e7e6ce5..d7f9f941 100644 --- a/keyvi/src/cpp/dictionary/fsa/generator.h +++ b/keyvi/src/cpp/dictionary/fsa/generator.h @@ -125,12 +125,18 @@ enum generator_state { */ typedef const internal::IValueStoreWriter::vs_param_t generator_param_t; +struct ValueHandle final { + uint64_t value_idx; + uint32_t weight; + bool no_minimization; +}; + template class Generator final { public: Generator(size_t memory_limit = 1073741824, - const generator_param_t& params = generator_param_t()) + const generator_param_t& params = generator_param_t(), ValueStoreT* value_store = NULL) : memory_limit_(memory_limit), params_(params) { // use 50% or limit minus 200MB for the memory limit of the hashtable @@ -148,11 +154,17 @@ final { persistence_ = new PersistenceT(memory_limit - memory_limit_minimization, params_[TEMPORARY_PATH_KEY]); - value_store_ = new ValueStoreT(params_); stack_ = new internal::UnpackedStateStack(persistence_, 30); builder_ = new internal::SparseArrayBuilder( memory_limit_minimization, persistence_, ValueStoreT::inner_weight, minimize_); + + if (value_store != NULL) { + value_store_ = value_store; + } else { + value_store_ = new ValueStoreT(params_); + } + } ~Generator() { @@ -170,46 +182,54 @@ final { Generator& operator=(Generator const&) = delete; Generator(const Generator& that) = delete; - void Reset(){ - // TODO: refactor for code reusage - // delete the old data structures and create new ones + /** + * Add a key-value pair to the generator. + * @param input_key The input key. + * @param value A value (depending on the Valuestore implementation). + */ + void Add(const std::string& input_key, typename ValueStoreT::value_t value = + ValueStoreT::no_value) { - delete persistence_; - delete value_store_; - if (stack_) { - delete stack_; - } + const char* key = input_key.c_str(); - if (builder_) { - delete builder_; + size_t commonPrefixLength = get_common_prefix_length(last_key_.c_str(), key); + + if (commonPrefixLength == input_key.size() && last_key_.size() == input_key.size()) { + last_key_ = key; + return; } - // use 50% or limit minus 200MB for the memory limit of the hashtable - size_t memory_limit_minimization = std::max( - memory_limit_ / 2, memory_limit_ - (200 * 1024 * 1024)); + // check which stack can be consumed (packed into the sparse array) + ConsumeStack(commonPrefixLength); - persistence_ = new PersistenceT(memory_limit_ - memory_limit_minimization, - params_[TEMPORARY_PATH_KEY]); - value_store_ = new ValueStoreT(params_); + // put everything that is not common between the two strings (the suffix) into the stack + FeedStack(commonPrefixLength, input_key.size(), key); - stack_ = new internal::UnpackedStateStack(persistence_, 30); - builder_ = new internal::SparseArrayBuilder( - memory_limit_minimization, persistence_, ValueStoreT::inner_weight, minimize_); + // get value and mark final state + bool no_minimization = false; + uint64_t value_idx = value_store_->GetValue(value, no_minimization); + + stack_->InsertFinalState(input_key.size(), value_idx, no_minimization); + + // count number of entries + ++number_of_keys_added_; + + // if inner weights are used update them + uint32_t weight = value_store_->GetWeightValue(value); + if (weight > 0){ + stack_->UpdateWeights(0, input_key.size() + 1, weight); + } - last_key_ = std::string(); - highest_stack_ = 0; - number_of_keys_added_ = 0; - state_ = generator_state::EMPTY; - start_state_ = 0; + last_key_ = key; + state_ = generator_state::FEEDING; } /** - * Add a key-value pair to the generator. + * Add a key and previously inserted value to the generator. * @param input_key The input key. - * @param value A value (depending on the Valuestore implementation). + * @param ValueHandle A handle returned by a previous call to RegisterValue */ - void Add(const std::string& input_key, typename ValueStoreT::value_t value = - ValueStoreT::no_value) { + void Add(const std::string& input_key, const ValueHandle& handle) { const char* key = input_key.c_str(); @@ -224,12 +244,16 @@ final { ConsumeStack(commonPrefixLength); // put everything that is not common between the two strings (the suffix) into the stack - FeedStack(commonPrefixLength, input_key.size(), key, value); + FeedStack(commonPrefixLength, input_key.size(), key); + + stack_->InsertFinalState(input_key.size(), handle.value_idx, handle.no_minimization); + + // count number of entries + ++number_of_keys_added_; // if inner weights are used update them - uint32_t weight = value_store_->GetWeightValue(value); - if (weight > 0){ - stack_->UpdateWeights(0, input_key.size() + 1, weight); + if (handle.weight > 0){ + stack_->UpdateWeights(0, input_key.size() + 1, handle.weight); } last_key_ = key; @@ -334,8 +358,7 @@ final { internal::SerializationUtils::WriteJsonRecord(stream, pt); } - inline void FeedStack(const size_t start, const size_t end, const char* key, - typename ValueStoreT::value_t value) { + inline void FeedStack(const size_t start, const size_t end, const char* key) { for (size_t i = start; i < end; ++i) { uint32_t ukey = static_cast(static_cast(key[i])); @@ -346,15 +369,6 @@ final { if (end > highest_stack_) { highest_stack_ = end; } - - // mark final state - bool no_minimization = false; - auto value_idx = value_store_->GetValue(value, no_minimization); - - stack_->InsertFinalState(end, value_idx, no_minimization); - - // count number of entries - ++number_of_keys_added_; } inline void ConsumeStack(const size_t end) { diff --git a/keyvi/src/cpp/dictionary/fsa/generator_adapter.h b/keyvi/src/cpp/dictionary/fsa/generator_adapter.h index 3eddea68..d240df4e 100644 --- a/keyvi/src/cpp/dictionary/fsa/generator_adapter.h +++ b/keyvi/src/cpp/dictionary/fsa/generator_adapter.h @@ -39,8 +39,8 @@ class GeneratorAdapterInterface { virtual void Add(const std::string& input_key, typename ValueStoreT::value_t value = ValueStoreT::no_value) {} + virtual void Add(const std::string& input_key, const fsa::ValueHandle& value) {} - virtual void Reset() {} virtual size_t GetFsaSize() const {return 0;} virtual void CloseFeeding() {} virtual void Write(std::ostream& stream) {} @@ -55,8 +55,8 @@ template { public: GeneratorAdapter(size_t memory_limit = 1073741824, - const generator_param_t& value_store_params = generator_param_t()): - generator_(memory_limit, value_store_params) + const generator_param_t& value_store_params = generator_param_t(), ValueStoreT* value_store = NULL): + generator_(memory_limit, value_store_params, value_store) {} void Add(const std::string& input_key, typename ValueStoreT::value_t value = @@ -64,8 +64,8 @@ class GeneratorAdapter final: public GeneratorAdapterInterfacePersist(); + // free up memory from hashtable + hash_.Clear(); + } + void Write(std::ostream& stream) { boost::property_tree::ptree pt; pt.put("size", std::to_string(values_buffer_size_)); diff --git a/keyvi/src/cpp/dictionary/fsa/internal/memory_map_manager.h b/keyvi/src/cpp/dictionary/fsa/internal/memory_map_manager.h index 05ed8596..da0a4278 100644 --- a/keyvi/src/cpp/dictionary/fsa/internal/memory_map_manager.h +++ b/keyvi/src/cpp/dictionary/fsa/internal/memory_map_manager.h @@ -176,24 +176,50 @@ final { } void Write (std::ostream& stream, size_t end) const { - size_t number_of_chunks = mappings_.size(); - if (number_of_chunks == 0){ + if (persisted_) { + for (int i =0; i < number_of_chunks_; i++) + { + std::ifstream data_file; + data_file.open (GetFilenameForChunk(i).native().c_str(), std::ios::binary); + stream << data_file.rdbuf(); + data_file.close(); + } + } else if (number_of_chunks_ == 0) { return; - } - - // write all but the last - for (size_t i = 0; i< number_of_chunks - 1; ++i){ - char *ptr = (char*) mappings_[i].region_->get_address(); - stream.write (ptr, chunk_size_); - } - char *ptr = (char*) mappings_[number_of_chunks - 1].region_->get_address(); - stream.write (ptr, end - ((number_of_chunks - 1) * chunk_size_)); + }else { + // write all but the last + for (size_t i = 0; i< number_of_chunks_ - 1; ++i){ + char *ptr = (char*) mappings_[i].region_->get_address(); + stream.write (ptr, chunk_size_); + } + char *ptr = (char*) mappings_[number_of_chunks_ - 1].region_->get_address(); + stream.write (ptr, end - ((number_of_chunks_ - 1) * chunk_size_)); + } } size_t GetSize() const { return tail_; } + /** + * Frees up all mmap's, should be called after everything has been written. + */ + void Persist() { + persisted_ = true; + for (auto& m : mappings_) { + m.region_->flush(); + delete m.region_; + delete m.mapping_; + } + + // truncate last file according to the written buffers + if (number_of_chunks_ > 1) { + boost::filesystem::resize_file(GetFilenameForChunk(number_of_chunks_ - 1), tail_ - ((number_of_chunks_ - 1) * chunk_size_)); + } + + mappings_.clear(); + } + private: struct mapping { boost::interprocess::file_mapping* mapping_; @@ -205,9 +231,19 @@ final { boost::filesystem::path directory_; boost::filesystem::path filename_pattern_; size_t tail_ = 0; + bool persisted_ = false; + size_t number_of_chunks_ = 0; + + boost::filesystem::path GetFilenameForChunk(int i) const { + boost::filesystem::path filename(directory_); + filename /= filename_pattern_; + filename += "_"; + filename += std::to_string(i); + return filename; + } void* GetChunk(size_t chunk_number) { - while (chunk_number >= mappings_.size()) { + while (chunk_number >= number_of_chunks_) { CreateMapping(); } @@ -217,10 +253,7 @@ final { void CreateMapping() { mapping new_mapping; - boost::filesystem::path filename(directory_); - filename /= filename_pattern_; - filename += "_"; - filename += std::to_string(mappings_.size()); + boost::filesystem::path filename = GetFilenameForChunk(number_of_chunks_); std::filebuf fbuf; fbuf.open( @@ -239,6 +272,7 @@ final { *new_mapping.mapping_, boost::interprocess::read_write); mappings_.push_back(new_mapping); + ++number_of_chunks_; } }; diff --git a/keyvi/src/cpp/dictionary/fsa/internal/null_value_store.h b/keyvi/src/cpp/dictionary/fsa/internal/null_value_store.h index 51ed0638..c50ba190 100644 --- a/keyvi/src/cpp/dictionary/fsa/internal/null_value_store.h +++ b/keyvi/src/cpp/dictionary/fsa/internal/null_value_store.h @@ -64,6 +64,13 @@ class NullValueStore final : public IValueStoreWriter { } void Write(std::ostream& stream) {} + + /** + * Close the value store, so no more updates; + */ + void CloseFeeding() { + } + }; class NullValueStoreReader final: public IValueStoreReader{ diff --git a/keyvi/src/cpp/dictionary/fsa/internal/string_value_store.h b/keyvi/src/cpp/dictionary/fsa/internal/string_value_store.h index 98130cb9..fb89333a 100644 --- a/keyvi/src/cpp/dictionary/fsa/internal/string_value_store.h +++ b/keyvi/src/cpp/dictionary/fsa/internal/string_value_store.h @@ -216,6 +216,14 @@ class StringValueStore final : public IValueStoreWriter { stream.write((const char*) &string_values_[0], string_values_.size()); } + /** + * Close the value store, so no more updates; + */ + void CloseFeeding() { + // free up memory from hashtable + hash_.Clear(); + } + private: std::vector string_values_; MinimizationHash hash_; diff --git a/keyvi/src/cpp/keyvicompiler/keyvicompiler.cpp b/keyvi/src/cpp/keyvicompiler/keyvicompiler.cpp index 1072a1dc..95835b01 100644 --- a/keyvi/src/cpp/keyvicompiler/keyvicompiler.cpp +++ b/keyvi/src/cpp/keyvicompiler/keyvicompiler.cpp @@ -95,40 +95,22 @@ void compile_multiple(CompilerType& compiler, std::function -void finalize_compile(CompilerType& compiler, std::string& output, - size_t partition_size = 0, const std::string& manifest = "") { - if (partition_size == 0) { - std::ofstream out_stream(output, std::ios::binary); - compiler.Compile(callback); - try { - compiler.SetManifestFromString(manifest); - } catch(boost::property_tree::json_parser::json_parser_error const& error) { - std::cout << "Failed to set manifest: " << manifest << std::endl; - std::cout << error.what() << std::endl; - } - compiler.Write(out_stream); - out_stream.close(); - } else { - std::string output_part_zero = output + ".0"; - int partition_number = 1; - std::ofstream out_stream(output_part_zero, std::ios::binary); - - while (compiler.CompileNext(partition_size, out_stream, 2, callback)) { - std::cout << "Finalize partition " << partition_number << std::endl; - - out_stream.close(); - out_stream.open(output + "." + std::to_string(partition_number), - std::ios::binary); - ++partition_number; - } - - out_stream.close(); +void finalize_compile(CompilerType& compiler, std::string& output, const std::string& manifest = "") { + std::ofstream out_stream(output, std::ios::binary); + compiler.Compile(callback); + try { + compiler.SetManifestFromString(manifest); + } catch(boost::property_tree::json_parser::json_parser_error const& error) { + std::cout << "Failed to set manifest: " << manifest << std::endl; + std::cout << error.what() << std::endl; } + compiler.Write(out_stream); + out_stream.close(); } template void compile_integer(std::vector& input, std::string& output, - size_t memory_limit, size_t partition_size = 0, + size_t memory_limit, const std::string& manifest = "", const vs_param_t& value_store_params = vs_param_t()) { keyvi::dictionary::DictionaryCompiler< @@ -156,13 +138,12 @@ void compile_integer(std::vector& input, std::string& output, }; compile_multiple(compiler, parser, input); - finalize_compile(compiler, output, partition_size, manifest); + finalize_compile(compiler, output, manifest); } template void compile_strings_inner(Compiler& compiler, std::vector& input, std::string& output, - size_t partition_size = 0, const std::string& manifest = "") { std::function(std::string)> parser = [] (std::string line) { size_t tab = line.find('\t'); @@ -176,24 +157,24 @@ void compile_strings_inner(Compiler& compiler, compile_multiple(compiler, parser, input); - finalize_compile(compiler, output, partition_size, manifest); + finalize_compile(compiler, output, manifest); } template void compile_strings(std::vector& input, std::string& output, - size_t memory_limit, size_t partition_size = 0, + size_t memory_limit, const std::string& manifest = "", const vs_param_t& value_store_params = vs_param_t()) { keyvi::dictionary::DictionaryCompiler< keyvi::dictionary::fsa::internal::SparseArrayPersistence, keyvi::dictionary::fsa::internal::StringValueStore> compiler( memory_limit, value_store_params); - compile_strings_inner(compiler, input, output, partition_size, manifest); + compile_strings_inner(compiler, input, output, manifest); } template void compile_key_only(std::vector& input, std::string& output, - size_t memory_limit, size_t partition_size = 0, + size_t memory_limit, const std::string& manifest = "", const vs_param_t& value_store_params = vs_param_t()) { keyvi::dictionary::DictionaryCompiler< @@ -214,19 +195,19 @@ void compile_key_only(std::vector& input, std::string& output, compile_multiple(compiler, parser, input); - finalize_compile(compiler, output, partition_size, manifest); + finalize_compile(compiler, output, manifest); } template void compile_json(std::vector& input, std::string& output, - size_t memory_limit, size_t partition_size = 0, + size_t memory_limit, const std::string& manifest = "", const vs_param_t& value_store_params = vs_param_t()) { keyvi::dictionary::DictionaryCompiler< keyvi::dictionary::fsa::internal::SparseArrayPersistence, keyvi::dictionary::fsa::internal::JsonValueStore> compiler( memory_limit, value_store_params); - compile_strings_inner(compiler, input, output, partition_size, manifest); + compile_strings_inner(compiler, input, output, manifest); } /** Extracts the value store parameters. */ @@ -268,9 +249,6 @@ int main(int argc, char** argv) { "dictionary-type,d", boost::program_options::value()->default_value("integer"), "type of dictionary (integer (default), string, key-only, json)"); - description.add_options()("partition-size,p", - boost::program_options::value(), - "create partitions with a maximum size"); description.add_options()("compact,c", "Compact Mode"); description.add_options()( "value-store-parameter,V", @@ -314,11 +292,6 @@ int main(int argc, char** argv) { compact = true; } - size_t partition_size = 0; - if (vm.count("partition-size")) { - partition_size = vm["partition-size"].as(); - } - std::string manifest = vm["manifest"].as(); std::cout << manifest << std::endl; @@ -340,33 +313,33 @@ int main(int argc, char** argv) { if (dictionary_type == "integer") { if (compact){ compile_integer(input_files, output_file, memory_limit, - partition_size, manifest, value_store_params); + manifest, value_store_params); } else { - compile_integer(input_files, output_file, memory_limit, partition_size, + compile_integer(input_files, output_file, memory_limit, manifest, value_store_params); } } else if (dictionary_type == "string") { if (compact){ compile_strings(input_files, output_file, memory_limit, - partition_size, manifest, value_store_params); + manifest, value_store_params); } else { - compile_strings(input_files, output_file, memory_limit, partition_size, + compile_strings(input_files, output_file, memory_limit, manifest, value_store_params); } } else if (dictionary_type == "key-only") { if (compact){ compile_key_only(input_files, output_file, memory_limit, - partition_size, manifest, value_store_params); + manifest, value_store_params); } else { - compile_key_only(input_files, output_file, memory_limit, partition_size, + compile_key_only(input_files, output_file, memory_limit, manifest, value_store_params); } } else if (dictionary_type == "json") { if (compact){ compile_json(input_files, output_file, memory_limit, - partition_size, manifest, value_store_params); + manifest, value_store_params); } else { - compile_json(input_files, output_file, memory_limit, partition_size, + compile_json(input_files, output_file, memory_limit, manifest, value_store_params); } } else {