Vowpal Wabbit
Public Member Functions | Public Attributes | List of all members
parser Struct Reference

#include <parser.h>

Public Member Functions

 parser (size_t ring_size, bool strict_parse_)
 
 ~parser ()
 

Public Attributes

v_array< substring > words
 
v_array< substring > name
 
VW::object_pool< example, example_initializer > example_pool
 
VW::ptr_queue< example > ready_parsed_examples
 
io_buf * input = nullptr
 
int(* reader )(vw *, v_array< example *> &examples)
 
void(* text_reader )(vw *, char *, size_t, v_array< example *> &)
 
hash_func_t hasher
 
bool resettable
 
io_buf * output = nullptr
 
bool write_cache = false
 
bool sort_features = false
 
bool sorted_cache = false
 
const size_t ring_size
 
uint64_t begin_parsed_examples = 0
 
uint64_t end_parsed_examples = 0
 
uint32_t in_pass_counter = 0
 
bool emptylines_separate_examples = false
 
std::mutex output_lock
 
std::condition_variable output_done
 
bool done = false
 
v_array< size_t > gram_mask
 
v_array< size_t > ids
 
v_array< size_t > counts
 
size_t finished_count
 
int label_sock = 0
 
int bound_sock = 0
 
int max_fd = 0
 
v_array< substring > parse_name
 
label_parser lp
 
bool audit = false
 
bool decision_service_json = false
 
bool strict_parse
 
std::exception_ptr exc_ptr
 

Detailed Description

Definition at line 38 of file parser.h.

Constructor & Destructor Documentation

◆ parser()

parser::parser ( size_t  ring_size,
bool  strict_parse_ 
)
inline

Definition at line 40 of file parser.h.

References simple_label.

41  : example_pool{ring_size}, ready_parsed_examples{ring_size}, ring_size{ring_size}, strict_parse{strict_parse_}
42  {
43  this->input = new io_buf{};
44  this->output = new io_buf{};
45  this->lp = simple_label;
46 
47  // Free parser must still be used for the following fields.
48  this->words = v_init<substring>();
49  this->name = v_init<substring>();
50  this->parse_name = v_init<substring>();
51  this->gram_mask = v_init<size_t>();
52  this->ids = v_init<size_t>();
53  this->counts = v_init<size_t>();
54  }
simple_label
label_parser simple_label
Definition: simple_label.cc:106
parser::example_pool
VW::object_pool< example, example_initializer > example_pool
Definition: parser.h:66
parser::words
v_array< substring > words
Definition: parser.h:63
parser::ready_parsed_examples
VW::ptr_queue< example > ready_parsed_examples
Definition: parser.h:67
parser::counts
v_array< size_t > counts
Definition: parser.h:94
parser::input
io_buf * input
Definition: parser.h:69
parser::name
v_array< substring > name
Definition: parser.h:64
parser::parse_name
v_array< substring > parse_name
Definition: parser.h:100
parser::strict_parse
bool strict_parse
Definition: parser.h:107
parser::ids
v_array< size_t > ids
Definition: parser.h:93
parser::gram_mask
v_array< size_t > gram_mask
Definition: parser.h:91
parser::ring_size
const size_t ring_size
Definition: parser.h:80
io_buf
Definition: io_buf.h:54
parser::output
io_buf * output
Definition: parser.h:75
parser::lp
label_parser lp
Definition: parser.h:102

◆ ~parser()

parser::~parser ( )
inline

Definition at line 56 of file parser.h.

57  {
58  delete input;
59  delete output;
60  }
parser::input
io_buf * input
Definition: parser.h:69
parser::output
io_buf * output
Definition: parser.h:75

Member Data Documentation

◆ audit

bool parser::audit = false

Definition at line 104 of file parser.h.

Referenced by enable_sources().

◆ begin_parsed_examples

uint64_t parser::begin_parsed_examples = 0

Definition at line 81 of file parser.h.

Referenced by VW::clean_example(), VW::get_unused_example(), and VW::new_unused_example().

◆ bound_sock

int parser::bound_sock = 0

Definition at line 97 of file parser.h.

Referenced by enable_sources(), and reset_source().

◆ counts

v_array<size_t> parser::counts

Definition at line 94 of file parser.h.

Referenced by free_parser().

◆ decision_service_json

bool parser::decision_service_json = false

Definition at line 105 of file parser.h.

Referenced by enable_sources(), and parse_line_json().

◆ done

bool parser::done = false

Definition at line 90 of file parser.h.

Referenced by lock_done(), and parse_dispatch().

◆ emptylines_separate_examples

bool parser::emptylines_separate_examples = false

Definition at line 84 of file parser.h.

Referenced by CSOAA::csldf_setup(), Search::setup(), and VW::setup_example().

◆ end_parsed_examples

uint64_t parser::end_parsed_examples = 0

Definition at line 82 of file parser.h.

Referenced by LEARNER::generic_driver_onethread(), VW::import_example(), TC_parser< audit >::parserWarning(), VW::read_example(), VW::setup_example(), and thread_dispatch().

◆ example_pool

VW::object_pool<example, example_initializer> parser::example_pool

Definition at line 66 of file parser.h.

Referenced by VW::clean_example(), free_parser(), VW::get_unused_example(), and VW::is_ring_example().

◆ exc_ptr

std::exception_ptr parser::exc_ptr

Definition at line 108 of file parser.h.

Referenced by parse_dispatch().

◆ finished_count

size_t parser::finished_count

Definition at line 95 of file parser.h.

◆ gram_mask

v_array<size_t> parser::gram_mask

Definition at line 91 of file parser.h.

Referenced by free_parser(), and generateGrams().

◆ hasher

hash_func_t parser::hasher

Definition at line 73 of file parser.h.

Referenced by VW::are_features_compatible(), VW::hash_feature(), VW::hash_feature_cstr(), VW::hash_space(), TC_parser< audit >::maybeFeature(), TC_parser< audit >::nameSpaceInfo(), and parse_feature_tweaks().

◆ ids

v_array<size_t> parser::ids

Definition at line 93 of file parser.h.

◆ in_pass_counter

uint32_t parser::in_pass_counter = 0

Definition at line 83 of file parser.h.

Referenced by end_pass_example(), parse_dispatch(), and VW::setup_example().

◆ input

io_buf* parser::input = nullptr

Definition at line 69 of file parser.h.

Referenced by enable_sources(), finalize_source(), parse_cache(), read_cached_features(), read_features(), reset_source(), and set_compressed().

◆ label_sock

int parser::label_sock = 0

Definition at line 96 of file parser.h.

Referenced by enable_sources().

◆ lp

label_parser parser::lp

Definition at line 102 of file parser.h.

Referenced by cb_adf_setup(), cb_algs_setup(), cbifyldf_setup(), CCB::ccb_explore_adf_setup(), cs_active_setup(), CSOAA::csldf_setup(), CSOAA::csoaa_setup(), end_pass_example(), DefaultState< audit >::EndObject(), explore_eval_setup(), free_parser(), ezexample::get_new_example(), VW::import_example(), VWReaderHandler< audit >::init(), LEARNER::init_cost_sensitive_learner(), LEARNER::init_multiclass_learner(), lda_setup(), learn(), memory_tree_setup(), ezexample::mini_setup_example(), multilabel_oaa_setup(), mwt_setup(), VW::new_unused_example(), parse_dictionary_argument(), parse_dispatch(), VW::parse_example_label(), read_cached_features(), Search::search::set_label_parser(), VW::cb_explore_adf::softmax::setup(), VW::cb_explore_adf::greedy::setup(), VW::cb_explore_adf::first::setup(), VW::cb_explore_adf::bag::setup(), VW::cb_explore_adf::cover::setup(), VW::cb_explore_adf::regcb::setup(), Search::setup(), VW::setup_example(), LabelObjectState< audit >::StartObject(), MultiState< audit >::StartObject(), SlotsState< audit >::StartObject(), substring_to_example(), and ezexample::train().

◆ max_fd

int parser::max_fd = 0

Definition at line 98 of file parser.h.

Referenced by enable_sources().

◆ name

v_array<substring> parser::name

Definition at line 64 of file parser.h.

Referenced by free_parser().

◆ output

io_buf* parser::output = nullptr

Definition at line 75 of file parser.h.

Referenced by finalize_source(), free_parser(), make_write_cache(), parse_cache(), reset_source(), set_compressed(), and VW::setup_example().

◆ output_done

std::condition_variable parser::output_done

Definition at line 88 of file parser.h.

Referenced by VW::finish_example(), and reset_source().

◆ output_lock

std::mutex parser::output_lock

Definition at line 87 of file parser.h.

Referenced by VW::finish_example(), and reset_source().

◆ parse_name

v_array<substring> parser::parse_name

Definition at line 100 of file parser.h.

Referenced by VW::finish(), MULTILABEL::parse_label(), CB::parse_label(), COST_SENSITIVE::parse_label(), and CCB::parse_label().

◆ reader

int(* parser::reader) (vw *, v_array< example * > &examples)

Definition at line 70 of file parser.h.

Referenced by enable_sources(), parse_cache(), parse_dispatch(), and reset_source().

◆ ready_parsed_examples

VW::ptr_queue<example> parser::ready_parsed_examples

Definition at line 67 of file parser.h.

Referenced by free_parser(), VW::get_example(), lock_done(), reset_source(), and thread_dispatch().

◆ resettable

bool parser::resettable

Definition at line 74 of file parser.h.

Referenced by enable_sources(), parse_cache(), and reset_source().

◆ ring_size

const size_t parser::ring_size

Definition at line 80 of file parser.h.

Referenced by lda_setup(), learn(), receive_result(), and sender_setup().

◆ sort_features

bool parser::sort_features = false

Definition at line 77 of file parser.h.

Referenced by parse_example_tweaks(), and VW::setup_example().

◆ sorted_cache

bool parser::sorted_cache = false

Definition at line 78 of file parser.h.

Referenced by enable_sources(), parse_cache(), and read_cached_features().

◆ strict_parse

bool parser::strict_parse

Definition at line 107 of file parser.h.

Referenced by lda_setup(), and TC_parser< audit >::parserWarning().

◆ text_reader

void(* parser::text_reader) (vw *, char *, size_t, v_array< example * > &)

Definition at line 71 of file parser.h.

Referenced by enable_sources().

◆ words

v_array<substring> parser::words

Definition at line 63 of file parser.h.

Referenced by free_parser(), and substring_to_example().

◆ write_cache

bool parser::write_cache = false

Definition at line 76 of file parser.h.

Referenced by enable_sources(), make_write_cache(), parse_cache(), reset_source(), and VW::setup_example().

The documentation for this struct was generated from the following file:
Generated on Thu Apr 9 2020 14:45:27 for Vowpal Wabbit by   doxygen 1.8.13