VW Namespace Reference

Namespaces
	cb_explore_adf
	config
	shared_feature_merger

Classes
struct	autolink
struct	cb_sample_data
struct	default_cleanup
struct	no_lock_object_pool
struct	object_pool
struct	primitive_feature_space
class	ptr_queue
class	SpanningTree
class	strict_parse_exception
class	topk
struct	v_array_allocator
struct	v_array_deleter
struct	value_object_pool
struct	version_struct
class	vw_argument_disagreement_exception
class	vw_argument_invalid_value_exception
class	vw_exception
class	vw_unrecognised_option_exception

Typedefs
typedef example &(*	example_factory_t) (void *)
template<typename T >
using	v_array_pool = VW::value_object_pool< v_array< T >, v_array_allocator< T >, v_array_deleter< T > >

Functions
uint32_t	convert (size_t number)
void	copy_example_label (example *dst, example *src, size_t, void(*copy_label)(void *, void *))
void	copy_example_metadata (bool, example *dst, example *src)
void	copy_example_data (bool audit, example *dst, example *src)
void	copy_example_data (bool audit, example *dst, example *src, size_t label_size, void(*copy_label)(void *, void *))
void	move_feature_namespace (example *dst, example *src, namespace_index c)
feature *	get_features (vw &all, example *ec, size_t &feature_map_len)
void	return_features (feature *f)
example *	alloc_examples (size_t, size_t count=1)
void	dealloc_example (void(*delete_label)(void *), example &ec, void(*delete_prediction)(void *))
void	finish_example (vw &, example &)
void	clean_example (vw &, example &, bool rewind)
void	finish_example (vw &all, multi_ex &ec_seq)
void	return_multiple_example (vw &all, v_array< example *> &examples)
const char *	are_features_compatible (vw &vw1, vw &vw2)
void	cmd_string_replace_value (std::stringstream *&ss, std::string flag_to_replace, std::string new_value)
char **	to_argv_escaped (std::string const &s, int &argc)
char **	to_argv (std::string const &s, int &argc)
char **	get_argv_from_string (std::string s, int &argc)
void	free_args (int argc, char *argv[])
vw *	initialize (options_i &options, io_buf *model, bool skipModelLoad, trace_message_t trace_listener, void *trace_context)
vw *	initialize (std::string s, io_buf *model, bool skipModelLoad, trace_message_t trace_listener, void *trace_context)
vw *	initialize_escaped (std::string const &s, io_buf *model, bool skipModelLoad, trace_message_t trace_listener, void *trace_context)
vw *	initialize (int argc, char *argv[], io_buf *model, bool skipModelLoad, trace_message_t trace_listener, void *trace_context)
vw *	seed_vw_model (vw *vw_model, const std::string extra_args, trace_message_t trace_listener, void *trace_context)
void	delete_dictionary_entry (substring ss, features *A)
void	sync_stats (vw &all)
void	finish (vw &all, bool delete_all)
void	read_line (vw &all, example *ex, char *line)
void	read_lines (vw *all, char *line, size_t, v_array< example *> &examples)
example &	get_unused_example (vw *all)
template<bool audit>
void	read_line_json (vw &all, v_array< example *> &examples, char *line, example_factory_t example_factory, void *ex_factory_context)
void	apply_pdrop (vw &all, float pdrop, v_array< example *> &examples)
template<bool audit>
void	read_line_decision_service_json (vw &all, v_array< example *> &examples, char *line, size_t length, bool copy_line, example_factory_t example_factory, void *ex_factory_context, DecisionServiceInteraction *data)
void	save_predictor (vw &all, std::string reg_name)
void	save_predictor (vw &all, io_buf &buf)
void	setup_examples (vw &all, v_array< example *> &examples)
void	setup_example (vw &all, example *ae)
example *	new_unused_example (vw &all)
example *	read_example (vw &all, char *example_line)
example *	read_example (vw &all, std::string example_line)
void	add_constant_feature (vw &vw, example *ec)
void	add_label (example *ec, float label, float weight, float base)
example *	import_example (vw &all, const std::string &label, primitive_feature_space *features, size_t len)
primitive_feature_space *	export_example (vw &all, example *ec, size_t &len)
void	releaseFeatureSpace (primitive_feature_space *features, size_t len)
void	parse_example_label (vw &all, example &ec, std::string label)
void	empty_example (vw &, example &ec)
example *	get_example (parser *p)
float	get_topic_prediction (example *ec, size_t i)
float	get_label (example *ec)
float	get_importance (example *ec)
float	get_initial (example *ec)
float	get_prediction (example *ec)
float	get_cost_sensitive_prediction (example *ec)
v_array< float > &	get_cost_sensitive_prediction_confidence_scores (example *ec)
uint32_t *	get_multilabel_predictions (example *ec, size_t &len)
float	get_action_score (example *ec, size_t i)
size_t	get_action_score_length (example *ec)
size_t	get_tag_length (example *ec)
const char *	get_tag (example *ec)
size_t	get_feature_number (example *ec)
float	get_confidence (example *ec)
void	start_parser (vw &all)
void	end_parser (vw &all)
bool	is_ring_example (vw &all, example *ae)
const version_struct	version (PACKAGE_VERSION)
const std::string	git_commit (COMMIT_VERSION)
void	clear_example_data (example &)
uint64_t	hash_space (vw &all, const std::string &s)
uint64_t	hash_space_static (const std::string &s, const std::string &hash)
uint64_t	hash_feature (vw &all, const std::string &s, uint64_t u)
uint64_t	hash_feature_static (const std::string &s, uint64_t u, const std::string &h, uint32_t num_bits)
uint64_t	hash_feature_cstr (vw &all, char *fstr, uint64_t u)
float	get_weight (vw &all, uint32_t index, uint32_t offset)
void	set_weight (vw &all, uint32_t index, uint32_t offset, float value)
uint32_t	num_weights (vw &all)
uint32_t	get_stride (vw &all)
void	init_features (primitive_feature_space &fs, size_t features_count)
void	set_feature (primitive_feature_space &fs, size_t index, uint64_t feature_hash, float value)
void	validate_version (vw &all)
void	validate_min_max_label (vw &all)
void	validate_default_bits (vw &all, uint32_t local_num_bits)
void	validate_num_bits (vw &all)

Typedef Documentation

example_factory_t

typedef example&(* VW::example_factory_t) (void *)

Definition at line 76 of file parse_primitives.h.

v_array_pool

template<typename T >

using VW::v_array_pool = typedef VW::value_object_pool<v_array<T>, v_array_allocator<T>, v_array_deleter<T> >

Definition at line 21 of file v_array_pool.h.

Function Documentation

add_constant_feature()

void VW::add_constant_feature	(	vw &	vw,
		example *	ec
	)

Definition at line 774 of file parser.cc.

References vw::audit, constant, constant_namespace, example_predict::feature_space, vw::hash_inv, example_predict::indices, example::num_features, v_array< T >::push_back(), and example::total_sum_feat_sq.

Referenced by ezexample::ezexample(), and setup_example().

{
  ec->indices.push_back(constant_namespace);
  ec->feature_space[constant_namespace].push_back(1, constant);
  ec->total_sum_feat_sq++;
  ec->num_features++;
  if (vw.audit || vw.hash_inv)
    ec->feature_space[constant_namespace].space_names.push_back(audit_strings_ptr(new audit_strings("", "Constant")));
}

add_label()

void VW::add_label	(	example *	ec,
		float	label,
		float	weight,
		float	base
	)

Definition at line 784 of file parser.cc.

References label_data::initial, example::l, label_data::label, polylabel::simple, and example::weight.

Referenced by VW_AddLabel().

{
  ec->l.simple.label = label;
  ec->l.simple.initial = base;
  ec->weight = weight;
}

alloc_examples()

example * VW::alloc_examples	(	size_t	,
		size_t	count = 1
	)

Definition at line 204 of file example.cc.

References example_predict::ft_offset, and example::in_use.

Referenced by add_to_vali(), baseline_setup(), ExpReplay::expreplay_setup(), init_adf_data(), EntityRelationTask::initialize(), DepParserTask::initialize(), SequenceTask_DemoLDF::initialize(), and parse_dictionary_argument().

{
  example* ec = calloc_or_throw<example>(count);
  if (ec == nullptr)
    return nullptr;
  for (size_t i = 0; i < count; i++)
  {
    ec[i].in_use = true;
    ec[i].ft_offset = 0;
    //  std::cerr << "  alloc_example.indices.begin()=" << ec->indices.begin() << " end=" << ec->indices.end() << " //
    //  ld = " << ec->ld << "\t|| me = " << ec << std::endl;
  }
  return ec;
}

apply_pdrop()

void VW::apply_pdrop ( vw &  all, float  pdrop, v_array< example *> &  examples  )

inline

Definition at line 1374 of file parse_example_json.h.

References label_type::cb, label_type::ccb, and vw::label_type.

Referenced by read_line_decision_service_json().

{
  if (all.label_type == label_type::label_type_t::cb)
  {
    for (auto& e : examples)
    {
      e->l.cb.weight = 1 - pdrop;
    }
  }
  else if (all.label_type == label_type::label_type_t::ccb)
  {
    for (auto& e : examples)
    {
      e->l.conditional_contextual_bandit.weight = 1 - pdrop;
    }
  }
}

are_features_compatible()

const char * VW::are_features_compatible	(	vw &	vw1,
		vw &	vw2
	)

Definition at line 501 of file parse_args.cc.

References vw::add_constant, vw::affix_features, vw::dictionary_path, parser::hasher, vw::ignore, vw::ignore_linear, vw::ignore_some, vw::ignore_some_linear, vw::interactions, vw::limit, vw::ngram, vw::num_bits, vw::p, vw::permutations, vw::redefine, vw::redefine_some, vw::skips, and vw::spelling_features.

{
  if (vw1.p->hasher != vw2.p->hasher)
    return "hasher";

  if (!std::equal(vw1.spelling_features.begin(), vw1.spelling_features.end(), vw2.spelling_features.begin()))
    return "spelling_features";

  if (!std::equal(vw1.affix_features.begin(), vw1.affix_features.end(), vw2.affix_features.begin()))
    return "affix_features";

  if (!std::equal(vw1.ngram.begin(), vw1.ngram.end(), vw2.ngram.begin()))
    return "ngram";

  if (!std::equal(vw1.skips.begin(), vw1.skips.end(), vw2.skips.begin()))
    return "skips";

  if (!std::equal(vw1.limit.begin(), vw1.limit.end(), vw2.limit.begin()))
    return "limit";

  if (vw1.num_bits != vw2.num_bits)
    return "num_bits";

  if (vw1.permutations != vw2.permutations)
    return "permutations";

  if (vw1.interactions.size() != vw2.interactions.size())
    return "interactions size";

  if (vw1.ignore_some != vw2.ignore_some)
    return "ignore_some";

  if (vw1.ignore_some && !std::equal(vw1.ignore.begin(), vw1.ignore.end(), vw2.ignore.begin()))
    return "ignore";

  if (vw1.ignore_some_linear != vw2.ignore_some_linear)
    return "ignore_some_linear";

  if (vw1.ignore_some_linear &&
      !std::equal(vw1.ignore_linear.begin(), vw1.ignore_linear.end(), vw2.ignore_linear.begin()))
    return "ignore_linear";

  if (vw1.redefine_some != vw2.redefine_some)
    return "redefine_some";

  if (vw1.redefine_some && !std::equal(vw1.redefine.begin(), vw1.redefine.end(), vw2.redefine.begin()))
    return "redefine";

  if (vw1.add_constant != vw2.add_constant)
    return "add_constant";

  if (vw1.dictionary_path.size() != vw2.dictionary_path.size())
    return "dictionary_path size";

  if (!std::equal(vw1.dictionary_path.begin(), vw1.dictionary_path.end(), vw2.dictionary_path.begin()))
    return "dictionary_path";

  for (auto i = std::begin(vw1.interactions), j = std::begin(vw2.interactions); i != std::end(vw1.interactions);
       ++i, ++j)
    if (*i != *j)
      return "interaction mismatch";

  return nullptr;
}

clean_example()

void VW::clean_example	(	vw &	all,
		example &	ec,
		bool	rewind
	)

Definition at line 867 of file parser.cc.

References parser::begin_parsed_examples, empty_example(), parser::example_pool, example::in_use, vw::p, and VW::object_pool< T, TInitializer, TCleanup >::return_object().

Referenced by dealloc_example(), finish_example(), and return_multiple_example().

{
  if (rewind)
  {
    assert(all.p->begin_parsed_examples > 0);
    all.p->begin_parsed_examples--;
  }

  empty_example(all, ec);
  assert(ec.in_use);
  ec.in_use = false;
  all.p->example_pool.return_object(&ec);
}

clear_example_data()

void VW::clear_example_data

(

example &

)

cmd_string_replace_value()

void VW::cmd_string_replace_value	(	std::stringstream *&	ss,
		std::string	flag_to_replace,
		std::string	new_value
	)

Definition at line 1570 of file parse_args.cc.

{
  flag_to_replace.append(
      " ");  // add a space to make sure we obtain the right flag in case 2 flags start with the same set of characters
  std::string cmd = ss->str();
  size_t pos = cmd.find(flag_to_replace);
  if (pos == std::string::npos)
    // flag currently not present in command std::string, so just append it to command std::string
    *ss << " " << flag_to_replace << new_value;
  else
  {
    // flag is present, need to replace old value with new value

    // compute position after flag_to_replace
    pos += flag_to_replace.size();

    // now pos is position where value starts
    // find position of next space
    size_t pos_after_value = cmd.find(" ", pos);
    if (pos_after_value == std::string::npos)
    {
      // we reach the end of the std::string, so replace the all characters after pos by new_value
      cmd.replace(pos, cmd.size() - pos, new_value);
    }
    else
    {
      // replace characters between pos and pos_after_value by new_value
      cmd.replace(pos, pos_after_value - pos, new_value);
    }

    ss->str(cmd);
  }
}

convert()

uint32_t VW::convert

(

size_t

number

)

Definition at line 211 of file cache.cc.

References THROW.

Referenced by CCB::cache_label(), and utf16_to_utf8().

{
  if (number > UINT32_MAX)
  {
    THROW("size_t value is out of bounds of uint32_t.")
  }
  return static_cast<uint32_t>(number);
}

copy_example_data() [1/2]

void VW::copy_example_data	(	bool	audit,
		example *	dst,
		example *	src
	)

Definition at line 72 of file example.cc.

References c, copy_array(), copy_example_metadata(), example_predict::feature_space, example_predict::indices, example_predict::interactions, example::num_features, and example::total_sum_feat_sq.

Referenced by add_to_vali(), memory_tree_ns::copy_example_data(), copy_example_data(), copy_example_to_adf(), EntityRelationTask::predict_entity(), ExpReplay::predict_or_learn(), EntityRelationTask::predict_relation(), SequenceTask_DemoLDF::run(), Search::search_predict(), Search::predictor::set_input_at(), and ezexample::train().

{
  // std::cerr << "copy_example_data dst = " << dst << std::endl;
  copy_example_metadata(audit, dst, src);

  // copy feature data
  copy_array(dst->indices, src->indices);
  for (namespace_index c : src->indices) dst->feature_space[c].deep_copy_from(src->feature_space[c]);
  // copy_array(dst->atomics[i], src->atomics[i]);
  dst->num_features = src->num_features;
  dst->total_sum_feat_sq = src->total_sum_feat_sq;
  dst->interactions = src->interactions;
}

copy_example_data() [2/2]

void VW::copy_example_data	(	bool	audit,
		example *	dst,
		example *	src,
		size_t	label_size,
		void(*)(void *, void *)	copy_label
	)

Definition at line 86 of file example.cc.

References copy_example_data(), copy_example_label(), and CB::copy_label().

{
  copy_example_data(audit, dst, src);
  copy_example_label(dst, src, label_size, copy_label);
}

copy_example_label()

void VW::copy_example_label	(	example *	dst,
		example *	src,
		size_t	,
		void(*)(void *, void *)	copy_label
	)

Definition at line 40 of file example.cc.

References CB::copy_label(), and example::l.

Referenced by copy_example_data().

{
  if (copy_label)
    copy_label(&dst->l, &src->l);  // TODO: we really need to delete_label on dst :(
  else
    dst->l = src->l;
}

copy_example_metadata()

void VW::copy_example_metadata	(	bool	,
		example *	dst,
		example *	src
	)

Definition at line 48 of file example.cc.

References example::confidence, copy_array(), features::deep_copy_from(), example::end_pass, example::example_counter, example_predict::ft_offset, example::in_use, example::loss, example::partial_prediction, example::passthrough, example::sorted, example::tag, example::test_only, and example::weight.

Referenced by copy_example_data(), predict_or_learn(), and sensitivity().

{
  copy_array(dst->tag, src->tag);
  dst->example_counter = src->example_counter;

  dst->ft_offset = src->ft_offset;

  dst->partial_prediction = src->partial_prediction;
  if (src->passthrough == nullptr)
    dst->passthrough = nullptr;
  else
  {
    dst->passthrough = new features;
    dst->passthrough->deep_copy_from(*src->passthrough);
  }
  dst->loss = src->loss;
  dst->weight = src->weight;
  dst->confidence = src->confidence;
  dst->test_only = src->test_only;
  dst->end_pass = src->end_pass;
  dst->sorted = src->sorted;
  dst->in_use = src->in_use;
}

dealloc_example()

void VW::dealloc_example	(	void(*)(void *)	delete_label,
		example &	ec,
		void(*)(void *)	delete_prediction
	)

Definition at line 219 of file example.cc.

References clean_example(), CB::delete_label(), v_array< T >::delete_v(), features::delete_v(), example_predict::feature_space, finish_example(), example_predict::indices, example::l, example::passthrough, example::pred, and example::tag.

Referenced by memory_tree_ns::diag_kronecker_product_test(), EntityRelationTask::finish(), DepParserTask::finish(), SequenceTask_DemoLDF::finish(), Search::predictor::free_ec(), memory_tree_ns::free_example(), free_parser(), parse_dictionary_argument(), baseline::~baseline(), cbify::~cbify(), ExpReplay::expreplay< lp >::~expreplay(), nn::~nn(), Search::search::~search(), and warm_cb::~warm_cb().

{
  if (delete_label)
    delete_label(&ec.l);

  if (delete_prediction)
    delete_prediction(&ec.pred);

  ec.tag.delete_v();

  if (ec.passthrough)
  {
    ec.passthrough->delete_v();
    delete ec.passthrough;
  }

  for (auto& j : ec.feature_space) j.delete_v();

  ec.indices.delete_v();
}

delete_dictionary_entry()

void VW::delete_dictionary_entry	(	substring	ss,
		features *	A
	)

Definition at line 1797 of file parse_args.cc.

References substring::begin, and features::delete_v().

Referenced by finish().

{
  free(ss.begin);
  A->delete_v();
  delete A;
}

empty_example()

void VW::empty_example	(	vw &	,
		example &	ec
	)

Definition at line 857 of file parser.cc.

Referenced by clean_example(), ezexample::finish(), and ezexample::setup_for_predict().

{
  for (features& fs : ec) fs.clear();

  ec.indices.clear();
  ec.tag.clear();
  ec.sorted = false;
  ec.end_pass = false;
}

end_parser()

void VW::end_parser

(

vw &

all

)

Definition at line 1007 of file parser.cc.

References vw::parse_thread.

Referenced by main(), VW_EndParser(), and VW_Finish_Passes().

{ all.parse_thread.join(); }

export_example()

primitive_feature_space * VW::export_example	(	vw &	all,
		example *	ec,
		size_t &	len
	)

Definition at line 812 of file parser.cc.

References f, example_predict::feature_space, VW::primitive_feature_space::fs, example_predict::indices, VW::primitive_feature_space::len, VW::primitive_feature_space::name, v_array< T >::size(), stride_shift(), parameters::stride_shift(), feature::weight_index, and vw::weights.

Referenced by VW_ExportExample().

{
  len = ec->indices.size();
  primitive_feature_space* fs_ptr = new primitive_feature_space[len];

  int fs_count = 0;

  for (size_t idx = 0; idx < len; ++idx)
  {
    namespace_index i = ec->indices[idx];
    fs_ptr[fs_count].name = i;
    fs_ptr[fs_count].len = ec->feature_space[i].size();
    fs_ptr[fs_count].fs = new feature[fs_ptr[fs_count].len];

    uint32_t stride_shift = all.weights.stride_shift();
    int f_count = 0;
    for (features::iterator& f : ec->feature_space[i])
    {
      feature t = {f.value(), f.index()};
      t.weight_index >>= stride_shift;
      fs_ptr[fs_count].fs[f_count] = t;
      f_count++;
    }
    fs_count++;
  }
  return fs_ptr;
}

finish()

void VW::finish	(	vw &	all,
		bool	delete_all
	)

Definition at line 1823 of file parse_args.cc.

References vw::all_reduce, v_array< T >::clear(), io_buf::close_file_or_socket(), vw::current_pass, delete_dictionary_entry(), v_array< T >::delete_v(), shared_data::example_number, vw::final_prediction_sink, vw::final_regressor_name, finalize_regressor(), finalize_source(), LEARNER::learner< T, E >::finish(), free_it(), free_parser(), get_best_constant(), shared_data::holdout_best_loss, shared_data::holdout_multiclass_log_loss, vw::holdout_set_off, vw::l, shared_data::ldict, vw::loaded_dictionaries, vw::loss, shared_data::multiclass_log_loss, vw::options, vw::p, parser::parse_name, shared_data::queries, vw::quiet, shared_data::report_multiclass_log_loss, vw::sd, parameters::seeded(), vw::should_delete_options, v_array< T >::size(), shared_data::sum_loss, shared_data::total_features, vw::trace_message, VW::config::options_i::was_supplied(), shared_data::weighted_examples(), shared_data::weighted_labeled_examples, shared_data::weighted_labels, vw::weights, and namedlabels::~namedlabels().

Referenced by audit_regressor_setup(), enable_sources(), initialize(), main(), parse_args(), and VW_Finish().

{
  // also update VowpalWabbit::PerformanceStatistics::get() (vowpalwabbit.cpp)
  if (!all.quiet && !all.options->was_supplied("audit_regressor"))
  {
    all.trace_message.precision(6);
    all.trace_message << std::fixed;
    all.trace_message << endl << "finished run";
    if (all.current_pass == 0 || all.current_pass == 1)
      all.trace_message << endl << "number of examples = " << all.sd->example_number;
    else
    {
      all.trace_message << endl << "number of examples per pass = " << all.sd->example_number / all.current_pass;
      all.trace_message << endl << "passes used = " << all.current_pass;
    }
    all.trace_message << endl << "weighted example sum = " << all.sd->weighted_examples();
    all.trace_message << endl << "weighted label sum = " << all.sd->weighted_labels;
    all.trace_message << endl << "average loss = ";
    if (all.holdout_set_off)
      if (all.sd->weighted_labeled_examples > 0)
        all.trace_message << all.sd->sum_loss / all.sd->weighted_labeled_examples;
      else
        all.trace_message << "n.a.";
    else if ((all.sd->holdout_best_loss == FLT_MAX) || (all.sd->holdout_best_loss == FLT_MAX * 0.5))
      all.trace_message << "undefined (no holdout)";
    else
      all.trace_message << all.sd->holdout_best_loss << " h";
    if (all.sd->report_multiclass_log_loss)
    {
      if (all.holdout_set_off)
        all.trace_message << endl
                          << "average multiclass log loss = "
                          << all.sd->multiclass_log_loss / all.sd->weighted_labeled_examples;
      else
        all.trace_message << endl
                          << "average multiclass log loss = "
                          << all.sd->holdout_multiclass_log_loss / all.sd->weighted_labeled_examples << " h";
    }

    float best_constant;
    float best_constant_loss;
    if (get_best_constant(all, best_constant, best_constant_loss))
    {
      all.trace_message << endl << "best constant = " << best_constant;
      if (best_constant_loss != FLT_MIN)
        all.trace_message << endl << "best constant's loss = " << best_constant_loss;
    }

    all.trace_message << endl << "total feature number = " << all.sd->total_features;
    if (all.sd->queries > 0)
      all.trace_message << endl << "total queries = " << all.sd->queries;
    all.trace_message << endl;
  }

  // implement finally.
  // finalize_regressor can throw if it can't write the file.
  // we still want to free up all the memory.
  vw_exception finalize_regressor_exception(__FILE__, __LINE__, "empty");
  bool finalize_regressor_exception_thrown = false;
  try
  {
    finalize_regressor(all, all.final_regressor_name);
  }
  catch (vw_exception& e)
  {
    finalize_regressor_exception = e;
    finalize_regressor_exception_thrown = true;
  }

  if (all.l != nullptr)
  {
    all.l->finish();
    free_it(all.l);
  }

  // Check if options object lifetime is managed internally.
  if (all.should_delete_options)
    delete all.options;

  // TODO: migrate all finalization into parser destructor
  if (all.p != nullptr)
  {
    free_parser(all);
    finalize_source(all.p);
    all.p->parse_name.clear();
    all.p->parse_name.delete_v();
    delete all.p;
  }

  bool seeded;
  if (all.weights.seeded() > 0)
    seeded = true;
  else
    seeded = false;
  if (!seeded)
  {
    if (all.sd->ldict)
    {
      all.sd->ldict->~namedlabels();
      free(all.sd->ldict);
    }
    free(all.sd);
  }
  for (size_t i = 0; i < all.final_prediction_sink.size(); i++)
    if (all.final_prediction_sink[i] != 1)
      io_buf::close_file_or_socket(all.final_prediction_sink[i]);
  all.final_prediction_sink.delete_v();
  for (size_t i = 0; i < all.loaded_dictionaries.size(); i++)
  {
    // Warning C6001 is triggered by the following:
    // (a) dictionary_info.name is allocated using 'calloc_or_throw<char>(strlen(s)+1)' and (b) freed using
    // 'free(all.loaded_dictionaries[i].name)'
    //
    // When the call to allocation is replaced by (a) 'new char[strlen(s)+1]' and deallocated using (b) 'delete []', the
    // warning goes away. Disable SDL warning.
    //    #pragma warning(disable:6001)
    free_it(all.loaded_dictionaries[i].name);
    //#pragma warning(default:6001)

    all.loaded_dictionaries[i].dict->iter(delete_dictionary_entry);
    all.loaded_dictionaries[i].dict->delete_v();
    free_it(all.loaded_dictionaries[i].dict);
  }
  delete all.loss;

  delete all.all_reduce;

  if (delete_all)
    delete &all;

  if (finalize_regressor_exception_thrown)
    throw finalize_regressor_exception;
}

finish_example() [1/2]

void VW::finish_example	(	vw &	,
		example &
	)

Definition at line 881 of file parser.cc.

References clean_example(), is_ring_example(), parser::output_done, parser::output_lock, and vw::p.

Referenced by audit_regressor_setup(), LEARNER::multi_example_handler< context_type >::complete_multi_ex(), dealloc_example(), LEARNER::drain_examples(), LEARNER::end_pass(), CB_ALGS::eval_finish_example(), ezexample::finish(), CB_ALGS::finish_example(), MULTICLASS::finish_example(), finish_example(), MWT::finish_example(), finish_example(), CB_EXPLORE::finish_example(), COST_SENSITIVE::finish_example(), finish_example_scores(), VW::cb_explore_adf::cb_explore_adf_base< ExploreType >::finish_multiline_example(), EXPLORE_EVAL::finish_multiline_example(), finish_multiline_example(), CB_ADF::finish_multiline_example(), CCB::finish_multiline_example(), CSOAA::finish_multiline_example(), Search::finish_multiline_example(), lda_setup(), learn(), multilabel_oaa_setup(), return_active_example(), return_confidence_example(), return_example(), no_label::return_no_label_example(), return_simple_example(), LEARNER::save(), sender_setup(), topk_setup(), VW_FinishExample(), and ezexample::~ezexample().

{
  // only return examples to the pool that are from the pool and not externally allocated
  if (!is_ring_example(all, &ec))
    return;

  clean_example(all, ec, false);

  {
    std::lock_guard<std::mutex> lock(all.p->output_lock);
    all.p->output_done.notify_one();
  }
}

finish_example() [2/2]

void VW::finish_example	(	vw &	all,
		multi_ex &	ec_seq
	)

Definition at line 243 of file example.cc.

References finish_example().

{
  if (!ec_seq.empty())
    for (example* ecc : ec_seq)
      if (ecc->in_use)
        VW::finish_example(all, *ecc);
}

free_args()

void VW::free_args	(	int	argc,
		char *	argv[]
	)

Definition at line 1648 of file parse_args.cc.

Referenced by initialize(), and initialize_escaped().

{
  for (int i = 0; i < argc; i++) free(argv[i]);
  free(argv);
}

get_action_score()

float VW::get_action_score	(	example *	ec,
		size_t	i
	)

Definition at line 932 of file parser.cc.

References polyprediction::a_s, example::pred, and v_array< T >::size().

Referenced by VW_GetActionScore().

{
  ACTION_SCORE::action_scores scores = ec->pred.a_s;

  if (i < scores.size())
  {
    return scores[i].score;
  }
  else
  {
    return 0.0;
  }
}

get_action_score_length()

size_t VW::get_action_score_length

(

example *

ec

)

Definition at line 946 of file parser.cc.

References polyprediction::a_s, example::pred, and v_array< T >::size().

Referenced by VW_GetActionScoreLength().

{ return ec->pred.a_s.size(); }

get_argv_from_string()

char ** VW::get_argv_from_string	(	std::string	s,
		int &	argc
	)

Definition at line 1646 of file parse_args.cc.

References to_argv().

Referenced by main().

{ return to_argv(s, argc); }

get_confidence()

float VW::get_confidence

(

example *

ec

)

Definition at line 954 of file parser.cc.

References example::confidence.

Referenced by VW_GetConfidence().

{ return ec->confidence; }

get_cost_sensitive_prediction()

float VW::get_cost_sensitive_prediction

(

example *

ec

)

Definition at line 921 of file parser.cc.

References polyprediction::multiclass, and example::pred.

Referenced by VW_GetCostSensitivePrediction(), and VW_PredictCostSensitive().

{ return (float)ec->pred.multiclass; }

get_cost_sensitive_prediction_confidence_scores()

v_array< float > & VW::get_cost_sensitive_prediction_confidence_scores

(

example *

ec

)

Definition at line 923 of file parser.cc.

References example::pred, and polyprediction::scalars.

{ return ec->pred.scalars; }

get_example()

example * VW::get_example

(

parser *

p

)

Definition at line 909 of file parser.cc.

References VW::ptr_queue< T >::pop(), and parser::ready_parsed_examples.

Referenced by LEARNER::drain_examples(), LEARNER::ready_examples_queue::pop(), and VW_GetExample().

{ return p->ready_parsed_examples.pop(); }

get_feature_number()

size_t VW::get_feature_number

(

example *

ec

)

Definition at line 952 of file parser.cc.

References example::num_features.

Referenced by VW_GetFeatureNumber().

{ return ec->num_features; }

get_features()

feature * VW::get_features	(	vw &	all,
		example *	ec,
		size_t &	feature_map_len
	)

Definition at line 125 of file example.cc.

References v_array< T >::begin(), features_and_source::feature_map, features_and_source::mask, parameters::mask(), v_array< T >::size(), features_and_source::stride_shift, parameters::stride_shift(), and vw::weights.

Referenced by VW_GetFeatures().

{
  features_and_source fs;
  fs.stride_shift = all.weights.stride_shift();
  fs.mask = (uint64_t)all.weights.mask() >> all.weights.stride_shift();
  fs.feature_map = v_init<feature>();
  GD::foreach_feature<features_and_source, uint64_t, vec_store>(all, *ec, fs);

  feature_map_len = fs.feature_map.size();
  return fs.feature_map.begin();
}

get_importance()

float VW::get_importance

(

example *

ec

)

Definition at line 915 of file parser.cc.

References example::weight.

Referenced by VW_GetImportance().

{ return ec->weight; }

get_initial()

float VW::get_initial

(

example *

ec

)

Definition at line 917 of file parser.cc.

References label_data::initial, example::l, and polylabel::simple.

Referenced by VW_GetInitial().

{ return ec->l.simple.initial; }

get_label()

float VW::get_label

(

example *

ec

)

Definition at line 913 of file parser.cc.

References example::l, label_data::label, and polylabel::simple.

Referenced by VW_GetLabel().

{ return ec->l.simple.label; }

get_multilabel_predictions()

uint32_t * VW::get_multilabel_predictions	(	example *	ec,
		size_t &	len
	)

Definition at line 925 of file parser.cc.

References v_array< T >::begin(), MULTILABEL::labels::label_v, polyprediction::multilabels, example::pred, and v_array< T >::size().

Referenced by VW_GetMultilabelPredictions().

{
  MULTILABEL::labels labels = ec->pred.multilabels;
  len = labels.label_v.size();
  return labels.label_v.begin();
}

get_prediction()

float VW::get_prediction

(

example *

ec

)

Definition at line 919 of file parser.cc.

References example::pred, and polyprediction::scalar.

Referenced by receive_result(), VW_GetPrediction(), VW_Learn(), and VW_Predict().

{ return ec->pred.scalar; }

get_stride()

uint32_t VW::get_stride ( vw &  all )

inline

Definition at line 189 of file vw.h.

References parameters::stride(), and vw::weights.

Referenced by VW_Get_Stride().

{ return all.weights.stride(); }

get_tag()

const char * VW::get_tag

(

example *

ec

)

Definition at line 950 of file parser.cc.

References v_array< T >::begin(), and example::tag.

Referenced by VW_GetTag().

{ return ec->tag.begin(); }

get_tag_length()

size_t VW::get_tag_length

(

example *

ec

)

Definition at line 948 of file parser.cc.

References v_array< T >::size(), and example::tag.

Referenced by VW_GetTagLength().

{ return ec->tag.size(); }

get_topic_prediction()

float VW::get_topic_prediction	(	example *	ec,
		size_t	i
	)

Definition at line 911 of file parser.cc.

References example::pred, and polyprediction::scalars.

Referenced by VW_GetTopicPrediction().

{ return ec->pred.scalars[i]; }

get_unused_example()

example & VW::get_unused_example

(

vw *

all

)

Definition at line 664 of file parser.cc.

References parser::begin_parsed_examples, parser::example_pool, VW::object_pool< T, TInitializer, TCleanup >::get_object(), example::in_use, and vw::p.

Referenced by import_example(), line_to_examples_json(), new_unused_example(), parse_dispatch(), parse_line_json(), prepare_for_learner(), read_example(), and read_lines().

{
  parser* p = all->p;
  auto ex = p->example_pool.get_object();
  ex->in_use = true;
  p->begin_parsed_examples++;
  return *ex;
}

get_weight()

float VW::get_weight ( vw &  all, uint32_t  index, uint32_t  offset  )

inline

Definition at line 177 of file vw.h.

References parameters::stride_shift(), and vw::weights.

Referenced by SVRG::learn(), and VW_Get_Weight().

{
  return (&all.weights[((uint64_t)index) << all.weights.stride_shift()])[offset];
}

git_commit()

const std::string VW::git_commit

(

COMMIT_VERSION

)

Referenced by parse_diagnostics().

hash_feature()

uint64_t VW::hash_feature ( vw &  all, const std::string &  s, uint64_t  u  )

inline

Definition at line 153 of file vw.h.

References substring::begin, substring::end, parser::hasher, vw::p, and vw::parse_mask.

Referenced by Namespace< audit >::AddFeature(), DefaultState< audit >::Float(), ezexample::hash(), CCB::inject_slot_id(), and VW_HashFeatureA().

{
  substring ss;
  ss.begin = (char*)s.c_str();
  ss.end = ss.begin + s.length();
  return all.p->hasher(ss, u) & all.parse_mask;
}

hash_feature_cstr()

uint64_t VW::hash_feature_cstr ( vw &  all, char *  fstr, uint64_t  u  )

inline

Definition at line 169 of file vw.h.

References substring::begin, substring::end, parser::hasher, vw::p, and vw::parse_mask.

Referenced by ezexample::hash().

{
  substring ss;
  ss.begin = fstr;
  ss.end = ss.begin + strlen(fstr);
  return all.p->hasher(ss, u) & all.parse_mask;
}

hash_feature_static()

uint64_t VW::hash_feature_static ( const std::string &  s, uint64_t  u, const std::string &  h, uint32_t  num_bits  )

inline

Definition at line 160 of file vw.h.

References substring::begin, substring::end, and getHasher().

Referenced by VW_HashFeatureStaticA().

{
  substring ss;
  ss.begin = (char*)s.c_str();
  ss.end = ss.begin + s.length();
  size_t parse_mark = (1 << num_bits) - 1;
  return getHasher(h)(ss, u) & parse_mark;
}

hash_space()

uint64_t VW::hash_space ( vw &  all, const std::string &  s  )

inline

Definition at line 138 of file vw.h.

References substring::begin, substring::end, vw::hash_seed, parser::hasher, and vw::p.

Referenced by ezexample::addns(), CCB::ccb_explore_adf_setup(), ezexample::ezexample(), ezexample::hash(), Context< audit >::PushNamespace(), and VW_HashSpaceA().

{
  substring ss;
  ss.begin = (char*)s.c_str();
  ss.end = ss.begin + s.length();
  return all.p->hasher(ss, all.hash_seed);
}

hash_space_static()

uint64_t VW::hash_space_static ( const std::string &  s, const std::string &  hash  )

inline

Definition at line 145 of file vw.h.

References substring::begin, substring::end, and getHasher().

Referenced by VW_HashSpaceStaticA().

{
  substring ss;
  ss.begin = (char*)s.c_str();
  ss.end = ss.begin + s.length();
  return getHasher(hash)(ss, 0);
}

import_example()

example * VW::import_example	(	vw &	all,
		const std::string &	label,
		primitive_feature_space *	features,
		size_t	len
	)

Definition at line 791 of file parser.cc.

References label_parser::default_label, parser::end_parsed_examples, example_predict::feature_space, VW::primitive_feature_space::fs, get_unused_example(), example_predict::indices, example::l, VW::primitive_feature_space::len, parser::lp, VW::primitive_feature_space::name, vw::p, parse_example_label(), v_array< T >::push_back(), setup_example(), feature::weight_index, and feature::x.

Referenced by VW_ImportExample().

{
  example* ret = &get_unused_example(&all);
  all.p->lp.default_label(&ret->l);

  if (label.length() > 0)
    parse_example_label(all, *ret, label);

  for (size_t i = 0; i < len; i++)
  {
    unsigned char index = features[i].name;
    ret->indices.push_back(index);
    for (size_t j = 0; j < features[i].len; j++)
      ret->feature_space[index].push_back(features[i].fs[j].x, features[i].fs[j].weight_index);
  }

  setup_example(all, ret);
  all.p->end_parsed_examples++;
  return ret;
}

init_features()

void VW::init_features ( primitive_feature_space &  fs, size_t  features_count  )

inline

Definition at line 191 of file vw.h.

References VW::primitive_feature_space::fs, and VW::primitive_feature_space::len.

Referenced by VW_InitFeatures().

{
  fs.fs = new feature[features_count];
  fs.len = features_count;
}

initialize() [1/3]

vw * VW::initialize	(	options_i &	options,
		io_buf *	model,
		bool	skipModelLoad,
		trace_message_t	trace_listener,
		void *	trace_context
	)

Definition at line 1654 of file parse_args.cc.

References VW::config::options_i::check_unregistered(), finish(), VW::config::options_i::get_typed_option(), VW::config::options_i::help(), LEARNER::learner< T, E >::init_driver(), vw::initial_regressors, vw::l, load_header_merge_options(), parse_args(), parse_dictionary_argument(), parse_modules(), parse_sources(), vw::per_feature_regularizer_input, read_regressor_file(), vw::trace_message, and VW::config::options_i::was_supplied().

Referenced by initialize(), initialize_escaped(), seed_vw_model(), setup(), VW_InitializeA(), and VW_InitializeWithModel().

{
  vw& all = parse_args(options, trace_listener, trace_context);

  try
  {
    // if user doesn't pass in a model, read from options
    io_buf localModel;
    if (!model)
    {
      std::vector<std::string> all_initial_regressor_files(all.initial_regressors);
      if (options.was_supplied("input_feature_regularizer"))
      {
        all_initial_regressor_files.push_back(all.per_feature_regularizer_input);
      }
      read_regressor_file(all, all_initial_regressor_files, localModel);
      model = &localModel;
    }

    // Loads header of model files and loads the command line options into the options object.
    load_header_merge_options(options, all, *model);

    std::vector<std::string> dictionary_nses;
    parse_modules(options, all, dictionary_nses);

    parse_sources(options, all, *model, skipModelLoad);

    // we must delay so parse_mask is fully defined.
    for (size_t id = 0; id < dictionary_nses.size(); id++) parse_dictionary_argument(all, dictionary_nses[id]);

    options.check_unregistered();

    // upon direct query for help -- spit it out to stdout;
    if (options.get_typed_option<bool>("help").value())
    {
      cout << options.help();
      exit(0);
    }

    all.l->init_driver();

    return &all;
  }
  catch (std::exception& e)
  {
    all.trace_message << "Error: " << e.what() << endl;
    finish(all);
    throw;
  }
  catch (...)
  {
    finish(all);
    throw;
  }
}

initialize() [2/3]

vw * VW::initialize	(	std::string	s,
		io_buf *	model,
		bool	skipModelLoad,
		trace_message_t	trace_listener,
		void *	trace_context
	)

Definition at line 1711 of file parse_args.cc.

References free_args(), initialize(), and to_argv().

{
  int argc = 0;
  char** argv = to_argv(s, argc);
  vw* ret = nullptr;

  try
  {
    ret = initialize(argc, argv, model, skipModelLoad, trace_listener, trace_context);
  }
  catch (...)
  {
    free_args(argc, argv);
    throw;
  }

  free_args(argc, argv);
  return ret;
}

initialize() [3/3]

vw * VW::initialize	(	int	argc,
		char *	argv[],
		io_buf *	model,
		bool	skipModelLoad,
		trace_message_t	trace_listener,
		void *	trace_context
	)

Definition at line 1752 of file parse_args.cc.

References initialize(), and vw::should_delete_options.

{
  options_i* options = new config::options_boost_po(argc, argv);
  vw* all = initialize(*options, model, skipModelLoad, trace_listener, trace_context);

  // When VW is deleted the options object will be cleaned up too.
  all->should_delete_options = true;
  return all;
}

initialize_escaped()

vw * VW::initialize_escaped	(	std::string const &	s,
		io_buf *	model,
		bool	skipModelLoad,
		trace_message_t	trace_listener,
		void *	trace_context
	)

Definition at line 1731 of file parse_args.cc.

References free_args(), initialize(), and to_argv_escaped().

Referenced by VW_InitializeEscapedA(), and VW_InitializeWithModelEscaped().

{
  int argc = 0;
  char** argv = to_argv_escaped(s, argc);
  vw* ret = nullptr;

  try
  {
    ret = initialize(argc, argv, model, skipModelLoad, trace_listener, trace_context);
  }
  catch (...)
  {
    free_args(argc, argv);
    throw;
  }

  free_args(argc, argv);
  return ret;
}

is_ring_example()

bool VW::is_ring_example	(	vw &	all,
		example *	ae
	)

Definition at line 1009 of file parser.cc.

References parser::example_pool, VW::object_pool< T, TInitializer, TCleanup >::is_from_pool(), and vw::p.

Referenced by finish_example(), and ezexample::~ezexample().

{ return all.p->example_pool.is_from_pool(ae); }

move_feature_namespace()

void VW::move_feature_namespace	(	example *	dst,
		example *	src,
		namespace_index	c
	)

Definition at line 92 of file example.cc.

References v_array< T >::begin(), c, v_array< T >::end(), example_predict::feature_space, recall_tree_ns::find(), example_predict::indices, example::num_features, v_array< T >::push_back(), and example::total_sum_feat_sq.

Referenced by predict_or_learn().

{
  if (std::find(src->indices.begin(), src->indices.end(), c) == src->indices.end())
    return;  // index not present in src
  if (std::find(dst->indices.begin(), dst->indices.end(), c) == dst->indices.end())
    dst->indices.push_back(c);

  auto& fdst = dst->feature_space[c];
  auto& fsrc = src->feature_space[c];

  src->num_features -= fsrc.size();
  src->total_sum_feat_sq -= fsrc.sum_feat_sq;
  std::swap(fdst, fsrc);
  dst->num_features += fdst.size();
  dst->total_sum_feat_sq += fdst.sum_feat_sq;
}

new_unused_example()

example * VW::new_unused_example

(

vw &

all

)

Definition at line 753 of file parser.cc.

References parser::begin_parsed_examples, label_parser::default_label, example::example_counter, get_unused_example(), example::l, parser::lp, and vw::p.

Referenced by ezexample::get_new_example().

{
  example* ec = &get_unused_example(&all);
  all.p->lp.default_label(&ec->l);
  all.p->begin_parsed_examples++;
  ec->example_counter = (size_t)all.p->begin_parsed_examples;
  return ec;
}

num_weights()

uint32_t VW::num_weights ( vw &  all )

inline

Definition at line 187 of file vw.h.

References vw::length().

Referenced by SVRG::learn(), vw_slim::vw_predict< W >::load(), and VW_Num_Weights().

{ return (uint32_t)all.length(); }

parse_example_label()

void VW::parse_example_label	(	vw &	all,
		example &	ec,
		std::string	label
	)

Definition at line 846 of file parser.cc.

References v_array< T >::clear(), v_array< T >::delete_v(), example::l, parser::lp, vw::p, label_parser::parse_label, vw::sd, and tokenize().

Referenced by import_example(), ezexample::set_label(), LabelState< audit >::String(), and VW_AddStringLabel().

{
  v_array<substring> words = v_init<substring>();
  char* cstr = (char*)label.c_str();
  substring str = {cstr, cstr + label.length()};
  tokenize(' ', str, words);
  all.p->lp.parse_label(all.p, all.sd, &ec.l, words);
  words.clear();
  words.delete_v();
}

read_example() [1/2]

example * VW::read_example	(	vw &	all,
		char *	example_line
	)

Definition at line 761 of file parser.cc.

References parser::end_parsed_examples, get_unused_example(), vw::p, read_line(), and setup_example().

Referenced by ezexample::finish(), read_example(), ezexample::setup_for_predict(), and VW_ReadExampleA().

{
  example* ret = &get_unused_example(&all);

  VW::read_line(all, ret, example_line);
  setup_example(all, ret);
  all.p->end_parsed_examples++;

  return ret;
}

read_example() [2/2]

example * VW::read_example	(	vw &	all,
		std::string	example_line
	)

Definition at line 772 of file parser.cc.

References read_example().

{ return read_example(all, (char*)example_line.c_str()); }

read_line()

void VW::read_line	(	vw &	all,
		example *	ex,
		char *	line
	)

Definition at line 487 of file parse_example.cc.

References substring::begin, substring::end, and substring_to_example().

Referenced by parse_dictionary_argument(), read_example(), and read_lines().

{
  substring ss = {line, line + strlen(line)};
  while ((ss.end >= ss.begin) && (*(ss.end - 1) == '\n')) ss.end--;
  substring_to_example(&all, ex, ss);
}

read_line_decision_service_json()

template<bool audit>

void VW::read_line_decision_service_json	(	vw &	all,
		v_array< example *> &	examples,
		char *	line,
		size_t	length,
		bool	copy_line,
		example_factory_t	example_factory,
		void *	ex_factory_context,
		DecisionServiceInteraction *	data
	)

Definition at line 1393 of file parse_example_json.h.

References apply_pdrop(), VWReaderHandler< audit >::ctx, VWReaderHandler< audit >::current_state(), VWReaderHandler< audit >::error(), json_parser< audit >::handler, VWReaderHandler< audit >::init(), BaseState< audit >::name, DecisionServiceInteraction::probabilityOfDrop, json_parser< audit >::reader, and THROW.

{
  std::vector<char> line_vec;
  if (copy_line)
  {
    line_vec.insert(line_vec.end(), line, line + length);
    line = &line_vec.front();
  }

  InsituStringStream ss(line);
  json_parser<audit> parser;

  VWReaderHandler<audit>& handler = parser.handler;
  handler.init(&all, &examples, &ss, line + length, example_factory, ex_factory_context);
  handler.ctx.SetStartStateToDecisionService(data);

  ParseResult result =
      parser.reader.template Parse<kParseInsituFlag, InsituStringStream, VWReaderHandler<audit>>(ss, handler);

  apply_pdrop(all, data->probabilityOfDrop, examples);

  if (!result.IsError())
    return;

  BaseState<audit>* current_state = handler.current_state();

  THROW("JSON parser error at " << result.Offset() << ": " << GetParseError_En(result.Code())
                                << ". "
                                   "Handler: "
                                << handler.error().str()
                                << "State: " << (current_state ? current_state->name : "null"));
}  // namespace VW

read_line_json()

template<bool audit>

void VW::read_line_json	(	vw &	all,
		v_array< example *> &	examples,
		char *	line,
		example_factory_t	example_factory,
		void *	ex_factory_context
	)

Definition at line 1348 of file parse_example_json.h.

References VWReaderHandler< audit >::current_state(), VWReaderHandler< audit >::error(), json_parser< audit >::handler, VWReaderHandler< audit >::init(), BaseState< audit >::name, json_parser< audit >::reader, and THROW.

{
  // string line_copy(line);
  // destructive parsing
  InsituStringStream ss(line);
  json_parser<audit> parser;

  VWReaderHandler<audit>& handler = parser.handler;
  handler.init(&all, &examples, &ss, line + strlen(line), example_factory, ex_factory_context);

  ParseResult result =
      parser.reader.template Parse<kParseInsituFlag, InsituStringStream, VWReaderHandler<audit>>(ss, handler);
  if (!result.IsError())
    return;

  BaseState<audit>* current_state = handler.current_state();

  THROW("JSON parser error at " << result.Offset() << ": " << GetParseError_En(result.Code())
                                << ". "
                                   "Handler: "
                                << handler.error().str()
                                << "State: " << (current_state ? current_state->name : "null"));  // <<
  // "Line: '"<< line_copy << "'");
}

read_lines()

void VW::read_lines	(	vw *	all,
		char *	line,
		size_t	,
		v_array< example *> &	examples
	)

Definition at line 494 of file parse_example.cc.

References get_unused_example(), v_array< T >::push_back(), read_line(), v_array< T >::size(), and split().

Referenced by enable_sources().

{
  auto lines = split(line, "\n");
  for (size_t i = 0; i < lines.size(); i++)
  {
    // Check if a new empty example needs to be added.
    if (examples.size() < i + 1)
    {
      examples.push_back(&VW::get_unused_example(all));
    }
    read_line(*all, examples[i], const_cast<char*>(lines[i].c_str()));
  }
}

releaseFeatureSpace()

void VW::releaseFeatureSpace	(	primitive_feature_space *	features,
		size_t	len
	)

Definition at line 840 of file parser.cc.

Referenced by VW_ReleaseFeatureSpace().

{
  for (size_t i = 0; i < len; i++) delete[] features[i].fs;
  delete (features);
}

return_features()

void VW::return_features

(

feature *

f

)

Definition at line 137 of file example.cc.

References free_it().

Referenced by VW_ReturnFeatures().

{ free_it(f); }

return_multiple_example()

void VW::return_multiple_example	(	vw &	all,
		v_array< example *> &	examples
	)

Definition at line 251 of file example.cc.

References clean_example().

Referenced by line_to_examples_json(), and parse_line_json().

{
  for (auto ec : examples)
  {
    clean_example(all, *ec, true);
  }
  examples.clear();
}

save_predictor() [1/2]

void VW::save_predictor	(	vw &	all,
		std::string	reg_name
	)

Definition at line 642 of file parse_regressor.cc.

References dump_regressor().

Referenced by predict_or_learn(), VW_CopyModelData(), and VW_SaveModel().

{ dump_regressor(all, reg_name, false); }

save_predictor() [2/2]

void VW::save_predictor	(	vw &	all,
		io_buf &	buf
	)

Definition at line 644 of file parse_regressor.cc.

References dump_regressor().

{ dump_regressor(all, buf, false); }

seed_vw_model()

vw * VW::seed_vw_model	(	vw *	vw_model,
		const std::string	extra_args,
		trace_message_t	trace_listener,
		void *	trace_context
	)

Definition at line 1765 of file parse_args.cc.

References VW::config::options_serializer_boost_po::add(), free_it(), VW::config::options_i::get_all_options(), initialize(), vw::options, vw::sd, parameters::shallow_copy(), VW::config::options_serializer_boost_po::str(), VW::config::options_i::was_supplied(), and vw::weights.

Referenced by VW_SeedWithModel().

{
  options_serializer_boost_po serializer;
  for (auto const& option : vw_model->options->get_all_options())
  {
    if (vw_model->options->was_supplied(option->m_name))
    {
      // ignore no_stdin since it will be added by vw::initialize, and ignore -i since we don't want to reload the
      // model.
      if (option->m_name == "no_stdin" || option->m_name == "initial_regressor")
      {
        continue;
      }

      serializer.add(*option);
    }
  }

  auto serialized_options = serializer.str();
  serialized_options = serialized_options + " " + extra_args;

  vw* new_model =
      VW::initialize(serialized_options.c_str(), nullptr, true /* skipModelLoad */, trace_listener, trace_context);
  free_it(new_model->sd);

  // reference model states stored in the specified VW instance
  new_model->weights.shallow_copy(vw_model->weights);  // regressor
  new_model->sd = vw_model->sd;                        // shared data

  return new_model;
}

set_feature()

void VW::set_feature ( primitive_feature_space &  fs, size_t  index, uint64_t  feature_hash, float  value  )

inline

Definition at line 197 of file vw.h.

References VW::primitive_feature_space::fs, feature::weight_index, and feature::x.

Referenced by VW_SetFeature().

{
  fs.fs[index].weight_index = feature_hash;
  fs.fs[index].x = value;
}

set_weight()

void VW::set_weight ( vw &  all, uint32_t  index, uint32_t  offset, float  value  )

inline

Definition at line 182 of file vw.h.

References parameters::stride_shift(), and vw::weights.

Referenced by SVRG::learn(), GD::save_load(), and VW_Set_Weight().

{
  (&all.weights[((uint64_t)index) << all.weights.stride_shift()])[offset] = value;
}

setup_example()

void VW::setup_example	(	vw &	all,
		example *	ae
	)

Definition at line 678 of file parser.cc.

References vw::add_constant, add_constant_feature(), v_array< T >::begin(), cache_features(), label_parser::cache_label, parser::emptylines_separate_examples, v_array< T >::end(), parser::end_parsed_examples, INTERACTIONS::eval_count_of_generated_ft(), example::example_counter, example_is_newline(), feature_limit(), example_predict::feature_space, generateGrams(), label_parser::get_weight, vw::holdout_after, vw::holdout_period, vw::holdout_set_off, vw::ignore, vw::ignore_some, parser::in_pass_counter, example_predict::indices, vw::interactions, is_test_only(), example::l, vw::limit_strings, example::loss, parser::lp, vw::ngram_strings, example::num_features, parser::output, vw::p, vw::parse_mask, example::partial_prediction, parser::sort_features, example::sorted, parameters::stride_shift(), label_parser::test_label, example::test_only, example::total_sum_feat_sq, unique_sort_features(), example::weight, vw::weights, vw::wpp, and parser::write_cache.

Referenced by import_example(), read_example(), and setup_examples().

{
  if (all.p->sort_features && ae->sorted == false)
    unique_sort_features(all.parse_mask, ae);

  if (all.p->write_cache)
  {
    all.p->lp.cache_label(&ae->l, *(all.p->output));
    cache_features(*(all.p->output), ae, all.parse_mask);
  }

  ae->partial_prediction = 0.;
  ae->num_features = 0;
  ae->total_sum_feat_sq = 0;
  ae->loss = 0.;

  ae->example_counter = (size_t)(all.p->end_parsed_examples);
  if (!all.p->emptylines_separate_examples)
    all.p->in_pass_counter++;

  ae->test_only = is_test_only(all.p->in_pass_counter, all.holdout_period, all.holdout_after, all.holdout_set_off,
      all.p->emptylines_separate_examples ? (all.holdout_period - 1) : 0);
  ae->test_only |= all.p->lp.test_label(&ae->l);

  if (all.p->emptylines_separate_examples && example_is_newline(*ae))
    all.p->in_pass_counter++;

  ae->weight = all.p->lp.get_weight(&ae->l);

  if (all.ignore_some)
    for (unsigned char* i = ae->indices.begin(); i != ae->indices.end(); i++)
      if (all.ignore[*i])
      {
        // delete namespace
        ae->feature_space[*i].clear();
        memmove(i, i + 1, (ae->indices.end() - (i + 1)) * sizeof(*i));
        ae->indices.end()--;
        i--;
      }

  if (!all.ngram_strings.empty())
    generateGrams(all, ae);

  if (all.add_constant)  // add constant feature
    VW::add_constant_feature(all, ae);

  if (!all.limit_strings.empty())
    feature_limit(all, ae);

  uint64_t multiplier = (uint64_t)all.wpp << all.weights.stride_shift();

  if (multiplier != 1)  // make room for per-feature information.
    for (features& fs : *ae)
      for (auto& j : fs.indicies) j *= multiplier;
  ae->num_features = 0;
  ae->total_sum_feat_sq = 0;
  for (features& fs : *ae)
  {
    ae->num_features += fs.size();
    ae->total_sum_feat_sq += fs.sum_feat_sq;
  }

  // Set the interactions for this example to the global set.
  ae->interactions = &all.interactions;

  size_t new_features_cnt;
  float new_features_sum_feat_sq;
  INTERACTIONS::eval_count_of_generated_ft(all, *ae, new_features_cnt, new_features_sum_feat_sq);
  ae->num_features += new_features_cnt;
  ae->total_sum_feat_sq += new_features_sum_feat_sq;
}

setup_examples()

void VW::setup_examples	(	vw &	all,
		v_array< example *> &	examples
	)

Definition at line 673 of file parser.cc.

References setup_example().

Referenced by parse_dispatch().

{
  for (example* ae : examples) setup_example(all, ae);
}

start_parser()

void VW::start_parser

(

vw &

all

)

Definition at line 974 of file parser.cc.

References main_parse_loop(), and vw::parse_thread.

Referenced by main(), VW_Finish_Passes(), and VW_StartParser().

{ all.parse_thread = std::thread(main_parse_loop, &all); }

sync_stats()

void VW::sync_stats

(

vw &

all

)

Definition at line 1804 of file parse_args.cc.

References accumulate_scalar(), vw::all_reduce, shared_data::example_number, loss(), vw::sd, shared_data::sum_loss, shared_data::total_features, shared_data::weighted_labeled_examples, shared_data::weighted_labels, and shared_data::weighted_unlabeled_examples.

Referenced by main().

{
  if (all.all_reduce != nullptr)
  {
    float loss = (float)all.sd->sum_loss;
    all.sd->sum_loss = (double)accumulate_scalar(all, loss);
    float weighted_labeled_examples = (float)all.sd->weighted_labeled_examples;
    all.sd->weighted_labeled_examples = (double)accumulate_scalar(all, weighted_labeled_examples);
    float weighted_labels = (float)all.sd->weighted_labels;
    all.sd->weighted_labels = (double)accumulate_scalar(all, weighted_labels);
    float weighted_unlabeled_examples = (float)all.sd->weighted_unlabeled_examples;
    all.sd->weighted_unlabeled_examples = (double)accumulate_scalar(all, weighted_unlabeled_examples);
    float example_number = (float)all.sd->example_number;
    all.sd->example_number = (uint64_t)accumulate_scalar(all, example_number);
    float total_features = (float)all.sd->total_features;
    all.sd->total_features = (uint64_t)accumulate_scalar(all, total_features);
  }
}

to_argv()

char ** VW::to_argv	(	std::string const &	s,
		int &	argc
	)

Definition at line 1623 of file parse_args.cc.

References c, and tokenize().

Referenced by get_argv_from_string(), and initialize().

{
  char* c = calloc_or_throw<char>(s.length() + 3);
  c[0] = 'b';
  c[1] = ' ';
  strcpy(c + 2, s.c_str());
  substring ss = {c, c + s.length() + 2};
  std::vector<substring> foo;
  tokenize(' ', ss, foo);

  char** argv = calloc_or_throw<char*>(foo.size());
  for (size_t i = 0; i < foo.size(); i++)
  {
    *(foo[i].end) = '\0';
    argv[i] = calloc_or_throw<char>(foo[i].end - foo[i].begin + 1);
    sprintf(argv[i], "%s", foo[i].begin);
  }

  argc = (int)foo.size();
  free(c);
  return argv;
}

to_argv_escaped()

char ** VW::to_argv_escaped	(	std::string const &	s,
		int &	argc
	)

Definition at line 1604 of file parse_args.cc.

References escaped_tokenize().

Referenced by initialize_escaped().

{
  substring ss = {const_cast<char*>(s.c_str()), const_cast<char*>(s.c_str() + s.length())};
  std::vector<substring> tokens = escaped_tokenize(' ', ss);
  char** argv = calloc_or_throw<char*>(tokens.size() + 1);
  argv[0] = calloc_or_throw<char>(2);
  argv[0][0] = 'b';
  argv[0][1] = '\0';

  for (size_t i = 0; i < tokens.size(); i++)
  {
    argv[i + 1] = calloc_or_throw<char>(tokens[i].end - tokens[i].begin + 1);
    sprintf(argv[i + 1], "%s", tokens[i].begin);
  }

  argc = static_cast<int>(tokens.size() + 1);
  return argv;
}

validate_default_bits()

void VW::validate_default_bits	(	vw &	all,
		uint32_t	local_num_bits
	)

Definition at line 26 of file vw_validate.cc.

References vw::default_bits, vw::num_bits, and THROW.

Referenced by save_load_header().

{
  if (all.default_bits != true && all.num_bits != local_num_bits)
    THROW("-b bits mismatch: command-line " << all.num_bits << " != " << local_num_bits << " stored in model");
}

validate_min_max_label()

void VW::validate_min_max_label

(

vw &

all

)

Definition at line 20 of file vw_validate.cc.

References shared_data::max_label, shared_data::min_label, vw::sd, and THROW.

{
  if (all.sd->max_label < all.sd->min_label)
    THROW("Max label cannot be less than min label.");
}

validate_num_bits()

void VW::validate_num_bits

(

vw &

all

)

Definition at line 32 of file vw_validate.cc.

References vw::num_bits, and THROW.

Referenced by parse_feature_tweaks(), and save_load_header().

{
  if (all.num_bits > sizeof(size_t) * 8 - 3)
    THROW("Only " << sizeof(size_t) * 8 - 3 << " or fewer bits allowed.  If this is a serious limit, speak up.");
}

validate_version()

void VW::validate_version

(

vw &

all

)

Definition at line 12 of file vw_validate.cc.

References LAST_COMPATIBLE_VERSION, vw::model_file_ver, THROW, and VW::version_struct::to_string().

Referenced by save_load_header().

{
  if (all.model_file_ver < LAST_COMPATIBLE_VERSION)
    THROW("Model has possibly incompatible version! " << all.model_file_ver.to_string());
  if (all.model_file_ver > PACKAGE_VERSION)
    std::cerr << "Warning: model version is more recent than VW version.  This may not work." << std::endl;
}

version()

const version_struct VW::version

(

PACKAGE_VERSION

)

Referenced by cache_numbits(), Program::Main(), make_write_cache(), parse_diagnostics(), and save_load_header().