CB_EXPLORE Namespace Reference

Classes
struct	cb_explore

Functions
template<bool is_learn>
void	predict_or_learn_first (cb_explore &data, single_learner &base, example &ec)
template<bool is_learn>
void	predict_or_learn_greedy (cb_explore &data, single_learner &base, example &ec)
template<bool is_learn>
void	predict_or_learn_bag (cb_explore &data, single_learner &base, example &ec)
void	get_cover_probabilities (cb_explore &data, single_learner &, example &ec, v_array< action_score > &probs)
template<bool is_learn>
void	predict_or_learn_cover (cb_explore &data, single_learner &base, example &ec)
void	print_update_cb_explore (vw &all, bool is_test, example &ec, std::stringstream &pred_string)
void	output_example (vw &all, cb_explore &data, example &ec, CB::label &ld)
void	finish_example (vw &all, cb_explore &c, example &ec)

Function Documentation

finish_example()

void CB_EXPLORE::finish_example	(	vw &	all,
		cb_explore &	c,
		example &	ec
	)

Definition at line 266 of file cb_explore.cc.

References polylabel::cb, VW::finish_example(), example::l, and output_example().

Referenced by cb_explore_setup().

{
  output_example(all, c, ec, ec.l.cb);
  VW::finish_example(all, ec);
}

get_cover_probabilities()

void CB_EXPLORE::get_cover_probabilities	(	cb_explore &	data,
		single_learner &	,
		example &	ec,
		v_array< action_score > &	probs
	)

Definition at line 125 of file cb_explore.cc.

References ACTION_SCORE::begin_scores(), CB_EXPLORE::cb_explore::cbcs, v_array< T >::clear(), CB_EXPLORE::cb_explore::counter, CB_EXPLORE::cb_explore::cover_size, CB_EXPLORE::cb_explore::cs, ACTION_SCORE::end_scores(), exploration::enforce_minimum_probability(), f, polyprediction::multiclass, GEN_CS::cb_to_cs::num_actions, example::pred, CB_EXPLORE::cb_explore::preds, and v_array< T >::push_back().

Referenced by predict_or_learn_cover().

{
  float additive_probability = 1.f / (float)data.cover_size;
  data.preds.clear();

  for (uint32_t i = 0; i < data.cbcs.num_actions; i++) probs.push_back({i, 0.});

  for (size_t i = 0; i < data.cover_size; i++)
  {
    // get predicted cost-sensitive predictions
    if (i == 0)
      data.cs->predict(ec, i);
    else
      data.cs->predict(ec, i + 1);
    uint32_t pred = ec.pred.multiclass;
    probs[pred - 1].score += additive_probability;
    data.preds.push_back((uint32_t)pred);
  }
  uint32_t num_actions = data.cbcs.num_actions;

  float min_prob = std::min(1.f / num_actions, 1.f / (float)std::sqrt(data.counter * num_actions));

  enforce_minimum_probability(min_prob * num_actions, false, begin_scores(probs), end_scores(probs));

  data.counter++;
}

output_example()

void CB_EXPLORE::output_example	(	vw &	all,
		cb_explore &	data,
		example &	ec,
		CB::label &	ld
	)

Definition at line 235 of file cb_explore.cc.

References polyprediction::a_s, c, CB::cb_label, CB_EXPLORE::cb_explore::cbcs, f, vw::final_prediction_sink, CB_ALGS::get_cost_estimate(), CB_ADF::get_observed_cost(), GEN_CS::cb_to_cs::known_cost, loss(), example::num_features, example::pred, GEN_CS::cb_to_cs::pred_scores, vw::print_text, print_update_cb_explore(), vw::sd, v_array< T >::size(), example::tag, test_label(), example::test_only, and shared_data::update().

Referenced by finish_example().

{
  float loss = 0.;

  cb_to_cs& c = data.cbcs;

  if ((c.known_cost = get_observed_cost(ld)) != nullptr)
    for (uint32_t i = 0; i < ec.pred.a_s.size(); i++)
      loss += get_cost_estimate(c.known_cost, c.pred_scores, i + 1) * ec.pred.a_s[i].score;

  all.sd->update(ec.test_only, get_observed_cost(ld) != nullptr, loss, 1.f, ec.num_features);

  std::stringstream ss;
  float maxprob = 0.;
  uint32_t maxid = 0;
  for (uint32_t i = 0; i < ec.pred.a_s.size(); i++)
  {
    ss << std::fixed << ec.pred.a_s[i].score << " ";
    if (ec.pred.a_s[i].score > maxprob)
    {
      maxprob = ec.pred.a_s[i].score;
      maxid = i + 1;
    }
  }
  for (int sink : all.final_prediction_sink) all.print_text(sink, ss.str(), ec.tag);

  std::stringstream sso;
  sso << maxid << ":" << std::fixed << maxprob;
  print_update_cb_explore(all, CB::cb_label.test_label(&ld), ec, sso);
}

predict_or_learn_bag()

template<bool is_learn>

void CB_EXPLORE::predict_or_learn_bag	(	cb_explore &	data,
		single_learner &	base,
		example &	ec
	)

Definition at line 101 of file cb_explore.cc.

References CB_EXPLORE::cb_explore::_random_state, polyprediction::a_s, CB_EXPLORE::cb_explore::bag_size, CB_EXPLORE::cb_explore::cbcs, v_array< T >::clear(), LEARNER::learner< T, E >::learn(), polyprediction::multiclass, GEN_CS::cb_to_cs::num_actions, example::pred, LEARNER::learner< T, E >::predict(), prediction_type::prob, v_array< T >::push_back(), and BS::weight_gen().

{
  // Randomize over predictions from a base set of predictors
  action_scores probs = ec.pred.a_s;
  probs.clear();

  for (uint32_t i = 0; i < data.cbcs.num_actions; i++) probs.push_back({i, 0.});
  float prob = 1.f / (float)data.bag_size;
  for (size_t i = 0; i < data.bag_size; i++)
  {
    uint32_t count = BS::weight_gen(data._random_state);
    if (is_learn && count > 0)
      base.learn(ec, i);
    else
      base.predict(ec, i);
    uint32_t chosen = ec.pred.multiclass - 1;
    probs[chosen].score += prob;
    if (is_learn)
      for (uint32_t j = 1; j < count; j++) base.learn(ec, i);
  }

  ec.pred.a_s = probs;
}

predict_or_learn_cover()

template<bool is_learn>

void CB_EXPLORE::predict_or_learn_cover	(	cb_explore &	data,
		single_learner &	base,
		example &	ec
	)

Definition at line 153 of file cb_explore.cc.

References polyprediction::a_s, polylabel::cb, CB_EXPLORE::cb_explore::cb_label, CB_EXPLORE::cb_explore::cbcs, v_array< T >::clear(), COST_SENSITIVE::label::costs, CB_EXPLORE::cb_explore::counter, CB_EXPLORE::cb_explore::cover_probs, CB_EXPLORE::cb_explore::cover_size, polylabel::cs, CB_EXPLORE::cb_explore::cs, CB_EXPLORE::cb_explore::cs_label, f, get_cover_probabilities(), CB_ADF::get_observed_cost(), GEN_CS::cb_to_cs::known_cost, example::l, LEARNER::learner< T, E >::learn(), GEN_CS::cb_to_cs::num_actions, example::pred, CB_EXPLORE::cb_explore::preds, CB_EXPLORE::cb_explore::psi, and CB_EXPLORE::cb_explore::second_cs_label.

{
  // Randomize over predictions from a base set of predictors
  // Use cost sensitive oracle to cover actions to form distribution.

  uint32_t num_actions = data.cbcs.num_actions;

  action_scores probs = ec.pred.a_s;
  probs.clear();
  data.cs_label.costs.clear();

  for (uint32_t j = 0; j < num_actions; j++) data.cs_label.costs.push_back({FLT_MAX, j + 1, 0., 0.});

  size_t cover_size = data.cover_size;
  size_t counter = data.counter;
  v_array<float>& probabilities = data.cover_probs;
  v_array<uint32_t>& predictions = data.preds;

  float additive_probability = 1.f / (float)cover_size;

  float min_prob = std::min(1.f / num_actions, 1.f / (float)std::sqrt(counter * num_actions));

  data.cb_label = ec.l.cb;

  ec.l.cs = data.cs_label;
  get_cover_probabilities(data, base, ec, probs);

  if (is_learn)
  {
    ec.l.cb = data.cb_label;
    base.learn(ec);

    // Now update oracles

    // 1. Compute loss vector
    data.cs_label.costs.clear();
    float norm = min_prob * num_actions;
    ec.l.cb = data.cb_label;
    data.cbcs.known_cost = get_observed_cost(data.cb_label);
    gen_cs_example<false>(data.cbcs, ec, data.cb_label, data.cs_label);
    for (uint32_t i = 0; i < num_actions; i++) probabilities[i] = 0;

    ec.l.cs = data.second_cs_label;
    // 2. Update functions
    for (size_t i = 0; i < cover_size; i++)
    {
      // Create costs of each action based on online cover
      for (uint32_t j = 0; j < num_actions; j++)
      {
        float pseudo_cost =
            data.cs_label.costs[j].x - data.psi * min_prob / (std::max(probabilities[j], min_prob) / norm) + 1;
        data.second_cs_label.costs[j].class_index = j + 1;
        data.second_cs_label.costs[j].x = pseudo_cost;
      }
      if (i != 0)
        data.cs->learn(ec, i + 1);
      if (probabilities[predictions[i] - 1] < min_prob)
        norm += std::max(0.f, additive_probability - (min_prob - probabilities[predictions[i] - 1]));
      else
        norm += additive_probability;
      probabilities[predictions[i] - 1] += additive_probability;
    }
  }

  ec.l.cb = data.cb_label;
  ec.pred.a_s = probs;
}

predict_or_learn_first()

template<bool is_learn>

void CB_EXPLORE::predict_or_learn_first	(	cb_explore &	data,
		single_learner &	base,
		example &	ec
	)

Definition at line 51 of file cb_explore.cc.

References polyprediction::a_s, polylabel::cb, CB_EXPLORE::cb_explore::cbcs, v_array< T >::clear(), CB::label::costs, example::l, LEARNER::learner< T, E >::learn(), polyprediction::multiclass, GEN_CS::cb_to_cs::num_actions, example::pred, LEARNER::learner< T, E >::predict(), prediction_type::prob, v_array< T >::push_back(), and CB_EXPLORE::cb_explore::tau.

{
  // Explore tau times, then act according to optimal.
  action_scores probs = ec.pred.a_s;

  if (is_learn && ec.l.cb.costs[0].probability < 1)
    base.learn(ec);
  else
    base.predict(ec);

  probs.clear();
  if (data.tau > 0)
  {
    float prob = 1.f / (float)data.cbcs.num_actions;
    for (uint32_t i = 0; i < data.cbcs.num_actions; i++) probs.push_back({i, prob});
    data.tau--;
  }
  else
  {
    uint32_t chosen = ec.pred.multiclass - 1;
    for (uint32_t i = 0; i < data.cbcs.num_actions; i++) probs.push_back({i, 0.});
    probs[chosen].score = 1.0;
  }

  ec.pred.a_s = probs;
}

predict_or_learn_greedy()

template<bool is_learn>

void CB_EXPLORE::predict_or_learn_greedy	(	cb_explore &	data,
		single_learner &	base,
		example &	ec
	)

Definition at line 79 of file cb_explore.cc.

References polyprediction::a_s, ACTION_SCORE::begin_scores(), CB_EXPLORE::cb_explore::cbcs, v_array< T >::clear(), ACTION_SCORE::end_scores(), CB_EXPLORE::cb_explore::epsilon, exploration::generate_epsilon_greedy(), LEARNER::learner< T, E >::learn(), polyprediction::multiclass, GEN_CS::cb_to_cs::num_actions, example::pred, LEARNER::learner< T, E >::predict(), v_array< T >::push_back(), and v_array< T >::resize().

{
  // Explore uniform random an epsilon fraction of the time.
  // TODO: pointers are copied here. What happens if base.learn/base.predict re-allocs?
  // ec.pred.a_s = probs; will restore the than free'd memory
  action_scores probs = ec.pred.a_s;
  probs.clear();

  if (is_learn)
    base.learn(ec);
  else
    base.predict(ec);

  // pre-allocate pdf
  probs.resize(data.cbcs.num_actions);
  for (uint32_t i = 0; i < data.cbcs.num_actions; i++) probs.push_back({i, 0});
  generate_epsilon_greedy(data.epsilon, ec.pred.multiclass - 1, begin_scores(probs), end_scores(probs));

  ec.pred.a_s = probs;
}

print_update_cb_explore()

void CB_EXPLORE::print_update_cb_explore	(	vw &	all,
		bool	is_test,
		example &	ec,
		std::stringstream &	pred_string
	)

Definition at line 221 of file cb_explore.cc.

References vw::bfgs, polylabel::cb, CB::label::costs, vw::current_pass, shared_data::dump_interval, vw::holdout_set_off, example::l, example::num_features, shared_data::print_update(), vw::progress_add, vw::progress_arg, vw::quiet, vw::sd, and shared_data::weighted_examples().

Referenced by output_example().

{
  if (all.sd->weighted_examples() >= all.sd->dump_interval && !all.quiet && !all.bfgs)
  {
    std::stringstream label_string;
    if (is_test)
      label_string << " unknown";
    else
      label_string << ec.l.cb.costs[0].action;
    all.sd->print_update(all.holdout_set_off, all.current_pass, label_string.str(), pred_string.str(), ec.num_features,
        all.progress_add, all.progress_arg);
  }
}