cbify.cc File Reference

#include "reductions.h"
#include "cb_algs.h"
#include "vw.h"
#include "hash.h"
#include "explore.h"
#include <vector>

Go to the source code of this file.

Classes
struct	cbify_adf_data
struct	cbify

Functions
float	loss (cbify &data, uint32_t label, uint32_t final_prediction)
float	loss_cs (cbify &data, v_array< COST_SENSITIVE::wclass > &costs, uint32_t final_prediction)
float	loss_csldf (cbify &data, std::vector< v_array< COST_SENSITIVE::wclass >> &cs_costs, uint32_t final_prediction)
void	copy_example_to_adf (cbify &data, example &ec)
template<bool is_learn, bool use_cs>
void	predict_or_learn (cbify &data, single_learner &base, example &ec)
template<bool is_learn, bool use_cs>
void	predict_or_learn_adf (cbify &data, multi_learner &base, example &ec)
void	init_adf_data (cbify &data, const size_t num_actions)
template<bool is_learn>
void	do_actual_learning_ldf (cbify &data, multi_learner &base, multi_ex &ec_seq)
void	output_example (vw &all, example &ec, bool &hit_loss, multi_ex *ec_seq)
void	output_example_seq (vw &all, multi_ex &ec_seq)
void	finish_multiline_example (vw &all, cbify &, multi_ex &ec_seq)
base_learner *	cbify_setup (options_i &options, vw &all)
base_learner *	cbifyldf_setup (options_i &options, vw &all)

Function Documentation

cbify_setup()

base_learner* cbify_setup	(	options_i &	options,
		vw &	all
	)

Definition at line 383 of file cbify.cc.

References VW::config::option_group_definition::add(), add(), VW::config::options_i::add_and_parse(), LEARNER::as_multiline(), LEARNER::as_singleline(), vw::delete_prediction, f, init_adf_data(), LEARNER::init_cost_sensitive_learner(), LEARNER::init_multiclass_learner(), VW::config::options_i::insert(), LEARNER::make_base(), VW::config::make_option(), vw::p, setup_base(), prediction_type::to_string(), uniform_hash(), and VW::config::options_i::was_supplied().

Referenced by parse_reductions().

{
  uint32_t num_actions = 0;
  auto data = scoped_calloc_or_throw<cbify>();
  bool use_cs;

  option_group_definition new_options("Make Multiclass into Contextual Bandit");
  new_options
      .add(make_option("cbify", num_actions)
               .keep()
               .help("Convert multiclass on <k> classes into a contextual bandit problem"))
      .add(make_option("cbify_cs", use_cs).help("consume cost-sensitive classification examples instead of multiclass"))
      .add(make_option("loss0", data->loss0).default_value(0.f).help("loss for correct label"))
      .add(make_option("loss1", data->loss1).default_value(1.f).help("loss for incorrect label"));
  options.add_and_parse(new_options);

  if (!options.was_supplied("cbify"))
    return nullptr;

  data->use_adf = options.was_supplied("cb_explore_adf");
  data->app_seed = uniform_hash("vw", 2, 0);
  data->a_s = v_init<action_score>();
  data->all = &all;

  if (data->use_adf)
    init_adf_data(*data, num_actions);

  if (!options.was_supplied("cb_explore") && !data->use_adf)
  {
    std::stringstream ss;
    ss << num_actions;
    options.insert("cb_explore", ss.str());
  }

  if (data->use_adf)
  {
    options.insert("cb_min_cost", std::to_string(data->loss0));
    options.insert("cb_max_cost", std::to_string(data->loss1));
  }

  if (options.was_supplied("baseline"))
  {
    std::stringstream ss;
    ss << std::max(std::abs(data->loss0), std::abs(data->loss1)) / (data->loss1 - data->loss0);
    options.insert("lr_multiplier", ss.str());
  }

  learner<cbify, example>* l;

  if (data->use_adf)
  {
    multi_learner* base = as_multiline(setup_base(options, all));
    if (use_cs)
      l = &init_cost_sensitive_learner(
          data, base, predict_or_learn_adf<true, true>, predict_or_learn_adf<false, true>, all.p, 1);
    else
      l = &init_multiclass_learner(
          data, base, predict_or_learn_adf<true, false>, predict_or_learn_adf<false, false>, all.p, 1);
  }
  else
  {
    single_learner* base = as_singleline(setup_base(options, all));
    if (use_cs)
      l = &init_cost_sensitive_learner(
          data, base, predict_or_learn<true, true>, predict_or_learn<false, true>, all.p, 1);
    else
      l = &init_multiclass_learner(data, base, predict_or_learn<true, false>, predict_or_learn<false, false>, all.p, 1);
  }
  all.delete_prediction = nullptr;

  return make_base(*l);
}

cbifyldf_setup()

base_learner* cbifyldf_setup	(	options_i &	options,
		vw &	all
	)

Definition at line 456 of file cbify.cc.

References VW::config::option_group_definition::add(), add(), VW::config::options_i::add_and_parse(), LEARNER::as_multiline(), COST_SENSITIVE::cs_label, vw::delete_prediction, f, finish_multiline_example(), LEARNER::init_learner(), VW::config::options_i::insert(), parser::lp, LEARNER::make_base(), VW::config::make_option(), prediction_type::multiclass, vw::p, LEARNER::learner< T, E >::set_finish_example(), setup_base(), prediction_type::to_string(), uniform_hash(), and VW::config::options_i::was_supplied().

Referenced by parse_reductions().

{
  auto data = scoped_calloc_or_throw<cbify>();
  bool cbify_ldf_option = false;

  option_group_definition new_options("Make csoaa_ldf into Contextual Bandit");
  new_options
      .add(make_option("cbify_ldf", cbify_ldf_option).keep().help("Convert csoaa_ldf into a contextual bandit problem"))
      .add(make_option("loss0", data->loss0).default_value(0.f).help("loss for correct label"))
      .add(make_option("loss1", data->loss1).default_value(1.f).help("loss for incorrect label"));
  options.add_and_parse(new_options);

  if (!options.was_supplied("cbify_ldf"))
    return nullptr;

  data->app_seed = uniform_hash("vw", 2, 0);
  data->all = &all;
  data->use_adf = true;

  if (!options.was_supplied("cb_explore_adf"))
  {
    options.insert("cb_explore_adf", "");
  }
  options.insert("cb_min_cost", std::to_string(data->loss0));
  options.insert("cb_max_cost", std::to_string(data->loss1));

  if (options.was_supplied("baseline"))
  {
    std::stringstream ss;
    ss << std::max(std::abs(data->loss0), std::abs(data->loss1)) / (data->loss1 - data->loss0);
    options.insert("lr_multiplier", ss.str());
  }

  multi_learner* base = as_multiline(setup_base(options, all));
  learner<cbify, multi_ex>& l = init_learner(
      data, base, do_actual_learning_ldf<true>, do_actual_learning_ldf<false>, 1, prediction_type::multiclass);

  l.set_finish_example(finish_multiline_example);
  all.p->lp = COST_SENSITIVE::cs_label;
  all.delete_prediction = nullptr;

  return make_base(l);
}

copy_example_to_adf()

void copy_example_to_adf	(	cbify &	data,
		example &	ec
	)

Definition at line 96 of file cbify.cc.

References a, cbify::adf_data, cbify::all, CB::cb_label, VW::copy_example_data(), label_parser::default_label, CB_ALGS::example_is_newline_not_header(), parameters::mask(), parameters::stride_shift(), label_parser::test_label, and vw::weights.

Referenced by predict_or_learn_adf().

{
  auto& adf_data = data.adf_data;
  const uint64_t ss = data.all->weights.stride_shift();
  const uint64_t mask = data.all->weights.mask();

  for (size_t a = 0; a < adf_data.num_actions; ++a)
  {
    auto& eca = *adf_data.ecs[a];
    // clear label
    auto& lab = eca.l.cb;
    CB::cb_label.default_label(&lab);

    // copy data
    VW::copy_example_data(false, &eca, &ec);

    // offset indices for given action
    for (features& fs : eca)
    {
      for (feature_index& idx : fs.indicies)
      {
        idx = ((((idx >> ss) * 28904713) + 4832917 * (uint64_t)a) << ss) & mask;
      }
    }

    // avoid empty example by adding a tag (hacky)
    if (CB_ALGS::example_is_newline_not_header(eca) && CB::cb_label.test_label(&eca.l))
    {
      eca.tag.push_back('n');
    }
  }
}

do_actual_learning_ldf()

template<bool is_learn>

void do_actual_learning_ldf	(	cbify &	data,
		multi_learner &	base,
		multi_ex &	ec_seq
	)

Definition at line 242 of file cbify.cc.

References CB::cb_class::action, cbify::app_seed, ACTION_SCORE::begin_scores(), cbify::cb_as, cbify::cb_costs, cbify::cb_label, CB::cb_class::cost, CB::label::costs, cbify::cs_costs, ACTION_SCORE::end_scores(), cbify::example_counter, LEARNER::learner< T, E >::learn(), loss_csldf(), LEARNER::learner< T, E >::predict(), CB::cb_class::probability, exploration::sample_after_normalizing(), and THROW.

{
  // change label and pred data for cb
  if (data.cs_costs.size() < ec_seq.size())
    data.cs_costs.resize(ec_seq.size());
  if (data.cb_costs.size() < ec_seq.size())
    data.cb_costs.resize(ec_seq.size());
  if (data.cb_as.size() < ec_seq.size())
    data.cb_as.resize(ec_seq.size());
  for (size_t i = 0; i < ec_seq.size(); ++i)
  {
    auto& ec = *ec_seq[i];
    data.cs_costs[i] = ec.l.cs.costs;
    data.cb_costs[i].clear();
    data.cb_as[i].clear();
    ec.l.cb.costs = data.cb_costs[i];
    ec.pred.a_s = data.cb_as[i];
  }

  base.predict(ec_seq);

  auto& out_ec = *ec_seq[0];

  uint32_t chosen_action;
  if (sample_after_normalizing(data.app_seed + data.example_counter++, begin_scores(out_ec.pred.a_s),
          end_scores(out_ec.pred.a_s), chosen_action))
    THROW("Failed to sample from pdf");

  CB::cb_class cl;
  cl.action = out_ec.pred.a_s[chosen_action].action + 1;
  cl.probability = out_ec.pred.a_s[chosen_action].score;

  if (!cl.action)
    THROW("No action with non-zero probability found!");

  cl.cost = loss_csldf(data, data.cs_costs, cl.action);

  // add cb label to chosen action
  data.cb_label.costs.clear();
  data.cb_label.costs.push_back(cl);
  data.cb_costs[cl.action - 1] = ec_seq[cl.action - 1]->l.cb.costs;
  ec_seq[cl.action - 1]->l.cb = data.cb_label;

  base.learn(ec_seq);

  // set cs prediction and reset cs costs
  for (size_t i = 0; i < ec_seq.size(); ++i)
  {
    auto& ec = *ec_seq[i];
    data.cb_as[i] = ec.pred.a_s;  // store action_score vector for later reuse.
    if (i == cl.action - 1)
      data.cb_label = ec.l.cb;
    else
      data.cb_costs[i] = ec.l.cb.costs;
    ec.l.cs.costs = data.cs_costs[i];
    if (i == cl.action - 1)
      ec.pred.multiclass = cl.action;
    else
      ec.pred.multiclass = 0;
  }
}

finish_multiline_example()

void finish_multiline_example	(	vw &	all,
		cbify &	,
		multi_ex &	ec_seq
	)

Definition at line 373 of file cbify.cc.

References VW::finish_example(), and output_example_seq().

Referenced by cbifyldf_setup(), CCB::ccb_explore_adf_setup(), CSOAA::csldf_setup(), explore_eval_setup(), VW::cb_explore_adf::softmax::setup(), VW::cb_explore_adf::greedy::setup(), VW::cb_explore_adf::first::setup(), VW::cb_explore_adf::bag::setup(), VW::cb_explore_adf::cover::setup(), VW::cb_explore_adf::regcb::setup(), and Search::setup().

{
  if (!ec_seq.empty())
  {
    output_example_seq(all, ec_seq);
    // global_print_newline(all);
  }
  VW::finish_example(all, ec_seq);
}

init_adf_data()

void init_adf_data	(	cbify &	data,
		const size_t	num_actions
	)

Definition at line 226 of file cbify.cc.

References a, cbify::adf_data, cbify::all, VW::alloc_examples(), CB::cb_label, label_parser::default_label, vw::interactions, and cbify_adf_data::num_actions.

Referenced by cbify_setup().

{
  auto& adf_data = data.adf_data;
  adf_data.num_actions = num_actions;

  adf_data.ecs.resize(num_actions);
  for (size_t a = 0; a < num_actions; ++a)
  {
    adf_data.ecs[a] = VW::alloc_examples(CB::cb_label.label_size, 1);
    auto& lab = adf_data.ecs[a]->l.cb;
    CB::cb_label.default_label(&lab);
    adf_data.ecs[a]->interactions = &data.all->interactions;
  }
}

loss()

float loss	(	cbify &	data,
		uint32_t	label,
		uint32_t	final_prediction
	)

Definition at line 60 of file cbify.cc.

References cbify::loss0, and cbify::loss1.

Referenced by MULTICLASS::finish_example(), MWT::finish_example(), Search::search::get_metatask_data(), EXPLORE_EVAL::output_example(), VW::cb_explore_adf::cb_explore_adf_base< ExploreType >::output_example(), CB_ALGS::output_example(), MULTILABEL::output_example(), CB_EXPLORE::output_example(), COST_SENSITIVE::output_example(), output_example(), CSOAA::output_example(), CCB::output_example(), CSOAA::output_rank_example(), Search::search::predict(), EntityRelationTask::predict_entity(), predict_or_learn(), predict_or_learn_adf(), EntityRelationTask::predict_relation(), Search::search::predictLDF(), ArgmaxTask::run(), Search::search_declare_loss(), VW::sync_stats(), shared_data::update(), CB_ADF::cb_adf::update_statistics(), and vw::vw().

{
  if (label != final_prediction)
    return data.loss1;
  else
    return data.loss0;
}

loss_cs()

float loss_cs	(	cbify &	data,
		v_array< COST_SENSITIVE::wclass > &	costs,
		uint32_t	final_prediction
	)

Definition at line 68 of file cbify.cc.

References cbify::loss0, and cbify::loss1.

Referenced by predict_or_learn(), and predict_or_learn_adf().

{
  float cost = 0.;
  for (auto wc : costs)
  {
    if (wc.class_index == final_prediction)
    {
      cost = wc.x;
      break;
    }
  }
  return data.loss0 + (data.loss1 - data.loss0) * cost;
}

loss_csldf()

float loss_csldf	(	cbify &	data,
		std::vector< v_array< COST_SENSITIVE::wclass >> &	cs_costs,
		uint32_t	final_prediction
	)

Definition at line 82 of file cbify.cc.

References cbify::loss0, and cbify::loss1.

Referenced by do_actual_learning_ldf().

{
  float cost = 0.;
  for (auto costs : cs_costs)
  {
    if (costs[0].class_index == final_prediction)
    {
      cost = costs[0].x;
      break;
    }
  }
  return data.loss0 + (data.loss1 - data.loss0) * cost;
}

output_example()

void output_example	(	vw &	all,
		example &	ec,
		bool &	hit_loss,
		multi_ex *	ec_seq
	)

Definition at line 304 of file cbify.cc.

References COST_SENSITIVE::label::costs, polylabel::cs, COST_SENSITIVE::cs_label, COST_SENSITIVE::ec_is_example_header(), example_is_newline(), vw::final_prediction_sink, example::l, loss(), polyprediction::multiclass, example::num_features, example::pred, vw::print, vw::print_text, COST_SENSITIVE::print_update(), vw::raw_prediction, vw::sd, v_array< T >::size(), shared_data::sum_loss, shared_data::sum_loss_since_last_dump, example::tag, test_label(), and shared_data::total_features.

Referenced by output_example_seq().

{
  COST_SENSITIVE::label& ld = ec.l.cs;
  v_array<COST_SENSITIVE::wclass> costs = ld.costs;

  if (example_is_newline(ec))
    return;
  if (COST_SENSITIVE::ec_is_example_header(ec))
    return;

  all.sd->total_features += ec.num_features;

  float loss = 0.;

  uint32_t predicted_class = ec.pred.multiclass;

  if (!COST_SENSITIVE::cs_label.test_label(&ec.l))
  {
    for (auto const& cost : costs)
    {
      if (hit_loss)
        break;
      if (predicted_class == cost.class_index)
      {
        loss = cost.x;
        hit_loss = true;
      }
    }

    all.sd->sum_loss += loss;
    all.sd->sum_loss_since_last_dump += loss;
  }

  for (int sink : all.final_prediction_sink) all.print(sink, (float)ec.pred.multiclass, 0, ec.tag);

  if (all.raw_prediction > 0)
  {
    std::string outputString;
    std::stringstream outputStringStream(outputString);
    for (size_t i = 0; i < costs.size(); i++)
    {
      if (i > 0)
        outputStringStream << ' ';
      outputStringStream << costs[i].class_index << ':' << costs[i].partial_prediction;
    }
    // outputStringStream << std::endl;
    all.print_text(all.raw_prediction, outputStringStream.str(), ec.tag);
  }

  COST_SENSITIVE::print_update(all, COST_SENSITIVE::cs_label.test_label(&ec.l), ec, ec_seq, false, predicted_class);
}

output_example_seq()

void output_example_seq	(	vw &	all,
		multi_ex &	ec_seq
	)

Definition at line 356 of file cbify.cc.

References shared_data::example_number, output_example(), vw::print_text, vw::raw_prediction, vw::sd, and shared_data::weighted_labeled_examples.

Referenced by EXPLORE_EVAL::finish_multiline_example(), finish_multiline_example(), CB_ADF::finish_multiline_example(), and CSOAA::finish_multiline_example().

{
  if (ec_seq.empty())
    return;
  all.sd->weighted_labeled_examples += ec_seq[0]->weight;
  all.sd->example_number++;

  bool hit_loss = false;
  for (example* ec : ec_seq) output_example(all, *ec, hit_loss, &(ec_seq));

  if (all.raw_prediction > 0)
  {
    v_array<char> empty = {nullptr, nullptr, nullptr, 0};
    all.print_text(all.raw_prediction, "", empty);
  }
}

predict_or_learn()

template<bool is_learn, bool use_cs>

void predict_or_learn	(	cbify &	data,
		single_learner &	base,
		example &	ec
	)

Definition at line 130 of file cbify.cc.

References cbify::a_s, polyprediction::a_s, CB::cb_class::action, cbify::app_seed, ACTION_SCORE::begin_scores(), polylabel::cb, cbify::cb_label, v_array< T >::clear(), CB::cb_class::cost, CB::label::costs, COST_SENSITIVE::label::costs, polylabel::cs, ACTION_SCORE::end_scores(), cbify::example_counter, example::l, MULTICLASS::label_t::label, LEARNER::learner< T, E >::learn(), loss(), loss_cs(), polylabel::multi, polyprediction::multiclass, example::pred, LEARNER::learner< T, E >::predict(), CB::cb_class::probability, exploration::sample_after_normalizing(), and THROW.

{
  // Store the multiclass or cost-sensitive input label
  MULTICLASS::label_t ld;
  COST_SENSITIVE::label csl;
  if (use_cs)
    csl = ec.l.cs;
  else
    ld = ec.l.multi;

  data.cb_label.costs.clear();
  ec.l.cb = data.cb_label;
  ec.pred.a_s = data.a_s;

  // Call the cb_explore algorithm. It returns a vector of probabilities for each action
  base.predict(ec);
  // data.probs = ec.pred.scalars;

  uint32_t chosen_action;
  if (sample_after_normalizing(
          data.app_seed + data.example_counter++, begin_scores(ec.pred.a_s), end_scores(ec.pred.a_s), chosen_action))
    THROW("Failed to sample from pdf");

  CB::cb_class cl;
  cl.action = chosen_action + 1;
  cl.probability = ec.pred.a_s[chosen_action].score;

  if (!cl.action)
    THROW("No action with non-zero probability found!");
  if (use_cs)
    cl.cost = loss_cs(data, csl.costs, cl.action);
  else
    cl.cost = loss(data, ld.label, cl.action);

  // Create a new cb label
  data.cb_label.costs.push_back(cl);
  ec.l.cb = data.cb_label;

  if (is_learn)
    base.learn(ec);

  data.a_s.clear();
  data.a_s = ec.pred.a_s;

  if (use_cs)
    ec.l.cs = csl;
  else
    ec.l.multi = ld;

  ec.pred.multiclass = cl.action;
}

predict_or_learn_adf()

template<bool is_learn, bool use_cs>

void predict_or_learn_adf	(	cbify &	data,
		multi_learner &	base,
		example &	ec
	)

Definition at line 183 of file cbify.cc.

References CB::cb_class::action, cbify::adf_data, cbify::app_seed, ACTION_SCORE::begin_scores(), copy_example_to_adf(), CB::cb_class::cost, COST_SENSITIVE::label::costs, polylabel::cs, cbify_adf_data::ecs, ACTION_SCORE::end_scores(), cbify::example_counter, example::l, MULTICLASS::label_t::label, LEARNER::learner< T, E >::learn(), loss(), loss_cs(), polylabel::multi, polyprediction::multiclass, example::pred, LEARNER::learner< T, E >::predict(), CB::cb_class::probability, exploration::sample_after_normalizing(), and THROW.

{
  // Store the multiclass or cost-sensitive input label
  MULTICLASS::label_t ld;
  COST_SENSITIVE::label csl;
  if (use_cs)
    csl = ec.l.cs;
  else
    ld = ec.l.multi;

  copy_example_to_adf(data, ec);
  base.predict(data.adf_data.ecs);

  auto& out_ec = *data.adf_data.ecs[0];

  uint32_t chosen_action;
  if (sample_after_normalizing(data.app_seed + data.example_counter++, begin_scores(out_ec.pred.a_s),
          end_scores(out_ec.pred.a_s), chosen_action))
    THROW("Failed to sample from pdf");

  CB::cb_class cl;
  cl.action = out_ec.pred.a_s[chosen_action].action + 1;
  cl.probability = out_ec.pred.a_s[chosen_action].score;

  if (!cl.action)
    THROW("No action with non-zero probability found!");

  if (use_cs)
    cl.cost = loss_cs(data, csl.costs, cl.action);
  else
    cl.cost = loss(data, ld.label, cl.action);

  // add cb label to chosen action
  auto& lab = data.adf_data.ecs[cl.action - 1]->l.cb;
  lab.costs.clear();
  lab.costs.push_back(cl);

  if (is_learn)
    base.learn(data.adf_data.ecs);

  ec.pred.multiclass = cl.action;
}