GEN_CS Namespace Reference

Classes
struct	cb_to_cs
struct	cb_to_cs_adf

Functions
constexpr bool	observed_cost (cb_class *cl)
cb_class *	get_observed_cost (CB::label &ld)
float	safe_probability (float prob)
void	gen_cs_example_ips (multi_ex &examples, COST_SENSITIVE::label &cs_labels, float clip_p)
void	gen_cs_example_dm (multi_ex &examples, COST_SENSITIVE::label &cs_labels)
void	gen_cs_test_example (multi_ex &examples, COST_SENSITIVE::label &cs_labels)
void	gen_cs_example_ips (cb_to_cs &c, CB::label &ld, COST_SENSITIVE::label &cs_ld, float clip_p)
void	gen_cs_example_mtr (cb_to_cs_adf &c, multi_ex &ec_seq, COST_SENSITIVE::label &cs_labels)
void	gen_cs_example_sm (multi_ex &, uint32_t chosen_action, float sign_offset, ACTION_SCORE::action_scores action_vals, COST_SENSITIVE::label &cs_labels)
template<bool is_learn>
void	gen_cs_example_dm (cb_to_cs &c, example &ec, COST_SENSITIVE::label &cs_ld)
template<bool is_learn>
void	gen_cs_label (cb_to_cs &c, example &ec, COST_SENSITIVE::label &cs_ld, uint32_t action, float clip_p=0.f)
template<bool is_learn>
void	gen_cs_example_dr (cb_to_cs &c, example &ec, CB::label &ld, COST_SENSITIVE::label &cs_ld, float=0.f)
template<bool is_learn>
void	gen_cs_example (cb_to_cs &c, example &ec, CB::label &ld, COST_SENSITIVE::label &cs_ld)
template<bool is_learn>
void	gen_cs_example_dr (cb_to_cs_adf &c, multi_ex &examples, COST_SENSITIVE::label &cs_labels, float clip_p=0.f)
template<bool is_learn>
void	gen_cs_example (cb_to_cs_adf &c, multi_ex &ec_seq, COST_SENSITIVE::label &cs_labels)
template<bool is_learn>
void	call_cs_ldf (LEARNER::multi_learner &base, multi_ex &examples, v_array< CB::label > &cb_labels, COST_SENSITIVE::label &cs_labels, v_array< COST_SENSITIVE::label > &prepped_cs_labels, uint64_t offset, size_t id=0)

Function Documentation

call_cs_ldf()

template<bool is_learn>

void GEN_CS::call_cs_ldf	(	LEARNER::multi_learner &	base,
		multi_ex &	examples,
		v_array< CB::label > &	cb_labels,
		COST_SENSITIVE::label &	cs_labels,
		v_array< COST_SENSITIVE::label > &	prepped_cs_labels,
		uint64_t	offset,
		size_t	id = 0
	)

Definition at line 251 of file gen_cs_example.h.

References v_array< T >::clear(), COST_SENSITIVE::label::costs, v_array< T >::end(), v_array< T >::end_array, LEARNER::learner< T, E >::learn(), LEARNER::learner< T, E >::predict(), v_array< T >::push_back(), v_array< T >::resize(), and v_array< T >::size().

{
  cb_labels.clear();
  if (prepped_cs_labels.size() < cs_labels.costs.size() + 1)
  {
    prepped_cs_labels.resize(cs_labels.costs.size() + 1);
    prepped_cs_labels.end() = prepped_cs_labels.end_array;
  }

  // 1st: save cb_label (into mydata) and store cs_label for each example, which will be passed into base.learn.
  // also save offsets
  uint64_t saved_offset = examples[0]->ft_offset;
  size_t index = 0;
  for (auto ec : examples)
  {
    cb_labels.push_back(ec->l.cb);
    prepped_cs_labels[index].costs.clear();
    prepped_cs_labels[index].costs.push_back(cs_labels.costs[index]);
    ec->l.cs = prepped_cs_labels[index++];
    ec->ft_offset = offset;
  }

  // 2nd: predict for each ex
  // // call base.predict for all examples
  if (is_learn)
    base.learn(examples, (int32_t)id);
  else
    base.predict(examples, (int32_t)id);

  // 3rd: restore cb_label for each example
  // (**ec).l.cb = array.element.
  // and restore offsets
  for (size_t i = 0; i < examples.size(); ++i)
  {
    examples[i]->l.cb = cb_labels[i];
    examples[i]->ft_offset = saved_offset;
  }
}

gen_cs_example() [1/2]

template<bool is_learn>

void GEN_CS::gen_cs_example	(	cb_to_cs &	c,
		example &	ec,
		CB::label &	ld,
		COST_SENSITIVE::label &	cs_ld
	)

Definition at line 167 of file gen_cs_example.h.

References c, GEN_CS::cb_to_cs::cb_type, CB_TYPE_DM, CB_TYPE_DR, CB_TYPE_IPS, f, gen_cs_example_dm(), gen_cs_example_ips(), gen_cs_example_mtr(), gen_cs_example_sm(), gen_cs_test_example(), and THROW.

{
  switch (c.cb_type)
  {
    case CB_TYPE_IPS:
      gen_cs_example_ips(c, ld, cs_ld);
      break;
    case CB_TYPE_DM:
      gen_cs_example_dm<is_learn>(c, ec, cs_ld);
      break;
    case CB_TYPE_DR:
      gen_cs_example_dr<is_learn>(c, ec, ld, cs_ld);
      break;
    default:
      THROW("Unknown cb_type specified for contextual bandit learning: " << c.cb_type);
  }
}

gen_cs_example() [2/2]

template<bool is_learn>

void GEN_CS::gen_cs_example	(	cb_to_cs_adf &	c,
		multi_ex &	ec_seq,
		COST_SENSITIVE::label &	cs_labels
	)

Definition at line 232 of file gen_cs_example.h.

References c, GEN_CS::cb_to_cs_adf::cb_type, CB_TYPE_DR, CB_TYPE_IPS, CB_TYPE_MTR, gen_cs_example_ips(), gen_cs_example_mtr(), and THROW.

{
  switch (c.cb_type)
  {
    case CB_TYPE_IPS:
      gen_cs_example_ips(ec_seq, cs_labels);
      break;
    case CB_TYPE_DR:
      gen_cs_example_dr<is_learn>(c, ec_seq, cs_labels);
      break;
    case CB_TYPE_MTR:
      gen_cs_example_mtr(c, ec_seq, cs_labels);
      break;
    default:
      THROW("Unknown cb_type specified for contextual bandit learning: " << c.cb_type);
  }
}

gen_cs_example_dm() [1/2]

template<bool is_learn>

void GEN_CS::gen_cs_example_dm	(	cb_to_cs &	c,
		example &	ec,
		COST_SENSITIVE::label &	cs_ld
	)

Definition at line 53 of file gen_cs_example.h.

References CB::cb_class::action, GEN_CS::cb_to_cs::avg_loss_regressors, polylabel::cb, CB::cb_class::cost, CB::label::costs, COST_SENSITIVE::label::costs, GEN_CS::cb_to_cs::known_cost, example::l, GEN_CS::cb_to_cs::last_correct_cost, GEN_CS::cb_to_cs::last_pred_reg, polyprediction::multiclass, GEN_CS::cb_to_cs::nb_ex_regressors, GEN_CS::cb_to_cs::num_actions, example::pred, GEN_CS::cb_to_cs::pred_scores, GEN_CS::cb_to_cs::scorer, and COST_SENSITIVE::wclass::x.

{  // this implements the direct estimation method, where costs are directly specified by the learned regressor.
  CB::label ld = ec.l.cb;

  float min = FLT_MAX;
  uint32_t argmin = 1;
  // generate cost sensitive example
  cs_ld.costs.clear();
  c.pred_scores.costs.clear();

  if (ld.costs.size() == 0 ||
      (ld.costs.size() == 1 &&
          ld.costs[0].cost != FLT_MAX))  // this is a typical example where we can perform all actions
  {                                      // in this case generate cost-sensitive example with all actions
    for (uint32_t i = 1; i <= c.num_actions; i++)
    {
      COST_SENSITIVE::wclass wc = {0., i, 0., 0.};
      // get cost prediction for this action
      wc.x = CB_ALGS::get_cost_pred<is_learn>(c.scorer, c.known_cost, ec, i, 0);
      if (wc.x < min)
      {
        min = wc.x;
        argmin = i;
      }

      c.pred_scores.costs.push_back(wc);

      if (c.known_cost != nullptr && c.known_cost->action == i)
      {
        c.nb_ex_regressors++;
        c.avg_loss_regressors += (1.0f / c.nb_ex_regressors) *
            ((c.known_cost->cost - wc.x) * (c.known_cost->cost - wc.x) - c.avg_loss_regressors);
        c.last_pred_reg = wc.x;
        c.last_correct_cost = c.known_cost->cost;
      }

      cs_ld.costs.push_back(wc);
    }
  }
  else  // this is an example where we can only perform a subset of the actions
  {     // in this case generate cost-sensitive example with only allowed actions
    for (auto& cl : ld.costs)
    {
      COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.};

      // get cost prediction for this action
      wc.x = CB_ALGS::get_cost_pred<is_learn>(c.scorer, c.known_cost, ec, cl.action, 0);
      if (wc.x < min || (wc.x == min && cl.action < argmin))
      {
        min = wc.x;
        argmin = cl.action;
      }
      c.pred_scores.costs.push_back(wc);

      if (c.known_cost != nullptr && c.known_cost->action == cl.action)
      {
        c.nb_ex_regressors++;
        c.avg_loss_regressors += (1.0f / c.nb_ex_regressors) *
            ((c.known_cost->cost - wc.x) * (c.known_cost->cost - wc.x) - c.avg_loss_regressors);
        c.last_pred_reg = wc.x;
        c.last_correct_cost = c.known_cost->cost;
      }

      cs_ld.costs.push_back(wc);
    }
  }

  ec.pred.multiclass = argmin;
}

gen_cs_example_dm() [2/2]

void GEN_CS::gen_cs_example_dm	(	multi_ex &	examples,
		COST_SENSITIVE::label &	cs_labels
	)

Definition at line 60 of file gen_cs_example.cc.

References CB::label::costs, COST_SENSITIVE::label::costs, and COST_SENSITIVE::wclass::x.

Referenced by gen_cs_example(), and CB_ADF::cb_adf::learn_DM().

{
  cs_labels.costs.clear();
  for (uint32_t i = 0; i < examples.size(); i++)
  {
    CB::label ld = examples[i]->l.cb;

    COST_SENSITIVE::wclass wc = {0., i, 0., 0.};
    if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX)
      wc.x = ld.costs[0].cost;
    cs_labels.costs.push_back(wc);
  }
}

gen_cs_example_dr() [1/2]

template<bool is_learn>

void GEN_CS::gen_cs_example_dr	(	cb_to_cs &	c,
		example &	ec,
		CB::label &	ld,
		COST_SENSITIVE::label &	cs_ld,
		float	= 0.f
	)

Definition at line 147 of file gen_cs_example.h.

References c, CB::label::costs, COST_SENSITIVE::label::costs, GEN_CS::cb_to_cs::num_actions, and GEN_CS::cb_to_cs::pred_scores.

{  // this implements the doubly robust method
  cs_ld.costs.clear();
  c.pred_scores.costs.clear();
  if (ld.costs.size() == 0)  // a test example
    for (uint32_t i = 1; i <= c.num_actions; i++)
    {  // Explicit declaration for a weak compiler.
      COST_SENSITIVE::wclass temp = {FLT_MAX, i, 0., 0.};
      cs_ld.costs.push_back(temp);
    }
  else if (ld.costs.size() == 0 || (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX))
    // this is a typical example where we can perform all actions
    // in this case generate cost-sensitive example with all actions
    for (uint32_t i = 1; i <= c.num_actions; i++) gen_cs_label<is_learn>(c, ec, cs_ld, i);
  else  // this is an example where we can only perform a subset of the actions
    // in this case generate cost-sensitive example with only allowed actions
    for (auto& cl : ld.costs) gen_cs_label<is_learn>(c, ec, cs_ld, cl.action);
}

gen_cs_example_dr() [2/2]

template<bool is_learn>

void GEN_CS::gen_cs_example_dr	(	cb_to_cs_adf &	c,
		multi_ex &	examples,
		COST_SENSITIVE::label &	cs_labels,
		float	clip_p = 0.f
	)

Definition at line 197 of file gen_cs_example.h.

References CB::cb_class::action, CB::cb_class::cost, COST_SENSITIVE::label::costs, CB_ALGS::example_is_newline_not_header(), GEN_CS::cb_to_cs_adf::known_cost, GEN_CS::cb_to_cs_adf::pred_scores, CB::cb_class::probability, GEN_CS::cb_to_cs_adf::scorer, and COST_SENSITIVE::wclass::x.

{  // size_t mysize = examples.size();
  c.pred_scores.costs.clear();

  cs_labels.costs.clear();
  for (size_t i = 0; i < examples.size(); i++)
  {
    if (CB_ALGS::example_is_newline_not_header(*examples[i]))
      continue;

    COST_SENSITIVE::wclass wc = {0., (uint32_t)i, 0., 0.};

    if (c.known_cost.action == i)
    {
      int known_index = c.known_cost.action;
      c.known_cost.action = 0;
      // get cost prediction for this label
      // num_actions should be 1 effectively.
      // my get_cost_pred function will use 1 for 'index-1+base'
      wc.x = CB_ALGS::get_cost_pred<is_learn>(c.scorer, &(c.known_cost), *(examples[i]), 0, 2);
      c.known_cost.action = known_index;
    }
    else
      wc.x = CB_ALGS::get_cost_pred<is_learn>(c.scorer, nullptr, *(examples[i]), 0, 2);

    c.pred_scores.costs.push_back(wc);  // done

    // add correction if we observed cost for this action and regressor is wrong
    if (c.known_cost.probability != -1 && c.known_cost.action == i)
      wc.x += (c.known_cost.cost - wc.x) / std::max(c.known_cost.probability, clip_p);
    cs_labels.costs.push_back(wc);
  }
}

gen_cs_example_ips() [1/2]

void GEN_CS::gen_cs_example_ips	(	multi_ex &	examples,
		COST_SENSITIVE::label &	cs_labels,
		float	clip_p
	)

Definition at line 45 of file gen_cs_example.cc.

References CB::label::costs, COST_SENSITIVE::label::costs, safe_probability(), and COST_SENSITIVE::wclass::x.

Referenced by gen_cs_example(), CB_ADF::cb_adf::learn_IPS(), CB_ADF::cb_adf::learn_MTR(), and VW::cb_explore_adf::cover::cb_explore_adf_cover::predict_or_learn_impl().

{
  cs_labels.costs.clear();
  for (uint32_t i = 0; i < examples.size(); i++)
  {
    CB::label ld = examples[i]->l.cb;

    COST_SENSITIVE::wclass wc = {0., i, 0., 0.};
    if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX)
      wc.x = ld.costs[0].cost / safe_probability(std::max(ld.costs[0].probability, clip_p));
    cs_labels.costs.push_back(wc);
  }
}

gen_cs_example_ips() [2/2]

void GEN_CS::gen_cs_example_ips	(	cb_to_cs &	c,
		CB::label &	ld,
		COST_SENSITIVE::label &	cs_ld,
		float	clip_p
	)

Definition at line 86 of file gen_cs_example.cc.

References CB::cb_class::action, GEN_CS::cb_to_cs::avg_loss_regressors, CB::cb_class::cost, CB::label::costs, COST_SENSITIVE::label::costs, GEN_CS::cb_to_cs::known_cost, GEN_CS::cb_to_cs::last_correct_cost, GEN_CS::cb_to_cs::last_pred_reg, GEN_CS::cb_to_cs::nb_ex_regressors, GEN_CS::cb_to_cs::num_actions, CB::cb_class::probability, safe_probability(), and COST_SENSITIVE::wclass::x.

{
  // this implements the inverse propensity score method, where cost are importance weighted by the probability of the
  // chosen action generate cost-sensitive example
  cs_ld.costs.clear();
  if (ld.costs.size() == 0 || (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX))
  // this is a typical example where we can perform all actions
  {
    // in this case generate cost-sensitive example with all actions
    for (uint32_t i = 1; i <= c.num_actions; i++)
    {
      COST_SENSITIVE::wclass wc = {0., i, 0., 0.};
      if (c.known_cost != nullptr && i == c.known_cost->action)
      {
        // use importance weighted cost for observed action, 0 otherwise
        wc.x = c.known_cost->cost / safe_probability(std::max(c.known_cost->probability, clip_p));

        // ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything
        // update the loss of this regressor
        c.nb_ex_regressors++;
        c.avg_loss_regressors +=
            (1.0f / c.nb_ex_regressors) * ((c.known_cost->cost) * (c.known_cost->cost) - c.avg_loss_regressors);
        c.last_pred_reg = 0;
        c.last_correct_cost = c.known_cost->cost;
      }

      cs_ld.costs.push_back(wc);
    }
  }
  else  // this is an example where we can only perform a subset of the actions
  {
    // in this case generate cost-sensitive example with only allowed actions
    for (auto& cl : ld.costs)
    {
      COST_SENSITIVE::wclass wc = {0., cl.action, 0., 0.};
      if (c.known_cost != nullptr && cl.action == c.known_cost->action)
      {
        // use importance weighted cost for observed action, 0 otherwise
        wc.x = c.known_cost->cost / safe_probability(std::max(c.known_cost->probability, clip_p));

        // ips can be thought as the doubly robust method with a fixed regressor that predicts 0 costs for everything
        // update the loss of this regressor
        c.nb_ex_regressors++;
        c.avg_loss_regressors +=
            (1.0f / c.nb_ex_regressors) * ((c.known_cost->cost) * (c.known_cost->cost) - c.avg_loss_regressors);
        c.last_pred_reg = 0;
        c.last_correct_cost = c.known_cost->cost;
      }

      cs_ld.costs.push_back(wc);
    }
  }
}

gen_cs_example_mtr()

void GEN_CS::gen_cs_example_mtr	(	cb_to_cs_adf &	c,
		multi_ex &	ec_seq,
		COST_SENSITIVE::label &	cs_labels
	)

Definition at line 140 of file gen_cs_example.cc.

References GEN_CS::cb_to_cs_adf::action_sum, CB::label::costs, COST_SENSITIVE::label::costs, GEN_CS::cb_to_cs_adf::event_sum, GEN_CS::cb_to_cs_adf::mtr_ec_seq, GEN_CS::cb_to_cs_adf::mtr_example, and COST_SENSITIVE::wclass::x.

Referenced by gen_cs_example(), and CB_ADF::cb_adf::learn_MTR().

{
  c.action_sum += ec_seq.size();
  c.event_sum++;

  c.mtr_ec_seq.clear();
  cs_labels.costs.clear();
  for (size_t i = 0; i < ec_seq.size(); i++)
  {
    CB::label ld = ec_seq[i]->l.cb;

    COST_SENSITIVE::wclass wc = {0, 0, 0, 0};

    if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX)
    {
      wc.x = ld.costs[0].cost;
      c.mtr_example = (uint32_t)i;
      c.mtr_ec_seq.push_back(ec_seq[i]);
      cs_labels.costs.push_back(wc);
    }
  }
}

gen_cs_example_sm()

void GEN_CS::gen_cs_example_sm	(	multi_ex &	,
		uint32_t	chosen_action,
		float	sign_offset,
		ACTION_SCORE::action_scores	action_vals,
		COST_SENSITIVE::label &	cs_labels
	)

Definition at line 163 of file gen_cs_example.cc.

References COST_SENSITIVE::label::costs, v_array< T >::size(), and COST_SENSITIVE::wclass::x.

Referenced by gen_cs_example(), and CB_ADF::cb_adf::learn_SM().

{
  cs_labels.costs.clear();
  for (uint32_t i = 0; i < action_vals.size(); i++)
  {
    uint32_t current_action = action_vals[i].action;
    COST_SENSITIVE::wclass wc = {0., current_action, 0., 0.};

    if (current_action == chosen_action)
      wc.x = action_vals[i].score + sign_offset;
    else
      wc.x = action_vals[i].score - sign_offset;

    // TODO: This clipping is conceptually unnecessary because the example weight for this instance should be close to
    // 0.
    if (wc.x > 100.)
      wc.x = 100.0;
    if (wc.x < -100.)
      wc.x = -100.0;

    cs_labels.costs.push_back(wc);
  }
}

gen_cs_label()

template<bool is_learn>

void GEN_CS::gen_cs_label	(	cb_to_cs &	c,
		example &	ec,
		COST_SENSITIVE::label &	cs_ld,
		uint32_t	action,
		float	clip_p = 0.f
	)

Definition at line 124 of file gen_cs_example.h.

References CB::cb_class::action, GEN_CS::cb_to_cs::avg_loss_regressors, CB::cb_class::cost, COST_SENSITIVE::label::costs, GEN_CS::cb_to_cs::known_cost, GEN_CS::cb_to_cs::last_correct_cost, GEN_CS::cb_to_cs::last_pred_reg, GEN_CS::cb_to_cs::nb_ex_regressors, GEN_CS::cb_to_cs::num_actions, GEN_CS::cb_to_cs::pred_scores, CB::cb_class::probability, GEN_CS::cb_to_cs::scorer, and COST_SENSITIVE::wclass::x.

{
  COST_SENSITIVE::wclass wc = {0., action, 0., 0.};

  // get cost prediction for this action
  wc.x = CB_ALGS::get_cost_pred<is_learn>(c.scorer, c.known_cost, ec, action, c.num_actions);

  c.pred_scores.costs.push_back(wc);
  // add correction if we observed cost for this action and regressor is wrong
  if (c.known_cost != nullptr && c.known_cost->action == action)
  {
    c.nb_ex_regressors++;
    c.avg_loss_regressors += (1.0f / c.nb_ex_regressors) *
        ((c.known_cost->cost - wc.x) * (c.known_cost->cost - wc.x) - c.avg_loss_regressors);
    c.last_pred_reg = wc.x;
    c.last_correct_cost = c.known_cost->cost;
    wc.x += (c.known_cost->cost - wc.x) / std::max(c.known_cost->probability, clip_p);
  }

  cs_ld.costs.push_back(wc);
}

gen_cs_test_example()

void GEN_CS::gen_cs_test_example	(	multi_ex &	examples,
		COST_SENSITIVE::label &	cs_labels
	)

Definition at line 75 of file gen_cs_example.cc.

References COST_SENSITIVE::label::costs.

Referenced by CB_ADF::cb_adf::do_actual_learning(), gen_cs_example(), and CB_ADF::cb_adf::learn_SM().

{
  cs_labels.costs.clear();
  for (uint32_t i = 0; i < examples.size(); i++)
  {
    COST_SENSITIVE::wclass wc = {FLT_MAX, i, 0., 0.};
    cs_labels.costs.push_back(wc);
  }
}

get_observed_cost()

CB::cb_class * GEN_CS::get_observed_cost

(

CB::label &

ld

)

Definition at line 25 of file gen_cs_example.cc.

References CB::label::costs, and observed_cost().

{
  for (auto& cl : ld.costs)
    if (observed_cost(&cl))
      return &cl;
  return nullptr;
}

observed_cost()

constexpr bool GEN_CS::observed_cost ( cb_class *  cl )

inline

Definition at line 19 of file gen_cs_example.cc.

References CB::cb_class::cost, and CB::cb_class::probability.

Referenced by get_observed_cost().

{
  // cost observed for this action if it has non zero probability and cost != FLT_MAX
  return (cl != nullptr && cl->cost != FLT_MAX && cl->probability > .0);
}

safe_probability()

float GEN_CS::safe_probability

(

float

prob

)

Definition at line 33 of file gen_cs_example.cc.

References f, and prediction_type::prob.

Referenced by gen_cs_example_ips(), and CB_ADF::cb_adf::learn_SM().

{
  if (prob <= 0.)
  {
    std::cout << "Probability " << prob << " is not possible, replacing with 1e-3.  Fix your dataset. " << std::endl;
    return 1e-3f;
  }
  else
    return prob;
}