CB_ADF::cb_adf Struct Reference

Public Member Functions
template<bool is_learn>
void	do_actual_learning (LEARNER::multi_learner &base, multi_ex &ec_seq)
bool	update_statistics (example &ec, multi_ex *ec_seq)
	cb_adf (shared_data *sd, size_t cb_type, VW::version_struct *model_file_ver, bool rank_all, float clip_p, bool no_predict)
void	set_scorer (LEARNER::single_learner *scorer)
bool	get_rank_all () const
const cb_to_cs_adf &	get_gen_cs () const
const VW::version_struct *	get_model_file_ver () const
	~cb_adf ()

Private Member Functions
void	learn_IPS (multi_learner &base, multi_ex &examples)
void	learn_DR (multi_learner &base, multi_ex &examples)
void	learn_DM (multi_learner &base, multi_ex &examples)
void	learn_SM (multi_learner &base, multi_ex &examples)
template<bool predict>
void	learn_MTR (multi_learner &base, multi_ex &examples)

Private Attributes
shared_data *	_sd
VW::version_struct *	_model_file_ver
cb_to_cs_adf	_gen_cs
v_array< CB::label >	_cb_labels
COST_SENSITIVE::label	_cs_labels
v_array< COST_SENSITIVE::label >	_prepped_cs_labels
action_scores	_a_s
action_scores	_a_s_mtr_cs
action_scores	_prob_s
v_array< uint32_t >	_backup_nf
v_array< float >	_backup_weights
uint64_t	_offset
const bool	_no_predict
const bool	_rank_all
const float	_clip_p

Detailed Description

Definition at line 30 of file cb_adf.cc.

Constructor & Destructor Documentation

cb_adf()

CB_ADF::cb_adf::cb_adf ( shared_data *  sd, size_t  cb_type, VW::version_struct *  model_file_ver, bool  rank_all, float  clip_p, bool  no_predict  )

inline

Definition at line 59 of file cb_adf.cc.

References GEN_CS::cb_to_cs_adf::cb_type.

      : _sd(sd), _model_file_ver(model_file_ver), _no_predict(no_predict), _rank_all(rank_all), _clip_p(clip_p)
  {
    _gen_cs.cb_type = cb_type;
  }

~cb_adf()

CB_ADF::cb_adf::~cb_adf ( )

inline

Definition at line 74 of file cb_adf.cc.

References COST_SENSITIVE::label::costs, v_array< T >::delete_v(), and GEN_CS::cb_to_cs_adf::pred_scores.

  {
    _cb_labels.delete_v();
    for (auto& prepped_cs_label : _prepped_cs_labels) prepped_cs_label.costs.delete_v();
    _prepped_cs_labels.delete_v();
    _cs_labels.costs.delete_v();
    _backup_weights.delete_v();
    _backup_nf.delete_v();
    _prob_s.delete_v();

    _a_s.delete_v();
    _a_s_mtr_cs.delete_v();
    _gen_cs.pred_scores.costs.delete_v();
  }

Member Function Documentation

do_actual_learning()

template<bool is_learn>

void CB_ADF::cb_adf::do_actual_learning	(	LEARNER::multi_learner &	base,
		multi_ex &	ec_seq
	)

Definition at line 295 of file cb_adf.cc.

References CB_TYPE_DM, CB_TYPE_DR, CB_TYPE_IPS, CB_TYPE_MTR, CB_TYPE_SM, GEN_CS::gen_cs_test_example(), CB_ADF::get_observed_cost(), CB_ADF::test_adf_sequence(), and THROW.

Referenced by CB_ADF::learn(), and CB_ADF::predict().

{
  _offset = ec_seq[0]->ft_offset;
  _gen_cs.known_cost = get_observed_cost(ec_seq);  // need to set for test case
  if (is_learn && test_adf_sequence(ec_seq) != nullptr)
  {
    /*  v_array<float> temp_scores;
    temp_scores = v_init<float>();
    do_actual_learning<false>(data,base);
    for (size_t i = 0; i < data.ec_seq[0]->pred.a_s.size(); i++)
    temp_scores.push_back(data.ec_seq[0]->pred.a_s[i].score);*/
    switch (_gen_cs.cb_type)
    {
      case CB_TYPE_IPS:
        learn_IPS(base, ec_seq);
        break;
      case CB_TYPE_DR:
        learn_DR(base, ec_seq);
        break;
      case CB_TYPE_DM:
        learn_DM(base, ec_seq);
        break;
      case CB_TYPE_MTR:
        if (_no_predict)
          learn_MTR<false>(base, ec_seq);
        else
          learn_MTR<true>(base, ec_seq);
        break;
      case CB_TYPE_SM:
        learn_SM(base, ec_seq);
        break;
      default:
        THROW("Unknown cb_type specified for contextual bandit learning: " << _gen_cs.cb_type);
    }

    /*      for (size_t i = 0; i < temp_scores.size(); i++)
    if (temp_scores[i] != data.ec_seq[0]->pred.a_s[i].score)
     std::cout << "problem! " << temp_scores[i] << " != " << data.ec_seq[0]->pred.a_s[i].score << " for " <<
    data.ec_seq[0]->pred.a_s[i].action << std::endl; temp_scores.delete_v();*/
  }
  else
  {
    gen_cs_test_example(ec_seq, _cs_labels);  // create test labels.
    call_cs_ldf<false>(base, ec_seq, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
  }
}

get_gen_cs()

const cb_to_cs_adf& CB_ADF::cb_adf::get_gen_cs ( ) const

inline

Definition at line 70 of file cb_adf.cc.

Referenced by CB_ADF::save_load().

{ return _gen_cs; }

get_model_file_ver()

const VW::version_struct* CB_ADF::cb_adf::get_model_file_ver ( ) const

inline

Definition at line 72 of file cb_adf.cc.

Referenced by CB_ADF::save_load().

{ return _model_file_ver; }

get_rank_all()

bool CB_ADF::cb_adf::get_rank_all ( ) const

inline

Definition at line 68 of file cb_adf.cc.

Referenced by CB_ADF::output_example_seq().

{ return _rank_all; }

learn_DM()

void CB_ADF::cb_adf::learn_DM ( multi_learner &  base, multi_ex &  examples  )

private

Definition at line 233 of file cb_adf.cc.

References GEN_CS::gen_cs_example_dm().

{
  gen_cs_example_dm(examples, _cs_labels);
  call_cs_ldf<true>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
}

learn_DR()

void CB_ADF::cb_adf::learn_DR ( multi_learner &  base, multi_ex &  examples  )

private

Definition at line 227 of file cb_adf.cc.

{
  gen_cs_example_dr<true>(_gen_cs, examples, _cs_labels, _clip_p);
  call_cs_ldf<true>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
}

learn_IPS()

void CB_ADF::cb_adf::learn_IPS ( multi_learner &  base, multi_ex &  examples  )

private

Definition at line 131 of file cb_adf.cc.

References GEN_CS::gen_cs_example_ips().

{
  gen_cs_example_ips(examples, _cs_labels, _clip_p);
  call_cs_ldf<true>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
}

learn_MTR()

template<bool predict>

void CB_ADF::cb_adf::learn_MTR ( multi_learner &  base, multi_ex &  examples  )

private

Definition at line 240 of file cb_adf.cc.

References GEN_CS::gen_cs_example_ips(), GEN_CS::gen_cs_example_mtr(), and CB_ADF::predict().

{
  // uint32_t action = 0;
  if (predict)  // first get the prediction to return
  {
    gen_cs_example_ips(examples, _cs_labels);
    call_cs_ldf<false>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
    std::swap(examples[0]->pred.a_s, _a_s);
  }
  // second train on _one_ action (which requires up to 3 examples).
  // We must go through the cost sensitive classifier layer to get
  // proper feature handling.
  gen_cs_example_mtr(_gen_cs, examples, _cs_labels);
  uint32_t nf = (uint32_t)examples[_gen_cs.mtr_example]->num_features;
  float old_weight = examples[_gen_cs.mtr_example]->weight;
  const float clipped_p = std::max(examples[_gen_cs.mtr_example]->l.cb.costs[0].probability, _clip_p);
  examples[_gen_cs.mtr_example]->weight *= 1.f / clipped_p * ((float)_gen_cs.event_sum / (float)_gen_cs.action_sum);

  std::swap(_gen_cs.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs);
  // TODO!!! cb_labels are not getting properly restored (empty costs are dropped)
  GEN_CS::call_cs_ldf<true>(base, _gen_cs.mtr_ec_seq, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);
  examples[_gen_cs.mtr_example]->num_features = nf;
  examples[_gen_cs.mtr_example]->weight = old_weight;
  std::swap(_gen_cs.mtr_ec_seq[0]->pred.a_s, _a_s_mtr_cs);
  std::swap(examples[0]->pred.a_s, _a_s);
}

learn_SM()

void CB_ADF::cb_adf::learn_SM ( multi_learner &  base, multi_ex &  examples  )

private

Definition at line 137 of file cb_adf.cc.

References ACTION_SCORE::action_score::action, ACTION_SCORE::begin_scores(), CB::label::costs, ACTION_SCORE::end_scores(), GEN_CS::gen_cs_example_sm(), GEN_CS::gen_cs_test_example(), exploration::generate_softmax(), GEN_CS::safe_probability(), and ACTION_SCORE::action_score::score.

{
  gen_cs_test_example(examples, _cs_labels);  // create test labels.
  call_cs_ldf<false>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);

  // Can probably do this more efficiently than 6 loops over the examples...
  //[1: initialize temporary storage;
  // 2: find chosen action;
  // 3: create cs_labels (gen_cs_example_sm);
  // 4: get probability of chosen action;
  // 5: backup example wts;
  // 6: restore example wts]
  _a_s.clear();
  _prob_s.clear();
  // TODO: Check that predicted scores are always stored with the first example
  for (uint32_t i = 0; i < examples[0]->pred.a_s.size(); i++)
  {
    _a_s.push_back({examples[0]->pred.a_s[i].action, examples[0]->pred.a_s[i].score});
    _prob_s.push_back({examples[0]->pred.a_s[i].action, 0.0});
  }

  float sign_offset = 1.0;  // To account for negative rewards/costs
  uint32_t chosen_action = 0;
  float example_weight = 1.0;

  for (uint32_t i = 0; i < examples.size(); i++)
  {
    CB::label ld = examples[i]->l.cb;
    if (ld.costs.size() == 1 && ld.costs[0].cost != FLT_MAX)
    {
      chosen_action = i;
      example_weight = ld.costs[0].cost / safe_probability(ld.costs[0].probability);

      // Importance weights of examples cannot be negative.
      // So we use a trick: set |w| as weight, and use sign(w) as an offset in the regression target.
      if (ld.costs[0].cost < 0.0)
      {
        sign_offset = -1.0;
        example_weight = -example_weight;
      }
      break;
    }
  }

  gen_cs_example_sm(examples, chosen_action, sign_offset, _a_s, _cs_labels);

  // Lambda is -1 in the call to generate_softmax because in vw, lower score is better; for softmax higher score is
  // better.
  generate_softmax(-1.0, begin_scores(_a_s), end_scores(_a_s), begin_scores(_prob_s), end_scores(_prob_s));

  // TODO: Check Marco's example that causes VW to report prob > 1.

  for (auto const& action_score : _prob_s)  // Scale example_wt by prob of chosen action
  {
    if (action_score.action == chosen_action)
    {
      example_weight *= action_score.score;
      break;
    }
  }

  _backup_weights.clear();
  _backup_nf.clear();
  for (auto const& action_score : _prob_s)
  {
    uint32_t current_action = action_score.action;
    _backup_weights.push_back(examples[current_action]->weight);
    _backup_nf.push_back((uint32_t)examples[current_action]->num_features);

    if (current_action == chosen_action)
      examples[current_action]->weight = example_weight * (1.0f - action_score.score);
    else
      examples[current_action]->weight = example_weight * action_score.score;

    if (examples[current_action]->weight <= 1e-15)
      examples[current_action]->weight = 0;
  }

  // Do actual training
  call_cs_ldf<true>(base, examples, _cb_labels, _cs_labels, _prepped_cs_labels, _offset);

  // Restore example weights and numFeatures
  for (size_t i = 0; i < _prob_s.size(); i++)
  {
    uint32_t current_action = _prob_s[i].action;
    examples[current_action]->weight = _backup_weights[i];
    examples[current_action]->num_features = _backup_nf[i];
  }
}

set_scorer()

void CB_ADF::cb_adf::set_scorer ( LEARNER::single_learner *  scorer )

inline

Definition at line 66 of file cb_adf.cc.

References GEN_CS::cb_to_cs_adf::scorer.

Referenced by cb_adf_setup().

{ _gen_cs.scorer = scorer; }

update_statistics()

bool CB_ADF::cb_adf::update_statistics	(	example &	ec,
		multi_ex *	ec_seq
	)

Definition at line 357 of file cb_adf.cc.

References polyprediction::a_s, CB_ALGS::get_cost_estimate(), loss(), example::num_features, example::pred, and example::weight.

Referenced by CB_ADF::output_example(), and CB_ADF::output_rank_example().

{
  size_t num_features = 0;

  uint32_t action = ec.pred.a_s[0].action;
  for (const auto& example : *ec_seq) num_features += example->num_features;

  float loss = 0.;

  bool labeled_example = true;
  if (_gen_cs.known_cost.probability > 0)
    loss = get_cost_estimate(&(_gen_cs.known_cost), _gen_cs.pred_scores, action);
  else
    labeled_example = false;

  bool holdout_example = labeled_example;
  for (auto const& i : *ec_seq) holdout_example &= i->test_only;

  _sd->update(holdout_example, labeled_example, loss, ec.weight, num_features);
  return labeled_example;
}

Member Data Documentation

_a_s

action_scores CB_ADF::cb_adf::_a_s

private

Definition at line 43 of file cb_adf.cc.

_a_s_mtr_cs

action_scores CB_ADF::cb_adf::_a_s_mtr_cs

private

Definition at line 44 of file cb_adf.cc.

_backup_nf

v_array<uint32_t> CB_ADF::cb_adf::_backup_nf

private

Definition at line 46 of file cb_adf.cc.

_backup_weights

v_array<float> CB_ADF::cb_adf::_backup_weights

private

Definition at line 47 of file cb_adf.cc.

_cb_labels

v_array<CB::label> CB_ADF::cb_adf::_cb_labels

private

Definition at line 39 of file cb_adf.cc.

_clip_p

const float CB_ADF::cb_adf::_clip_p

private

Definition at line 52 of file cb_adf.cc.

_cs_labels

COST_SENSITIVE::label CB_ADF::cb_adf::_cs_labels

private

Definition at line 40 of file cb_adf.cc.

_gen_cs

cb_to_cs_adf CB_ADF::cb_adf::_gen_cs

private

Definition at line 38 of file cb_adf.cc.

_model_file_ver

VW::version_struct* CB_ADF::cb_adf::_model_file_ver

private

Definition at line 36 of file cb_adf.cc.

_no_predict

const bool CB_ADF::cb_adf::_no_predict

private

Definition at line 50 of file cb_adf.cc.

_offset

uint64_t CB_ADF::cb_adf::_offset

private

Definition at line 49 of file cb_adf.cc.

_prepped_cs_labels

v_array<COST_SENSITIVE::label> CB_ADF::cb_adf::_prepped_cs_labels

private

Definition at line 41 of file cb_adf.cc.

_prob_s

action_scores CB_ADF::cb_adf::_prob_s

private

Definition at line 45 of file cb_adf.cc.

_rank_all

const bool CB_ADF::cb_adf::_rank_all

private

Definition at line 51 of file cb_adf.cc.

_sd

shared_data* CB_ADF::cb_adf::_sd

private

Definition at line 33 of file cb_adf.cc.

---

The documentation for this struct was generated from the following file:

• /mnt/c/w/linux/vowpal_wabbit/vowpalwabbit/cb_adf.cc