CCB Namespace Reference

Classes
struct	conditional_contextual_bandit_outcome
struct	label

Typedefs
typedef v_array< ACTION_SCORE::action_scores >	decision_scores_t

Enumerations
enum	example_type : uint8_t { unset = 0, shared = 1, action = 2, slot = 3 }

Functions
void	default_label (void *v)
size_t	read_cached_label (shared_data *, void *v, io_buf &cache)
float	ccb_weight (void *v)
void	cache_label (void *v, io_buf &cache)
bool	test_label (void *v)
void	delete_label (void *v)
void	copy_label (void *dst, void *src)
ACTION_SCORE::action_score	convert_to_score (const substring &action_id_str, const substring &probability_str)
CCB::conditional_contextual_bandit_outcome *	parse_outcome (substring &outcome)
void	parse_explicit_inclusions (CCB::label *ld, v_array< substring > &split_inclusions)
void	parse_label (parser *p, shared_data *, void *v, v_array< substring > &words)
void	clear_all (ccb &data)
bool	split_multi_example_and_stash_labels (const multi_ex &examples, ccb &data)
template<bool is_learn>
bool	sanity_checks (ccb &data)
void	create_cb_labels (ccb &data)
void	delete_cb_labels (ccb &data)
void	attach_label_to_example (uint32_t action_index_one_based, example *example, conditional_contextual_bandit_outcome *outcome, ccb &data)
void	save_action_scores (ccb &data, decision_scores_t &decision_scores)
void	clear_pred_and_label (ccb &data)
bool	has_action (multi_ex &cb_ex)
void	inject_slot_features (example *shared, example *slot)
template<bool audit>
void	inject_slot_id (ccb &data, example *shared, size_t id)
template<bool audit>
void	remove_slot_id (example *shared)
void	remove_slot_features (example *shared, example *slot)
void	calculate_and_insert_interactions (example *shared, std::vector< example *> actions, std::vector< std::string > &generated_interactions)
template<bool is_learn>
void	build_cb_example (multi_ex &cb_ex, example *slot, ccb &data)
template<bool is_learn>
void	learn_or_predict (ccb &data, multi_learner &base, multi_ex &examples)
void	print_decision_scores (int f, decision_scores_t &decision_scores)
void	print_update (vw &all, std::vector< example *> &slots, decision_scores_t &decision_scores, size_t num_features)
void	output_example (vw &all, ccb &, multi_ex &ec_seq)
void	finish_multiline_example (vw &all, ccb &data, multi_ex &ec_seq)
base_learner *	ccb_explore_adf_setup (options_i &options, vw &all)
bool	ec_is_example_header (example const &ec)

Variables
label_parser	ccb_label_parser
static constexpr uint32_t	SHARED_EX_INDEX = 0
static constexpr uint32_t	TOP_ACTION_INDEX = 0

Typedef Documentation

decision_scores_t

typedef v_array<ACTION_SCORE::action_scores> CCB::decision_scores_t

Definition at line 28 of file conditional_contextual_bandit.h.

Enumeration Type Documentation

example_type

enum CCB::example_type : uint8_t

Enumerator
unset
shared
action
slot

Definition at line 22 of file ccb_label.h.

                  : uint8_t
{
  unset = 0,
  shared = 1,
  action = 2,
  slot = 3
};

Function Documentation

attach_label_to_example()

void CCB::attach_label_to_example	(	uint32_t	action_index_one_based,
		example *	example,
		conditional_contextual_bandit_outcome *	outcome,
		ccb &	data
	)

Definition at line 147 of file conditional_contextual_bandit.cc.

References CB::cb_class::action, polylabel::cb, ccb::cb_label, CCB::conditional_contextual_bandit_outcome::cost, CB::cb_class::cost, CB::label::costs, example::l, CCB::conditional_contextual_bandit_outcome::probabilities, and CB::cb_class::probability.

Referenced by build_cb_example().

{
  // save the cb label
  // Action is unused in cb
  data.cb_label.action = action_index_one_based;
  data.cb_label.probability = outcome->probabilities[0].score;
  data.cb_label.cost = outcome->cost;

  example->l.cb.costs.push_back(data.cb_label);
}

build_cb_example()

template<bool is_learn>

void CCB::build_cb_example	(	multi_ex &	cb_ex,
		example *	slot,
		ccb &	data
	)

Definition at line 320 of file conditional_contextual_bandit.cc.

References polyprediction::a_s, ccb::action_score_pool, ccb::action_with_label, ccb::actions, attach_label_to_example(), polylabel::conditional_contextual_bandit, ccb::exclude_list, CCB::label::explicit_included_actions, ccb::include_list, inject_slot_features(), example::l, ccb::origin_index, CCB::label::outcome, example::pred, CCB::conditional_contextual_bandit_outcome::probabilities, ccb::shared, and example::tag.

{
  bool slot_has_label = slot->l.conditional_contextual_bandit.outcome != nullptr;

  // Merge the slot features with the shared example and set it in the cb multi-example
  // TODO is it imporant for total_sum_feat_sq and num_features to be correct at this point?
  inject_slot_features(data.shared, slot);
  cb_ex.push_back(data.shared);

  // Retrieve the action index whitelist (if the list is empty, then all actions are white-listed)
  auto& explicit_includes = slot->l.conditional_contextual_bandit.explicit_included_actions;
  if (explicit_includes.size() != 0)
  {
    // First time seeing this, initialize the vector with falses so we can start setting each included action.
    if (data.include_list.size() == 0)
    {
      data.include_list.assign(data.actions.size(), false);
    }

    for (uint32_t included_action_id : explicit_includes)
    {
      data.include_list[included_action_id] = true;
    }
  }

  // set the available actions in the cb multi-example
  uint32_t index = 0;
  data.origin_index.clear();
  data.origin_index.resize(data.actions.size(), 0);
  for (size_t i = 0; i < data.actions.size(); i++)
  {
    // Filter actions that are not explicitly included. If the list is empty though, everything is included.
    if (!data.include_list.empty() && !data.include_list[i])
      continue;

    // Filter actions chosen by previous slots
    if (data.exclude_list[i])
      continue;

    // Select the action
    cb_ex.push_back(data.actions[i]);

    // Save the original index from the root multi-example
    data.origin_index[index++] = (uint32_t)i;

    // Remember the index of the chosen action
    if (is_learn && slot_has_label && i == slot->l.conditional_contextual_bandit.outcome->probabilities[0].action)
    {
      // This is used to remove the label later.
      data.action_with_label = (uint32_t)i;
      attach_label_to_example(index, data.actions[i], slot->l.conditional_contextual_bandit.outcome, data);
    }
  }

  // Must provide a prediction that cb can write into, this will be saved into the decision scores object later.
  data.shared->pred.a_s = data.action_score_pool.get_object();

  // Tag can be used for specifying the sampling seed per slot. For it to be used it must be inserted into the shared
  // example.
  std::swap(data.shared->tag, slot->tag);
}

cache_label()

void CCB::cache_label	(	void *	v,
		io_buf &	cache
	)

Definition at line 116 of file ccb_label.cc.

References io_buf::buf_write(), c, VW::convert(), CCB::conditional_contextual_bandit_outcome::cost, CCB::label::explicit_included_actions, CCB::label::outcome, CCB::conditional_contextual_bandit_outcome::probabilities, v_array< T >::size(), CCB::label::type, and CCB::label::weight.

{
  char* c;
  CCB::label* ld = static_cast<CCB::label*>(v);
  size_t size = sizeof(uint8_t)  // type
      + sizeof(bool)             // outcome exists?
      + (ld->outcome == nullptr ? 0
                                : sizeof(ld->outcome->cost)                                    // cost
                    + sizeof(uint32_t)                                                         // probabilities size
                    + sizeof(ACTION_SCORE::action_score) * ld->outcome->probabilities.size())  // probabilities
      + sizeof(uint32_t)  // explicit_included_actions size
      + sizeof(uint32_t) * ld->explicit_included_actions.size() + sizeof(ld->weight);

  cache.buf_write(c, size);

  *(uint8_t*)c = static_cast<uint8_t>(ld->type);
  c += sizeof(ld->type);

  *(bool*)c = ld->outcome != nullptr;
  c += sizeof(bool);

  if (ld->outcome != nullptr)
  {
    *(float*)c = ld->outcome->cost;
    c += sizeof(ld->outcome->cost);

    *(uint32_t*)c = convert(ld->outcome->probabilities.size());
    c += sizeof(uint32_t);

    for (const auto& score : ld->outcome->probabilities)
    {
      *(ACTION_SCORE::action_score*)c = score;
      c += sizeof(ACTION_SCORE::action_score);
    }
  }

  *(uint32_t*)c = convert(ld->explicit_included_actions.size());
  c += sizeof(uint32_t);

  for (const auto& included_action : ld->explicit_included_actions)
  {
    *(uint32_t*)c = included_action;
    c += sizeof(included_action);
  }

  *(float*)c = ld->weight;
  c += sizeof(ld->weight);
}

calculate_and_insert_interactions()

void CCB::calculate_and_insert_interactions	(	example *	shared,
		std::vector< example *>	actions,
		std::vector< std::string > &	generated_interactions
	)

Definition at line 281 of file conditional_contextual_bandit.cc.

References ccb_id_namespace, example_predict::indices, INTERACTIONS::is_printable_namespace(), and INTERACTIONS::printable_start.

Referenced by learn_or_predict().

{
  std::bitset<INTERACTIONS::printable_ns_size> found_namespaces;

  const auto original_size = generated_interactions.size();
  for (size_t i = 0; i < original_size; i++)
  {
    auto interaction_copy = generated_interactions[i];
    interaction_copy.push_back((char)ccb_id_namespace);
    generated_interactions.push_back(interaction_copy);
  }

  for (const auto& action : actions)
  {
    for (const auto& action_index : action->indices)
    {
      if (INTERACTIONS::is_printable_namespace(action_index) &&
          !found_namespaces[action_index - INTERACTIONS::printable_start])
      {
        found_namespaces[action_index - INTERACTIONS::printable_start] = true;
        generated_interactions.push_back({(char)action_index, (char)ccb_id_namespace});
      }
    }
  }

  for (const auto& shared_index : shared->indices)
  {
    if (INTERACTIONS::is_printable_namespace(shared_index) &&
        !found_namespaces[shared_index - INTERACTIONS::printable_start])
    {
      found_namespaces[shared_index - INTERACTIONS::printable_start] = true;
      generated_interactions.push_back({(char)shared_index, (char)ccb_id_namespace});
    }
  }
}

ccb_explore_adf_setup()

LEARNER::base_learner * CCB::ccb_explore_adf_setup	(	options_i &	options,
		vw &	all
	)

Definition at line 626 of file conditional_contextual_bandit.cc.

References VW::config::option_group_definition::add(), VW::config::options_i::add_and_parse(), LEARNER::as_multiline(), label_type::ccb, ccb_id_namespace, ccb_label_parser, prediction_type::decision_probs, ACTION_SCORE::delete_action_scores(), vw::delete_prediction, finish_multiline_example(), VW::hash_space(), LEARNER::init_learner(), VW::config::options_i::insert(), vw::interactions, vw::label_type, parser::lp, LEARNER::make_base(), VW::config::make_option(), vw::p, LEARNER::learner< T, E >::set_finish_example(), setup_base(), parameters::stride_shift(), THROW, VW::config::options_i::was_supplied(), and vw::weights.

Referenced by parse_reductions().

{
  auto data = scoped_calloc_or_throw<ccb>();
  bool ccb_explore_adf_option = false;
  bool slate = false;
  option_group_definition new_options(
      "EXPERIMENTAL: Conditional Contextual Bandit Exploration with Action Dependent Features");
  new_options.add(
      make_option("ccb_explore_adf", ccb_explore_adf_option)
          .keep()
          .help("EXPERIMENTAL: Do Conditional Contextual Bandit learning with multiline action dependent features."));
  new_options.add(make_option("slate", slate).keep().help("EXPERIMENTAL - MAY CHANGE: Enable slate mode in CCB."));
  options.add_and_parse(new_options);

  if (!ccb_explore_adf_option)
    return nullptr;

  if (!options.was_supplied("cb_explore_adf"))
  {
    options.insert("cb_explore_adf", "");
    options.add_and_parse(new_options);
  }

  if (options.was_supplied("cb_sample") && slate)
  {
    THROW("--slate and --cb_sample cannot be supplied together");
  }

  if (!options.was_supplied("cb_sample") && !slate)
  {
    options.insert("cb_sample", "");
    options.add_and_parse(new_options);
  }

  auto base = as_multiline(setup_base(options, all));
  all.p->lp = CCB::ccb_label_parser;
  all.label_type = label_type::ccb;

  // Stash the base learners stride_shift so we can properly add a feature later.
  data->base_learner_stride_shift = all.weights.stride_shift();

  // Extract from lower level reductions
  data->default_cb_label = {FLT_MAX, 0, -1.f, 0.f};
  data->shared = nullptr;
  data->original_interactions = &all.interactions;
  data->all = &all;

  data->id_namespace_str.push_back((char)ccb_id_namespace);
  data->id_namespace_str.append("_id");
  data->id_namespace_hash = VW::hash_space(all, data->id_namespace_str);

  learner<ccb, multi_ex>& l =
      init_learner(data, base, learn_or_predict<true>, learn_or_predict<false>, 1, prediction_type::decision_probs);

  all.delete_prediction = ACTION_SCORE::delete_action_scores;

  l.set_finish_example(finish_multiline_example);
  return make_base(l);
}

ccb_weight()

float CCB::ccb_weight

(

void *

v

)

Definition at line 110 of file ccb_label.cc.

References CCB::label::weight.

{
  CCB::label* ld = (CCB::label*)v;
  return ld->weight;
}

clear_all()

void CCB::clear_all

(

ccb &

data

)

Definition at line 60 of file conditional_contextual_bandit.cc.

References ccb::action_with_label, ccb::actions, ccb::shared, ccb::slots, and ccb::stored_labels.

Referenced by learn_or_predict().

{
  // data.include_list and data.exclude_list aren't cleared here but are assigned in the predict/learn function
  data.shared = nullptr;
  data.actions.clear();
  data.slots.clear();
  data.action_with_label = 0;
  data.stored_labels.clear();
}

clear_pred_and_label()

void CCB::clear_pred_and_label

(

ccb &

data

)

Definition at line 175 of file conditional_contextual_bandit.cc.

References ccb::action_with_label, and ccb::actions.

Referenced by learn_or_predict().

{
  // Don't need to return to pool, as that will be done when the example is output.

  // This just needs to be cleared as it is reused.
  data.actions[data.action_with_label]->l.cb.costs.clear();
}

convert_to_score()

ACTION_SCORE::action_score CCB::convert_to_score	(	const substring &	action_id_str,
		const substring &	probability_str
	)

Definition at line 219 of file ccb_label.cc.

References float_of_substring(), int_of_substring(), and THROW.

Referenced by parse_outcome().

{
  auto action_id = static_cast<uint32_t>(int_of_substring(action_id_str));
  auto probability = float_of_substring(probability_str);
  if (std::isnan(probability))
    THROW("error NaN probability: " << probability_str);

  if (probability > 1.0)
  {
    std::cerr << "invalid probability > 1 specified for an outcome, resetting to 1.\n";
    probability = 1.0;
  }
  if (probability < 0.0)
  {
    std::cerr << "invalid probability < 0 specified for an outcome, resetting to 0.\n";
    probability = .0;
  }

  return {action_id, probability};
}

copy_label()

void CCB::copy_label	(	void *	dst,
		void *	src
	)

Definition at line 200 of file ccb_label.cc.

References copy_array(), CCB::conditional_contextual_bandit_outcome::cost, CCB::label::explicit_included_actions, CCB::label::outcome, CCB::conditional_contextual_bandit_outcome::probabilities, CCB::label::type, and CCB::label::weight.

{
  CCB::label* ldDst = static_cast<CCB::label*>(dst);
  CCB::label* ldSrc = static_cast<CCB::label*>(src);

  if (ldSrc->outcome)
  {
    ldDst->outcome = new CCB::conditional_contextual_bandit_outcome();
    ldDst->outcome->probabilities = v_init<ACTION_SCORE::action_score>();

    ldDst->outcome->cost = ldSrc->outcome->cost;
    copy_array(ldDst->outcome->probabilities, ldSrc->outcome->probabilities);
  }

  copy_array(ldDst->explicit_included_actions, ldSrc->explicit_included_actions);
  ldDst->type = ldSrc->type;
  ldDst->weight = ldSrc->weight;
}

create_cb_labels()

void CCB::create_cb_labels

(

ccb &

data

)

Definition at line 125 of file conditional_contextual_bandit.cc.

References ccb::actions, polylabel::cb, ccb::cb_label_pool, CB::label::costs, ccb::default_cb_label, example::l, ccb::shared, and CB::label::weight.

Referenced by learn_or_predict().

{
  data.shared->l.cb.costs = data.cb_label_pool.get_object();
  data.shared->l.cb.costs.push_back(data.default_cb_label);
  for (example* action : data.actions)
  {
    action->l.cb.costs = data.cb_label_pool.get_object();
  }
  data.shared->l.cb.weight = 1.0;
}

default_label()

void CCB::default_label

(

void *

v

)

Definition at line 165 of file ccb_label.cc.

References v_array< T >::clear(), v_array< T >::delete_v(), CCB::label::explicit_included_actions, CCB::label::outcome, CCB::conditional_contextual_bandit_outcome::probabilities, CCB::label::type, unset, and CCB::label::weight.

Referenced by read_cached_label().

{
  CCB::label* ld = static_cast<CCB::label*>(v);

  // This is tested against nullptr, so unfortunately as things are this must be deleted when not used.
  if (ld->outcome)
  {
    ld->outcome->probabilities.delete_v();
    delete ld->outcome;
    ld->outcome = nullptr;
  }

  ld->explicit_included_actions.clear();
  ld->type = example_type::unset;
  ld->weight = 1.0;
}

delete_cb_labels()

void CCB::delete_cb_labels

(

ccb &

data

)

Definition at line 137 of file conditional_contextual_bandit.cc.

References ccb::actions, polylabel::cb, ccb::cb_label_pool, CB::label::costs, example::l, return_v_array(), and ccb::shared.

Referenced by learn_or_predict().

{
  return_v_array(data.shared->l.cb.costs, data.cb_label_pool);

  for (example* action : data.actions)
  {
    return_v_array(action->l.cb.costs, data.cb_label_pool);
  }
}

delete_label()

void CCB::delete_label

(

void *

v

)

Definition at line 188 of file ccb_label.cc.

References v_array< T >::delete_v(), CCB::label::explicit_included_actions, CCB::label::outcome, and CCB::conditional_contextual_bandit_outcome::probabilities.

{
  CCB::label* ld = static_cast<CCB::label*>(v);
  if (ld->outcome)
  {
    ld->outcome->probabilities.delete_v();
    delete ld->outcome;
    ld->outcome = nullptr;
  }
  ld->explicit_included_actions.delete_v();
}

ec_is_example_header()

bool CCB::ec_is_example_header

(

example const &

ec

)

Definition at line 686 of file conditional_contextual_bandit.cc.

References polylabel::conditional_contextual_bandit, example::l, shared, and CCB::label::type.

Referenced by LEARNER::example_is_newline_not_header().

{ return ec.l.conditional_contextual_bandit.type == example_type::shared; }

finish_multiline_example()

void CCB::finish_multiline_example	(	vw &	all,
		ccb &	data,
		multi_ex &	ec_seq
	)

Definition at line 609 of file conditional_contextual_bandit.cc.

References ccb::action_score_pool, vw::final_prediction_sink, VW::finish_example(), CB_ADF::global_print_newline(), output_example(), and return_v_array().

{
  if (ec_seq.size() > 0)
  {
    output_example(all, data, ec_seq);
    CB_ADF::global_print_newline(all.final_prediction_sink);
  }

  for (auto& a_s : ec_seq[0]->pred.decision_scores)
  {
    return_v_array(a_s, data.action_score_pool);
  }
  ec_seq[0]->pred.decision_scores.clear();

  VW::finish_example(all, ec_seq);
}

has_action()

bool CCB::has_action

(

multi_ex &

cb_ex

)

Definition at line 184 of file conditional_contextual_bandit.cc.

Referenced by learn_or_predict().

{ return !cb_ex.empty(); }

inject_slot_features()

void CCB::inject_slot_features	(	example *	shared,
		example *	slot
	)

Definition at line 190 of file conditional_contextual_bandit.cc.

References LabelDict::add_example_namespace(), ccb_slot_namespace, constant_namespace, default_namespace, example_predict::feature_space, and example_predict::indices.

Referenced by build_cb_example().

{
  for (auto index : slot->indices)
  {
    // constant namespace should be ignored, as it already exists and we don't want to double it up.
    if (index == constant_namespace)
    {
      continue;
    }
    else if (index == default_namespace)  // slot default namespace has a special namespace in shared
    {
      LabelDict::add_example_namespace(*shared, ccb_slot_namespace, slot->feature_space[default_namespace]);
    }
    else
    {
      LabelDict::add_example_namespace(*shared, index, slot->feature_space[index]);
    }
  }
}

inject_slot_id()

template<bool audit>

void CCB::inject_slot_id	(	ccb &	data,
		example *	shared,
		size_t	id
	)

Definition at line 211 of file conditional_contextual_bandit.cc.

References ccb::all, ccb::base_learner_stride_shift, ccb_id_namespace, example_predict::feature_space, VW::hash_feature(), id(), ccb::id_namespace_hash, ccb::id_namespace_str, example_predict::indices, v_array< T >::push_back(), ccb::slot_id_hashes, prediction_type::to_string(), and vw::wpp.

{
  // id is zero based, so the vector must be of size id + 1
  if (id + 1 > data.slot_id_hashes.size())
  {
    data.slot_id_hashes.resize(id + 1, 0);
  }

  uint64_t index;
  if (data.slot_id_hashes[id] == 0)
  {
    auto current_index_str = "index" + std::to_string(id);
    index = VW::hash_feature(*data.all, current_index_str, data.id_namespace_hash);

    // To maintain indicies consistent with what the parser does we must scale.
    index *= static_cast<uint64_t>(data.all->wpp) << data.base_learner_stride_shift;
    data.slot_id_hashes[id] = index;
  }
  else
  {
    index = data.slot_id_hashes[id];
  }

  shared->feature_space[ccb_id_namespace].push_back(1., index);
  shared->indices.push_back(ccb_id_namespace);

  if (audit)
  {
    auto current_index_str = "index" + std::to_string(id);
    shared->feature_space[ccb_id_namespace].space_names.push_back(
        audit_strings_ptr(new audit_strings(data.id_namespace_str, current_index_str)));
  }
}

learn_or_predict()

template<bool is_learn>

void CCB::learn_or_predict	(	ccb &	data,
		multi_learner &	base,
		multi_ex &	examples
	)

Definition at line 385 of file conditional_contextual_bandit.cc.

References ccb::actions, ccb::all, vw::audit, calculate_and_insert_interactions(), ccb::cb_ex, clear_all(), clear_pred_and_label(), create_cb_labels(), delete_cb_labels(), ccb::exclude_list, ccb::generated_interactions, has_action(), ccb::include_list, example_predict::interactions, ccb::original_interactions, remove_slot_features(), save_action_scores(), ccb::shared, slot, ccb::slots, split_multi_example_and_stash_labels(), ccb::stored_labels, and example::tag.

{
  clear_all(data);
  if (!split_multi_example_and_stash_labels(examples, data))  // split shared, actions and slots
    return;

#ifndef NDEBUG
  if (!sanity_checks<is_learn>(data))
    return;
#endif

  // This will overwrite the labels with CB.
  create_cb_labels(data);

  // Reset exclusion list for this example.
  data.exclude_list.assign(data.actions.size(), false);

  auto decision_scores = examples[0]->pred.decision_scores;

  // for each slot, re-build the cb example and call cb_explore_adf
  size_t slot_id = 0;
  for (example* slot : data.slots)
  {
    // Namespace crossing for slot features.
    data.generated_interactions.clear();
    std::copy(data.original_interactions->begin(), data.original_interactions->end(),
        std::back_inserter(data.generated_interactions));
    calculate_and_insert_interactions(data.shared, data.actions, data.generated_interactions);
    data.shared->interactions = &data.generated_interactions;
    for (auto ex : data.actions)
    {
      ex->interactions = &data.generated_interactions;
    }

    data.include_list.clear();
    build_cb_example<is_learn>(data.cb_ex, slot, data);

    if (data.all->audit)
      inject_slot_id<true>(data, data.shared, slot_id);
    else
      inject_slot_id<false>(data, data.shared, slot_id);

    if (has_action(data.cb_ex))
    {
      // the cb example contains at least 1 action
      multiline_learn_or_predict<is_learn>(base, data.cb_ex, examples[0]->ft_offset);
      save_action_scores(data, decision_scores);
      clear_pred_and_label(data);
    }
    else
    {
      // the cb example contains no action => cannot decide
      decision_scores.push_back(data.action_score_pool.get_object());
    }

    data.shared->interactions = data.original_interactions;
    for (auto ex : data.actions)
    {
      ex->interactions = data.original_interactions;
    }
    remove_slot_features(data.shared, slot);

    if (data.all->audit)
      remove_slot_id<true>(data.shared);
    else
      remove_slot_id<false>(data.shared);

    // Put back the original shared example tag.
    std::swap(data.shared->tag, slot->tag);
    slot_id++;
    data.cb_ex.clear();
  }

  delete_cb_labels(data);

  // Restore ccb labels to the example objects.
  for (size_t i = 0; i < examples.size(); i++)
  {
    examples[i]->l.conditional_contextual_bandit = {
        data.stored_labels[i].type, data.stored_labels[i].outcome, data.stored_labels[i].explicit_included_actions, 0.};
  }

  // Save the predictions
  examples[0]->pred.decision_scores = decision_scores;
}

output_example()

void CCB::output_example	(	vw &	all,
		ccb &	,
		multi_ex &	ec_seq
	)

Definition at line 557 of file conditional_contextual_bandit.cc.

References vw::final_prediction_sink, CB_ALGS::get_cost_estimate(), loss(), print_decision_scores(), print_update(), vw::sd, slot, TOP_ACTION_INDEX, and shared_data::update().

Referenced by finish_multiline_example().

{
  if (ec_seq.size() <= 0)
    return;

  std::vector<example*> slots;
  size_t num_features = 0;
  float loss = 0.;

  // Should this be done for shared, action and slot?
  for (auto ec : ec_seq)
  {
    num_features += ec->num_features;

    if (ec->l.conditional_contextual_bandit.type == CCB::example_type::slot)
    {
      slots.push_back(ec);
    }
  }

  // Is it hold out?
  size_t num_labelled = 0;
  auto preds = ec_seq[0]->pred.decision_scores;
  for (size_t i = 0; i < slots.size(); i++)
  {
    auto outcome = slots[i]->l.conditional_contextual_bandit.outcome;
    if (outcome != nullptr)
    {
      num_labelled++;
      float l = CB_ALGS::get_cost_estimate(
          outcome->probabilities[TOP_ACTION_INDEX], outcome->cost, preds[i][TOP_ACTION_INDEX].action);
      loss += l * preds[i][TOP_ACTION_INDEX].score;
    }
  }

  if (num_labelled > 0 && num_labelled < slots.size())
  {
    std::cerr << "Warning: Unlabeled example in train set, was this intentional?\n";
  }

  bool holdout_example = num_labelled > 0;
  for (size_t i = 0; i < ec_seq.size(); i++) holdout_example &= ec_seq[i]->test_only;

  // TODO what does weight mean here?
  all.sd->update(holdout_example, num_labelled > 0, loss, ec_seq[SHARED_EX_INDEX]->weight, num_features);

  for (auto sink : all.final_prediction_sink)
    print_decision_scores(sink, ec_seq[SHARED_EX_INDEX]->pred.decision_scores);

  CCB::print_update(all, slots, preds, num_features);
}

parse_explicit_inclusions()

void CCB::parse_explicit_inclusions	(	CCB::label *	ld,
		v_array< substring > &	split_inclusions
	)

Definition at line 277 of file ccb_label.cc.

References CCB::label::explicit_included_actions, int_of_substring(), and v_array< T >::push_back().

Referenced by parse_label().

{
  for (const auto& inclusion : split_inclusions)
  {
    ld->explicit_included_actions.push_back(int_of_substring(inclusion));
  }
}

parse_label()

void CCB::parse_label	(	parser *	p,
		shared_data *	,
		void *	v,
		v_array< substring > &	words
	)

Definition at line 285 of file ccb_label.cc.

References accumulate(), v_array< T >::begin(), v_array< T >::end(), f, recall_tree_ns::find(), CCB::label::outcome, parse_explicit_inclusions(), parser::parse_name, parse_outcome(), CCB::conditional_contextual_bandit_outcome::probabilities, shared, v_array< T >::size(), slot, substring_equal(), THROW, tokenize(), CCB::label::type, and CCB::label::weight.

{
  CCB::label* ld = static_cast<CCB::label*>(v);
  ld->weight = 1.0;

  if (words.size() < 2)
    THROW("ccb labels may not be empty");
  if (!substring_equal(words[0], "ccb"))
  {
    THROW("ccb labels require the first word to be ccb");
  }

  auto type = words[1];
  if (substring_equal(type, "shared"))
  {
    if (words.size() > 2)
      THROW("shared labels may not have a cost");
    ld->type = CCB::example_type::shared;
  }
  else if (substring_equal(type, "action"))
  {
    if (words.size() > 2)
      THROW("action labels may not have a cost");
    ld->type = CCB::example_type::action;
  }
  else if (substring_equal(type, "slot"))
  {
    if (words.size() > 4)
      THROW("ccb slot label can only have a type cost and exclude list");
    ld->type = CCB::example_type::slot;

    // Skip the first two words "ccb <type>"
    for (size_t i = 2; i < words.size(); i++)
    {
      auto is_outcome = std::find(words[i].begin, words[i].end, ':');
      if (is_outcome != words[i].end)
      {
        if (ld->outcome != nullptr)
        {
          THROW("There may be only 1 outcome associated with a slot.")
        }

        ld->outcome = parse_outcome(words[i]);
      }
      else
      {
        tokenize(',', words[i], p->parse_name);
        parse_explicit_inclusions(ld, p->parse_name);
      }
    }

    // If a full distribution has been given, check if it sums to 1, otherwise throw.
    if (ld->outcome && ld->outcome->probabilities.size() > 1)
    {
      float total_pred = std::accumulate(ld->outcome->probabilities.begin(), ld->outcome->probabilities.end(), 0.f,
          [](float result_so_far, ACTION_SCORE::action_score action_pred) {
            return result_so_far + action_pred.score;
          });

      // TODO do a proper comparison here.
      if (total_pred > 1.1f || total_pred < 0.9f)
      {
        THROW("When providing all predicition probabilties they must add up to 1.f");
      }
    }
  }
  else
  {
    THROW("unknown label type: " << type);
  }
}

parse_outcome()

CCB::conditional_contextual_bandit_outcome* CCB::parse_outcome

(

substring &

outcome

)

Definition at line 241 of file ccb_label.cc.

References convert_to_score(), float_of_substring(), THROW, and tokenize().

Referenced by parse_label().

{
  auto& ccb_outcome = *(new CCB::conditional_contextual_bandit_outcome());

  auto split_commas = v_init<substring>();
  tokenize(',', outcome, split_commas);

  auto split_colons = v_init<substring>();
  tokenize(':', split_commas[0], split_colons);

  if (split_colons.size() != 3)
    THROW("Malformed ccb label");

  ccb_outcome.probabilities = v_init<ACTION_SCORE::action_score>();
  ccb_outcome.probabilities.push_back(convert_to_score(split_colons[0], split_colons[2]));

  ccb_outcome.cost = float_of_substring(split_colons[1]);
  if (std::isnan(ccb_outcome.cost))
    THROW("error NaN cost: " << split_colons[1]);

  split_colons.clear();

  for (size_t i = 1; i < split_commas.size(); i++)
  {
    tokenize(':', split_commas[i], split_colons);
    if (split_colons.size() != 2)
      THROW("Must be action probability pairs");
    ccb_outcome.probabilities.push_back(convert_to_score(split_colons[0], split_colons[1]));
  }

  split_colons.delete_v();
  split_commas.delete_v();

  return &ccb_outcome;
}

print_decision_scores()

void CCB::print_decision_scores	(	int	f,
		decision_scores_t &	decision_scores
	)

Definition at line 471 of file conditional_contextual_bandit.cc.

References slot, and io_buf::write_file_or_socket().

Referenced by output_example().

{
  if (f >= 0)
  {
    std::stringstream ss;
    for (auto slot : decision_scores)
    {
      std::string delimiter = "";
      for (auto action_score : slot)
      {
        ss << delimiter << action_score.action << ':' << action_score.score;
        delimiter = ",";
      }
      ss << '\n';
    }
    ssize_t len = ss.str().size();
    ssize_t t = io_buf::write_file_or_socket(f, ss.str().c_str(), (unsigned int)len);
    if (t != len)
      std::cerr << "write error: " << strerror(errno) << std::endl;
  }
}

print_update()

void CCB::print_update	(	vw &	all,
		std::vector< example *> &	slots,
		decision_scores_t &	decision_scores,
		size_t	num_features
	)

Definition at line 493 of file conditional_contextual_bandit.cc.

References vw::bfgs, shared_data::col_current_label, shared_data::col_current_predict, vw::current_pass, shared_data::dump_interval, vw::holdout_set_off, shared_data::print_update(), vw::progress_add, vw::progress_arg, vw::quiet, vw::sd, slot, prediction_type::to_string(), and shared_data::weighted_examples().

Referenced by output_example().

{
  if (all.sd->weighted_examples() >= all.sd->dump_interval && !all.quiet && !all.bfgs)
  {
    std::string label_str = "";
    std::string delim = "";
    int counter = 0;
    for (auto slot : slots)
    {
      counter++;

      auto outcome = slot->l.conditional_contextual_bandit.outcome;
      if (outcome == nullptr)
      {
        label_str += delim;
        label_str += "?";
      }
      else
      {
        label_str += delim;
        label_str += std::to_string(outcome->probabilities[0].action);
        label_str += ":";
        label_str += std::to_string(outcome->cost);
      }

      delim = ",";

      // Stop after 2...
      if (counter > 1 && slots.size() > 2)
      {
        label_str += delim;
        label_str += "...";
        break;
      }
    }
    std::ostringstream label_buf;
    label_buf << std::setw(all.sd->col_current_label) << std::right << std::setfill(' ') << label_str;

    std::string pred_str = "";
    delim = "";
    counter = 0;
    for (auto slot : decision_scores)
    {
      counter++;
      pred_str += delim;
      pred_str += std::to_string(slot[0].action);
      delim = ",";

      // Stop after 3...
      if (counter > 2)
      {
        pred_str += delim;
        pred_str += "...";
        break;
      }
    }
    std::ostringstream pred_buf;
    pred_buf << std::setw(all.sd->col_current_predict) << std::right << std::setfill(' ') << pred_str;

    all.sd->print_update(all.holdout_set_off, all.current_pass, label_buf.str(), pred_buf.str(), num_features,
        all.progress_add, all.progress_arg);
  }
}

read_cached_label()

size_t CCB::read_cached_label	(	shared_data *	,
		void *	v,
		io_buf &	cache
	)

Definition at line 26 of file ccb_label.cc.

References io_buf::buf_read(), v_array< T >::clear(), CCB::conditional_contextual_bandit_outcome::cost, default_label(), CCB::label::explicit_included_actions, CCB::label::outcome, CCB::conditional_contextual_bandit_outcome::probabilities, v_array< T >::push_back(), CCB::label::type, and CCB::label::weight.

{
  // Since read_cached_features doesn't default the label we must do it here.
  default_label(v);
  CCB::label* ld = static_cast<CCB::label*>(v);

  if (ld->outcome)
  {
    ld->outcome->probabilities.clear();
  }
  ld->explicit_included_actions.clear();

  size_t read_count = 0;
  char* read_ptr;

  size_t next_read_size = sizeof(ld->type);
  if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
    return 0;
  ld->type = *(CCB::example_type*)read_ptr;
  read_count += sizeof(ld->type);

  bool is_outcome_present;
  next_read_size = sizeof(bool);
  if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
    return 0;
  is_outcome_present = *(bool*)read_ptr;
  read_count += sizeof(is_outcome_present);

  if (is_outcome_present)
  {
    ld->outcome = new CCB::conditional_contextual_bandit_outcome();
    ld->outcome->probabilities = v_init<ACTION_SCORE::action_score>();

    next_read_size = sizeof(ld->outcome->cost);
    if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
      return 0;
    ld->outcome->cost = *(float*)read_ptr;
    read_count += sizeof(ld->outcome->cost);

    uint32_t size_probs;
    next_read_size = sizeof(size_probs);
    if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
      return 0;
    size_probs = *(uint32_t*)read_ptr;
    read_count += sizeof(size_probs);

    for (uint32_t i = 0; i < size_probs; i++)
    {
      ACTION_SCORE::action_score a_s;
      next_read_size = sizeof(a_s);
      if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
        return 0;
      a_s = *(ACTION_SCORE::action_score*)read_ptr;
      read_count += sizeof(a_s);

      ld->outcome->probabilities.push_back(a_s);
    }
  }

  uint32_t size_includes;
  next_read_size = sizeof(size_includes);
  if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
    return 0;
  size_includes = *(uint32_t*)read_ptr;
  read_count += sizeof(size_includes);

  for (uint32_t i = 0; i < size_includes; i++)
  {
    uint32_t include;
    next_read_size = sizeof(include);
    if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
      return 0;
    include = *(uint32_t*)read_ptr;
    read_count += sizeof(include);
    ld->explicit_included_actions.push_back(include);
  }

  next_read_size = sizeof(ld->weight);
  if (cache.buf_read(read_ptr, next_read_size) < next_read_size)
    return 0;
  ld->weight = *(float*)read_ptr;
  return read_count;
}

remove_slot_features()

void CCB::remove_slot_features	(	example *	shared,
		example *	slot
	)

Definition at line 259 of file conditional_contextual_bandit.cc.

References ccb_slot_namespace, constant_namespace, default_namespace, LabelDict::del_example_namespace(), example_predict::feature_space, and example_predict::indices.

Referenced by learn_or_predict().

{
  for (auto index : slot->indices)
  {
    // constant namespace should be ignored, as it already exists and we don't want to double it up.
    if (index == constant_namespace)
    {
      continue;
    }

    else if (index == default_namespace)  // slot default namespace has a special namespace in shared
    {
      LabelDict::del_example_namespace(*shared, ccb_slot_namespace, slot->feature_space[32]);
    }
    else
    {
      LabelDict::del_example_namespace(*shared, index, slot->feature_space[index]);
    }
  }
}

remove_slot_id()

template<bool audit>

void CCB::remove_slot_id

(

example *

shared

)

Definition at line 247 of file conditional_contextual_bandit.cc.

References ccb_id_namespace, example_predict::feature_space, example_predict::indices, and v_array< T >::pop().

{
  shared->feature_space[ccb_id_namespace].indicies.pop();
  shared->feature_space[ccb_id_namespace].values.pop();
  shared->indices.pop();

  if (audit)
  {
    shared->feature_space[ccb_id_namespace].space_names.pop();
  }
}

sanity_checks()

template<bool is_learn>

bool CCB::sanity_checks

(

ccb &

data

)

Definition at line 100 of file conditional_contextual_bandit.cc.

References ccb::actions, slot, and ccb::slots.

{
  if (data.slots.size() > data.actions.size())
  {
    std::cerr << "ccb_adf_explore: badly formatted example - number of actions " << data.actions.size()
              << " must be greater than the number of slots " << data.slots.size();
    return false;
  }

  if (is_learn)
  {
    for (auto slot : data.slots)
    {
      if (slot->l.conditional_contextual_bandit.outcome != nullptr &&
          slot->l.conditional_contextual_bandit.outcome->probabilities.size() == 0)
      {
        std::cerr << "ccb_adf_explore: badly formatted example - missing label probability";
        return false;
      }
    }
  }
  return true;
}

save_action_scores()

void CCB::save_action_scores	(	ccb &	data,
		decision_scores_t &	decision_scores
	)

Definition at line 159 of file conditional_contextual_bandit.cc.

References polyprediction::a_s, ccb::exclude_list, ccb::origin_index, example::pred, v_array< T >::push_back(), and ccb::shared.

Referenced by learn_or_predict().

{
  auto& pred = data.shared->pred.a_s;
  decision_scores.push_back(pred);

  // correct indices: we want index relative to the original ccb multi-example, with no actions filtered
  for (auto& action_score : pred)
  {
    action_score.action = data.origin_index[action_score.action];
  }

  // Exclude the chosen action from next slots.
  auto original_index_of_chosen_action = pred[0].action;
  data.exclude_list[original_index_of_chosen_action] = true;
}

split_multi_example_and_stash_labels()

bool CCB::split_multi_example_and_stash_labels	(	const multi_ex &	examples,
		ccb &	data
	)

Definition at line 71 of file conditional_contextual_bandit.cc.

References ccb::actions, shared, ccb::shared, slot, ccb::slots, and ccb::stored_labels.

Referenced by learn_or_predict().

{
  for (auto ex : examples)
  {
    switch (ex->l.conditional_contextual_bandit.type)
    {
      case example_type::shared:
        data.shared = ex;
        break;
      case example_type::action:
        data.actions.push_back(ex);
        break;
      case example_type::slot:
        data.slots.push_back(ex);
        break;
      default:
        std::cout << "ccb_adf_explore: badly formatted example - invalid example type";
        return false;
    }

    // Stash the CCB labels before rewriting them.
    data.stored_labels.push_back({ex->l.conditional_contextual_bandit.type, ex->l.conditional_contextual_bandit.outcome,
        ex->l.conditional_contextual_bandit.explicit_included_actions, 0.});
  }

  return true;
}

test_label()

bool CCB::test_label

(

void *

v

)

Definition at line 182 of file ccb_label.cc.

References CCB::label::outcome.

{
  CCB::label* ld = static_cast<CCB::label*>(v);
  return ld->outcome == nullptr;
}

Variable Documentation

ccb_label_parser

label_parser CCB::ccb_label_parser

Initial value:

= {default_label, parse_label, cache_label, read_cached_label, delete_label, ccb_weight,
    copy_label, test_label, sizeof(CCB::label)}

Definition at line 358 of file ccb_label.cc.

Referenced by ccb_explore_adf_setup().

SHARED_EX_INDEX

constexpr uint32_t CCB::SHARED_EX_INDEX = 0

static

Definition at line 57 of file conditional_contextual_bandit.cc.

TOP_ACTION_INDEX

constexpr uint32_t CCB::TOP_ACTION_INDEX = 0

static

Definition at line 58 of file conditional_contextual_bandit.cc.

Referenced by output_example().