ExpReplay Namespace Reference

Classes
struct	expreplay

Functions
template<bool is_learn, label_parser & lp>
void	predict_or_learn (expreplay< lp > &er, LEARNER::single_learner &base, example &ec)
template<label_parser & lp>
void	multipredict (expreplay< lp > &, LEARNER::single_learner &base, example &ec, size_t count, size_t step, polyprediction *pred, bool finalize_predictions)
template<label_parser & lp>
void	end_pass (expreplay< lp > &er)
template<char er_level, label_parser & lp>
LEARNER::base_learner *	expreplay_setup (VW::config::options_i &options, vw &all)

Function Documentation

end_pass()

template<label_parser & lp>

void ExpReplay::end_pass

(

expreplay< lp > &

er

)

Definition at line 69 of file expreplay.h.

References ExpReplay::expreplay< lp >::base, ExpReplay::expreplay< lp >::buf, ExpReplay::expreplay< lp >::filled, LEARNER::learner< T, E >::learn(), and ExpReplay::expreplay< lp >::N.

{  // we need to go through and learn on everyone who remains
  // also need to clean up remaining examples
  for (size_t n = 0; n < er.N; n++)
    if (er.filled[n])
    {  // TODO: if er.replay_count > 1 do we need to play these more?
      er.base->learn(er.buf[n]);
      er.filled[n] = false;
    }
}

expreplay_setup()

template<char er_level, label_parser & lp>

LEARNER::base_learner* ExpReplay::expreplay_setup	(	VW::config::options_i &	options,
		vw &	all
	)

Definition at line 81 of file expreplay.h.

References add(), VW::config::options_i::add_and_parse(), ExpReplay::expreplay< lp >::all, VW::alloc_examples(), LEARNER::as_singleline(), vw::get_random_state(), LEARNER::init_learner(), vw::interactions, LEARNER::make_base(), VW::config::make_option(), vw::quiet, LEARNER::learner< T, E >::set_end_pass(), setup_base(), and VW::config::options_i::was_supplied().

{
  std::string replay_string = "replay_";
  replay_string += er_level;
  std::string replay_count_string = replay_string;
  replay_count_string += "_count";

  auto er = scoped_calloc_or_throw<expreplay<lp>>();
  VW::config::option_group_definition new_options("Experience Replay");
  new_options
      .add(VW::config::make_option(replay_string, er->N)
               .keep()
               .help("use experience replay at a specified level [b=classification/regression, m=multiclass, c=cost "
                     "sensitive] with specified buffer size"))
      .add(VW::config::make_option(replay_count_string, er->replay_count)
               .default_value(1)
               .help("how many times (in expectation) should each example be played (default: 1 = permuting)"));
  options.add_and_parse(new_options);

  if (!options.was_supplied(replay_string) || er->N == 0)
    return nullptr;

  er->all = &all;
  er->_random_state = all.get_random_state();
  er->buf = VW::alloc_examples(1, er->N);
  er->buf->interactions = &all.interactions;

  if (er_level == 'c')
    for (size_t n = 0; n < er->N; n++) er->buf[n].l.cs.costs = v_init<COST_SENSITIVE::wclass>();

  er->filled = calloc_or_throw<bool>(er->N);

  if (!all.quiet)
    std::cerr << "experience replay level=" << er_level << ", buffer=" << er->N << ", replay count=" << er->replay_count
              << std::endl;

  er->base = LEARNER::as_singleline(setup_base(options, all));
  LEARNER::learner<expreplay<lp>, example>* l =
      &init_learner(er, er->base, predict_or_learn<true, lp>, predict_or_learn<false, lp>);
  l->set_end_pass(end_pass<lp>);

  return make_base(*l);
}

multipredict()

template<label_parser & lp>

void ExpReplay::multipredict	(	expreplay< lp > &	,
		LEARNER::single_learner &	base,
		example &	ec,
		size_t	count,
		size_t	step,
		polyprediction *	pred,
		bool	finalize_predictions
	)

Definition at line 62 of file expreplay.h.

References LEARNER::learner< T, E >::multipredict().

{
  base.multipredict(ec, count, step, pred, finalize_predictions);
}

predict_or_learn()

template<bool is_learn, label_parser & lp>

void ExpReplay::predict_or_learn	(	expreplay< lp > &	er,
		LEARNER::single_learner &	base,
		example &	ec
	)

Definition at line 35 of file expreplay.h.

References ExpReplay::expreplay< lp >::_random_state, ExpReplay::expreplay< lp >::all, vw::audit, ExpReplay::expreplay< lp >::buf, VW::copy_example_data(), ExpReplay::expreplay< lp >::filled, example::l, LEARNER::learner< T, E >::learn(), ExpReplay::expreplay< lp >::N, LEARNER::learner< T, E >::predict(), and ExpReplay::expreplay< lp >::replay_count.

{  // regardless of what happens, we must predict
  base.predict(ec);
  // if we're not learning, that's all that has to happen
  if (!is_learn || lp.get_weight(&ec.l) == 0.)
    return;

  for (size_t replay = 1; replay < er.replay_count; replay++)
  {
    size_t n = (size_t)(er._random_state->get_and_update_random() * (float)er.N);
    if (er.filled[n])
      base.learn(er.buf[n]);
  }

  size_t n = (size_t)(er._random_state->get_and_update_random() * (float)er.N);
  if (er.filled[n])
    base.learn(er.buf[n]);

  er.filled[n] = true;
  VW::copy_example_data(er.all->audit, &er.buf[n], &ec);  // don't copy the label
  if (lp.copy_label)
    lp.copy_label(&er.buf[n].l, &ec.l);
  else
    er.buf[n].l = ec.l;
}