SVRG Namespace Reference

Classes
struct	svrg
struct	update

Functions
template<int offset>
void	vec_add (float &p, const float x, float &w)
template<int offset>
float	inline_predict (vw &all, example &ec)
float	predict_stable (const svrg &s, example &ec)
void	predict (svrg &s, single_learner &, example &ec)
float	gradient_scalar (const svrg &s, const example &ec, float pred)
void	update_inner_feature (update &u, float x, float &w)
void	update_stable_feature (float &g_scalar, float x, float &w)
void	update_inner (const svrg &s, example &ec)
void	update_stable (const svrg &s, example &ec)
void	learn (svrg &s, single_learner &base, example &ec)
void	save_load (svrg &s, io_buf &model_file, bool read, bool text)

Function Documentation

gradient_scalar()

float SVRG::gradient_scalar	(	const svrg &	s,
		const example &	ec,
		float	pred
	)

Definition at line 61 of file svrg.cc.

References SVRG::svrg::all, loss_function::first_derivative(), example::l, label_data::label, vw::loss, vw::sd, polylabel::simple, and example::weight.

Referenced by update_inner(), and update_stable().

{
  return s.all->loss->first_derivative(s.all->sd, pred, ec.l.simple.label) * ec.weight;
}

inline_predict()

template<int offset>

float SVRG::inline_predict ( vw &  all, example &  ec  )

inline

Definition at line 41 of file svrg.cc.

References label_data::initial, example::l, and polylabel::simple.

{
  float acc = ec.l.simple.initial;
  GD::foreach_feature<float, vec_add<offset> >(all, ec, acc);
  return acc;
}

learn()

void SVRG::learn	(	svrg &	s,
		single_learner &	base,
		example &	ec
	)

Definition at line 105 of file svrg.cc.

References SVRG::svrg::all, VW::get_weight(), example::in_use, VW::num_weights(), vw::passes_complete, predict(), SVRG::svrg::prev_pass, vw::quiet, VW::set_weight(), SVRG::svrg::stable_grad_count, SVRG::svrg::stage_size, update_inner(), update_stable(), W_INNER, W_STABLE, and W_STABLEGRAD.

Referenced by svrg_setup().

{
  assert(ec.in_use);

  predict(s, base, ec);

  const int pass = (int)s.all->passes_complete;

  if (pass % (s.stage_size + 1) == 0)  // Compute exact gradient
  {
    if (s.prev_pass != pass && !s.all->quiet)
    {
      std::cout << "svrg pass " << pass << ": committing stable point" << std::endl;
      for (uint32_t j = 0; j < VW::num_weights(*s.all); j++)
      {
        float w = VW::get_weight(*s.all, j, W_INNER);
        VW::set_weight(*s.all, j, W_STABLE, w);
        VW::set_weight(*s.all, j, W_STABLEGRAD, 0.f);
      }
      s.stable_grad_count = 0;
      std::cout << "svrg pass " << pass << ": computing exact gradient" << std::endl;
    }
    update_stable(s, ec);
    s.stable_grad_count++;
  }
  else  // Perform updates
  {
    if (s.prev_pass != pass && !s.all->quiet)
    {
      std::cout << "svrg pass " << pass << ": taking steps" << std::endl;
    }
    update_inner(s, ec);
  }

  s.prev_pass = pass;
}

predict()

void SVRG::predict	(	svrg &	s,
		single_learner &	,
		example &	ec
	)

Definition at line 55 of file svrg.cc.

References SVRG::svrg::all, GD::finalize_prediction(), example::partial_prediction, example::pred, polyprediction::scalar, and vw::sd.

Referenced by learn(), and svrg_setup().

{
  ec.partial_prediction = inline_predict<W_INNER>(*s.all, ec);
  ec.pred.scalar = GD::finalize_prediction(s.all->sd, ec.partial_prediction);
}

predict_stable()

float SVRG::predict_stable	(	const svrg &	s,
		example &	ec
	)

Definition at line 50 of file svrg.cc.

References SVRG::svrg::all, GD::finalize_prediction(), and vw::sd.

Referenced by update_inner(), and update_stable().

{
  return GD::finalize_prediction(s.all->sd, inline_predict<W_STABLE>(*s.all, ec));
}

save_load()

void SVRG::save_load	(	svrg &	s,
		io_buf &	model_file,
		bool	read,
		bool	text
	)

Definition at line 142 of file svrg.cc.

References SVRG::svrg::all, bin_text_read_write_fixed(), v_array< T >::empty(), io_buf::files, initialize_regressor(), GD::save_load_online_state(), GD::save_load_regressor(), and vw::save_resume.

Referenced by svrg_setup().

{
  if (read)
  {
    initialize_regressor(*s.all);
  }

  if (!model_file.files.empty())
  {
    bool resume = s.all->save_resume;
    std::stringstream msg;
    msg << ":" << resume << "\n";
    bin_text_read_write_fixed(model_file, (char*)&resume, sizeof(resume), "", read, msg, text);

    double temp = 0.;
    if (resume)
      GD::save_load_online_state(*s.all, model_file, read, text, temp);
    else
      GD::save_load_regressor(*s.all, model_file, read, text);
  }
}

update_inner()

void SVRG::update_inner	(	const svrg &	s,
		example &	ec
	)

Definition at line 88 of file svrg.cc.

References SVRG::svrg::all, SVRG::update::eta, vw::eta, SVRG::update::g_scalar_inner, SVRG::update::g_scalar_stable, gradient_scalar(), SVRG::update::norm, example::pred, predict_stable(), polyprediction::scalar, and SVRG::svrg::stable_grad_count.

Referenced by learn().

{
  update u;
  // |ec| already has prediction according to inner weights.
  u.g_scalar_inner = gradient_scalar(s, ec, ec.pred.scalar);
  u.g_scalar_stable = gradient_scalar(s, ec, predict_stable(s, ec));
  u.eta = s.all->eta;
  u.norm = (float)s.stable_grad_count;
  GD::foreach_feature<update, update_inner_feature>(*s.all, ec, u);
}

update_inner_feature()

void SVRG::update_inner_feature ( update &  u, float  x, float &  w  )

inline

Definition at line 76 of file svrg.cc.

References SVRG::update::eta, SVRG::update::g_scalar_inner, SVRG::update::g_scalar_stable, SVRG::update::norm, and W_STABLEGRAD.

{
  float* ws = &w;
  w -= u.eta * ((u.g_scalar_inner - u.g_scalar_stable) * x + ws[W_STABLEGRAD] / u.norm);
}

update_stable()

void SVRG::update_stable	(	const svrg &	s,
		example &	ec
	)

Definition at line 99 of file svrg.cc.

References SVRG::svrg::all, gradient_scalar(), and predict_stable().

Referenced by learn().

{
  float g = gradient_scalar(s, ec, predict_stable(s, ec));
  GD::foreach_feature<float, update_stable_feature>(*s.all, ec, g);
}

update_stable_feature()

void SVRG::update_stable_feature ( float &  g_scalar, float  x, float &  w  )

inline

Definition at line 82 of file svrg.cc.

References W_STABLEGRAD.

{
  float* ws = &w;
  ws[W_STABLEGRAD] += g_scalar * x;
}

vec_add()

template<int offset>

void SVRG::vec_add ( float &  p, const float  x, float &  w  )

inline

Definition at line 34 of file svrg.cc.

{
  float* ws = &w;
  p += x * ws[offset];
}