accumulate.h File Reference

#include "global_data.h"

Go to the source code of this file.

Functions
void	accumulate (vw &all, parameters &weights, size_t o)
float	accumulate_scalar (vw &all, float local_sum)
void	accumulate_weighted_avg (vw &all, parameters &weights)
void	accumulate_avg (vw &all, parameters &weights, size_t o)

Function Documentation

accumulate()

void accumulate	(	vw &	all,
		parameters &	weights,
		size_t	o
	)

Definition at line 20 of file accumulate.cc.

References parameters::dense_weights, vw::num_bits, parameters::sparse, parameters::sparse_weights, dense_parameters::stride_shift(), sparse_parameters::stride_shift(), and UINT64_ONE.

Referenced by average_diff(), bs_predict_mean(), calculate_sd(), ldamath::expdigammify(), exploration::generate_bag(), CCB::parse_label(), and process_pass().

{
  uint64_t length = UINT64_ONE << all.num_bits;  // This is size of gradient
  float* local_grad = new float[length];

  if (weights.sparse)
    for (uint64_t i = 0; i < length; i++)
      local_grad[i] = (&(weights.sparse_weights[i << weights.sparse_weights.stride_shift()]))[offset];
  else
    for (uint64_t i = 0; i < length; i++)
      local_grad[i] = (&(weights.dense_weights[i << weights.dense_weights.stride_shift()]))[offset];

  all_reduce<float, add_float>(all, local_grad, length);  // TODO: modify to not use first()

  if (weights.sparse)
    for (uint64_t i = 0; i < length; i++)
      (&(weights.sparse_weights[i << weights.sparse_weights.stride_shift()]))[offset] = local_grad[i];
  else
    for (uint64_t i = 0; i < length; i++)
      (&(weights.dense_weights[i << weights.dense_weights.stride_shift()]))[offset] = local_grad[i];

  delete[] local_grad;
}

accumulate_avg()

void accumulate_avg	(	vw &	all,
		parameters &	weights,
		size_t	o
	)

Definition at line 51 of file accumulate.cc.

References vw::all_reduce, parameters::dense_weights, vw::num_bits, parameters::sparse, parameters::sparse_weights, dense_parameters::stride_shift(), sparse_parameters::stride_shift(), and AllReduce::total.

Referenced by GD::end_pass().

{
  uint32_t length = 1 << all.num_bits;  // This is size of gradient
  float numnodes = (float)all.all_reduce->total;
  float* local_grad = new float[length];

  if (weights.sparse)
    for (uint64_t i = 0; i < length; i++)
      local_grad[i] = (&(weights.sparse_weights[i << weights.sparse_weights.stride_shift()]))[offset];
  else
    for (uint64_t i = 0; i < length; i++)
      local_grad[i] = (&(weights.dense_weights[i << weights.dense_weights.stride_shift()]))[offset];

  all_reduce<float, add_float>(all, local_grad, length);  // TODO: modify to not use first()

  if (weights.sparse)
    for (uint64_t i = 0; i < length; i++)
      (&(weights.sparse_weights[i << weights.sparse_weights.stride_shift()]))[offset] = local_grad[i] / numnodes;
  else
    for (uint64_t i = 0; i < length; i++)
      (&(weights.dense_weights[i << weights.dense_weights.stride_shift()]))[offset] = local_grad[i] / numnodes;

  delete[] local_grad;
}

accumulate_scalar()

float accumulate_scalar	(	vw &	all,
		float	local_sum
	)

Definition at line 44 of file accumulate.cc.

Referenced by end_pass(), process_pass(), summarize_holdout_set(), and VW::sync_stats().

{
  float temp = local_sum;
  all_reduce<float, add_float>(all, &temp, 1);
  return temp;
}

accumulate_weighted_avg()

void accumulate_weighted_avg	(	vw &	all,
		parameters &	weights
	)

Definition at line 117 of file accumulate.cc.

References parameters::adaptive, parameters::dense_weights, do_weighting(), dense_parameters::first(), vw::num_bits, parameters::sparse, parameters::sparse_weights, dense_parameters::stride_shift(), sparse_parameters::stride_shift(), parameters::stride_shift(), and vw::trace_message.

Referenced by GD::end_pass().

{
  if (!weights.adaptive)
  {
    all.trace_message << "Weighted averaging is implemented only for adaptive gradient, use accumulate_avg instead\n";
    return;
  }

  uint32_t length = 1 << all.num_bits;  // This is the number of parameters
  float* local_weights = new float[length];

  if (weights.sparse)
    for (uint64_t i = 0; i < length; i++)
      local_weights[i] = (&(weights.sparse_weights[i << weights.sparse_weights.stride_shift()]))[1];
  else
    for (uint64_t i = 0; i < length; i++)
      local_weights[i] = (&(weights.dense_weights[i << weights.dense_weights.stride_shift()]))[1];

  // First compute weights for averaging
  all_reduce<float, add_float>(all, local_weights, length);

  if (weights.sparse)
    do_weighting(all, length, local_weights, weights.sparse_weights);
  else
    do_weighting(all, length, local_weights, weights.dense_weights);

  if (weights.sparse)
    std::cout << "sparse parameters not supported with parallel computation!" << std::endl;
  else
    all_reduce<float, add_float>(
        all, weights.dense_weights.first(), ((size_t)length) * (1ull << weights.stride_shift()));
  delete[] local_weights;
}