nn.cc File Reference

#include <cfloat>
#include <cmath>
#include <cstdio>
#include <sstream>
#include <memory>
#include "reductions.h"
#include "rand48.h"
#include "gd.h"
#include "vw.h"

Go to the source code of this file.

Classes
struct	nn

Macros
#define	cast_uint32_t   static_cast<uint32_t>

Functions
static float	fastpow2 (float p)
static float	fastexp (float p)
static float	fasttanh (float p)
void	finish_setup (nn &n, vw &all)
void	end_pass (nn &n)
template<bool is_learn, bool recompute_hidden>
void	predict_or_learn_multi (nn &n, single_learner &base, example &ec)
void	multipredict (nn &n, single_learner &base, example &ec, size_t count, size_t step, polyprediction *pred, bool finalize_predictions)
void	finish_example (vw &all, nn &, example &ec)
base_learner *	nn_setup (options_i &options, vw &all)

Variables
constexpr float	hidden_min_activation = -3
constexpr float	hidden_max_activation = 3
constexpr uint64_t	nn_constant = 533357803

Macro Definition Documentation

cast_uint32_t

#define cast_uint32_t   static_cast<uint32_t>

Definition at line 62 of file nn.cc.

Referenced by fastpow2().

Function Documentation

end_pass()

void end_pass

(

nn &

n

)

Definition at line 146 of file nn.cc.

References nn::all, vw::bfgs, nn::save_xsubi, and nn::xsubi.

{
  if (n.all->bfgs)
    n.xsubi = n.save_xsubi;
}

fastexp()

static float fastexp ( float  p )

inlinestatic

Definition at line 79 of file nn.cc.

References f, and fastpow2().

Referenced by ldamath::exponential< float, USE_FAST_APPROX >(), ldamath::fastdigamma(), and fasttanh().

{ return fastpow2(1.442695040f * p); }

fastpow2()

static float fastpow2 ( float  p )

inlinestatic

Definition at line 64 of file nn.cc.

References cast_uint32_t, and f.

Referenced by fastexp(), ldamath::fastexp(), and ldamath::fastpow().

{
  float offset = (p < 0) ? 1.0f : 0.0f;
  float clipp = (p < -126) ? -126.0f : p;
  int w = (int)clipp;
  float z = clipp - w + offset;
  union
  {
    uint32_t i;
    float f;
  } v = {cast_uint32_t((1 << 23) * (clipp + 121.2740575f + 27.7280233f / (4.84252568f - z) - 1.49012907f * z))};

  return v.f;
}

fasttanh()

static float fasttanh ( float  p )

inlinestatic

Definition at line 81 of file nn.cc.

References f, and fastexp().

Referenced by predict_or_learn_multi().

{ return -1.0f + 2.0f / (1.0f + fastexp(-2.0f * p)); }

finish_example()

void finish_example	(	vw &	all,
		nn &	,
		example &	ec
	)

Definition at line 409 of file nn.cc.

References vw::raw_prediction, and return_simple_example().

Referenced by nn_setup().

{
  int save_raw_prediction = all.raw_prediction;
  all.raw_prediction = -1;
  return_simple_example(all, nullptr, ec);
  all.raw_prediction = save_raw_prediction;
}

finish_setup()

void finish_setup	(	nn &	n,
		vw &	all
	)

Definition at line 83 of file nn.cc.

References vw::audit, constant, constant_namespace, example_predict::feature_space, nn::finished_setup, vw::hash_inv, nn::hiddenbias, example::in_use, nn::increment, example_predict::indices, features::indicies, nn::inpass, example_predict::interactions, vw::interactions, nn::k, example::l, label_data::label, nn_constant, nn_output_namespace, example::num_features, nn::output_layer, nn::outputweight, v_array< T >::push_back(), features::push_back(), polylabel::simple, features::space_names, parameters::stride_shift(), example::total_sum_feat_sq, features::values, example::weight, and vw::weights.

Referenced by predict_or_learn_multi().

{
  // TODO: output_layer audit

  memset(&n.output_layer, 0, sizeof(n.output_layer));
  n.output_layer.interactions = &all.interactions;
  n.output_layer.indices.push_back(nn_output_namespace);
  uint64_t nn_index = nn_constant << all.weights.stride_shift();

  features& fs = n.output_layer.feature_space[nn_output_namespace];
  for (unsigned int i = 0; i < n.k; ++i)
  {
    fs.push_back(1., nn_index);
    if (all.audit || all.hash_inv)
    {
      std::stringstream ss;
      ss << "OutputLayer" << i;
      fs.space_names.push_back(audit_strings_ptr(new audit_strings("", ss.str())));
    }
    nn_index += (uint64_t)n.increment;
  }
  n.output_layer.num_features += n.k;

  if (!n.inpass)
  {
    fs.push_back(1., nn_index);
    if (all.audit || all.hash_inv)
      fs.space_names.push_back(audit_strings_ptr(new audit_strings("", "OutputLayerConst")));
    ++n.output_layer.num_features;
  }

  n.output_layer.in_use = true;

  // TODO: not correct if --noconstant
  memset(&n.hiddenbias, 0, sizeof(n.hiddenbias));
  n.hiddenbias.interactions = &all.interactions;
  n.hiddenbias.indices.push_back(constant_namespace);
  n.hiddenbias.feature_space[constant_namespace].push_back(1, (uint64_t)constant);
  if (all.audit || all.hash_inv)
    n.hiddenbias.feature_space[constant_namespace].space_names.push_back(
        audit_strings_ptr(new audit_strings("", "HiddenBias")));
  n.hiddenbias.total_sum_feat_sq++;
  n.hiddenbias.l.simple.label = FLT_MAX;
  n.hiddenbias.weight = 1;
  n.hiddenbias.in_use = true;

  memset(&n.outputweight, 0, sizeof(n.outputweight));
  n.outputweight.interactions = &all.interactions;
  n.outputweight.indices.push_back(nn_output_namespace);
  features& outfs = n.output_layer.feature_space[nn_output_namespace];
  n.outputweight.feature_space[nn_output_namespace].push_back(outfs.values[0], outfs.indicies[0]);
  if (all.audit || all.hash_inv)
    n.outputweight.feature_space[nn_output_namespace].space_names.push_back(
        audit_strings_ptr(new audit_strings("", "OutputWeight")));
  n.outputweight.feature_space[nn_output_namespace].values[0] = 1;
  n.outputweight.total_sum_feat_sq++;
  n.outputweight.l.simple.label = FLT_MAX;
  n.outputweight.weight = 1;
  n.outputweight.in_use = true;

  n.finished_setup = true;
}

multipredict()

void multipredict	(	nn &	n,
		single_learner &	base,
		example &	ec,
		size_t	count,
		size_t	step,
		polyprediction *	pred,
		bool	finalize_predictions
	)

Definition at line 391 of file nn.cc.

References c, example_predict::ft_offset, example::partial_prediction, example::pred, and polyprediction::scalar.

Referenced by nn_setup().

{
  for (size_t c = 0; c < count; c++)
  {
    if (c == 0)
      predict_or_learn_multi<false, true>(n, base, ec);
    else
      predict_or_learn_multi<false, false>(n, base, ec);
    if (finalize_predictions)
      pred[c] = ec.pred;
    else
      pred[c].scalar = ec.partial_prediction;
    ec.ft_offset += (uint64_t)step;
  }
  ec.ft_offset -= (uint64_t)(step * count);
}

nn_setup()

base_learner* nn_setup	(	options_i &	options,
		vw &	all
	)

Definition at line 417 of file nn.cc.

References VW::config::option_group_definition::add(), add(), VW::config::options_i::add_and_parse(), LEARNER::as_singleline(), LEARNER::end_pass(), finish_example(), vw::get_random_state(), getLossFunction(), LEARNER::init_learner(), LEARNER::make_base(), VW::config::make_option(), multipredict(), nn::multitask, vw::quiet, vw::random_seed, LEARNER::learner< T, E >::set_end_pass(), LEARNER::learner< T, E >::set_finish_example(), LEARNER::learner< T, E >::set_multipredict(), setup_base(), vw::training, and VW::config::options_i::was_supplied().

Referenced by parse_reductions().

{
  auto n = scoped_calloc_or_throw<nn>();
  bool meanfield = false;
  option_group_definition new_options("Neural Network");
  new_options.add(make_option("nn", n->k).keep().help("Sigmoidal feedforward network with <k> hidden units"))
      .add(make_option("inpass", n->inpass)
               .keep()
               .help("Train or test sigmoidal feedforward network with input passthrough."))
      .add(make_option("multitask", n->multitask).keep().help("Share hidden layer across all reduced tasks."))
      .add(make_option("dropout", n->dropout).keep().help("Train or test sigmoidal feedforward network using dropout."))
      .add(make_option("meanfield", meanfield).help("Train or test sigmoidal feedforward network using mean field."));
  options.add_and_parse(new_options);

  if (!options.was_supplied("nn"))
    return nullptr;

  n->all = &all;
  n->_random_state = all.get_random_state();

  if (n->multitask && !all.quiet)
    std::cerr << "using multitask sharing for neural network " << (all.training ? "training" : "testing") << std::endl;

  if (options.was_supplied("meanfield"))
  {
    n->dropout = false;
    if (!all.quiet)
      std::cerr << "using mean field for neural network " << (all.training ? "training" : "testing") << std::endl;
  }

  if (n->dropout && !all.quiet)
    std::cerr << "using dropout for neural network " << (all.training ? "training" : "testing") << std::endl;

  if (n->inpass && !all.quiet)
    std::cerr << "using input passthrough for neural network " << (all.training ? "training" : "testing") << std::endl;

  n->finished_setup = false;
  n->squared_loss = getLossFunction(all, "squared", 0);

  n->xsubi = all.random_seed;

  n->save_xsubi = n->xsubi;

  n->hidden_units = calloc_or_throw<float>(n->k);
  n->dropped_out = calloc_or_throw<bool>(n->k);
  n->hidden_units_pred = calloc_or_throw<polyprediction>(n->k);
  n->hiddenbias_pred = calloc_or_throw<polyprediction>(n->k);

  auto base = as_singleline(setup_base(options, all));
  n->increment = base->increment;  // Indexing of output layer is odd.
  nn& nv = *n.get();
  learner<nn, example>& l =
      init_learner(n, base, predict_or_learn_multi<true, true>, predict_or_learn_multi<false, true>, n->k + 1);
  if (nv.multitask)
    l.set_multipredict(multipredict);
  l.set_finish_example(finish_example);
  l.set_end_pass(end_pass);

  return make_base(l);
}

predict_or_learn_multi()

template<bool is_learn, bool recompute_hidden>

void predict_or_learn_multi	(	nn &	n,
		single_learner &	base,
		example &	ec
	)

Definition at line 153 of file nn.cc.

References nn::_random_state, add_passthrough_feature, nn::all, nn::dropout, nn::dropped_out, f, fasttanh(), example_predict::feature_space, GD::finalize_prediction(), finish_setup(), nn::finished_setup, loss_function::first_derivative(), example_predict::ft_offset, hidden_max_activation, hidden_min_activation, nn::hidden_units_pred, nn::hiddenbias, nn::hiddenbias_pred, example_predict::indices, features::indicies, nn::inpass, nn::k, example::l, label_data::label, LEARNER::learner< T, E >::learn(), example::loss, vw::loss, shared_data::max_label, merand48(), shared_data::min_label, LEARNER::learner< T, E >::multipredict(), nn::multitask, nn_output_namespace, noop_mm(), nn::output_layer, nn::outputweight, example::partial_prediction, example::passthrough, v_array< T >::pop(), example::pred, LEARNER::learner< T, E >::predict(), nn::prediction, vw::print_text, v_array< T >::push_back(), vw::raw_prediction, prediction_type::scalar, polyprediction::scalar, vw::sd, vw::set_minmax, polylabel::simple, nn::squared_loss, features::sum_feat_sq, example::tag, example::total_sum_feat_sq, vw::training, LEARNER::learner< T, E >::update(), features::values, example::weight, and nn::xsubi.

{
  bool shouldOutput = n.all->raw_prediction > 0;
  if (!n.finished_setup)
    finish_setup(n, *(n.all));
  shared_data sd;
  memcpy(&sd, n.all->sd, sizeof(shared_data));
  shared_data* save_sd = n.all->sd;
  n.all->sd = &sd;

  label_data ld = ec.l.simple;
  void (*save_set_minmax)(shared_data*, float) = n.all->set_minmax;
  float save_min_label;
  float save_max_label;
  float dropscale = n.dropout ? 2.0f : 1.0f;
  loss_function* save_loss = n.all->loss;

  polyprediction* hidden_units = n.hidden_units_pred;
  polyprediction* hiddenbias_pred = n.hiddenbias_pred;
  bool* dropped_out = n.dropped_out;

  std::ostringstream outputStringStream;

  n.all->set_minmax = noop_mm;
  n.all->loss = n.squared_loss;
  save_min_label = n.all->sd->min_label;
  n.all->sd->min_label = hidden_min_activation;
  save_max_label = n.all->sd->max_label;
  n.all->sd->max_label = hidden_max_activation;

  uint64_t save_ft_offset = ec.ft_offset;

  if (n.multitask)
    ec.ft_offset = 0;

  n.hiddenbias.ft_offset = ec.ft_offset;

  if (recompute_hidden)
  {
    base.multipredict(n.hiddenbias, 0, n.k, hiddenbias_pred, true);

    for (unsigned int i = 0; i < n.k; ++i)
      // avoid saddle point at 0
      if (hiddenbias_pred[i].scalar == 0)
      {
        n.hiddenbias.l.simple.label = (float)(n._random_state->get_and_update_random() - 0.5);
        base.learn(n.hiddenbias, i);
        n.hiddenbias.l.simple.label = FLT_MAX;
      }

    base.multipredict(ec, 0, n.k, hidden_units, true);

    for (unsigned int i = 0; i < n.k; ++i) dropped_out[i] = (n.dropout && merand48(n.xsubi) < 0.5);

    if (ec.passthrough)
      for (unsigned int i = 0; i < n.k; ++i)
      {
        add_passthrough_feature(ec, i * 2, hiddenbias_pred[i].scalar);
        add_passthrough_feature(ec, i * 2 + 1, hidden_units[i].scalar);
      }
  }

  if (shouldOutput)
    for (unsigned int i = 0; i < n.k; ++i)
    {
      if (i > 0)
        outputStringStream << ' ';
      outputStringStream << i << ':' << hidden_units[i].scalar << ','
                         << fasttanh(hidden_units[i].scalar);  // TODO: huh, what was going on here?
    }

  n.all->loss = save_loss;
  n.all->set_minmax = save_set_minmax;
  n.all->sd->min_label = save_min_label;
  n.all->sd->max_label = save_max_label;
  ec.ft_offset = save_ft_offset;

  bool converse = false;
  float save_partial_prediction = 0;
  float save_final_prediction = 0;
  float save_ec_loss = 0;

CONVERSE:  // That's right, I'm using goto.  So sue me.

  n.output_layer.total_sum_feat_sq = 1;
  n.output_layer.feature_space[nn_output_namespace].sum_feat_sq = 1;

  n.outputweight.ft_offset = ec.ft_offset;

  n.all->set_minmax = noop_mm;
  n.all->loss = n.squared_loss;
  save_min_label = n.all->sd->min_label;
  n.all->sd->min_label = -1;
  save_max_label = n.all->sd->max_label;
  n.all->sd->max_label = 1;

  for (unsigned int i = 0; i < n.k; ++i)
  {
    float sigmah = (dropped_out[i]) ? 0.0f : dropscale * fasttanh(hidden_units[i].scalar);
    features& out_fs = n.output_layer.feature_space[nn_output_namespace];
    out_fs.values[i] = sigmah;

    n.output_layer.total_sum_feat_sq += sigmah * sigmah;
    out_fs.sum_feat_sq += sigmah * sigmah;

    n.outputweight.feature_space[nn_output_namespace].indicies[0] = out_fs.indicies[i];
    base.predict(n.outputweight, n.k);
    float wf = n.outputweight.pred.scalar;

    // avoid saddle point at 0
    if (wf == 0)
    {
      float sqrtk = std::sqrt((float)n.k);
      n.outputweight.l.simple.label = (float)(n._random_state->get_and_update_random() - 0.5) / sqrtk;
      base.update(n.outputweight, n.k);
      n.outputweight.l.simple.label = FLT_MAX;
    }
  }

  n.all->loss = save_loss;
  n.all->set_minmax = save_set_minmax;
  n.all->sd->min_label = save_min_label;
  n.all->sd->max_label = save_max_label;

  if (n.inpass)
  {
    // TODO: this is not correct if there is something in the
    // nn_output_namespace but at least it will not leak memory
    // in that case
    ec.indices.push_back(nn_output_namespace);
    features save_nn_output_namespace = ec.feature_space[nn_output_namespace];
    ec.feature_space[nn_output_namespace] = n.output_layer.feature_space[nn_output_namespace];
    ec.total_sum_feat_sq += n.output_layer.feature_space[nn_output_namespace].sum_feat_sq;
    if (is_learn)
      base.learn(ec, n.k);
    else
      base.predict(ec, n.k);
    n.output_layer.partial_prediction = ec.partial_prediction;
    n.output_layer.loss = ec.loss;
    ec.total_sum_feat_sq -= n.output_layer.feature_space[nn_output_namespace].sum_feat_sq;
    ec.feature_space[nn_output_namespace].sum_feat_sq = 0;
    ec.feature_space[nn_output_namespace] = save_nn_output_namespace;
    ec.indices.pop();
  }
  else
  {
    n.output_layer.ft_offset = ec.ft_offset;
    n.output_layer.l = ec.l;
    n.output_layer.weight = ec.weight;
    n.output_layer.partial_prediction = 0;
    if (is_learn)
      base.learn(n.output_layer, n.k);
    else
      base.predict(n.output_layer, n.k);
    ec.l = n.output_layer.l;
  }

  n.prediction = GD::finalize_prediction(n.all->sd, n.output_layer.partial_prediction);

  if (shouldOutput)
  {
    outputStringStream << ' ' << n.output_layer.partial_prediction;
    n.all->print_text(n.all->raw_prediction, outputStringStream.str(), ec.tag);
  }

  if (is_learn && n.all->training && ld.label != FLT_MAX)
  {
    float gradient = n.all->loss->first_derivative(n.all->sd, n.prediction, ld.label);

    if (fabs(gradient) > 0)
    {
      n.all->loss = n.squared_loss;
      n.all->set_minmax = noop_mm;
      save_min_label = n.all->sd->min_label;
      n.all->sd->min_label = hidden_min_activation;
      save_max_label = n.all->sd->max_label;
      n.all->sd->max_label = hidden_max_activation;
      save_ft_offset = ec.ft_offset;

      if (n.multitask)
        ec.ft_offset = 0;

      for (unsigned int i = 0; i < n.k; ++i)
      {
        if (!dropped_out[i])
        {
          float sigmah = n.output_layer.feature_space[nn_output_namespace].values[i] / dropscale;
          float sigmahprime = dropscale * (1.0f - sigmah * sigmah);
          n.outputweight.feature_space[nn_output_namespace].indicies[0] =
              n.output_layer.feature_space[nn_output_namespace].indicies[i];
          base.predict(n.outputweight, n.k);
          float nu = n.outputweight.pred.scalar;
          float gradhw = 0.5f * nu * gradient * sigmahprime;

          ec.l.simple.label = GD::finalize_prediction(n.all->sd, hidden_units[i].scalar - gradhw);
          ec.pred.scalar = hidden_units[i].scalar;
          if (ec.l.simple.label != hidden_units[i].scalar)
            base.update(ec, i);
        }
      }

      n.all->loss = save_loss;
      n.all->set_minmax = save_set_minmax;
      n.all->sd->min_label = save_min_label;
      n.all->sd->max_label = save_max_label;
      ec.ft_offset = save_ft_offset;
    }
  }

  ec.l.simple.label = ld.label;

  if (!converse)
  {
    save_partial_prediction = n.output_layer.partial_prediction;
    save_final_prediction = n.prediction;
    save_ec_loss = n.output_layer.loss;
  }

  if (n.dropout && !converse)
  {
    for (unsigned int i = 0; i < n.k; ++i)
    {
      dropped_out[i] = !dropped_out[i];
    }

    converse = true;
    goto CONVERSE;
  }

  ec.partial_prediction = save_partial_prediction;
  ec.pred.scalar = save_final_prediction;
  ec.loss = save_ec_loss;

  n.all->sd = save_sd;
  n.all->set_minmax(n.all->sd, sd.min_label);
  n.all->set_minmax(n.all->sd, sd.max_label);
}

Variable Documentation

hidden_max_activation

constexpr float hidden_max_activation = 3

Definition at line 21 of file nn.cc.

Referenced by predict_or_learn_multi().

hidden_min_activation

constexpr float hidden_min_activation = -3

Definition at line 20 of file nn.cc.

Referenced by predict_or_learn_multi().

nn_constant

constexpr uint64_t nn_constant = 533357803

Definition at line 22 of file nn.cc.

Referenced by finish_setup().