cache.cc File Reference

#include "cache.h"
#include "unique_sort.h"
#include "global_data.h"
#include "vw.h"

Go to the source code of this file.

Classes
struct	one_float

Functions
char *	run_len_decode (char *p, uint64_t &i)
char *	run_len_encode (char *p, uint64_t i)
int64_t	ZigZagDecode (uint64_t n)
size_t	read_cached_tag (io_buf &cache, example *ae)
struct one_float	__attribute__ ((packed))
int	read_cached_features (vw *all, v_array< example *> &examples)
uint64_t	ZigZagEncode (int64_t n)
void	output_byte (io_buf &cache, unsigned char s)
void	output_features (io_buf &cache, unsigned char index, features &fs, uint64_t mask)
void	cache_tag (io_buf &cache, v_array< char > tag)
void	cache_features (io_buf &cache, example *ae, uint64_t mask)

Variables
constexpr size_t	int_size = 11
constexpr size_t	char_size = 2
constexpr size_t	neg_1 = 1
constexpr size_t	general = 2
float	f

Function Documentation

__attribute__()

struct one_float __attribute__

(

(packed)

)

cache_features()

void cache_features	(	io_buf &	cache,
		example *	ae,
		uint64_t	mask
	)

Definition at line 203 of file cache.cc.

References cache_tag(), example_predict::feature_space, example_predict::indices, output_byte(), output_features(), v_array< T >::size(), and example::tag.

Referenced by VW::setup_example().

{
  cache_tag(cache, ae->tag);
  output_byte(cache, (unsigned char)ae->indices.size());

  for (namespace_index ns : ae->indices) output_features(cache, ns, ae->feature_space[ns], mask);
}

cache_tag()

void cache_tag	(	io_buf &	cache,
		v_array< char >	tag
	)

Definition at line 192 of file cache.cc.

References v_array< T >::begin(), io_buf::buf_write(), c, io_buf::set(), and v_array< T >::size().

Referenced by cache_features(), and learn().

{
  char* c;
  cache.buf_write(c, sizeof(size_t) + tag.size());
  *(size_t*)c = tag.size();
  c += sizeof(size_t);
  memcpy(c, tag.begin(), tag.size());
  c += tag.size();
  cache.set(c);
}

output_byte()

void output_byte	(	io_buf &	cache,
		unsigned char	s
	)

Definition at line 144 of file cache.cc.

References io_buf::buf_write(), c, and io_buf::set().

Referenced by cache_features(), and send_features().

{
  char* c;

  cache.buf_write(c, 1);
  *(c++) = s;
  cache.set(c);
}

output_features()

void output_features	(	io_buf &	cache,
		unsigned char	index,
		features &	fs,
		uint64_t	mask
	)

Definition at line 153 of file cache.cc.

References io_buf::buf_write(), c, one_float::f, general, int_size, neg_1, run_len_encode(), io_buf::set(), features::size(), features::values, and ZigZagEncode().

Referenced by cache_features(), and send_features().

{
  char* c;
  size_t storage = fs.size() * int_size;
  for (feature_value f : fs.values)
    if (f != 1. && f != -1.)
      storage += sizeof(feature_value);

  cache.buf_write(c, sizeof(index) + storage + sizeof(size_t));
  *reinterpret_cast<unsigned char*>(c) = index;
  c += sizeof(index);

  char* storage_size_loc = c;
  c += sizeof(size_t);

  uint64_t last = 0;
  for (features::iterator& f : fs)
  {
    feature_index fi = f.index() & mask;
    int64_t s_diff = (fi - last);
    uint64_t diff = ZigZagEncode(s_diff) << 2;
    last = fi;

    if (f.value() == 1.)
      c = run_len_encode(c, diff);
    else if (f.value() == -1.)
      c = run_len_encode(c, diff | neg_1);
    else
    {
      c = run_len_encode(c, diff | general);
      memcpy(c, &f.value(), sizeof(feature_value));
      c += sizeof(feature_value);
    }
  }

  cache.set(c);
  *(size_t*)storage_size_loc = c - storage_size_loc - sizeof(size_t);
}

read_cached_features()

int read_cached_features	(	vw *	all,
		v_array< example *> &	examples
	)

Definition at line 65 of file cache.cc.

References io_buf::buf_read(), c, char_size, example_predict::feature_space, general, example_predict::indices, parser::input, example::l, parser::lp, neg_1, vw::p, v_array< T >::push_back(), features::push_back(), label_parser::read_cached_label, read_cached_tag(), run_len_decode(), vw::sd, io_buf::set(), example::sorted, parser::sorted_cache, vw::trace_message, and ZigZagDecode().

Referenced by enable_sources(), parse_cache(), and reset_source().

{
  example* ae = examples[0];
  ae->sorted = all->p->sorted_cache;
  io_buf* input = all->p->input;

  size_t total = all->p->lp.read_cached_label(all->sd, &ae->l, *input);
  if (total == 0)
    return 0;
  if (read_cached_tag(*input, ae) == 0)
    return 0;
  char* c;
  unsigned char num_indices = 0;
  if (input->buf_read(c, sizeof(num_indices)) < sizeof(num_indices))
    return 0;
  num_indices = *(unsigned char*)c;
  c += sizeof(num_indices);

  all->p->input->set(c);
  for (; num_indices > 0; num_indices--)
  {
    size_t temp;
    unsigned char index = 0;
    if ((temp = input->buf_read(c, sizeof(index) + sizeof(size_t))) < sizeof(index) + sizeof(size_t))
    {
      all->trace_message << "truncated example! " << temp << " " << char_size + sizeof(size_t) << std::endl;
      return 0;
    }

    index = *(unsigned char*)c;
    c += sizeof(index);
    ae->indices.push_back((size_t)index);
    features& ours = ae->feature_space[index];
    size_t storage = *(size_t*)c;
    c += sizeof(size_t);
    all->p->input->set(c);
    total += storage;
    if (input->buf_read(c, storage) < storage)
    {
      all->trace_message << "truncated example! wanted: " << storage << " bytes" << std::endl;
      return 0;
    }

    char* end = c + storage;

    uint64_t last = 0;

    for (; c != end;)
    {
      feature_index i = 0;
      c = run_len_decode(c, i);
      feature_value v = 1.f;
      if (i & neg_1)
        v = -1.;
      else if (i & general)
      {
        v = ((one_float*)c)->f;
        c += sizeof(float);
      }
      uint64_t diff = i >> 2;
      int64_t s_diff = ZigZagDecode(diff);
      if (s_diff < 0)
        ae->sorted = false;
      i = last + s_diff;
      last = i;
      ours.push_back(v, i);
    }
    all->p->input->set(c);
  }

  return (int)total;
}

read_cached_tag()

size_t read_cached_tag	(	io_buf &	cache,
		example *	ae
	)

Definition at line 39 of file cache.cc.

Referenced by read_cached_features().

{
  char* c;
  size_t tag_size;
  if (cache.buf_read(c, sizeof(tag_size)) < sizeof(tag_size))
    return 0;
  tag_size = *(size_t*)c;
  c += sizeof(tag_size);
  cache.set(c);
  if (cache.buf_read(c, tag_size) < tag_size)
    return 0;

  ae->tag.clear();
  push_many(ae->tag, c, tag_size);
  return tag_size + sizeof(tag_size);
}

run_len_decode()

char* run_len_decode ( char *  p, uint64_t &  i  )

inline

Definition at line 16 of file cache.cc.

Referenced by read_cached_features().

{
  // read an int 7 bits at a time.
  size_t count = 0;
  while (*p & 128) i = i | ((uint64_t)(*(p++) & 127) << 7 * count++);
  i = i | ((uint64_t)(*(p++)) << 7 * count);
  return p;
}

run_len_encode()

char* run_len_encode ( char *  p, uint64_t  i  )

inline

Definition at line 25 of file cache.cc.

Referenced by output_features().

{
  // store an int 7 bits at a time.
  while (i >= 128)
  {
    *(p++) = (i & 127) | 128;
    i = i >> 7;
  }
  *(p++) = (i & 127);
  return p;
}

ZigZagDecode()

int64_t ZigZagDecode ( uint64_t  n )

inline

Definition at line 37 of file cache.cc.

Referenced by read_cached_features().

{ return (n >> 1) ^ -static_cast<int64_t>(n & 1); }

ZigZagEncode()

uint64_t ZigZagEncode ( int64_t  n )

inline

Definition at line 138 of file cache.cc.

Referenced by output_features().

{
  uint64_t ret = (n << 1) ^ (n >> 63);
  return ret;
}

Variable Documentation

char_size

constexpr size_t char_size = 2

Definition at line 12 of file cache.cc.

Referenced by read_cached_features().

f

float f

Definition at line 40 of file cache.cc.

Referenced by active_cover_setup(), DepParserTask::add_all_features(), DepParserTask::add_feature(), addbufs(), all_reduce(), AllReduceThreads::all_reduce(), AllReduceSockets::all_reduce_init(), Search::allowed_actions_to_label(), BASELINE::baseline_enabled(), bfgs_iter_middle(), VW::cb_explore_adf::regcb::cb_explore_adf_regcb::binary_search(), binarySearch(), boosting_setup(), bs_predict_vote(), cb_adf_setup(), cb_data_epsilon_0_skype_jb_test_runner(), cb_explore_setup(), cbify_setup(), cbifyldf_setup(), collision_cleanup(), Beam::beam< T >::compact(), compute_coherence_metrics(), OjaNewton::compute_delta(), recall_tree_ns::compute_recall_lbest(), cs_active_setup(), memory_tree_ns::diag_kronecker_prod_fs_test(), CSOAA::do_actual_learning_wap(), CB::ec_is_example_header(), enable_sources(), end_examples(), VW::export_example(), memory_tree_ns::F1_score_for_two_examples(), ldamath::fastdigamma(), fastexp(), ldamath::fastexp(), ldamath::fastlgamma(), fastpow2(), ldamath::fastpow2(), fasttanh(), finalize_preconditioner(), finalize_source(), find_cost_range(), find_in_path(), FloatToFloatState< audit >::Float(), float_of_substring(), Search::BaseTask::foreach_action(), GD::foreach_feature(), polar_normal_weights_wrapper< T >::func(), gd_mf_setup(), GEN_CS::gen_cs_example(), generate_cb_data_5(), get_active_coin_bias(), CLASSWEIGHTS::classweights::get_class_weight(), CB_EXPLORE::get_cover_probabilities(), get_pmin(), vw::get_random_state(), get_threshold(), hingeloss::getLoss(), logloss::getLoss(), poisson_loss::getLoss(), squaredloss::getUpdate(), glf1(), CB_ADF::global_print_newline(), CSOAA::global_print_newline(), Search::handle_condition_options(), EntityRelationTask::initialize(), ArgmaxTask::initialize(), OjaNewton::initialize_Z(), CSOAA::inner_loop(), inner_update_pistol_state_and_predict(), Beam::beam< T >::insert(), is_number(), kernel_svm_setup(), lda_loop(), lda_setup(), learn(), learn_batch(), learn_with_metrics(), memory_tree_ns::linear_kernel(), linear_kernel(), MARGINAL::make_marginal(), make_write_cache(), Search::BaseTask::maybe_override_prediction(), memory_tree_setup(), mf_print_offset_features(), OjaNewton_setup(), output_and_account_confidence_example(), output_and_account_example(), no_label::output_and_account_no_label_example(), MULTILABEL::output_example(), CB_EXPLORE::output_example(), parse_cache(), parse_example_tweaks(), CB::parse_label(), COST_SENSITIVE::parse_label(), CCB::parse_label(), parse_output_preds(), Beam::beam< T >::pop_best_item(), Search::BaseTask::post_prediction(), preconditioner_to_regularizer(), vw_slim::vw_predict< W >::predict(), MWT::predict_or_learn(), predict_or_learn(), predict_or_learn_active(), predict_or_learn_active_cover(), CB_EXPLORE::predict_or_learn_cover(), VW::cb_explore_adf::greedy::cb_explore_adf_greedy::predict_or_learn_impl(), VW::cb_explore_adf::bag::cb_explore_adf_bag::predict_or_learn_impl(), VW::cb_explore_adf::cover::cb_explore_adf_cover::predict_or_learn_impl(), predict_or_learn_multi(), GD::print_features(), GD::print_lda_features(), process_pass(), query_decision(), Search::random_policy(), Search::read_allowed_transitions(), comp_io_buf::read_file(), io_buf::read_file(), recall_tree_setup(), BASELINE::reset_baseline_disabled(), comp_io_buf::reset_file(), reset_source(), return_example(), ACTION_SCORE::reverse_order(), VW::SpanningTree::Run(), GraphTask::run(), run_predict_in_memory(), GEN_CS::safe_probability(), Search::safediv(), save_load(), save_load_sampling(), BASELINE::set_baseline_enabled(), LEARNER::learner< CB_EXPLORE::cb_explore, example >::set_end_examples(), LEARNER::learner< CB_EXPLORE::cb_explore, example >::set_end_pass(), LEARNER::learner< CB_EXPLORE::cb_explore, example >::set_finish(), LEARNER::learner< CB_EXPLORE::cb_explore, example >::set_finish_example(), LEARNER::learner< CB_EXPLORE::cb_explore, example >::set_init_driver(), VW::cb_explore_adf::softmax::setup(), VW::cb_explore_adf::greedy::setup(), VW::cb_explore_adf::cover::setup(), VW::cb_explore_adf::regcb::setup(), GD::setup(), Search::setup(), setup_lambdas(), sign(), memory_tree_ns::single_query_and_learn(), MultiState< audit >::StartArray(), vw_slim::stride_shift_guard::stride_shift_guard(), summarize_holdout_set(), TEST(), TEST_P(), recall_tree_ns::to_prob(), DepParserTask::transition_eager(), DepParserTask::transition_hybrid(), OjaNewton::update_eigenvalues(), VW_GetFeatureSpace(), VW_ImportExample(), VW_ReleaseFeatureSpace(), VW_SetFeatureSpace(), warm_cb_setup(), Search::BaseTask::with_output_string(), and vw_slim::stride_shift_guard::~stride_shift_guard().

general

constexpr size_t general = 2

Definition at line 14 of file cache.cc.

Referenced by output_features(), and read_cached_features().

int_size

constexpr size_t int_size = 11

Definition at line 11 of file cache.cc.

Referenced by output_features().

neg_1

constexpr size_t neg_1 = 1

Definition at line 13 of file cache.cc.

Referenced by output_features(), and read_cached_features().