cache.h File Reference

#include "v_array.h"
#include "io_buf.h"
#include "example.h"

Go to the source code of this file.

Namespaces
	VW

Functions
char *	run_len_decode (char *p, size_t &i)
char *	run_len_encode (char *p, size_t i)
int	read_cached_features (vw *all, v_array< example *> &examples)
void	cache_tag (io_buf &cache, v_array< char > tag)
void	cache_features (io_buf &cache, example *ae, uint64_t mask)
void	output_byte (io_buf &cache, unsigned char s)
void	output_features (io_buf &cache, unsigned char index, features &fs, uint64_t mask)
uint32_t	VW::convert (size_t number)

Function Documentation

cache_features()

void cache_features	(	io_buf &	cache,
		example *	ae,
		uint64_t	mask
	)

Definition at line 203 of file cache.cc.

References cache_tag(), example_predict::feature_space, example_predict::indices, output_byte(), output_features(), v_array< T >::size(), and example::tag.

Referenced by VW::setup_example().

{
  cache_tag(cache, ae->tag);
  output_byte(cache, (unsigned char)ae->indices.size());

  for (namespace_index ns : ae->indices) output_features(cache, ns, ae->feature_space[ns], mask);
}

cache_tag()

void cache_tag	(	io_buf &	cache,
		v_array< char >	tag
	)

Definition at line 192 of file cache.cc.

References v_array< T >::begin(), io_buf::buf_write(), c, io_buf::set(), and v_array< T >::size().

Referenced by cache_features(), and learn().

{
  char* c;
  cache.buf_write(c, sizeof(size_t) + tag.size());
  *(size_t*)c = tag.size();
  c += sizeof(size_t);
  memcpy(c, tag.begin(), tag.size());
  c += tag.size();
  cache.set(c);
}

output_byte()

void output_byte	(	io_buf &	cache,
		unsigned char	s
	)

Definition at line 144 of file cache.cc.

References io_buf::buf_write(), c, and io_buf::set().

Referenced by cache_features(), and send_features().

{
  char* c;

  cache.buf_write(c, 1);
  *(c++) = s;
  cache.set(c);
}

output_features()

void output_features	(	io_buf &	cache,
		unsigned char	index,
		features &	fs,
		uint64_t	mask
	)

Definition at line 153 of file cache.cc.

References io_buf::buf_write(), c, one_float::f, general, int_size, neg_1, run_len_encode(), io_buf::set(), features::size(), features::values, and ZigZagEncode().

Referenced by cache_features(), and send_features().

{
  char* c;
  size_t storage = fs.size() * int_size;
  for (feature_value f : fs.values)
    if (f != 1. && f != -1.)
      storage += sizeof(feature_value);

  cache.buf_write(c, sizeof(index) + storage + sizeof(size_t));
  *reinterpret_cast<unsigned char*>(c) = index;
  c += sizeof(index);

  char* storage_size_loc = c;
  c += sizeof(size_t);

  uint64_t last = 0;
  for (features::iterator& f : fs)
  {
    feature_index fi = f.index() & mask;
    int64_t s_diff = (fi - last);
    uint64_t diff = ZigZagEncode(s_diff) << 2;
    last = fi;

    if (f.value() == 1.)
      c = run_len_encode(c, diff);
    else if (f.value() == -1.)
      c = run_len_encode(c, diff | neg_1);
    else
    {
      c = run_len_encode(c, diff | general);
      memcpy(c, &f.value(), sizeof(feature_value));
      c += sizeof(feature_value);
    }
  }

  cache.set(c);
  *(size_t*)storage_size_loc = c - storage_size_loc - sizeof(size_t);
}

read_cached_features()

int read_cached_features	(	vw *	all,
		v_array< example *> &	examples
	)

Definition at line 65 of file cache.cc.

References io_buf::buf_read(), c, char_size, example_predict::feature_space, general, example_predict::indices, parser::input, example::l, parser::lp, neg_1, vw::p, v_array< T >::push_back(), features::push_back(), label_parser::read_cached_label, read_cached_tag(), run_len_decode(), vw::sd, io_buf::set(), example::sorted, parser::sorted_cache, vw::trace_message, and ZigZagDecode().

Referenced by enable_sources(), parse_cache(), and reset_source().

{
  example* ae = examples[0];
  ae->sorted = all->p->sorted_cache;
  io_buf* input = all->p->input;

  size_t total = all->p->lp.read_cached_label(all->sd, &ae->l, *input);
  if (total == 0)
    return 0;
  if (read_cached_tag(*input, ae) == 0)
    return 0;
  char* c;
  unsigned char num_indices = 0;
  if (input->buf_read(c, sizeof(num_indices)) < sizeof(num_indices))
    return 0;
  num_indices = *(unsigned char*)c;
  c += sizeof(num_indices);

  all->p->input->set(c);
  for (; num_indices > 0; num_indices--)
  {
    size_t temp;
    unsigned char index = 0;
    if ((temp = input->buf_read(c, sizeof(index) + sizeof(size_t))) < sizeof(index) + sizeof(size_t))
    {
      all->trace_message << "truncated example! " << temp << " " << char_size + sizeof(size_t) << std::endl;
      return 0;
    }

    index = *(unsigned char*)c;
    c += sizeof(index);
    ae->indices.push_back((size_t)index);
    features& ours = ae->feature_space[index];
    size_t storage = *(size_t*)c;
    c += sizeof(size_t);
    all->p->input->set(c);
    total += storage;
    if (input->buf_read(c, storage) < storage)
    {
      all->trace_message << "truncated example! wanted: " << storage << " bytes" << std::endl;
      return 0;
    }

    char* end = c + storage;

    uint64_t last = 0;

    for (; c != end;)
    {
      feature_index i = 0;
      c = run_len_decode(c, i);
      feature_value v = 1.f;
      if (i & neg_1)
        v = -1.;
      else if (i & general)
      {
        v = ((one_float*)c)->f;
        c += sizeof(float);
      }
      uint64_t diff = i >> 2;
      int64_t s_diff = ZigZagDecode(diff);
      if (s_diff < 0)
        ae->sorted = false;
      i = last + s_diff;
      last = i;
      ours.push_back(v, i);
    }
    all->p->input->set(c);
  }

  return (int)total;
}

run_len_decode()

char* run_len_decode	(	char *	p,
		size_t &	i
	)

run_len_encode()

char* run_len_encode	(	char *	p,
		size_t	i
	)