parse_example.cc File Reference

#include <cmath>
#include <math.h>
#include <cctype>
#include "parse_example.h"
#include "hash.h"
#include "unique_sort.h"
#include "global_data.h"
#include "constant.h"

Go to the source code of this file.

Classes
class	TC_parser< audit >

Namespaces
	VW

Functions
size_t	read_features (vw *all, char *&line, size_t &num_chars)
int	read_features_string (vw *all, v_array< example *> &examples)
void	substring_to_example (vw *all, example *ae, substring example)
std::vector< std::string >	split (char *phrase, const std::string &delimiter)
void	VW::read_line (vw &all, example *ex, char *line)
void	VW::read_lines (vw *all, char *line, size_t, v_array< example *> &examples)

Function Documentation

read_features()

size_t read_features	(	vw *	all,
		char *&	line,
		size_t &	num_chars
	)

Definition at line 16 of file parse_example.cc.

References parser::input, vw::p, and readto().

Referenced by read_features_json(), and read_features_string().

{
  line = nullptr;
  size_t num_chars_initial = readto(*(all->p->input), line, '\n');
  if (num_chars_initial < 1)
    return num_chars_initial;
  num_chars = num_chars_initial;
  if (line[0] == '\xef' && num_chars >= 3 && line[1] == '\xbb' && line[2] == '\xbf')
  {
    line += 3;
    num_chars -= 3;
  }
  if (num_chars > 0 && line[num_chars - 1] == '\n')
    num_chars--;
  if (num_chars > 0 && line[num_chars - 1] == '\r')
    num_chars--;
  return num_chars_initial;
}

read_features_string()

int read_features_string	(	vw *	all,
		v_array< example *> &	examples
	)

Definition at line 35 of file parse_example.cc.

References read_features(), and substring_to_example().

Referenced by enable_sources(), and reset_source().

{
  char* line;
  size_t num_chars;
  size_t num_chars_initial = read_features(all, line, num_chars);
  if (num_chars_initial < 1)
    return (int)num_chars_initial;

  substring example = {line, line + num_chars};
  substring_to_example(all, examples[0], example);

  return (int)num_chars_initial;
}

split()

std::vector<std::string> split	(	char *	phrase,
		const std::string &	delimiter
	)

Definition at line 469 of file parse_example.cc.

Referenced by VW::read_lines().

{
  std::vector<std::string> list;
  std::string s = std::string(phrase);
  size_t pos = 0;
  std::string token;
  while ((pos = s.find(delimiter)) != std::string::npos)
  {
    token = s.substr(0, pos);
    list.push_back(token);
    s.erase(0, pos + delimiter.length());
  }
  list.push_back(s);
  return list;
}

substring_to_example()

void substring_to_example	(	vw *	all,
		example *	ae,
		substring	example
	)

Definition at line 426 of file parse_example.cc.

References vw::audit, substring::begin, v_array< T >::clear(), label_parser::default_label, v_array< T >::empty(), substring::end, vw::hash_inv, example::l, v_array< T >::last(), parser::lp, vw::p, label_parser::parse_label, v_array< T >::pop(), push_many(), safe_index(), vw::sd, example::tag, tokenize(), and parser::words.

Referenced by prepare_for_learner(), read_features_string(), and VW::read_line().

{
  all->p->lp.default_label(&ae->l);
  char* bar_location = safe_index(example.begin, '|', example.end);
  char* tab_location = safe_index(example.begin, '\t', bar_location);
  substring label_space;
  if (tab_location != bar_location)
  {
    label_space.begin = tab_location + 1;
  }
  else
  {
    label_space.begin = example.begin;
  }
  label_space.end = bar_location;

  if (*example.begin == '|')
  {
    all->p->words.clear();
  }
  else
  {
    tokenize(' ', label_space, all->p->words);
    if (!all->p->words.empty() &&
        (all->p->words.last().end == label_space.end ||
            *(all->p->words.last().begin) == '\''))  // The last field is a tag, so record and strip it off
    {
      substring tag = all->p->words.pop();
      if (*tag.begin == '\'')
        tag.begin++;
      push_many(ae->tag, tag.begin, tag.end - tag.begin);
    }
  }

  if (!all->p->words.empty())
    all->p->lp.parse_label(all->p, all->sd, &ae->l, all->p->words);

  if (all->audit || all->hash_inv)
    TC_parser<true> parser_line(bar_location, example.end, *all, ae);
  else
    TC_parser<false> parser_line(bar_location, example.end, *all, ae);
}