parse_primitives.h File Reference

#include <cmath>
#include <string>
#include <vector>
#include <iostream>
#include <stdint.h>
#include <math.h>
#include "v_array.h"
#include "hashstring.h"

Go to the source code of this file.

Namespaces
	VW

Typedefs
typedef example &(*	VW::example_factory_t) (void *)
typedef uint64_t(*	hash_func_t) (substring, uint64_t)

Functions
std::ostream &	operator<< (std::ostream &os, const substring &ss)
std::ostream &	operator<< (std::ostream &os, const v_array< substring > &ss)
template<typename ContainerT >
void	tokenize (char delim, substring s, ContainerT &ret, bool allow_empty=false)
bool	substring_equal (const substring &a, const substring &b)
bool	substring_equal (const substring &ss, const char *str)
bool	operator== (const substring &ss, const char *str)
bool	operator== (const char *str, const substring &ss)
bool	operator== (const substring &ss1, const substring &ss2)
bool	operator!= (const substring &ss, const char *str)
bool	operator!= (const char *str, const substring &ss)
bool	operator!= (const substring &ss1, const substring &ss2)
size_t	substring_len (substring &s)
char *	safe_index (char *start, char v, char *max)
std::vector< substring >	escaped_tokenize (char delim, substring s, bool allow_empty=false)
void	print_substring (substring s)
hash_func_t	getHasher (const std::string &s)
float	parseFloat (char *p, char **end, char *endLine=nullptr)
float	float_of_substring (substring s)
int	int_of_substring (substring s)

Typedef Documentation

hash_func_t

typedef uint64_t(* hash_func_t) (substring, uint64_t)

Definition at line 79 of file parse_primitives.h.

Function Documentation

escaped_tokenize()

std::vector<substring> escaped_tokenize	(	char	delim,
		substring	s,
		bool	allow_empty = false
	)

Definition at line 58 of file parse_primitives.cc.

References substring::begin, and substring::end.

Referenced by safe_index(), and VW::to_argv_escaped().

{
  std::vector<substring> tokens;
  substring current;
  current.begin = s.begin;
  bool in_escape = false;
  char* reading_head = s.begin;
  char* writing_head = s.begin;

  while (reading_head < s.end)
  {
    char current_character = *reading_head++;

    if (in_escape)
    {
      *writing_head++ = current_character;
      in_escape = false;
    }
    else
    {
      if (current_character == delim)
      {
        current.end = writing_head++;
        *current.end = '\0';
        if (current.begin != current.end || allow_empty)
        {
          tokens.push_back(current);
        }

        // Regardless of whether the token was saved, we need to reset the current token.
        current.begin = writing_head;
        current.end = writing_head;
      }
      else if (current_character == '\\')
      {
        in_escape = !in_escape;
      }
      else
      {
        *writing_head++ = current_character;
      }
    }
  }

  current.end = writing_head;
  *current.end = '\0';
  if (current.begin != current.end || allow_empty)
  {
    tokens.push_back(current);
  }

  return tokens;
}

float_of_substring()

float float_of_substring ( substring  s )

inline

Definition at line 146 of file parse_primitives.h.

References substring::begin, substring::end, f, and parseFloat().

Referenced by CCB::convert_to_score(), COST_SENSITIVE::name_value(), MULTICLASS::parse_label(), CB::parse_label(), COST_SENSITIVE::parse_label(), CCB::parse_outcome(), and parse_simple_label().

{
  char* endptr = s.end;
  float f = parseFloat(s.begin, &endptr);
  if ((endptr == s.begin && s.begin != s.end) || std::isnan(f))
  {
    std::cout << "warning: " << std::string(s.begin, s.end - s.begin).c_str()
              << " is not a good float, replacing with 0" << std::endl;
    f = 0;
  }
  return f;
}

getHasher()

hash_func_t getHasher

(

const std::string &

s

)

Definition at line 36 of file parse_primitives.cc.

References hashall(), hashstring(), and THROW.

Referenced by VW::hash_feature_static(), VW::hash_space_static(), and parse_feature_tweaks().

{
  if (s == "strings")
    return hashstring;
  else if (s == "all")
    return hashall;
  else
    THROW("Unknown hash function: " << s);
}

int_of_substring()

int int_of_substring ( substring  s )

inline

Definition at line 159 of file parse_primitives.h.

References substring::begin, and substring::end.

Referenced by CCB::convert_to_score(), CCB::parse_explicit_inclusions(), MULTICLASS::parse_label(), and Search::parse_neighbor_features().

{
  char* endptr = s.end;
  int i = strtol(s.begin, &endptr, 10);
  if (endptr == s.begin && s.begin != s.end)
  {
    std::cout << "warning: " << std::string(s.begin, s.end - s.begin).c_str() << " is not a good int, replacing with 0"
              << std::endl;
    i = 0;
  }

  return i;
}

operator!=() [1/3]

bool operator!=	(	const substring &	ss,
		const char *	str
	)

Definition at line 52 of file parse_primitives.cc.

Referenced by tokenize().

{ return !(ss == str); }

operator!=() [2/3]

bool operator!=	(	const char *	str,
		const substring &	ss
	)

Definition at line 54 of file parse_primitives.cc.

{ return !(ss == str); }

operator!=() [3/3]

bool operator!=	(	const substring &	ss1,
		const substring &	ss2
	)

Definition at line 56 of file parse_primitives.cc.

{ return !(ss1 == ss2); }

operator<<() [1/2]

std::ostream& operator<<	(	std::ostream &	os,
		const substring &	ss
	)

Definition at line 112 of file parse_primitives.cc.

References substring::begin, and substring::end.

{
  std::string s(ss.begin, ss.end - ss.begin);
  return os << s;
}

operator<<() [2/2]

std::ostream& operator<<	(	std::ostream &	os,
		const v_array< substring > &	ss
	)

Definition at line 118 of file parse_primitives.cc.

{
  substring* it = ss.cbegin();

  if (it == ss.cend())
  {
    return os;
  }

  os << *it;

  for (it++; it != ss.cend(); it++)
  {
    os << ",";
    os << *it;
  }

  return os;
}

operator==() [1/3]

bool operator==	(	const substring &	ss,
		const char *	str
	)

Definition at line 46 of file parse_primitives.cc.

References substring_equal().

Referenced by tokenize().

{ return substring_equal(ss, str); }

operator==() [2/3]

bool operator==	(	const char *	str,
		const substring &	ss
	)

Definition at line 48 of file parse_primitives.cc.

References substring_equal().

{ return substring_equal(ss, str); }

operator==() [3/3]

bool operator==	(	const substring &	ss1,
		const substring &	ss2
	)

Definition at line 50 of file parse_primitives.cc.

References substring_equal().

{ return substring_equal(ss1, ss2); }

parseFloat()

float parseFloat ( char *  p, char **  end, char *  endLine = nullptr  )

inline

Definition at line 88 of file parse_primitives.h.

References ldamath::powf().

Referenced by TC_parser< audit >::featureValue(), float_of_substring(), is_number(), and TC_parser< audit >::nameSpaceInfoValue().

{
  char* start = p;
  bool endLine_is_null = endLine == nullptr;

  if (!*p)
  {
    *end = p;
    return 0;
  }
  int s = 1;
  while ((*p == ' ') && (endLine_is_null || p < endLine)) p++;

  if (*p == '-')
  {
    s = -1;
    p++;
  }

  float acc = 0;
  while (*p >= '0' && *p <= '9' && (endLine_is_null || p < endLine)) acc = acc * 10 + *p++ - '0';

  int num_dec = 0;
  if (*p == '.')
  {
    while (*(++p) >= '0' && *p <= '9' && (endLine_is_null || p < endLine))
    {
      if (num_dec < 35)
      {
        acc = acc * 10 + (*p - '0');
        num_dec++;
      }
    }
  }

  int exp_acc = 0;
  if ((*p == 'e' || *p == 'E') && (endLine_is_null || p < endLine))
  {
    p++;
    int exp_s = 1;
    if (*p == '-' && (endLine_is_null || p < endLine))
    {
      exp_s = -1;
      p++;
    }
    while (*p >= '0' && *p <= '9' && (endLine_is_null || p < endLine)) exp_acc = exp_acc * 10 + *p++ - '0';
    exp_acc *= exp_s;
  }
  if (*p == ' ' || *p == '\n' || *p == '\t' || p == endLine)  // easy case succeeded.
  {
    acc *= powf(10, (float)(exp_acc - num_dec));
    *end = p;
    return s * acc;
  }
  else
    return (float)strtod(start, end);
}

print_substring()

void print_substring ( substring  s )

inline

Definition at line 70 of file parse_primitives.h.

References substring::begin, and substring::end.

Referenced by no_label::parse_no_label(), and parse_simple_label().

{ std::cout.write(s.begin, s.end - s.begin); }

safe_index()

char* safe_index ( char *  start, char  v, char *  max  )

inline

Definition at line 61 of file parse_primitives.h.

References escaped_tokenize().

Referenced by substring_to_example().

{
  while (start != max && *start != v) start++;
  return start;
}

substring_equal() [1/2]

bool substring_equal	(	const substring &	a,
		const substring &	b
	)

Definition at line 19 of file parse_primitives.cc.

References substring::begin, and substring::end.

Referenced by namedlabels::namedlabels(), operator==(), parse_dictionary_argument(), CB::parse_label(), COST_SENSITIVE::parse_label(), CCB::parse_label(), and tokenize().

{
  return (a.end - a.begin == b.end - b.begin)  // same length
      && (strncmp(a.begin, b.begin, a.end - a.begin) == 0);
}

substring_equal() [2/2]

bool substring_equal	(	const substring &	ss,
		const char *	str
	)

Definition at line 25 of file parse_primitives.cc.

References substring::begin, and substring::end.

{
  size_t len_ss = ss.end - ss.begin;
  size_t len_str = strlen(str);
  if (len_ss != len_str)
    return false;
  return (strncmp(ss.begin, str, len_ss) == 0);
}

substring_len()

size_t substring_len

(

substring &

s

)

Definition at line 34 of file parse_primitives.cc.

References substring::begin, and substring::end.

Referenced by VW::cb_sample_data::learn_or_predict(), and tokenize().

{ return s.end - s.begin; }

tokenize()

template<typename ContainerT >

void tokenize	(	char	delim,
		substring	s,
		ContainerT &	ret,
		bool	allow_empty = false
	)

Definition at line 27 of file parse_primitives.h.

References a, substring::begin, substring::end, operator!=(), operator==(), substring_equal(), and substring_len().

Referenced by COST_SENSITIVE::name_value(), namedlabels::namedlabels(), VW::parse_example_label(), MULTILABEL::parse_label(), CB::parse_label(), CCB::parse_label(), Search::parse_neighbor_features(), CCB::parse_outcome(), substring_to_example(), and VW::to_argv().

{
  ret.clear();
  char* last = s.begin;
  for (; s.begin != s.end; s.begin++)
  {
    if (*s.begin == delim)
    {
      if (allow_empty || (s.begin != last))
      {
        substring temp = {last, s.begin};
        ret.push_back(temp);
      }
      last = s.begin + 1;
    }
  }
  if (allow_empty || (s.begin != last))
  {
    substring final_substring = {last, s.begin};
    ret.push_back(final_substring);
  }
}