TC_parser< audit > Class Template Reference

Public Member Functions
	~TC_parser ()
void	parserWarning (const char *message, char *begin, char *pos, const char *message2)
float	featureValue ()
substring	read_name ()
void	maybeFeature ()
void	nameSpaceInfoValue ()
void	nameSpaceInfo ()
void	listFeatures ()
void	nameSpace ()
void	listNameSpace ()
	TC_parser (char *reading_head, char *endLine, vw &all, example *ae)

Public Attributes
char *	beginLine
char *	reading_head
char *	endLine
float	cur_channel_v
bool	new_index
size_t	anon
uint64_t	channel_hash
char *	base
unsigned char	index
float	v
bool	redefine_some
std::array< unsigned char, NUM_NAMESPACES > *	redefine
parser *	p
example *	ae
std::array< uint64_t, NUM_NAMESPACES > *	affix_features
std::array< bool, NUM_NAMESPACES > *	spelling_features
v_array< char >	spelling
uint32_t	hash_seed
uint64_t	parse_mask
std::array< std::vector< feature_dict * >, NUM_NAMESPACES > *	namespace_dictionaries

Detailed Description

template<bool audit>
class TC_parser< audit >

Definition at line 50 of file parse_example.cc.

Constructor & Destructor Documentation

~TC_parser()

template<bool audit>

TC_parser< audit >::~TC_parser ( )

inline

Definition at line 75 of file parse_example.cc.

{}

TC_parser()

template<bool audit>

TC_parser< audit >::TC_parser ( char *  reading_head, char *  endLine, vw &  all, example *  ae  )

inline

Definition at line 401 of file parse_example.cc.

References TC_parser< audit >::ae, vw::affix_features, TC_parser< audit >::endLine, vw::hash_seed, TC_parser< audit >::listNameSpace(), vw::namespace_dictionaries, vw::p, vw::parse_mask, TC_parser< audit >::reading_head, vw::redefine, vw::redefine_some, and vw::spelling_features.

  {
    spelling = v_init<char>();
    if (endLine != reading_head)
    {
      this->beginLine = reading_head;
      this->reading_head = reading_head;
      this->endLine = endLine;
      this->p = all.p;
      this->redefine_some = all.redefine_some;
      this->redefine = &all.redefine;
      this->ae = ae;
      this->affix_features = &all.affix_features;
      this->spelling_features = &all.spelling_features;
      this->namespace_dictionaries = &all.namespace_dictionaries;
      this->base = nullptr;
      this->hash_seed = all.hash_seed;
      this->parse_mask = all.parse_mask;
      listNameSpace();
      if (base != nullptr)
        free(base);
    }
  }

Member Function Documentation

featureValue()

template<bool audit>

float TC_parser< audit >::featureValue ( )

inline

Definition at line 93 of file parse_example.cc.

References parseFloat(), TC_parser< audit >::parserWarning(), TC_parser< audit >::reading_head, and TC_parser< audit >::v.

Referenced by TC_parser< audit >::maybeFeature().

  {
    if (*reading_head == ' ' || *reading_head == '\t' || *reading_head == '|' || reading_head == endLine ||
        *reading_head == '\r')
      return 1.;
    else if (*reading_head == ':')
    {
      // featureValue --> ':' 'Float'
      ++reading_head;
      char* end_read = nullptr;
      v = parseFloat(reading_head, &end_read, endLine);
      if (end_read == reading_head)
      {
        parserWarning("malformed example! Float expected after : \"", beginLine, reading_head, "\"");
      }
      if (std::isnan(v))
      {
        v = 0.f;
        parserWarning("warning: invalid feature value:\"", reading_head, end_read, "\" read as NaN. Replacing with 0.");
      }
      reading_head = end_read;
      return v;
    }
    else
    {
      // syntax error
      parserWarning("malformed example! '|', ':', space, or EOL expected after : \"", beginLine, reading_head, "\"");
      return 0.f;
    }
  }

listFeatures()

template<bool audit>

void TC_parser< audit >::listFeatures ( )

inline

Definition at line 333 of file parse_example.cc.

References TC_parser< audit >::maybeFeature(), TC_parser< audit >::parserWarning(), and TC_parser< audit >::reading_head.

Referenced by TC_parser< audit >::nameSpace().

  {
    while ((*reading_head == ' ' || *reading_head == '\t') && (reading_head < endLine))
    {
      // listFeatures --> ' ' MaybeFeature ListFeatures
      ++reading_head;
      maybeFeature();
    }
    if (!(*reading_head == '|' || reading_head == endLine || *reading_head == '\r'))
    {
      // syntax error
      parserWarning("malformed example! '|',space, or EOL expected after : \"", beginLine, reading_head, "\"");
    }
  }

listNameSpace()

template<bool audit>

void TC_parser< audit >::listNameSpace ( )

inline

Definition at line 387 of file parse_example.cc.

References TC_parser< audit >::nameSpace(), TC_parser< audit >::parserWarning(), and TC_parser< audit >::reading_head.

Referenced by TC_parser< audit >::TC_parser().

  {
    while ((*reading_head == '|') && (reading_head < endLine))  // ListNameSpace --> '|' NameSpace ListNameSpace
    {
      ++reading_head;
      nameSpace();
    }
    if (reading_head != endLine && *reading_head != '\r')
    {
      // syntax error
      parserWarning("malformed example! '|' or EOL expected after : \"", beginLine, reading_head, "\"");
    }
  }

maybeFeature()

template<bool audit>

void TC_parser< audit >::maybeFeature ( )

inline

Definition at line 136 of file parse_example.cc.

References affix_constant, affix_namespace, substring::begin, v_array< T >::begin(), c, v_array< T >::clear(), v_array< T >::delete_v(), dictionary_namespace, substring::end, v_array< T >::end(), example_predict::feature_space, TC_parser< audit >::featureValue(), parser::hasher, hashstring(), id(), TC_parser< audit >::index, example_predict::indices, features::indicies, TC_parser< audit >::parse_mask, v_array< T >::push_back(), features::push_back(), push_many(), quadratic_constant, TC_parser< audit >::read_name(), v_array< T >::size(), features::size(), features::space_names, spelling_namespace, features::sum_feat_sq, uniform_hash(), and features::values.

Referenced by TC_parser< audit >::listFeatures().

  {
    if (*reading_head == ' ' || *reading_head == '\t' || *reading_head == '|' || reading_head == endLine ||
        *reading_head == '\r')
    {
      // maybeFeature --> ø
    }
    else
    {
      // maybeFeature --> 'String' FeatureValue
      substring feature_name = read_name();
      v = cur_channel_v * featureValue();
      uint64_t word_hash;
      if (feature_name.end != feature_name.begin)
        word_hash = (p->hasher(feature_name, channel_hash) & parse_mask);
      else
        word_hash = channel_hash + anon++;
      if (v == 0)
        return;  // dont add 0 valued features to list of features
      features& fs = ae->feature_space[index];
      fs.push_back(v, word_hash);
      if (audit)
      {
        v_array<char> feature_v = v_init<char>();
        push_many(feature_v, feature_name.begin, feature_name.end - feature_name.begin);
        feature_v.push_back('\0');
        fs.space_names.push_back(audit_strings_ptr(new audit_strings(base, feature_v.begin())));
        feature_v.delete_v();
      }
      if ((*affix_features)[index] > 0 && (feature_name.end != feature_name.begin))
      {
        features& affix_fs = ae->feature_space[affix_namespace];
        if (affix_fs.size() == 0)
          ae->indices.push_back(affix_namespace);
        uint64_t affix = (*affix_features)[index];
        while (affix > 0)
        {
          bool is_prefix = affix & 0x1;
          uint64_t len = (affix >> 1) & 0x7;
          substring affix_name = {feature_name.begin, feature_name.end};
          if (affix_name.end > affix_name.begin + len)
          {
            if (is_prefix)
              affix_name.end = affix_name.begin + len;
            else
              affix_name.begin = affix_name.end - len;
          }
          word_hash =
              p->hasher(affix_name, (uint64_t)channel_hash) * (affix_constant + (affix & 0xF) * quadratic_constant);
          affix_fs.push_back(v, word_hash);
          if (audit)
          {
            v_array<char> affix_v = v_init<char>();
            if (index != ' ')
              affix_v.push_back(index);
            affix_v.push_back(is_prefix ? '+' : '-');
            affix_v.push_back('0' + (char)len);
            affix_v.push_back('=');
            push_many(affix_v, affix_name.begin, affix_name.end - affix_name.begin);
            affix_v.push_back('\0');
            affix_fs.space_names.push_back(audit_strings_ptr(new audit_strings("affix", affix_v.begin())));
          }
          affix >>= 4;
        }
      }
      if ((*spelling_features)[index])
      {
        features& spell_fs = ae->feature_space[spelling_namespace];
        if (spell_fs.size() == 0)
          ae->indices.push_back(spelling_namespace);
        // v_array<char> spelling;
        spelling.clear();
        for (char* c = feature_name.begin; c != feature_name.end; ++c)
        {
          char d = 0;
          if ((*c >= '0') && (*c <= '9'))
            d = '0';
          else if ((*c >= 'a') && (*c <= 'z'))
            d = 'a';
          else if ((*c >= 'A') && (*c <= 'Z'))
            d = 'A';
          else if (*c == '.')
            d = '.';
          else
            d = '#';
          // if ((spelling.size() == 0) || (spelling.last() != d))
          spelling.push_back(d);
        }
        substring spelling_ss = {spelling.begin(), spelling.end()};
        uint64_t word_hash = hashstring(spelling_ss, (uint64_t)channel_hash);
        spell_fs.push_back(v, word_hash);
        if (audit)
        {
          v_array<char> spelling_v = v_init<char>();
          if (index != ' ')
          {
            spelling_v.push_back(index);
            spelling_v.push_back('_');
          }
          push_many(spelling_v, spelling_ss.begin, spelling_ss.end - spelling_ss.begin);
          spelling_v.push_back('\0');
          spell_fs.space_names.push_back(audit_strings_ptr(new audit_strings("spelling", spelling_v.begin())));
        }
      }
      if ((*namespace_dictionaries)[index].size() > 0)
      {
        for (auto map : (*namespace_dictionaries)[index])
        {
          uint64_t hash = uniform_hash(feature_name.begin, feature_name.end - feature_name.begin, quadratic_constant);
          features* feats = map->get(feature_name, hash);
          if ((feats != nullptr) && (feats->values.size() > 0))
          {
            features& dict_fs = ae->feature_space[dictionary_namespace];
            if (dict_fs.size() == 0)
              ae->indices.push_back(dictionary_namespace);
            push_many(dict_fs.values, feats->values.begin(), feats->values.size());
            push_many(dict_fs.indicies, feats->indicies.begin(), feats->indicies.size());
            dict_fs.sum_feat_sq += feats->sum_feat_sq;
            if (audit)
              for (size_t i = 0; i < feats->indicies.size(); ++i)
              {
                uint64_t id = feats->indicies[i];
                std::stringstream ss;
                ss << index << '_';
                for (char* fc = feature_name.begin; fc != feature_name.end; ++fc) ss << *fc;
                ss << '=' << id;
                dict_fs.space_names.push_back(audit_strings_ptr(new audit_strings("dictionary", ss.str())));
              }
          }
        }
      }
    }
  }

nameSpace()

template<bool audit>

void TC_parser< audit >::nameSpace ( )

inline

Definition at line 348 of file parse_example.cc.

References example_predict::feature_space, example_predict::indices, TC_parser< audit >::listFeatures(), TC_parser< audit >::nameSpaceInfo(), TC_parser< audit >::parserWarning(), v_array< T >::push_back(), and uniform_hash().

Referenced by TC_parser< audit >::listNameSpace().

  {
    cur_channel_v = 1.0;
    index = 0;
    new_index = false;
    anon = 0;
    if (*reading_head == ' ' || *reading_head == '\t' || reading_head == endLine || *reading_head == '|' ||
        *reading_head == '\r')
    {
      // NameSpace --> ListFeatures
      index = (unsigned char)' ';
      if (ae->feature_space[index].size() == 0)
        new_index = true;
      if (audit)
      {
        if (base != nullptr)
          free(base);
        base = calloc_or_throw<char>(2);
        base[0] = ' ';
        base[1] = '\0';
      }
      channel_hash = this->hash_seed == 0 ? 0 : uniform_hash("", 0, this->hash_seed);
      listFeatures();
    }
    else if (*reading_head != ':')
    {
      // NameSpace --> NameSpaceInfo ListFeatures
      nameSpaceInfo();
      listFeatures();
    }
    else
    {
      // syntax error
      parserWarning("malformed example! '|',String,space, or EOL expected after : \"", beginLine, reading_head, "\"");
    }
    if (new_index && ae->feature_space[index].size() > 0)
      ae->indices.push_back(index);
  }

nameSpaceInfo()

template<bool audit>

void TC_parser< audit >::nameSpaceInfo ( )

inline

Definition at line 302 of file parse_example.cc.

References substring::begin, v_array< T >::begin(), substring::end, example_predict::feature_space, parser::hasher, TC_parser< audit >::index, TC_parser< audit >::nameSpaceInfoValue(), TC_parser< audit >::parserWarning(), v_array< T >::push_back(), push_many(), and TC_parser< audit >::read_name().

Referenced by TC_parser< audit >::nameSpace().

  {
    if (reading_head == endLine || *reading_head == '|' || *reading_head == ' ' || *reading_head == '\t' ||
        *reading_head == ':' || *reading_head == '\r')
    {
      // syntax error
      parserWarning("malformed example! String expected after : \"", beginLine, reading_head, "\"");
    }
    else
    {
      // NameSpaceInfo --> 'String' NameSpaceInfoValue
      index = (unsigned char)(*reading_head);
      if (redefine_some)
        index = (*redefine)[index];  // redefine index
      if (ae->feature_space[index].size() == 0)
        new_index = true;
      substring name = read_name();
      if (audit)
      {
        v_array<char> base_v_array = v_init<char>();
        push_many(base_v_array, name.begin, name.end - name.begin);
        base_v_array.push_back('\0');
        if (base != nullptr)
          free(base);
        base = base_v_array.begin();
      }
      channel_hash = p->hasher(name, this->hash_seed);
      nameSpaceInfoValue();
    }
  }

nameSpaceInfoValue()

template<bool audit>

void TC_parser< audit >::nameSpaceInfoValue ( )

inline

Definition at line 270 of file parse_example.cc.

References parseFloat(), TC_parser< audit >::parserWarning(), and TC_parser< audit >::reading_head.

Referenced by TC_parser< audit >::nameSpaceInfo().

  {
    if (*reading_head == ' ' || *reading_head == '\t' || reading_head == endLine || *reading_head == '|' ||
        *reading_head == '\r')
    {
      // nameSpaceInfoValue -->  ø
    }
    else if (*reading_head == ':')
    {
      // nameSpaceInfoValue --> ':' 'Float'
      ++reading_head;
      char* end_read = nullptr;
      cur_channel_v = parseFloat(reading_head, &end_read);
      if (end_read == reading_head)
      {
        parserWarning("malformed example! Float expected after : \"", beginLine, reading_head, "\"");
      }
      if (std::isnan(cur_channel_v))
      {
        cur_channel_v = 1.f;
        parserWarning(
            "warning: invalid namespace value:\"", reading_head, end_read, "\" read as NaN. Replacing with 1.");
      }
      reading_head = end_read;
    }
    else
    {
      // syntax error
      parserWarning("malformed example! '|',':', space, or EOL expected after : \"", beginLine, reading_head, "\"");
    }
  }

parserWarning()

template<bool audit>

void TC_parser< audit >::parserWarning ( const char *  message, char *  begin, char *  pos, const char *  message2  )

inline

Definition at line 77 of file parse_example.cc.

References parser::end_parsed_examples, parser::strict_parse, and THROW_EX.

Referenced by TC_parser< audit >::featureValue(), TC_parser< audit >::listFeatures(), TC_parser< audit >::listNameSpace(), TC_parser< audit >::nameSpace(), TC_parser< audit >::nameSpaceInfo(), and TC_parser< audit >::nameSpaceInfoValue().

  {
    std::stringstream ss;
    ss << message << std::string(begin, pos - begin).c_str() << message2 << "in Example #"
       << this->p->end_parsed_examples << ": \"" << std::string(this->beginLine, this->endLine).c_str() << "\""
       << std::endl;
    if (p->strict_parse)
    {
      THROW_EX(VW::strict_parse_exception, ss.str());
    }
    else
    {
      std::cerr << ss.str();
    }
  }

read_name()

template<bool audit>

substring TC_parser< audit >::read_name ( )

inline

Definition at line 124 of file parse_example.cc.

References substring::begin, substring::end, and TC_parser< audit >::reading_head.

Referenced by TC_parser< audit >::maybeFeature(), and TC_parser< audit >::nameSpaceInfo().

  {
    substring ret;
    ret.begin = reading_head;
    while (!(*reading_head == ' ' || *reading_head == ':' || *reading_head == '\t' || *reading_head == '|' ||
        reading_head == endLine || *reading_head == '\r'))
      ++reading_head;
    ret.end = reading_head;

    return ret;
  }

Member Data Documentation

ae

template<bool audit>

example* TC_parser< audit >::ae

Definition at line 66 of file parse_example.cc.

Referenced by TC_parser< audit >::TC_parser().

affix_features

template<bool audit>

std::array<uint64_t, NUM_NAMESPACES>* TC_parser< audit >::affix_features

Definition at line 67 of file parse_example.cc.

anon

template<bool audit>

size_t TC_parser< audit >::anon

Definition at line 58 of file parse_example.cc.

base

template<bool audit>

char* TC_parser< audit >::base

Definition at line 60 of file parse_example.cc.

beginLine

template<bool audit>

char* TC_parser< audit >::beginLine

Definition at line 53 of file parse_example.cc.

channel_hash

template<bool audit>

uint64_t TC_parser< audit >::channel_hash

Definition at line 59 of file parse_example.cc.

cur_channel_v

template<bool audit>

float TC_parser< audit >::cur_channel_v

Definition at line 56 of file parse_example.cc.

endLine

template<bool audit>

char* TC_parser< audit >::endLine

Definition at line 55 of file parse_example.cc.

Referenced by TC_parser< audit >::TC_parser().

hash_seed

template<bool audit>

uint32_t TC_parser< audit >::hash_seed

Definition at line 70 of file parse_example.cc.

index

template<bool audit>

unsigned char TC_parser< audit >::index

Definition at line 61 of file parse_example.cc.

Referenced by TC_parser< audit >::maybeFeature(), and TC_parser< audit >::nameSpaceInfo().

namespace_dictionaries

template<bool audit>

std::array<std::vector<feature_dict*>, NUM_NAMESPACES>* TC_parser< audit >::namespace_dictionaries

Definition at line 73 of file parse_example.cc.

new_index

template<bool audit>

bool TC_parser< audit >::new_index

Definition at line 57 of file parse_example.cc.

p

template<bool audit>

parser* TC_parser< audit >::p

Definition at line 65 of file parse_example.cc.

parse_mask

template<bool audit>

uint64_t TC_parser< audit >::parse_mask

Definition at line 71 of file parse_example.cc.

Referenced by TC_parser< audit >::maybeFeature().

reading_head

template<bool audit>

char* TC_parser< audit >::reading_head

Definition at line 54 of file parse_example.cc.

Referenced by TC_parser< audit >::featureValue(), TC_parser< audit >::listFeatures(), TC_parser< audit >::listNameSpace(), TC_parser< audit >::nameSpaceInfoValue(), TC_parser< audit >::read_name(), and TC_parser< audit >::TC_parser().

redefine

template<bool audit>

std::array<unsigned char, NUM_NAMESPACES>* TC_parser< audit >::redefine

Definition at line 64 of file parse_example.cc.

redefine_some

template<bool audit>

bool TC_parser< audit >::redefine_some

Definition at line 63 of file parse_example.cc.

spelling

template<bool audit>

v_array<char> TC_parser< audit >::spelling

Definition at line 69 of file parse_example.cc.

spelling_features

template<bool audit>

std::array<bool, NUM_NAMESPACES>* TC_parser< audit >::spelling_features

Definition at line 68 of file parse_example.cc.

v

template<bool audit>

float TC_parser< audit >::v

Definition at line 62 of file parse_example.cc.

Referenced by TC_parser< audit >::featureValue().

---

The documentation for this class was generated from the following file:

• /mnt/c/w/linux/vowpal_wabbit/vowpalwabbit/parse_example.cc