io_buf Class Reference

#include <io_buf.h>

Inheritance diagram for io_buf:

Public Member Functions
void	verify_hash (bool verify)
uint32_t	hash ()
virtual int	open_file (const char *name, bool stdin_off)
virtual int	open_file (const char *name, bool stdin_off, int flag=READ)
virtual void	reset_file (int f)
	io_buf ()
virtual	~io_buf ()
void	set (char *p)
virtual size_t	num_files ()
virtual ssize_t	read_file (int f, void *buf, size_t nbytes)
ssize_t	fill (int f)
virtual ssize_t	write_file (int f, const void *buf, size_t nbytes)
virtual void	flush ()
virtual bool	close_file ()
virtual bool	compressed ()
void	close_files ()
void	buf_write (char *&pointer, size_t n)
size_t	buf_read (char *&pointer, size_t n)
size_t	bin_read_fixed (char *data, size_t len, const char *read_message)
size_t	bin_write_fixed (const char *data, size_t len)

Static Public Member Functions
static ssize_t	read_file_or_socket (int f, void *buf, size_t nbytes)
static ssize_t	write_file_or_socket (int f, const void *buf, size_t nbytes)
static void	close_file_or_socket (int f)
static bool	is_socket (int f)

Public Attributes
v_array< char >	space
v_array< int >	files
size_t	count
size_t	current
char *	head
v_array< char >	currentname
v_array< char >	finalname

Static Public Attributes
static constexpr int	READ = 1
static constexpr int	WRITE = 2

Private Attributes
bool	_verify_hash
uint32_t	_hash

Static Private Attributes
static constexpr size_t	INITIAL_BUFF_SIZE = 1 << 16

Detailed Description

Definition at line 54 of file io_buf.h.

Constructor & Destructor Documentation

io_buf()

io_buf::io_buf ( )

inline

Definition at line 147 of file io_buf.h.

References v_array< T >::begin(), and v_array< T >::resize().

           : _verify_hash{false}, _hash{0}, count{0}, current{0}
  {
    space = v_init<char>();
    files = v_init<int>();
    currentname = v_init<char>();
    finalname = v_init<char>();
    space.resize(INITIAL_BUFF_SIZE);
    head = space.begin();
  }

~io_buf()

virtual io_buf::~io_buf ( )

inlinevirtual

Definition at line 157 of file io_buf.h.

References v_array< T >::delete_v().

  {
    files.delete_v();
    space.delete_v();
  }

Member Function Documentation

bin_read_fixed()

size_t io_buf::bin_read_fixed ( char *  data, size_t  len, const char *  read_message  )

inline

Definition at line 230 of file io_buf.h.

References buf_read(), THROW, and uniform_hash().

Referenced by bin_read(), bin_text_read_write_fixed(), save_load(), save_load_flat_example(), save_load_header(), GD::save_load_online_state(), GD::save_load_regressor(), save_load_regularizer(), and save_load_sampling().

  {
    if (len > 0)
    {
      char* p;
      // if the model is corrupt the number of bytes can be less then specified (as there isn't enought data available
      // in the file)
      len = buf_read(p, len);

      // compute hash for check-sum
      if (_verify_hash)
        _hash = (uint32_t)uniform_hash(p, len, _hash);

      if (*read_message == '\0')
        memcpy(data, p, len);
      else if (memcmp(data, p, len) != 0)
        THROW(read_message);
      return len;
    }
    return 0;
  }

bin_write_fixed()

size_t io_buf::bin_write_fixed ( const char *  data, size_t  len  )

inline

Definition at line 252 of file io_buf.h.

References buf_write(), isbinary(), readto(), and uniform_hash().

Referenced by audit_regressor_feature(), audit_regressor_lda(), bin_text_write(), bin_text_write_fixed(), bin_write(), and save_load_flat_example().

  {
    if (len > 0)
    {
      char* p;
      buf_write(p, len);

      memcpy(p, data, len);

      // compute hash for check-sum
      if (_verify_hash)
        _hash = (uint32_t)uniform_hash(p, len, _hash);
    }
    return len;
  }

buf_read()

size_t io_buf::buf_read	(	char *&	pointer,
		size_t	n
	)

Definition at line 12 of file io_buf.cc.

References v_array< T >::begin(), current, v_array< T >::end(), files, fill(), head, v_array< T >::size(), and space.

Referenced by bin_read_fixed(), MULTILABEL::bufread_label(), CB::bufread_label(), COST_SENSITIVE::bufread_label(), close_files(), read_cached_features(), MULTICLASS::read_cached_label(), CCB::read_cached_label(), MULTILABEL::read_cached_label(), CB::read_cached_label(), COST_SENSITIVE::read_cached_label(), CB_EVAL::read_cached_label(), and read_cached_simple_label().

{
  // return a pointer to the next n bytes.  n must be smaller than the maximum size.
  if (head + n <= space.end())
  {
    pointer = head;
    head += n;
    return n;
  }
  else  // out of bytes, so refill.
  {
    if (head != space.begin())  // There exists room to shift.
    {
      // Out of buffer so swap to beginning.
      size_t left = space.end() - head;
      memmove(space.begin(), head, left);
      head = space.begin();
      space.end() = space.begin() + left;
    }
    if (fill(files[current]) > 0)   // read more bytes from current file if present
      return buf_read(pointer, n);  // more bytes are read.
    else if (++current < files.size())
      return buf_read(pointer, n);  // No more bytes, so go to next file and try again.
    else
    {
      // no more bytes to read, return all that we have left.
      pointer = head;
      head = space.end();
      return space.end() - pointer;
    }
  }
}

buf_write()

void io_buf::buf_write	(	char *&	pointer,
		size_t	n
	)

Definition at line 94 of file io_buf.cc.

References v_array< T >::begin(), v_array< T >::end(), v_array< T >::end_array, flush(), head, v_array< T >::resize(), and space.

Referenced by bin_write_fixed(), MULTICLASS::cache_label(), MULTILABEL::cache_label(), CB::cache_label(), COST_SENSITIVE::cache_label(), CCB::cache_label(), CB_EVAL::cache_label(), cache_simple_label(), cache_tag(), close_files(), output_byte(), and output_features().

{
  // return a pointer to the next n bytes to write into.
  if (head + n <= space.end_array)
  {
    pointer = head;
    head += n;
  }
  else  // Time to dump the file
  {
    if (head != space.begin())
      flush();
    else  // Array is short, so increase size.
    {
      space.resize(2 * (space.end_array - space.begin()));
      space.end() = space.begin();
      head = space.begin();
    }
    buf_write(pointer, n);
  }
}

close_file()

virtual bool io_buf::close_file ( )

inlinevirtual

Reimplemented in memory_io_buf, and comp_io_buf.

Definition at line 204 of file io_buf.h.

References close_file_or_socket(), v_array< T >::empty(), and v_array< T >::pop().

Referenced by close_files(), dump_regressor(), end_examples(), load_input_model(), parse_cache(), parse_dictionary_argument(), parse_mask_regressor_args(), parse_sources(), and reset_source().

  {
    if (!files.empty())
    {
      close_file_or_socket(files.pop());
      return true;
    }
    return false;
  }

close_file_or_socket()

void io_buf::close_file_or_socket ( int  f )

static

Definition at line 152 of file io_buf.cc.

References is_socket().

Referenced by close_file(), compressed(), VW::finish(), and reset_source().

{
#ifdef _WIN32
  if (io_buf::is_socket(f))
    closesocket(f);
  else
    _close(f);
#else
  close(f);
#endif
}

close_files()

void io_buf::close_files ( )

inline

Definition at line 218 of file io_buf.h.

References buf_read(), buf_write(), close_file(), and is_socket().

Referenced by finalize_source().

  {
    while (close_file())
      ;
  }

compressed()

virtual bool io_buf::compressed ( )

inlinevirtual

Reimplemented in comp_io_buf.

Definition at line 214 of file io_buf.h.

References close_file_or_socket().

Referenced by reset_source().

{ return false; }

fill()

ssize_t io_buf::fill ( int  f )

inline

Definition at line 171 of file io_buf.h.

References v_array< T >::begin(), v_array< T >::end(), v_array< T >::end_array, read_file(), and v_array< T >::resize().

Referenced by buf_read(), isbinary(), and readto().

  {  // if the loaded values have reached the allocated space
    if (space.end_array - space.end() == 0)
    {  // reallocate to twice as much space
      size_t head_loc = head - space.begin();
      space.resize(2 * (space.end_array - space.begin()));
      head = space.begin() + head_loc;
    }
    // read more bytes from file up to the remaining allocated space
    ssize_t num_read = read_file(f, space.end(), space.end_array - space.end());
    if (num_read >= 0)
    {  // if some bytes were actually loaded, update the end of loaded values
      space.end() = space.end() + num_read;
      return num_read;
    }
    else
      return 0;
  }

flush()

virtual void io_buf::flush ( )

inlinevirtual

Reimplemented in comp_io_buf.

Definition at line 194 of file io_buf.h.

References v_array< T >::begin(), v_array< T >::empty(), and write_file().

Referenced by buf_write(), dump_regressor(), end_examples(), reset_source(), and send_features().

  {
    if (!files.empty())
    {
      if (write_file(files[0], space.begin(), head - space.begin()) != (int)(head - space.begin()))
        std::cerr << "error, failed to write example\n";
      head = space.begin();
    }
  }

hash()

uint32_t io_buf::hash ( )

inline

Definition at line 83 of file io_buf.h.

References _hash, and THROW.

Referenced by save_load_header().

  {
    if (!_verify_hash)
      THROW("HASH WAS NOT CALCULATED");
    return _hash;
  }

is_socket()

bool io_buf::is_socket ( int  f )

static

Definition at line 116 of file io_buf.cc.

Referenced by close_file_or_socket(), close_files(), read_file_or_socket(), and write_file_or_socket().

{
  // this appears to work in practice, but could probably be done in a cleaner fashion
#ifdef _WIN32
  const int _nhandle = _getmaxstdio() / 2;
  return f >= _nhandle;
#else
  const int _nhandle = 32;
  return f >= _nhandle;
#endif
}

num_files()

virtual size_t io_buf::num_files ( )

inlinevirtual

Reimplemented in comp_io_buf.

Definition at line 165 of file io_buf.h.

References v_array< T >::size().

Referenced by reset_source().

{ return files.size(); }

open_file() [1/2]

virtual int io_buf::open_file ( const char *  name, bool  stdin_off  )

inlinevirtual

Definition at line 90 of file io_buf.h.

References open_file().

Referenced by dump_regressor(), enable_sources(), make_write_cache(), open_file(), parse_cache(), parse_dictionary_argument(), parse_mask_regressor_args(), read_regressor_file(), and reset_source().

{ return open_file(name, stdin_off, READ); }

open_file() [2/2]

virtual int io_buf::open_file ( const char *  name, bool  stdin_off, int  flag = READ  )

inlinevirtual

Reimplemented in comp_io_buf.

Definition at line 92 of file io_buf.h.

References O_LARGEFILE, v_array< T >::push_back(), READ, THROWERRNO, and WRITE.

  {
    int ret = -1;
    switch (flag)
    {
      case READ:
        if (*name != '\0')
        {
#ifdef _WIN32
          // _O_SEQUENTIAL hints to OS that we'll be reading sequentially, so cache aggressively.
          _sopen_s(&ret, name, _O_RDONLY | _O_BINARY | _O_SEQUENTIAL, _SH_DENYWR, 0);
#else
          ret = open(name, O_RDONLY | O_LARGEFILE);
#endif
        }
        else if (!stdin_off)
#ifdef _WIN32
          ret = _fileno(stdin);
#else
          ret = fileno(stdin);
#endif
        if (ret != -1)
          files.push_back(ret);
        break;

      case WRITE:
#ifdef _WIN32
        _sopen_s(&ret, name, _O_CREAT | _O_WRONLY | _O_BINARY | _O_TRUNC, _SH_DENYWR, _S_IREAD | _S_IWRITE);
#else
        ret = open(name, O_CREAT | O_WRONLY | O_LARGEFILE | O_TRUNC, 0666);
#endif
        if (ret != -1)
          files.push_back(ret);
        break;

      default:
        std::cerr << "Unknown file operation. Something other than READ/WRITE specified" << std::endl;
        ret = -1;
    }
    if (ret == -1 && *name != '\0')
      THROWERRNO("can't open: " << name);
    return ret;
  }

read_file()

virtual ssize_t io_buf::read_file ( int  f, void *  buf, size_t  nbytes  )

inlinevirtual

Reimplemented in memory_io_buf, and comp_io_buf.

Definition at line 167 of file io_buf.h.

References f, and read_file_or_socket().

Referenced by cache_numbits(), fill(), hash_file_contents(), and parse_dictionary_argument().

{ return read_file_or_socket(f, buf, nbytes); }

read_file_or_socket()

ssize_t io_buf::read_file_or_socket ( int  f, void *  buf, size_t  nbytes  )

static

Definition at line 128 of file io_buf.cc.

References is_socket().

Referenced by read_file().

{
#ifdef _WIN32
  if (is_socket(f))
    return recv(f, reinterpret_cast<char*>(buf), static_cast<int>(nbytes), 0);
  else
    return _read(f, buf, (unsigned int)nbytes);
#else
  return read(f, buf, (unsigned int)nbytes);
#endif
}

reset_file()

virtual void io_buf::reset_file ( int  f )

inlinevirtual

Reimplemented in comp_io_buf.

Definition at line 136 of file io_buf.h.

References v_array< T >::begin(), and v_array< T >::end().

Referenced by reset_source().

  {
#ifdef _WIN32
    _lseek(f, 0, SEEK_SET);
#else
    lseek(f, 0, SEEK_SET);
#endif
    space.end() = space.begin();
    head = space.begin();
  }

set()

void io_buf::set ( char *  p )

inline

Definition at line 163 of file io_buf.h.

Referenced by cache_tag(), output_byte(), output_features(), and read_cached_features().

{ head = p; }

verify_hash()

void io_buf::verify_hash ( bool  verify )

inline

Definition at line 74 of file io_buf.h.

Referenced by save_load_header().

  {
    _verify_hash = verify;
    // reset the hash so that the io_buf can be re-used for loading
    // as it is done for Reload()
    if (!verify)
      _hash = 0;
  }

write_file()

virtual ssize_t io_buf::write_file ( int  f, const void *  buf, size_t  nbytes  )

inlinevirtual

Reimplemented in memory_io_buf, and comp_io_buf.

Definition at line 190 of file io_buf.h.

References write_file_or_socket().

Referenced by flush(), and make_write_cache().

{ return write_file_or_socket(f, buf, nbytes); }

write_file_or_socket()

ssize_t io_buf::write_file_or_socket ( int  f, const void *  buf, size_t  nbytes  )

static

Definition at line 140 of file io_buf.cc.

References is_socket().

Referenced by active_print_result(), confidence_print_result(), CB_ADF::global_print_newline(), CSOAA::global_print_newline(), ACTION_SCORE::print_action_score(), CCB::print_decision_scores(), print_raw_text(), print_result(), MWT::print_scalars(), and write_file().

{
#ifdef _WIN32
  if (is_socket(f))
    return send(f, reinterpret_cast<const char*>(buf), static_cast<int>(nbytes), 0);
  else
    return _write(f, buf, (unsigned int)nbytes);
#else
  return write(f, buf, (unsigned int)nbytes);
#endif
}

Member Data Documentation

_hash

uint32_t io_buf::_hash

private

Definition at line 58 of file io_buf.h.

Referenced by hash().

_verify_hash

bool io_buf::_verify_hash

private

Definition at line 57 of file io_buf.h.

count

size_t io_buf::count

Definition at line 65 of file io_buf.h.

Referenced by enable_sources().

current

size_t io_buf::current

Definition at line 66 of file io_buf.h.

Referenced by buf_read(), enable_sources(), isbinary(), readto(), and reset_source().

currentname

v_array<char> io_buf::currentname

Definition at line 68 of file io_buf.h.

Referenced by free_parser(), make_write_cache(), and reset_source().

files

v_array<int> io_buf::files

Definition at line 64 of file io_buf.h.

Referenced by buf_read(), comp_io_buf::close_file(), enable_sources(), end_examples(), finalize_source(), isbinary(), make_write_cache(), comp_io_buf::open_file(), open_sockets(), readto(), reset_source(), SVRG::save_load(), MWT::save_load(), MARGINAL::save_load(), GD::save_load(), save_load(), save_load_header(), memory_tree_ns::save_load_memory_tree(), save_load_sampling(), save_load_svm_model(), save_load_tree(), recall_tree_ns::save_load_tree(), and sender::~sender().

finalname

v_array<char> io_buf::finalname

Definition at line 69 of file io_buf.h.

Referenced by free_parser(), make_write_cache(), and reset_source().

head

char* io_buf::head

Definition at line 67 of file io_buf.h.

Referenced by buf_read(), buf_write(), comp_io_buf::flush(), isbinary(), readto(), comp_io_buf::reset_file(), and sync_queries().

INITIAL_BUFF_SIZE

constexpr size_t io_buf::INITIAL_BUFF_SIZE = 1 << 16

staticprivate

Definition at line 59 of file io_buf.h.

READ

constexpr int io_buf::READ = 1

static

Definition at line 71 of file io_buf.h.

Referenced by enable_sources(), comp_io_buf::open_file(), open_file(), parse_cache(), parse_dictionary_argument(), parse_mask_regressor_args(), read_regressor_file(), and reset_source().

space

v_array<char> io_buf::space

Definition at line 62 of file io_buf.h.

Referenced by buf_read(), buf_write(), comp_io_buf::flush(), isbinary(), parse_cache(), readto(), comp_io_buf::reset_file(), save_load_header(), sync_queries(), and sender::~sender().

WRITE

constexpr int io_buf::WRITE = 2

static

Definition at line 72 of file io_buf.h.

Referenced by audit_regressor_setup(), dump_regressor(), make_write_cache(), comp_io_buf::open_file(), and open_file().

---

The documentation for this class was generated from the following files:

• /mnt/c/w/linux/vowpal_wabbit/vowpalwabbit/io_buf.h
• /mnt/c/w/linux/vowpal_wabbit/vowpalwabbit/io_buf.cc