icarus::ParsingToolkit Struct Reference

Utilities for text parsing. More...

#include <ParsingToolkit.h>

Classes
struct	CCTypeAdapter
struct	Error
struct	Params_t
	All parsing parameters. More...
struct	SplitView_t
	Record of a split token: pre-separator, separator and post-separator. More...

Public Types
using	QuotSpec_t = std::pair< std::string, std::string >
	Specification of quotation: opening and closing. More...

Public Member Functions
	ParsingToolkit ()
	Default parsing parameters. More...
	ParsingToolkit (Params_t params)
	Creates a parser with the specified parsing parameters. More...
Params_t const &	params () const noexcept
	Returns the current parameters of parsing. More...
template<typename BIter , typename EIter >
std::string_view	findFirstUnquoted (std::string_view sv, BIter beginKey, EIter endKey) const
	Finds the first of the specified keys in the unquoted part of sv. More...
template<typename Words >
std::vector< std::string >	removeEscapes (Words const &words) const
	Returns a copy of words with all escape characters removed. More...
template<typename Words >
std::vector< std::string >	removeQuotations (Words const &words) const
	Returns a copy of words with no quotation starts and ends. More...
template<typename Iter >
bool	isCharacterEscaped (Iter begin, Iter itCh) const
Input
std::pair< std::string, unsigned int >	readMultiline (std::istream &in) const
	Returns a single line of text from the input stream. More...
Tokenization
template<typename Delim >
std::vector< std::string_view >	splitWords (std::string const &s, Delim isDelimiter) const
	Splits a string into words. More...
std::vector< std::string_view >	splitWords (std::string const &s) const
	Helper version of splitWords(std::string const&, Delim). More...
template<typename Iter >
Iter	findCommentWord (Iter beginWord, Iter endWord) const
	Finds the first word starting with a comment marker. More...
template<typename WordType >
void	removeCommentLine (std::vector< WordType > &words) const
	Removes all the words from the one starting with a comment marker. More...
std::pair< std::string_view, QuotSpec_t const * >	findQuotationStart (std::string_view sv) const
	Finds the start of the next quotation in sv. More...
std::string_view	findQuotationEnd (std::string_view sv, std::string const &quotEnd) const
	Finds the quotation end in sv. More...
bool	isQuotationUnclosed (std::string_view sv) const
	Returns if the sequence sv has unclosed quotation at its end. More...
template<typename BIter , typename EIter >
std::string_view	findFirstUnescaped (std::string_view sv, BIter beginKey, EIter endKey) const
	Finds the first of the specified keys in sv. More...
template<typename Keys >
std::string_view	findFirstUnescaped (std::string_view sv, Keys const &keys) const
	Finds the first of the specified keys in sv. More...
template<typename Key >
std::string_view	findFirstUnescaped (std::string_view sv, std::initializer_list< Key > keys) const
template<typename Keys >
std::string_view	findFirstUnquoted (std::string_view sv, Keys const &keys) const
	Finds the first of the specified keys in the unquoted part of sv. More...
template<typename Key >
std::string_view	findFirstUnquoted (std::string_view sv, std::initializer_list< Key > keys) const
Characters
bool	isEscape (char ch) const
	Returns whether ch is an escape character. More...
template<typename BIter >
bool	isCharacterEscaped (BIter begin, BIter itCh) const
	Returns whether the character pointed by itCh is escaped or not. More...
template<typename Sel >
std::string_view::const_iterator	findNextCharacter (std::string_view s, Sel select) const
	Finds the next character satisfying the specified criterion. More...
std::string_view::const_iterator	findNextBlank (std::string_view s) const
	Helper function for findNextCharacter(std::string_view, Sel). More...
template<typename CType >
std::string_view	removeTrailingCharacters (std::string_view s, CType charType) const
	Consumes the blank characters a the beginning of s. More...
std::string_view	removeTrailingBlanks (std::string_view s) const
	Consumes the blank characters a the beginning of s. More...
std::string	removeWordEscapes (std::string &&w) const
	Returns a copy of w with all escape characters removed. More...
std::string	removeWordEscapes (std::string_view w) const
std::string	removeWordEscapes (const char *w) const
std::string	removeWordQuotations (std::string &&w) const
	Returns a copy of w with no quotation starts and ends. More...
std::string	removeWordQuotations (std::string_view w) const
std::string	removeWordQuotations (const char *w) const

Static Public Member Functions
static SplitView_t	splitOn (std::string_view sv, std::string_view sep)
	Splits the view sv in three: before sep, sep and after sep. More...
static std::string_view	make_view (std::string const &s)
	Creates a std::string_view from an entire string s. More...
template<typename BIter , typename EIter >
static std::string_view	make_view (BIter b, EIter e)
	Creates a std::string_view from two string iterators b and e. More...

Static Public Attributes
static constexpr CCTypeAdapter <&std::isblank >	isBlank {}
	Adapter for determining if a character is a blank (see std::isblank()). More...
static Params_t const	DefaultParameters

Private Member Functions
void	adoptParams (Params_t params)
	Initializes the parameters and caches. More...

Private Attributes
Params_t	fParams
	Parsing parameters. More...
std::string	fQuoteStarts
	Start characters of all supported quotations. More...

Detailed Description

Utilities for text parsing.

This "class" is a glorified namespace with some configuration inside.

Quotation

A quoted string is the content in between an opening quoting sequence and the matching closing sequence. Each sequence may be any string, including but not limited to a one-character long string. Escaping the first character of an opening or closing quotation string will turn it in common string data carrying no quotation meaning.

Escaping rules

Any single character following the escape character is "escaped". The escaped characters lose their standard function and are replaced by a substitute character. For example, escaping the first character of a opening quotation makes that a standard character. An escaped escape character is always replaced by the character itself, without its escape function.

Definition at line 54 of file ParsingToolkit.h.

Member Typedef Documentation

using icarus::ParsingToolkit::QuotSpec_t = std::pair<std::string, std::string>

Specification of quotation: opening and closing.

Definition at line 63 of file ParsingToolkit.h.

Constructor & Destructor Documentation

icarus::ParsingToolkit::ParsingToolkit ( )

inline

Default parsing parameters.

Creates a parser with the default parsing parameters.

Definition at line 100 of file ParsingToolkit.h.

{ adoptParams(DefaultParameters); }

icarus::ParsingToolkit::ParsingToolkit ( Params_t  params )

inline

Creates a parser with the specified parsing parameters.

Definition at line 103 of file ParsingToolkit.h.

{ adoptParams(std::move(params)); }

Member Function Documentation

void icarus::ParsingToolkit::adoptParams ( Params_t  params )

private

Initializes the parameters and caches.

Definition at line 220 of file ParsingToolkit.cxx.

                                                      {
  
  fParams = std::move(params);
  
  // sort the quotations by length
  auto const byOpeningLength = [](QuotSpec_t const& a, QuotSpec_t const& b)
    {
      std::size_t const al = a.first.length(), bl = b.first.length();
      return (al != bl)? (al > bl): (a < b);
    };
  std::sort(fParams.quotes.begin(), fParams.quotes.end(), byOpeningLength);
  
  // collect the first character of each of the opening quotes
  // (sorted and with no duplicates)
  for (QuotSpec_t const& quotSpec: fParams.quotes)
    fQuoteStarts += quotSpec.first.front();
  std::sort(fQuoteStarts.begin(), fQuoteStarts.end());
  fQuoteStarts.erase
    (std::unique(fQuoteStarts.begin(), fQuoteStarts.end()), fQuoteStarts.end());
  
} // icarus::ParsingToolkit::adoptParams()

template<typename Iter >

Iter icarus::ParsingToolkit::findCommentWord	(	Iter	beginWord,
		Iter	endWord
	)		const

Finds the first word starting with a comment marker.

Template Parameters

Iter	type of iterator to the words

Parameters

beginWord	iterator to the first word to consider
endWord	iterator past the lasy word to consider

Returns

an iterator to the comment word, or endWord if not found

The original list is modified, the word starting with a comment marker and all the following ones are removed.

Definition at line 748 of file ParsingToolkit.h.

{
  for (auto it = beginWord; it != endWord; ++it) {
    if (std::equal(fParams.comment.begin(), fParams.comment.end(), begin(*it)))
      return it;
  } // for
  return endWord;
} // icarus::ParsingToolkit::findCommentWord()

template<typename BIter , typename EIter >

std::string_view icarus::ParsingToolkit::findFirstUnescaped	(	std::string_view	sv,
		BIter	beginKey,
		EIter	endKey
	)		const

Finds the first of the specified keys in sv.

Template Parameters

BIter	type of iterator to the keys
EIter	type of key end-iterator

Parameters

sv	string to be parsed
beginKey	iterator to the first key
endKey	iterator past the last key

Returns

a view of the key found within sv, empty if none

The keys are required to be sorted, longest first, since they are tested in order and the first match is kept (e.g. if the first key is = and the second is ==, the second key is never matched since the first one matches first). The first character of the key must not be escaped. Escaped characters in the key are not supported.

If no key is found, the returned view is zero-length and pointing to the end of sv.

The quoting in sv is ignored.

Definition at line 634 of file ParsingToolkit.h.

{
  
  typename std::iterator_traits<BIter>::value_type const* key = nullptr;
  std::size_t keyPos = std::string_view::npos;
  
  for (auto iKey = beginKey; iKey != endKey; ++iKey) {
    // find where this key is (unescaped)
    std::size_t pos = 0;
    while (pos < sv.length()) {
      pos = sv.find(*iKey, pos);
      if (!isCharacterEscaped(sv.begin(), sv.begin() + pos)) break;
      ++pos;
    }
    // is this the first among the keys?
    if (pos >= std::min(keyPos, sv.length())) continue;
    key = &*iKey;
    keyPos = pos;
  } // for keys
  
  // return a substring of sv, not key
  if (key) {
    using std::begin, std::end;
    std::size_t const keyLength = make_view(*key).length();
    return { sv.data() + keyPos, keyLength };
  }
  else return { sv.data() + sv.length(), 0 };
} // icarus::ParsingToolkit::findFirstUnescaped()

template<typename Keys >

std::string_view icarus::ParsingToolkit::findFirstUnescaped	(	std::string_view	sv,
		Keys const &	keys
	)		const

Finds the first of the specified keys in sv.

Template Parameters

BIter	type of iterator to the keys
EIter	type of key end-iterator

Parameters

sv	string to be parsed
beginKey	iterator to the first key
endKey	iterator past the last key

Returns

a view of the key found within sv, empty if none

The keys are required to be sorted, longest first, since they are tested in order and the first match is kept (e.g. if the first key is = and the second is ==, the second key is never matched since the first one matches first). The first character of the key must not be escaped. Escaped characters in the key are not supported.

If no key is found, the returned view is zero-length and pointing to the end of sv.

The quoting in sv is ignored.

Definition at line 667 of file ParsingToolkit.h.

{
  using std::begin, std::end;
  return findFirstUnescaped(sv, begin(keys), end(keys)); 
} // icarus::ParsingToolkit::findFirstUnescaped(Keys)

template<typename Key >

std::string_view icarus::ParsingToolkit::findFirstUnescaped	(	std::string_view	sv,
		std::initializer_list< Key >	keys
	)		const

Definition at line 677 of file ParsingToolkit.h.

  { return findFirstUnescaped(sv, keys.begin(), keys.end()); }

template<typename BIter , typename EIter >

std::string_view icarus::ParsingToolkit::findFirstUnquoted	(	std::string_view	sv,
		BIter	beginKey,
		EIter	endKey
	)		const

Finds the first of the specified keys in the unquoted part of sv.

Template Parameters

BIter	type of iterator to the keys
EIter	type of key end-iterator

Parameters

sv	string to be parsed
beginKey	iterator to the first key
endKey	iterator past the last key

Returns

the view pointing to the key in sv, or empty to its end if none

The keys are required to be sorted, longest first, since they are tested in order and the first match is kept (e.g. if the first key is = and the second is ==, the second key is never matched since the first one matches first).

If no key is found, the returned view is zero-length and pointing to the end of sv.

Definition at line 684 of file ParsingToolkit.h.

{
  
  // if a key is found between `b` and `e`, returns `sv` split around the key;
  // otherwise, all `sv` is in post
  auto findKey = [this,beginKey,endKey]
    (std::string_view::const_iterator b, std::string_view::const_iterator e)
    { return findFirstUnescaped(make_view(b, e), beginKey, endKey); };
  
  std::string_view key{ sv.data() + sv.length(), 0 };
  while (!sv.empty()) {
    
    // find the next quotation
    auto const [ fromQ, qptr ] = findQuotationStart(sv);
    
    // search in the unquoted part
    key = findKey(sv.begin(), fromQ.begin());
    if (!key.empty()) break;
    
    // skip the quotation; if there is no quotation, we are done
    if (!qptr) break;
    
    sv = fromQ;
    sv.remove_prefix(qptr->first.length()); // skip the quotation start
    
    // find the end of quotation
    std::string_view const afterQ = findQuotationEnd(sv, qptr->second);
    
    if (afterQ.empty()) { // begin of quotation, but no end: no good
      // so we don't consider this as quotation: search in the "quoted" part
      key = findKey(fromQ.begin(), fromQ.end());
      break;
    } // if

    // skip the quoted material, and the quotation end too
    sv = afterQ;
    sv.remove_prefix(qptr->second.length());
    
  } // while
  
  return key;
  
} // icarus::ParsingToolkit::findFirstUnquoted(Iter)

template<typename Keys >

std::string_view icarus::ParsingToolkit::findFirstUnquoted	(	std::string_view	sv,
		Keys const &	keys
	)		const

Finds the first of the specified keys in the unquoted part of sv.

Template Parameters

BIter	type of iterator to the keys
EIter	type of key end-iterator

Parameters

sv	string to be parsed
beginKey	iterator to the first key
endKey	iterator past the last key

Returns

the view pointing to the key in sv, or empty to its end if none

The keys are required to be sorted, longest first, since they are tested in order and the first match is kept (e.g. if the first key is = and the second is ==, the second key is never matched since the first one matches first).

If no key is found, the returned view is zero-length and pointing to the end of sv.

Definition at line 732 of file ParsingToolkit.h.

{
  using std::begin, std::end;
  return findFirstUnquoted(sv, begin(keys), end(keys)); 
} // icarus::ParsingToolkit::findFirstUnquoted(Keys)

template<typename Key >

std::string_view icarus::ParsingToolkit::findFirstUnquoted	(	std::string_view	sv,
		std::initializer_list< Key >	keys
	)		const

Definition at line 742 of file ParsingToolkit.h.

  { return findFirstUnquoted(sv, keys.begin(), keys.end()); }

std::string_view::const_iterator icarus::ParsingToolkit::findNextBlank ( std::string_view  s ) const

inline

Helper function for findNextCharacter(std::string_view, Sel).

Definition at line 405 of file ParsingToolkit.h.

    { return findNextCharacter(s, isBlank); }

template<typename Sel >

std::string_view::const_iterator icarus::ParsingToolkit::findNextCharacter	(	std::string_view	s,
		Sel	select
	)		const

Finds the next character satisfying the specified criterion.

Template Parameters

Sel	type of functor determining which character to consider blank

Parameters

s	view of the string to be parsed
select	functor determining which character(s) to look for

Returns

an iterator to the first character, s.end() if none

By default, the selected character is a blank character ch, which has std::isblank(ch) true.

Definition at line 778 of file ParsingToolkit.h.

{
  auto const sbegin = s.begin(), send = s.end();
  auto it = sbegin;
  while (it != send) {
    it = std::find_if(it, send, selector);
    if (!isCharacterEscaped(sbegin, it)) return it;
    ++it; // skip the escaped character and move on
  } // while
  return send;
} // icarus::ParsingToolkit::findNextCharacter()

std::string_view icarus::ParsingToolkit::findQuotationEnd	(	std::string_view	sv,
		std::string const &	quotEnd
	)		const

Finds the quotation end in sv.

Parameters

sv	the buffer to look the quotation end into
quotEnd	the quotation end to be searched

Returns

a view of sv from the quotation end, included, empty if not found

Note that sv should not include the quotation start.

Definition at line 107 of file ParsingToolkit.cxx.

{
  while (!sv.empty()) {
    
    std::size_t const pos = sv.find(quotEnd);
    if (pos == std::string_view::npos) break;
    
    if (!isCharacterEscaped(sv.begin(), sv.begin() + pos)) {
      sv.remove_prefix(pos);
      return sv;
    }
    
    sv.remove_prefix(pos + 1);
    
  } // while
  
  return make_view(sv.end(), sv.end());
} // icarus::ParsingToolkit::findQuotationEnd()

auto icarus::ParsingToolkit::findQuotationStart

(

std::string_view

sv

)

const

Finds the start of the next quotation in sv.

Parameters

sv	the buffer to look the quotation start into

Returns

a subview of sv starting from the quotation found, empty if none

Definition at line 66 of file ParsingToolkit.cxx.

{
  
  while (!sv.empty()) {
    
    // look for a character that could start a quotation opening
    std::size_t const startPos = sv.find_first_of(fQuoteStarts);
    
    // no such character found:
    if (startPos == std::string_view::npos) break;
    
    // if the character is escaped, this is not a quotation opening:
    if (isCharacterEscaped(sv.begin(), sv.begin() + startPos)) {
      sv.remove_prefix(std::min(startPos + 1, sv.length()));
      continue;
    }
    
    sv.remove_prefix(std::min(startPos, sv.length()));
    
    // try all the opening quotes
    // (may be optimized by grouping them by first character)
    for (auto const& qSpec: fParams.quotes) {
      
//       if (sv.starts_with(qSpec.first)) return { sv, &qSpec }; // C++20
      if (sv.compare(0, qSpec.first.length(), qSpec.first) == 0)
        return { sv, &qSpec };
      
    } // for quotes
    
    // nope, just a character; remove it and keep looking
    sv.remove_prefix(1);
    
  } // while sv
  
  return { make_view(sv.end(), sv.end()), nullptr };
} // icarus::ParsingToolkit::findQuotationStart()

template<typename BIter >

bool icarus::ParsingToolkit::isCharacterEscaped	(	BIter	begin,
		BIter	itCh
	)		const

Returns whether the character pointed by itCh is escaped or not.

Template Parameters

BIter

iterator type

Parameters

begin	iterator to the beginning of the string
itCh	iterator to the character to be investigated.

Returns

whether there is an unescaped escape character before itCh

Note that itCh may be a end iterator (for an empty string, the result is false).

template<typename Iter >

bool icarus::ParsingToolkit::isCharacterEscaped	(	Iter	begin,
		Iter	itCh
	)		const

Definition at line 760 of file ParsingToolkit.h.

{
  unsigned int nEscapes = 0U;
  while (itCh-- != begin) {
    
    if (!isEscape(*itCh)) break;
    ++nEscapes;
    
  } // while
  
  return (nEscapes & 1) == 1; // odd number of escapes means escaped
  
} // icarus::ParsingToolkit::isCharacterEscaped()

bool icarus::ParsingToolkit::isEscape ( char  ch ) const

inline

Returns whether ch is an escape character.

Definition at line 375 of file ParsingToolkit.h.

{ return ch == fParams.escape; }

bool icarus::ParsingToolkit::isQuotationUnclosed

(

std::string_view

sv

)

const

Returns if the sequence sv has unclosed quotation at its end.

Definition at line 128 of file ParsingToolkit.cxx.

                                                                      {
  
  while (!sv.empty()) {
    
    auto [ qsv, qptr ] = findQuotationStart(sv);
    if (!qptr) return false;
    
    qsv.remove_prefix(qptr->first.length()); // remove the opening quote
    
    qsv = findQuotationEnd(qsv, qptr->second);
    if (qsv.empty()) return true;
    
    qsv.remove_prefix(qptr->second.length());
    sv = qsv;
  } // while
  
  return false;
} // icarus::ParsingToolkit::isQuotationUnclosed()

static std::string_view icarus::ParsingToolkit::make_view ( std::string const &  s )

inlinestatic

Creates a std::string_view from an entire string s.

Definition at line 510 of file ParsingToolkit.h.

    { return make_view(s.begin(), s.end()); }

template<typename BIter , typename EIter >

static std::string_view icarus::ParsingToolkit::make_view ( BIter  b, EIter  e  )

inlinestatic

Creates a std::string_view from two string iterators b and e.

Definition at line 515 of file ParsingToolkit.h.

    { return { &*b, static_cast<std::size_t>(std::distance(b, e)) }; }

Params_t const& icarus::ParsingToolkit::params ( ) const

inlinenoexcept

Returns the current parameters of parsing.

Definition at line 111 of file ParsingToolkit.h.

{ return fParams; }

std::pair< std::string, unsigned int > icarus::ParsingToolkit::readMultiline

(

std::istream &

in

)

const

Returns a single line of text from the input stream.

Parameters

in	the input stream

Returns

the string read, and the number of lines read

Exceptions

Error

on fatal parsing errors

This function reads entire lines from in, where a line is defined as in std::getline(). If the line ends with an unescaped escape character, another line is read and appended (the escape character is dropped). The return value is the merged string with no end-of-line characters, and the number of lines read. If there is no string to be read, it returns an empty string and 0U.

Special behaviour

• If the line ends while a quotation is still open, the next line is also merged, and the line break is kept; to merge quoted lines without preserving the line break character, end the quote on the first line, immediately break the line escaping it, and then next line should immediately start with opening a quotation.
• If the line ends while a quotation is still open, it is a parsing error to have the line break character escaped (an exception will be thrown) merged, and the line break is kept.
• If the file ends while a quotation is still open, the line is preserved as such.

Definition at line 27 of file ParsingToolkit.cxx.

{
  
  std::string fullLine;
  std::string openQuoteLine;
  unsigned int nLines = 0U;
  while (in) {
    
    std::string line;
    std::getline(in, line, in.widen(fParams.EOL));
    bool const isEOF = in.eof();
    if (!isEOF || !line.empty()) ++nLines;
    openQuoteLine.append(line);
    
    if (isQuotationUnclosed(make_view(openQuoteLine))) {
      if (isCharacterEscaped(line.begin(), line.end())) {
        fullLine.append(openQuoteLine);
        throw Error{ "Parser error: escaped end-of-line inside a quotation:\n"
          + fullLine + "\n" };
      }
      // if the newline is quoted, it's preserved
      if (!isEOF) openQuoteLine += fParams.EOL;
      continue;
    }
    fullLine.append(openQuoteLine);
    openQuoteLine.clear();
    
    if (!isCharacterEscaped(fullLine.begin(), fullLine.end())) break;
    
    fullLine.pop_back(); // remove the escape character
    
  } // while
  fullLine.append(openQuoteLine); // usually empty
  
  return { std::move(fullLine), nLines };
} // icarus::ParsingToolkit::readMultiline()

template<typename WordType >

void icarus::ParsingToolkit::removeCommentLine ( std::vector< WordType > &  words ) const

inline

Removes all the words from the one starting with a comment marker.

Parameters

words

list of words

The original list is modified, the word starting with a comment marker and all the following ones are removed.

Definition at line 216 of file ParsingToolkit.h.

    { words.erase(findCommentWord(words.begin(), words.end()), words.end()); }

template<typename Words >

std::vector< std::string > icarus::ParsingToolkit::removeEscapes

(

Words const &

words

)

const

Returns a copy of words with all escape characters removed.

Template Parameters

Words

type of list of words

Parameters

words

the list of words to change

Returns

the list of words without escaping

See Also

removeEscapes(std::string)

The escaping is removed from each of the words in the list, which are treated as independent. See removeEscapes(std::string) for the details.

Definition at line 810 of file ParsingToolkit.h.

{
  using std::size;
  std::vector<std::string> nv;
  nv.reserve(size(words));
  for (auto const& word: words) nv.push_back(removeWordEscapes(word));
  return nv;
} // icarus::ParsingToolkit::removeEscapes()

template<typename Words >

std::vector< std::string > icarus::ParsingToolkit::removeQuotations

(

Words const &

words

)

const

Returns a copy of words with no quotation starts and ends.

Template Parameters

Words

type of list of words

Parameters

words

the list of words to change

Returns

the list of words without quotations

See Also

removeQuotations(std::string)

The substitution is applied on each of the words in the list, which are treated as independent. See removeQuotations(std::string) for the details.

Definition at line 823 of file ParsingToolkit.h.

{
  using std::size;
  std::vector<std::string> nv;
  nv.reserve(size(words));
  for (auto const& word: words) nv.push_back(removeWordQuotations(word));
  return nv;
} // icarus::ParsingToolkit::removeEscapes()

std::string_view icarus::ParsingToolkit::removeTrailingBlanks ( std::string_view  s ) const

inline

Consumes the blank characters a the beginning of s.

See Also

removeTrailingCharacters()

Definition at line 422 of file ParsingToolkit.h.

    { return removeTrailingCharacters(s, isBlank); }

template<typename CType >

std::string_view icarus::ParsingToolkit::removeTrailingCharacters	(	std::string_view	s,
		CType	charType
	)		const

Consumes the blank characters a the beginning of s.

Template Parameters

CType

type of functor determining which type of character to remove

Parameters

s	view of the string to be parsed
charType	functor determining which characters to remove

Returns

a view of s starting after its trailing charType characters

See Also

removeTrailingBlanks()

Definition at line 794 of file ParsingToolkit.h.

{
  // REQUIREMENT: escape character must not be classified as delimiter
  assert(!charType(fParams.escape));
  
  while (!s.empty()) {
    if (!charType(s.front())) break; // escape character triggers this too
    s.remove_prefix(1U);
  } // while
  return s;
} // icarus::ParsingToolkit::removeTrailingCharacters()

std::string icarus::ParsingToolkit::removeWordEscapes

(

std::string &&

w

)

const

Returns a copy of w with all escape characters removed.

Parameters

w	the string to change

Returns

a copy of w without escaping

See Also

removeEscapes(Word const&)

The escaping scheme that is applied is just to remove the escape character (no replacement table supported here). An unescaped escape character at the end of the string will not be removed.

It is recommended that this be done as the last step of the parsing, since it changes the meaning of the parsing elements like quotations, comments etc.

Note that applying removeEscapes() more than once will keep removing characters that in the earlier passes were not considered escapes (for example, four escape characters become two in the first pass, one in the second and disappear in the following passes).

Definition at line 158 of file ParsingToolkit.cxx.

                                                                     {
  
  // replace in place
  std::string::const_iterator iSrc = s.begin(), send = s.end();
  std::string::iterator iDest = s.begin();
  
  // if the last character is an escape, it's kept
  while (iSrc != send) {
    char const ch = *iSrc++;
    *iDest++ = (isEscape(ch) && (iSrc != send))? *iSrc++: ch;
  } // while
  
  s.erase(iDest, send);
  return std::move(s);
} // icarus::ParsingToolkit::removeWordEscapes()

std::string icarus::ParsingToolkit::removeWordEscapes ( std::string_view  w ) const

inline

Definition at line 447 of file ParsingToolkit.h.

    { return removeWordEscapes(std::string{ w }); }

std::string icarus::ParsingToolkit::removeWordEscapes ( const char *  w ) const

inline

Definition at line 449 of file ParsingToolkit.h.

    { return removeWordEscapes(std::string{ w }); }

std::string icarus::ParsingToolkit::removeWordQuotations

(

std::string &&

w

)

const

Returns a copy of w with no quotation starts and ends.

Parameters

w	the string to change

Returns

the word without quotations

See Also

removeQuotations(Words const&)

Escaping is still honored (if present).

Note that applying removeQuotations more than once will keep removing quotation markings that in the earlier passes were not considered such (for example, `a1 << "b1 << 'c1 << " or " << c2' << b2" << a2will become first a1 << b1 << 'c1 << or << c2' << b2 << a2, and eventually a1 << b1 << c1 << or << c2 << b2 << a2`).

Definition at line 176 of file ParsingToolkit.cxx.

{
  std::string_view sv = make_view(s);
  std::string::iterator iDest = s.begin();
  
  while (!sv.empty()) {
    
    // find the next quotation
    auto const [ fromQ, qptr ] = findQuotationStart(sv);
    
    // copy the material until the next quotation
    iDest = std::copy(sv.begin(), fromQ.begin(), iDest);
    sv = fromQ;
    
    if (!qptr) break; // if there is no quotation, we are done
    
    sv.remove_prefix(qptr->first.length()); // skip the quotation start
    
    // find the end of quotation
    std::string_view const afterQ = findQuotationEnd(sv, qptr->second);
    
    if (afterQ.empty()) { // begin of quotation, but no end: no good
      // leave the "begin of quotation" as is
      iDest = std::copy(fromQ.begin(), fromQ.end(), iDest);
      sv.remove_prefix(sv.length()); // note: quote start was already removed
      break;
    }

    // copy the quoted material
    iDest = std::copy(sv.begin(), afterQ.begin(), iDest);
    sv = afterQ;
    
    sv.remove_prefix(qptr->second.length()); // skip the quotation end
    
  } // while
  
  assert(sv.empty());
  
  s.erase(iDest, s.end());
  return std::move(s);
} // icarus::ParsingToolkit::removeWordQuotations()

std::string icarus::ParsingToolkit::removeWordQuotations ( std::string_view  w ) const

inline

Definition at line 485 of file ParsingToolkit.h.

    { return removeWordQuotations(std::string{ w }); }

std::string icarus::ParsingToolkit::removeWordQuotations ( const char *  w ) const

inline

Definition at line 487 of file ParsingToolkit.h.

    { return removeWordQuotations(std::string{ w }); }

auto icarus::ParsingToolkit::splitOn ( std::string_view  sv, std::string_view  sep  )

static

Splits the view sv in three: before sep, sep and after sep.

Parameters

sv	view of the string to split
sep	a subview of sv to split at

Returns

a SplitView_t object with the three parts split, empty if needed

The view sep is required to be a subview of sv: it's not enough for it to have as content a substring of sv. For example, splitOn("a:1", ":") will not work, because the string "a:1" does not share data in memory with ":".

Even if sep is empty, it's still required to point with both begin() and end() within sv, and sv will be split according to that point.

Definition at line 149 of file ParsingToolkit.cxx.

{
  return
    { make_view(sv.begin(), sep.begin()), sep, make_view(sep.end(), sv.end()) };
} // icarus::ParsingToolkit::splitOn()

template<typename Delim >

std::vector< std::string_view > icarus::ParsingToolkit::splitWords	(	std::string const &	s,
		Delim	isDelimiter
	)		const

Splits a string into words.

Template Parameters

Delim

type of delimiter functor

Parameters

s	the string to be split
isDelimiter	(default: isblank()) determines if a character is a word delimiter

Returns

a sequence of views, one per word

The splitter algorithm defines a word separator as a sequence of one or more unescaped, unquoted delimiter characters, where a delimiter is a character ch for which isDelimiter(ch) is true.

Note that this function does not change the content of the data, and in particular it does not remove escaping nor quoting (although it interprets both).

A character used as delimiter can appear in a word only if escaped or within quotation. Contiguous non-delimiter elements of a string, including quoted strings, belong to the same word (for example, a" and "b is a single word when delimitation is by blank characters). An empty word can be introduced only in quotations (e.g. "").

The Delim type is a functor so that isDelimiter(ch) returns something convertible to bool, true if the ch character should be considered a delimiter. Note that no context is provided for the answer, so the use of each character as delimiter is fixed, and modified only by the hard-coded quotation and escaping rules.

The first characters of quotation starts and the escape characters must not be classified as delimiters, or the algorithm will give wrong results.

Definition at line 545 of file ParsingToolkit.h.

{
  // REQUIREMENT: escape character must not be classified as delimiter
  assert(!isDelimiter(fParams.escape));
  // REQUIREMENT: the first character of no quotation start must be classified
  //              as delimiter
  assert(
    std::count_if(fQuoteStarts.cbegin(), fQuoteStarts.cend(), isDelimiter) == 0
    );
  
  
  // helper class:
  // stores the word as collected so far, updates `sv` and starts new words
  class WordTracker {
    ParsingToolkit const& tk;
    Delim const& isDelimiter;
    std::string_view& sv;
    std::vector<std::string_view> words;
    std::string_view::const_iterator wStart;
      public:
    WordTracker(ParsingToolkit const& tk, Delim const& d, std::string_view& sv)
      : tk{ tk }, isDelimiter{ d }, sv{ consumeDelim(sv) }, wStart{ sv.begin() }
      {}
    void startNew()
      {
        words.push_back(make_view(wStart, sv.begin()));
        wStart = consumeDelim().begin();
      }
    void moveEndTo(std::string_view::const_iterator it)
      { moveEndBy(it - sv.begin()); }
    void moveEndBy(std::size_t n) { sv.remove_prefix(n); }
    std::vector<std::string_view> finish()
      { if (wStart != sv.begin()) startNew(); return std::move(words); }
    std::string_view& consumeDelim(std::string_view& s) const
      { return s = tk.removeTrailingCharacters(s, isDelimiter); }
    std::string_view& consumeDelim() { return consumeDelim(sv); }
  }; // WordTracker
  
  std::string_view sv = make_view(s);
  WordTracker words { *this, isDelimiter, sv }; // shares sv management
  
  // sv.begin() is kept updated to the candidate end of word;
  // the beginning of the current word is always cached as words.wStart
  while (!sv.empty()) {

    // process up to the next quotation
    auto const [ qsv, qptr ] = findQuotationStart(sv);
    
    // parse and split until the quotation start:
    auto const qstart = qsv.begin();
    while(true) {
      
      // find next space;
      // if next space is past the quotation, stop to the quotation instead
      words.moveEndTo
        (findNextCharacter(make_view(sv.begin(), qstart), isDelimiter));
      
      if (sv.begin() == qstart) break;
      
      // not the quote? it's a delimiter! new word found:
      words.startNew();
      
    } // while(true)
    
    // handle the quoted part
    if (qptr) {
      assert(sv.substr(0, qptr->first.length()) == qptr->first);
      
      words.moveEndBy(qptr->first.length());
      
      // find the end of the quote, and swallow it into the current word
      std::string_view const quotEnd = findQuotationEnd(sv, qptr->second);
      words.moveEndTo(quotEnd.begin());
      
      // if we have found a end of quote, swallow it too (otherwise it's over)
      if (!quotEnd.empty()) words.moveEndBy(qptr->second.length());
      
    } // if quotation found
    
  } // while
  
  return words.finish();

} // icarus::ParsingToolkit::splitWords()

std::vector<std::string_view> icarus::ParsingToolkit::splitWords ( std::string const &  s ) const

inline

Helper version of splitWords(std::string const&, Delim).

Definition at line 190 of file ParsingToolkit.h.

    { return splitWords(s, isBlank); }

Member Data Documentation

icarus::ParsingToolkit::Params_t const icarus::ParsingToolkit::DefaultParameters

static

Definition at line 97 of file ParsingToolkit.h.

Params_t icarus::ParsingToolkit::fParams

private

Parsing parameters.

Definition at line 519 of file ParsingToolkit.h.

std::string icarus::ParsingToolkit::fQuoteStarts

private

Start characters of all supported quotations.

Definition at line 524 of file ParsingToolkit.h.

constexpr CCTypeAdapter<&std::isblank> icarus::ParsingToolkit::isBlank {}

static

Adapter for determining if a character is a blank (see std::isblank()).

Definition at line 92 of file ParsingToolkit.h.

---

The documentation for this struct was generated from the following files:

• ParsingToolkit.h
• ParsingToolkit.cxx