ctpg
ctpg copied to clipboard
peter-winter
[question] Cant Parameterize and "Seperate out" Parser
I am trying to split apart (and parameterize) my parser, something like this..
template<
auto ConvertToTypeConstantParameter,
typename OperhandTypeParameterType,
auto DataTypeNameParameterConstant,
auto& LiteralRegexParameterConstant,
// = defualt_constant_expression_literal_regex,
auto& IdentifierRegexParameterConstant,
// = defualt_constant_expression_identifier_regex,
auto PlusCharacterParameterConstant = '+',
auto MinusCharacterParameterConstant = '-',
auto MultiplyCharacterParameterConstant = '*',
auto DivideCharacterParameterConstant = '/',
auto LeftParanethesisCharacterParameterConstant = '(',
auto RightParanethesisCharacterParameterConstant = ')',
auto& IdentifierNameParameterConstant
= identifier_name_string,
auto& FactorNameParameterConstant
= factor_name_string,
auto& SumNameParameterConstant
= sum_name_string,
auto& ParanthesisScopeParameterConstant
= parenthesis_scope_name_string
>
struct ConstantExpression
{
using DataType = OperhandTypeParameterType;
constexpr static auto data_type_name = DataTypeNameParameterConstant;
constexpr static auto literal_term = ctpg::regex_term< LiteralRegexParameterConstant >{
"num"
};
constexpr static auto identifier = ctpg::regex_term< IdentifierRegexParameterConstant >{
"id"
};
constexpr static auto factor = ctpg::nterm< OperhandTypeParameterType >{ FactorNameParameterConstant };
constexpr static auto sum = ctpg::nterm< OperhandTypeParameterType >{ SumNameParameterConstant };
constexpr static auto parenthesis_scope = ctpg::nterm< OperhandTypeParameterType >{ ParanthesisScopeParameterConstant };
constexpr static auto nterms = ctpg::nterms(
factor, sum, parenthesis_scope
);
constexpr static auto plus_term = ctpg::char_term{
PlusCharacterParameterConstant,
1,
ctpg::associativity::ltor
};
constexpr static auto minus_term = ctpg::char_term{
MinusCharacterParameterConstant,
1,
ctpg::associativity::ltor
};
constexpr static auto multiply_term = ctpg::char_term{
MultiplyCharacterParameterConstant,
2,
ctpg::associativity::ltor
};
constexpr static auto divide_term = ctpg::char_term{
DivideCharacterParameterConstant,
2,
ctpg::associativity::ltor
};
constexpr static auto left_parenthesis_term = ctpg::char_term{
LeftParanethesisCharacterParameterConstant,
3,
ctpg::associativity::ltor
};
constexpr static auto right_parenthesis_term = ctpg::char_term{
RightParanethesisCharacterParameterConstant,
3,
ctpg::associativity::ltor
};
constexpr static auto terms = ctpg::terms(
plus_term,
minus_term,
multiply_term,
divide_term,
left_parenthesis_term,
right_parenthesis_term
);
constexpr static auto rules = ctpg::rules(
factor( literal_term ) >= ConvertToTypeConstantParameter,
factor( factor, multiply_term, literal_term )
>= []( size_t current_factor, auto, const auto& next_token ) {
return current_factor * ConvertToTypeConstantParameter( next_token );
},
//More rules...
);
};
This way, for multiple data-types I can make more constant expresssion regexes for different data-types (and for different types of literals with different regexes e.g natural number [0-9][0-9]* vs float [0-9]*.[0-9]+).
The problem comes when I try to use a literal, it seems to have trouble accepting the string for the regex in the literal_term
I have tried multiple methods, at first all the parameters were just plain auto, and I tried serializing the strings into templates, then injecting them back into an array type with the size known at compile time (as a bunch of errors with the size of the type not being known at compile time come up otherwise).
template< auto FirstConstantParameter, auto... SeriesConstantParameters >
struct TakeOneFromTemplateSeries {
constexpr static auto first = FirstConstantParameter;
using NextType = TakeOneFromTemplateSeries< SeriesConstantParameters... >;
};
template< auto... ElementParameterConstants >
struct RawTemplateArray
{
using ElementType = decltype(
TakeOneFromTemplateSeries< ElementParameterConstants... >::first
);
constexpr static auto size = sizeof...( ElementParameterConstants );
constexpr static ElementType array[ size ] = { ElementParameterConstants... };
constexpr static ElementType* pointer = array;
};
template<
auto ArrayParameterConstant,
size_t IndexParameterConstant,
size_t ArrayLengthParameterConstant,
auto... ElementParameterConstants
>
struct ToRawTemplateArrayImplementation
{
using ResultType = typename ToRawTemplateArrayImplementation<
ArrayParameterConstant,
IndexParameterConstant + 1,
ArrayLengthParameterConstant,
ElementParameterConstants...,
ArrayParameterConstant[ IndexParameterConstant % ArrayLengthParameterConstant ]
>::ResultType;
};
template<
auto ArrayParameterConstant,
size_t IndexParameterConstant,
auto... ElementParameterConstants
>
struct ToRawTemplateArrayImplementation<
ArrayParameterConstant,
IndexParameterConstant,
IndexParameterConstant,
ElementParameterConstants...
>
{
using ResultType = RawTemplateArray<
ElementParameterConstants...
>;
};
template< auto ArrayParameterConstant >
struct ToRawTemplateArray
{
using ResultType = typename ToRawTemplateArrayImplementation<
ArrayParameterConstant,
0,
// std::strlen( ArrayParameterConstant )
ctpg::utils::str_len( ArrayParameterConstant ) + 1
>::ResultType;
};
constexpr static const char natural_number_regex_string[] = "[0-9][0-9]*";
constexpr static auto natural_number_regex = ToRawTemplateArray< natural_number_regex_string >::ResultType{};
constexpr static auto defualt_constant_expression_literal_regex = natural_number_regex;
I would then pass natural_number_regex into ConstantExpression
If I do it like this
template<
auto ConvertToTypeConstantParameter,
typename OperhandTypeParameterType,
auto& DataTypeNameParameterConstant = decltype( defualt_constant_expression_data_type_name )::array,
auto& LiteralRegexParameterConstant = decltype( defualt_constant_expression_literal_regex )::array,
auto& IdentifierRegexParameterConstant = decltype( defualt_constant_expression_identifier_regex )::array,
auto& IdentifierNameParameterConstant = decltype( defualt_constant_expression_identifier_term_name )::array,
auto& FactorNameParameterConstant = decltype( defualt_constant_expression_factor_nterm_name )::array,
auto& SumNameParameterConstant = decltype( defualt_constant_expression_sum_nterm_name )::array,
auto& ParanthesisScopeParameterConstant = decltype( defualt_constant_expression_parenthesis_scope_nterm_name )::array,
auto PlusCharacterParameterConstant = '+',
auto MinusCharacterParameterConstant = '-',
auto MultiplyCharacterParameterConstant = '*',
auto DivideCharacterParameterConstant = '/',
auto LeftParanethesisCharacterParameterConstant = '(',
auto RightParanethesisCharacterParameterConstant = ')',
>
The compiler would spit out a whole bunch of template barf I have spent a while formatting and trying to make some sense of, clang is giving me something GCC wont
ctpg.hpp:180:25: warning: ISO C++20 considers use of overloaded operator '==' (with operand types 'ctpg::stdex::cvector<unsigned short, 13>::iterator' and 'ctpg::stdex::cvector<unsigned short, 13>::iterator') to be ambiguous despite there being a unique best viable function [-Wambiguous-reversed-operator]
while (!(it == end()))
~~ ^ ~~~~~
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:2546:25: note: in instantiation of member function 'ctpg::stdex::cvector<unsigned short, 13>::erase' requested here
ps.cursor_stack.erase(ps.cursor_stack.end() - ri.r_elements, ps.cursor_stack.end());
ng_buffer<12>, ctpg::detail::no_stream>' requested here
auto res = p.parse(
^
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:3285:43: note: in instantiation of function template specialization 'ctpg::regex::analyze_dfa_size<12UL>' requested here
static const size_t dfa_size = regex::analyze_dfa_size(Pattern);
^
/root/workdir/Include/Warp/Expression.hpp:134:42: note: in instantiation of template class 'ctpg::regex_term<natural_number_regex_string>' requested here
constexpr static auto literal_term = ctpg::regex_term< LiteralRegexParameterConstant >{
^
/root/workdir/Source/Main.cpp:19:9: note: in instantiation of template class 'ConstantExpression<&to_size_t, unsigned long, natural_number_name_string, natural_number_regex_string, identifier_regex_string, '+', '-', '*', '/', '(', ')', identifier_name_string, factor_name_string, sum_name_string, parenthesis_scope_name_string>' requested here
NaturalNumberConstantExpression::factor,
^
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:127:28: note: ambiguity is between a regular call to this operator and a call with the argument order reversed
constexpr bool operator == (const it_type& other) const { return cast()->ptr == other.ptr; }
^
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:180:25: warning: ISO C++20 considers use of overloaded operator '==' (with operand types 'ctpg::stdex::cvector<unsigned short, 26>::iterator' and 'ctpg::stdex::cvector<unsigned short, 26>::iterator') to be ambiguous despite there being a unique best viable function [-Wambiguous-reversed-operator]
while (!(it == end()))
~~ ^ ~~~~~
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:2546:25: note: in instantiation of member function 'ctpg::stdex::cvector<unsigned short, 26>::erase' requested here
ps.cursor_stack.erase(ps.cursor_stack.end() - ri.r_elements, ps.cursor_stack.end());
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:127:28: note: ambiguity is between a regular call to this operator and a call with the argument order reversed
constexpr bool operator == (const it_type& other) const { return cast()->ptr == other.ptr; }
^
In file included from /root/workdir/Source/Main.cpp:1:
In file included from /root/workdir/Include/Warp/Expression.hpp:10:
In file included from /root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:12:
/usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/variant:1100:7: error: static_assert failed due to requirement '__detail::__variant::__exactly_once<ctpg::term_value<std::basic_string_view<char, std::char_traits<char>>>, std::nullptr_t, ctpg::no_type, unsigned long, ctpg::term_value<char>>' "T must occur exactly once in alternatives"
static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
These are just a few errors/warnings/notes I thought may be relevant
I have also tried specifying auto& (because I saw it in ctpg::buffers::cstring_buffer) instead of just auto -- no dice
With either, if I specify raw literals in the parameters I get something like
/root/workdir/Source/Main.cpp:6:9: error: pointer to subobject of string literal is not allowed in a template argument
"[0-9][0-9]*",
If I try to substitute constants such as
constexpr static const char natural_number_regex_string[] = "[0-9][0-9]*";
I get right back to the errors I had before
I have no idea how to make this work properly and think I may be forced to go back to a "monolithic" parser. Anyone know how to make this work
Thank you,
- Chris P.s Here is a gist of a full compiler output and here is my header file and this is a Main.cpp using it, if you comment out the second rule, it works because it does not have the literal term P.s.s If I substitute the mock rules with the actual ones I want too use, I get
/root/.conan/data/ctpg/1.3.6/_/_/package/5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9/include/ctpg.hpp:127:28: note: ambiguity is between a regular call to this operator and a call with the argument order reversed
constexpr bool operator == (const it_type& other) const { return cast()->ptr == other.ptr; }
^
/root/workdir/Source/Main.cpp:16:9: error: excess elements in struct initializer
NaturalNumberConstantExpression::factor,
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4 warnings and 1 error generated.
This warning is showing up quite persistently.
Hi Because regex_term accepts auto& as template argument it needs a static linkage object. If at some point ctpg will support c++20 some form of string_literal class could be accepted, and you would be able to parametrize (similar thing is done in ctre library). But these are plans for future.
Did you consider putting whole (with ctpg::parser call) parser definitions into specialized classes? Create specizlizations for different data types (like int and float) and make them inherit from a base class which will hold common terms, nterms, rules etc.
Hey thanks for the quick response
Hi Because regex_term accepts auto& as template argument it needs a static linkage object.
I suppose I don't quite understand why there is not static linkage? All my strings/arrays are declared static?
If at some point ctpg will support c++20 some form of string_literal class could be accepted, and you would be able to parametrize (similar thing is done in ctre library). But these are plans for future.
Is it possible to use CTRE in combination with your library?
Did you consider putting whole (with ctpg::parser call) parser definitions into specialized classes? Create specizlizations for different data types (like int and float) and make them inherit from a base class which will hold common terms, nterms, rules etc.
A bit, but I am not sure how I would "merge/cat" the parsers together, if I have in the same bit of code an expression that has a float, and an expression that has a integer, but those are not the only things in the code (function declarations, etc.) don't I want to run all that through the same parser, rather than doing a pass for integers and a pass for floats with 2 different parsers?
I suppose I can call additional parsers from within a parser, but that first parser would need awareness of all the possible types, I would also have to know when to stop, when to continue, and when to pass back too the "parent" parser.
UPDATE @peter-winter :
I managed to make a little progress using template specialization. It works if I call construct ctpg::rules, ctpg::terms, and ctpg::nterms in ctpg::parser directly, but not if I try to input them after the fact (this holds if I move their construction into consteval functions)
#include <iostream>
#include <functional>
#include <array>
#include <string>
#include <sstream>
#include <math.h>
#include <string.h>
#include <ctpg.hpp>
// This function was "yoiked" directly from https://github.com/peter-winter/ctpg //
constexpr size_t to_size_t( std::string_view integer_token )
{
size_t sum = 0;
for( auto digit : integer_token )
sum = ( sum * 10 ) + digit - '0';
return sum;
}
constexpr static const char parenthesis_scope_name_string[] = "ParenthesisScope";
constexpr static const char factor_name_string[] = "Factor";
constexpr static const char sum_name_string[] = "Sum";
enum class RegexLiteralTerms {
NaturalNumber = 0,
Identifier = 1
};
template< auto LiteralParameterType >
struct LiteralTerm
{
static_assert( true, "Error::Unkown literal type -- no valid specification" );
constexpr const static char regex[] = "";
constexpr const static char name[] = "Nothing";
constexpr const static auto term = ctpg::regex_term< regex >{ name };
constexpr const static auto literal_type = LiteralParameterType;
};
template<>
struct LiteralTerm< RegexLiteralTerms::NaturalNumber >
{
constexpr const static char regex[] = "[0-9][0-9]*";
constexpr const static char name[] = "NaturalNumber";
constexpr const static auto term = ctpg::regex_term< regex >{ name };
constexpr const static auto literal_type = RegexLiteralTerms::NaturalNumber;
};
template<>
struct LiteralTerm< RegexLiteralTerms::Identifier >
{
constexpr const static char regex[] = "[a-zA-Z\\_][a-zA-Z0-9\\_]*";
constexpr const static char name[] = "Identifier";
constexpr const static auto term = ctpg::regex_term< regex >{ name };
constexpr const static auto literal_type = RegexLiteralTerms::Identifier;
};
template< auto TypeParameterConstant
= RegexLiteralTerms::NaturalNumber >
struct DefaultTypes {
using Type = size_t;
};
template<>
struct DefaultTypes< RegexLiteralTerms::Identifier > {
using Type = std::string;
};
template<
auto TypeParameterConstant = RegexLiteralTerms::NaturalNumber,
typename CPlusPlusTypeParameterConstant
= DefaultTypes< TypeParameterConstant >::Type
>
struct Converter {
constexpr const static auto converter = to_size_t;
};
template<
typename OperhandTypeParameterType,
auto ConvertToTypeConstantParameter = Converter< RegexLiteralTerms::NaturalNumber >::converter,
typename LiteralRegexTermParameterConstant = LiteralTerm< RegexLiteralTerms::NaturalNumber >,
typename IdentifierRegexTermParameterConstant = LiteralTerm< RegexLiteralTerms::Identifier >,
auto FactorNameParameterConstant = parenthesis_scope_name_string,
auto SumNameParameterConstant = factor_name_string,
auto ParanthesisScopeParameterConstant = sum_name_string,
auto PlusCharacterParameterConstant = '+',
auto MinusCharacterParameterConstant = '-',
auto MultiplyCharacterParameterConstant = '*',
auto DivideCharacterParameterConstant = '/',
auto LeftParanethesisCharacterParameterConstant = '(',
auto RightParanethesisCharacterParameterConstant = ')'
>
struct ConstantExpression
{
constexpr static auto factor = ctpg::nterm< OperhandTypeParameterType >{ FactorNameParameterConstant };
constexpr static auto sum = ctpg::nterm< OperhandTypeParameterType >{ SumNameParameterConstant };
constexpr static auto parenthesis_scope = ctpg::nterm< OperhandTypeParameterType >{ ParanthesisScopeParameterConstant };
constexpr static auto nterms = ctpg::nterms(
factor, sum, parenthesis_scope
);
constexpr static auto plus_term = ctpg::char_term{
PlusCharacterParameterConstant,
1,
ctpg::associativity::ltor
};
constexpr static auto minus_term = ctpg::char_term{
MinusCharacterParameterConstant,
1,
ctpg::associativity::ltor
};
constexpr static auto multiply_term = ctpg::char_term{
MultiplyCharacterParameterConstant,
2,
ctpg::associativity::ltor
};
constexpr static auto divide_term = ctpg::char_term{
DivideCharacterParameterConstant,
2,
ctpg::associativity::ltor
};
constexpr static auto left_parenthesis_term = ctpg::char_term{
LeftParanethesisCharacterParameterConstant,
3,
ctpg::associativity::ltor
};
constexpr static auto right_parenthesis_term = ctpg::char_term{
RightParanethesisCharacterParameterConstant,
3,
ctpg::associativity::ltor
};
constexpr static auto terms = ctpg::terms(
plus_term,
minus_term,
multiply_term,
divide_term,
left_parenthesis_term,
right_parenthesis_term,
LiteralRegexTermParameterConstant::term,
IdentifierRegexTermParameterConstant::term
);
constexpr static auto rules = ctpg::rules(
factor( LiteralRegexTermParameterConstant::term ) >= ConvertToTypeConstantParameter,
factor( factor, multiply_term, IdentifierRegexTermParameterConstant::term )
>= []( size_t current_factor, auto, const auto& next_token ) {
return current_factor * ConvertToTypeConstantParameter( next_token );
},
factor( factor, divide_term, IdentifierRegexTermParameterConstant::term )
>= []( size_t current_factor, auto, const auto& next_token ) {
return current_factor / ConvertToTypeConstantParameter( next_token );
},
parenthesis_scope( left_parenthesis_term, factor, right_parenthesis_term )
>= [] ( auto, auto factor, auto ) { return factor; },
factor( parenthesis_scope ) >= []( auto parenthesis_scope ) { return parenthesis_scope; },
factor( factor, multiply_term, parenthesis_scope )
>= []( auto factor, auto, auto parenthesis_scope ) {
return factor * parenthesis_scope;
},
factor( factor, divide_term, parenthesis_scope )
>= []( auto factor, auto, auto parenthesis_scope ) {
return factor / parenthesis_scope;
},
factor( sum ) >= []( auto sum ) { return sum; },
sum( factor, plus_term, factor )
>= []( auto current_sum, auto, const auto& next_token ) {
return current_sum + next_token;
},
sum( factor, minus_term, factor )
>= []( auto current_sum, auto, const auto& next_token ) {
return current_sum - next_token;
}
);
constexpr const static auto parser = ctpg::parser<
decltype( factor ),
decltype( terms ),
decltype( nterms ),
decltype( rules ),
ctpg::use_generated_lexer
>{
factor,
terms,
nterms,
rules,
};
};
I can now parameterize... I think, however, I still cant enter the necessary components into the parser at the end there, it keeps telling me there are too many initializers.
Here is a gist of the output (I ran it through clang-format :smiley: )
My goal is to std::tuple_cat the rules, terms, and nterms together outside of any of these classes/modules I write
Well, I tink there are, I don't think you can initialize parser using {} initializer with 4 arguments. It is not a trivially initalized class, you have to use a constructor with the () syntax. You don't specify the template parameters, the deduction guides take care of this.
Wouldn't this work:
constexpr const static auto parser = ctpg::parser( factor, terms, nterms, rules);
You might have to std::move(rules) instead, I think costructor expects an rvalue referrence.
Well, I tink there are, I don't think you can initialize parser using {} initializer with 4 arguments. trivially initalized class, you have to use a constructor with the () syntax. parameters, the deduction guides take care of this. I tried a few different ways, thanks for the tip with {} I didnt know that! :)
constexpr const static auto parser = ctpg::parser( factor, terms, nterms, rules); You might have to std::move(rules) instead, I think costructor expects an rvalue referrence.
Just tried it with/without std::move and or with lexer where
constexpr const static auto lexer = ctpg::use_generated_lexer();
e.g
constexpr const static auto parser = ctpg::parser( factor, terms, nterms, std::move( rules ), lexer );
Compiler keeps complaining that there are too many initializes.
Also Compiler/Platform:
g++ (Ubuntu 11.1.0-1ubuntu1~20.04) 11.1.0 (OCI container)
I was able to resolve this by creating a member function that returns ctpg::rules I can even use std::tuple_cat to merge rules from multiple parsers and additional functions to generate rules.
For example (NOTE: The following was not compiled/tested):
struct MyParser
{
//... define terms etc.
template<auto OperationParameterConstant>
constexpr static const auto binary_operation_rules(auto to, auto left_term, auto operator, auto right_term) {
return ctpg::rules(
to(left_term, operator, right_term) >= OperationParameterConstant
);
}
consteval static const auto rules()
{
return ctpg::rules(
math_term(integer) >= [](auto integer) { return term{integer}; },
std::tuple_cat(
binary_operation_rules<[](auto left, auto add, auto right) { return left + right; }>(
sum, sum, plus, math_term
),
binary_operation_rules<[](auto left, auto add, auto right) { return left + right; }>(
sum, sum, plus, math_term
),
)
);
}
};
rules() can then be called when creating a ctpg;:parser like
constexpr static const auto my_parser = ctpg::parser(
MyParser::expression,
MyParser::my_terms,
MyParser::my_non_terminal_terms,
MyParser::rules()
);
Cool! Can I suggest not to call nonterms 'non terminal terms' ? Terms is short for 'terminal symbols' and nonterms is short for 'non terminal symbols'. Non terminal terms makes no sense.