Parser for byeol language source code. More...

#include <parser.hpp>

Inheritance diagram for by::parser:

Public Member Functions
node *	getSubPod ()
	Retrieves the root node of the parsed sub-pod.

const node *	getSubPod () const BY_CONST_FUNC(getSubPod()) srcSupplies &getSrcSupplies()

const srcSupplies &	getSrcSupplies () const BY_CONST_FUNC(getSrcSupplies()) me &addSupply(const srcSupply &new 1)

me &	addSupply (const tucontainable< srcSupply > &new1)

me &	relSupplies ()

tokenDispatcher &	getDispatcher ()

std::vector< ncnt > &	getIndents ()

nbool	isInit () const

template<typename T >
void	setScan ()
	Sets the scanning mode for the lexer.

void	rel () override

int	pushState (int newState)
	Pushes a new Flex scanner state onto the state stack.

int	popState ()
	Pops a Flex scanner state from the state stack.

nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner, nbool &isBypass) override
	Handles scanning for the lexer, overriding the base `tokenScanable` behavior.

nint	onTokenEndOfFile ()
	Handles the end-of-file token.

nint	onTokenColon (nint tok)
	Handles the colon token.

nint	onTokenNewLine (nint tok)
	Handles the newline token.

nint	onTokenLParan (nint tok)
	Handles the left parenthesis token.

nint	onTokenRParan (nint tok)
	Handles the right parenthesis token.

nint	onTokenComma (nint tok)
	Handles the comma token.

nbool	onTokenEndOfBraces ()
	Handles the end of a braces block (e.g., `}`).

void	onTokenStartOfStrTemplateBrace ()
	Handles the start of a string template brace (e.g., `${`).

void	onTokenBeginStr (nchar token)
	Handles the start of a string literal.

nchar	getOpenStrToken ()
	returns quote character which used to start current string sequence.

nint	onIndent (ncnt col, nint tok)
	Handles an increase in indentation.

nint	onDedent (ncnt col, nint tok)
	Handles a decrease in indentation.

nint	onIgnoreIndent (nint tok)
	Handles ignoring indentation (e.g., for inline blocks).

nchar	onScanUnexpected (const nchar *token)
	Handles an unexpected token encountered during scanning.

void	onSrcArea (const area &area)
	Callback for processing source area information from the lexer.

node *	onGet (const std::string &name)

node *	onGet (const std::string &name, const narr &args)

node *	onGet (node &from, const std::string &name)

node *	onGet (node &from, node &it)

node *	onGet (const std::string &name, const std::string &name2)

node *	onCallAccess (node &it, const narr &args)

FBOExpr *	onUnaryMinus (const node &it)

FBOExpr *	onUnaryNot (const node &it)

FUOExpr *	onUnaryBitwiseNot (const node &it)

node *	onUnaryDoublePlus (node &it)

node *	onUnaryDoubleMinus (node &it)

FUOExpr *	onUnaryPostfixDoublePlus (const node &it)

FUOExpr *	onUnaryPostfixDoubleMinus (const node &it)

FBOExpr *	onAdd (const node &lhs, const node &rhs)

FBOExpr *	onSub (const node &lhs, const node &rhs)

FBOExpr *	onMul (const node &lhs, const node &rhs)

FBOExpr *	onDiv (const node &lhs, const node &rhs)

FBOExpr *	onMod (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseAnd (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseXor (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseOr (const node &lhs, const node &rhs)

FBOExpr *	onLShift (const node &lhs, const node &rhs)

FBOExpr *	onRShift (const node &lhs, const node &rhs)

node *	onAssign (node &lhs, node &rhs)

node *	onAddAssign (node &lhs, node &rhs)

node *	onSubAssign (node &lhs, node &rhs)

node *	onMulAssign (node &lhs, node &rhs)

node *	onDivAssign (node &lhs, node &rhs)

node *	onModAssign (node &lhs, node &rhs)

FBOExpr *	onGt (const node &lhs, const node &rhs)

FBOExpr *	onGe (const node &lhs, const node &rhs)

FBOExpr *	onLt (const node &lhs, const node &rhs)

FBOExpr *	onLe (const node &lhs, const node &rhs)

FBOExpr *	onEq (const node &lhs, const node &rhs)

FBOExpr *	onNe (const node &lhs, const node &rhs)

FBOExpr *	onAnd (const node &lhs, const node &rhs)

FBOExpr *	onOr (const node &lhs, const node &rhs)

node *	onGetGeneric (const std::string &orgName, const args &typeParams)

node *	onGetElem (const node &arr, const node &idx)

node *	onGetArray (node &elemType)

obj *	onPod (const node &path)

obj *	onPod (const std::string &path)

obj *	onPod ()

obj *	onSubPod (obj &subpod)

endExpr *	onEnd (const blockExpr &blk)

blockExpr *	onBlock (const node *stmt)

blockExpr *	onBlock (blockExpr blk, const node stmt)

blockExpr *	onBlock ()

defBlock *	onDefBlock (node *stmt)

defBlock *	onDefBlock (defBlock blk, node stmt)

defBlock *	onDefBlock ()

node *	onFor (const std::string &iterName, const node &expr, const blockExpr &blk)

node *	onWhile (const node &condition, const blockExpr &blk)

node *	onParanthesisAsTuple (narr &tuple)

args *	onTuple ()

args *	onTuple (const node &elem)

args *	onTuple (args &as, const node &elem)

args *	onFuncCallTuple ()

args *	onFuncCallTuple (const node &elem)

args *	onFuncCallTuple (args &as, const node &elem)

args *	onTypeNames (const node &param)

args *	onTypeNames (args &params, const node &param)

template<typename T , typename Arg >
T *	onPrimitive (const Arg &arg)

template<typename T >
T *	onPrimitive ()

node *	onDefProp (const modifier &mod, const std::string &name, const node &rhs)

node *	onDefProp (const std::string &name, const node &rhs)

node *	onDefProp (const node &rhs)

node *	onDefProp (const modifier &mod, const node &rhs)

node *	onDefAssign (const modifier &mod, const std::string &name, const node *rhs)

node *	onDefAssign (const defPropExpr prop, const node rhs)

node *	onDefAssign (const std::string &name, const node *rhs)

obj *	onDefOrigin (const std::string &name, defBlock &blk)

obj *	onDefOrigin (const std::string &name, const narr &args, defBlock &blk)

genericOrigin *	onDefObjGeneric (const std::string &name, const args &typeParams, defBlock &blk)

genericOrigin *	onDefObjGeneric (const std::string &name, const args &typeParams, const narr &args, defBlock &blk)

node *	onDefArray (const narr &items)

node *	onDefSeq (const node &start, const node &end)

void	onCompilationUnit (obj *subpod)

void	onCompilationUnit (obj subpod, defBlock blk)

func *	onFuncSignature (const modifier &mod, const getExpr &access, const node *retType)

func *	onFuncSignature (const getExpr &access, const node *retType)

func *	onFuncSignature (const modifier &mod, node &it, const node *retType)

func *	onFuncSignature (node &it, const node *retType)

func *	onAbstractFunc (func &f)

node *	onFunc (func &func, const blockExpr &blk)

defNestedFuncExpr *	onLambda (const narr &params, const node &retType, const blockExpr &blk)

ctor *	onCtor (const modifier &mod, const narr &args, const blockExpr &blk)

ctor *	onCtor (const narr &args, const blockExpr &blk)

ctor *	onCtor (const modifier &mod, const blockExpr &blk)

ctor *	onCtor (const blockExpr &blk)

narr *	onParams ()

narr *	onParams (const defPropExpr *elem)

narr *	onParams (narr &it, const defPropExpr *elem)

modifier *	onModifier (nbool isPublic, nbool isOverriden)

node *	onDeclStmt (const narr &dotnames)

retExpr *	onRet ()

retExpr *	onRet (node &exp)

breakExpr *	onBreak ()

nextExpr *	onNext ()

evalExpr *	onFillFromOfFuncCall (const node &me, evalExpr &to)

evalExpr *	onEvalExpr (node &type, const narr &a)

evalExpr *	onEvalExpr (node &type, const args &a)

isExpr *	onIs (const node &me, const node &to)

asExpr *	onAs (const node &me, const node &as)

ifExpr *	onIf (const node &condition, const blockExpr &then)

ifExpr *	onIf (const node &condition, const blockExpr &then, const blockExpr &elseBlk)

ifExpr *	onIf (const node &condition, const blockExpr &then, const ifExpr &elseIf)

evalExpr *	onIn (const node &it, const node &container)

void	onEndFunc ()
	Callback invoked at the end of parsing a function definition.

void	onParseErr (const std::string &msg, const nchar *symbolName)
	Reports a parsing error.

nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner)
	Default entry point for token scanning.

virtual nint	onScan (parser &ps, YYSTYPE yylval, YYLTYPE loc, yyscan_t yyscanner, nbool &isBypass)=0
	Pure virtual method for concrete token scanning implementations.

Public Member Functions inherited from by::tworker< str, pod >
	tworker (const errReport &rpt)

errReport &	getReport ()

const errReport &	getReport () const BY_CONST_FUNC(getReport()) me &setReport(errReport &rpt)

me &	setFlag (nint newFlag)

me &	addFlag (nint flag)

me &	delFlag (nint clear)

nbool	isFlag (nint flag) const

nint	getFlag () const

me &	setTask (const pod &root)

me &	setTask (const pod *it) BY_SIDE_FUNC(it

pod *	getTask ()

const pod *	getTask () const BY_CONST_FUNC(getTask()) virtual void rel()

str	work ()

const area &	getArea () const BY_CONST_FUNC(_getArea()) nbool isOk() const

Public Member Functions inherited from by::tokenScanable
nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner)
	Default entry point for token scanning.

Protected Member Functions
void *	_scanString (const std::string &src, void *scanner)

exprMaker &	_getMaker ()

void	_prepare () override
	Protected virtual method for preparation before starting the work.

void	_onEndWork () override
	Protected virtual method called after the work is completed.

str	_onWork () override

Protected Member Functions inherited from by::tworker< str, pod >
void	_report (baseErr *e)

area &	_getArea ()

void	_onEndErrReport (const errReport &rpt) const

Friends
class	coreInternal

Additional Inherited Members
Public Types inherited from by::tworker< str, pod >
enum	logFlag

Public Attributes inherited from by::tworker< str, pod >
me setTask	it

me setTask *	this

Detailed Description

Parser for byeol language source code.

Entry point for the byeol parsing component, inheriting from worker. work() returns the parsed result as pod.

Similar Structure to Stela Parser

Since the stela language itself is part of the byeol language, its parser is also based on the byeol language parser. The stela parser is less complex compared to the byeol parser, so it's recommended to examine the stela parser code first before looking at the core module.

Scanner - Bison - Parser Structure

Uses flex and bison, naming flex as lowscanner and bison as lowparser. These low-level scanner and parser only exist inside the parser component and are not exposed externally. When parser::work() executes, it runs lowscanner, which tokenizes and passes tokens to lowparser. Lowparser matches rules and passes those events back to parser. Therefore, parser's functions starting with on are event handling functions that define how to create nodes and construct the AST.

lowparser.y Lifecycle Management

In each rule, calling parser::onXXXX() functions often creates new objects on the heap using the new keyword (not all functions do this). If these arguments are placed directly into tbicontainer, that's fine, but otherwise they must be bound with tstr beforehand to avoid memory leaks.

// - lowparser.y
pod: POD name-access NEWLINE {
    $$ = PS.onPod(*$2); // onpod() returns new pod()
}
 
compilation-unit: pod defblock {
    tstr<obj> pak($1); // Without binding with tstr like this,
 
    PS.onCompilationUnit(pak.get()); // if the pak value has issues inside onCompilationUnit()
                                     // and the operation is cancelled, the pod object
                                     // created on the heap becomes a memory leak.
}

This applies not only to onXXXX() functions but also to string tokens. See the following code:

// -- lowscanner.l
<stateString>\"       { // When string scan ends
    if(!yylval->asStr) yylval->asStr = new std::string(); // Creates string object with new
    ....
    return STRVAL;
}
 
// -- lowparser.y
    ...
   | STRVAL { // When STRVAL comes from scanner
    $$ = PS.onPrimitive<nStr>(*$1); // Pass it first. String is always immutable in this project
                                    // so the object is always copied.
    delete $1; // Since the object is copied, delete the original object.

Indentation Rule

The byeol language applies the offside rule, making it very sensitive to indentation. Unlike typical languages, it must be able to count how many spaces exist immediately after a newline. Once indentation is determined and the code line's scope is confirmed, subsequent spaces should be ignored. Consider the following byeol language example:

def A
    foo(val int) void
        if val > 0
          if val < 5
                  print("0 < val < 5")
        print("end of func") # 1)

After parsing print("0 < val < 5"), when parsing the next line print("end of func"), the parser must count the leading spaces and compare with the space counts for each scope to determine which scope this code line belongs to. Looking at the example, we can intuitively see that print("end of func") has the same spacing as if val > 0, meaning it belongs directly inside the foo() body. This means at this point, both the scope inside if val > 0 and the scope inside if val < 5 are closed. So lowscanner must add 2 DEDENT tokens (signifying scope termination) before lowparser recognizes print("end of func").

tokenDispatcher

Flex uses a separately designated stream called yyin to get characters and define them as tokens. As shown in the example above, adding tokens during parsing means adding specific characters to this stream. Typically unput is used for this, but to flexibly handle multiple unputs or pushing to the front, lowscanner internally uses tokenDispatcher.

tokenScan

As explained earlier, detecting indentation is crucial. Accurate space counting is required, so a strategy pattern is applied, switching between normalScan and indentScan. When a newline is detected, it switches to indentScan to count spaces accurately and determine scope.

Member Function Documentation

◆ _onEndWork()

void by::parser::_onEndWork ( )

overrideprotectedvirtual

Protected virtual method called after the work is completed.

This hook allows derived classes to perform any necessary cleanup or finalization after _onWork() has finished.

Reimplemented from by::tworker< str, pod >.

◆ _onWork()

str by::parser::_onWork ( )

overrideprotectedvirtual

parse with given srcSupply instances.

Parameters

script is null terminated cstring.

Returns: last parsed sub pod.

Implements by::tworker< str, pod >.

◆ _prepare()

void by::parser::_prepare ( )

overrideprotectedvirtual

Protected virtual method for preparation before starting the work.

This hook allows derived classes to perform any necessary setup or initialization before _onWork() is invoked.

Reimplemented from by::tworker< str, pod >.

◆ getOpenStrToken()

nchar by::parser::getOpenStrToken ( )

returns quote character which used to start current string sequence.

Returns: quote or double quote to be used to open string literal.

◆ getSubPod()

node * by::parser::getSubPod ( )

Retrieves the root node of the parsed sub-pod.

This method provides access to the top-level AST node representing the entire parsed sub-pod.

Returns: A pointer to the root node of the sub-pod.

◆ onDedent()

nint by::parser::onDedent	(	ncnt	col,
		nint	tok )

Handles a decrease in indentation.

Parameters

col	The column count of the new indentation level.
tok	The token ID that triggered the dedentation.

Returns: The token ID for DEDENT.

◆ onEndFunc()

void by::parser::onEndFunc ( )

Callback invoked at the end of parsing a function definition.

This method handles any cleanup or finalization required after a function's body has been parsed.

◆ onIgnoreIndent()

nint by::parser::onIgnoreIndent ( nint tok )

Handles ignoring indentation (e.g., for inline blocks).

Parameters

tok	The token ID.

Returns: The token ID.

◆ onIndent()

nint by::parser::onIndent	(	ncnt	col,
		nint	tok )

Handles an increase in indentation.

Parameters

col	The column count of the new indentation level.
tok	The token ID that triggered the indentation.

Returns: The token ID for INDENT.

◆ onParseErr()

void by::parser::onParseErr	(	const std::string &	msg,
		const nchar *	symbolName )

Reports a parsing error.

Parameters

msg	The error message.
symbolName	The name of the symbol related to the error.

◆ onScan() [1/3]

nint by::tokenScanable::onScan	(	parser &	ps,
		YYSTYPE *	val,
		YYLTYPE *	loc,
		yyscan_t	scanner )

Default entry point for token scanning.

This non-virtual method acts as a wrapper that internally dispatches to the pure virtual onScan method, which must be implemented by concrete scanner strategies.

Parameters

ps	The parser instance.
val	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
scanner	The Flex scanner instance.

Returns: The token ID.

◆ onScan() [2/3]

nint by::parser::onScan	(	parser &	ps,
		YYSTYPE *	val,
		YYLTYPE *	loc,
		yyscan_t	scanner,
		nbool &	isBypass )

overridevirtual

Handles scanning for the lexer, overriding the base tokenScanable behavior.

Parameters

ps	The `parser` instance.
val	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
scanner	The Flex scanner instance.
isBypass	Flag indicating if indentation bypass is active.

Returns: The token ID.

Implements by::tokenScanable.

◆ onScan() [3/3]

virtual nint by::tokenScanable::onScan	(	parser &	ps,
		YYSTYPE *	yylval,
		YYLTYPE *	loc,
		yyscan_t	yyscanner,
		nbool &	isBypass )

virtual

Pure virtual method for concrete token scanning implementations.

Parameters

ps	The parser instance.
yylval	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
yyscanner	The Flex scanner instance.
isBypass	Flag indicating if indentation bypass is active.

Returns: The token ID.

Implements by::tokenScanable.

◆ onScanUnexpected()

nchar by::parser::onScanUnexpected ( const nchar * token )

Handles an unexpected token encountered during scanning.

Parameters

token The unexpected token string.

Returns: The token ID for an error.

◆ onSrcArea()

void by::parser::onSrcArea ( const area & area )

Callback for processing source area information from the lexer.

Parameters

area	The source area (e.g., line, column) information.

◆ onTokenBeginStr()

void by::parser::onTokenBeginStr ( nchar token )

Handles the start of a string literal.

Stores the opening quote character ('"' or '\'') to distinguish between double-quoted and single-quoted strings during parsing. This enables support for both quote types using the same scanner state.

Parameters

token The opening quote character ('"' or '\'')

◆ onTokenColon()

nint by::parser::onTokenColon ( nint tok )

Handles the colon token.

Parameters

tok	The token ID of the colon.

Returns: The token ID.

◆ onTokenComma()

nint by::parser::onTokenComma ( nint tok )

Handles the comma token.

Parameters

tok	The token ID of the comma.

Returns: The token ID.

◆ onTokenEndOfBraces()

nbool by::parser::onTokenEndOfBraces ( )

Handles the end of a braces block (e.g., }).

Returns: true if the end of braces was handled, false otherwise.

◆ onTokenEndOfFile()

nint by::parser::onTokenEndOfFile ( )

Handles the end-of-file token.

Returns: The token ID for end-of-file.

◆ onTokenLParan()

nint by::parser::onTokenLParan ( nint tok )

Handles the left parenthesis token.

Parameters

tok	The token ID of the left parenthesis.

Returns: The token ID.

◆ onTokenNewLine()

nint by::parser::onTokenNewLine ( nint tok )

Handles the newline token.

Parameters

tok	The token ID of the newline.

Returns: The token ID.

◆ onTokenRParan()

nint by::parser::onTokenRParan ( nint tok )

Handles the right parenthesis token.

Parameters

tok	The token ID of the right parenthesis.

Returns: The token ID.

◆ onTokenStartOfStrTemplateBrace()

void by::parser::onTokenStartOfStrTemplateBrace ( )

Handles the start of a string template brace (e.g., ${).

Note: This method is likely related to parsing interpolated strings.

◆ popState()

int by::parser::popState ( )

Pops a Flex scanner state from the state stack.

Returns: The previous state.

◆ pushState()

int by::parser::pushState ( int newState )

Pushes a new Flex scanner state onto the state stack.

Parameters

newState The new state to push.

Returns: The new state.

◆ setScan()

template<typename T >

void by::parser::setScan ( )

inline

Sets the scanning mode for the lexer.

Template Parameters

T	The type of the scanner strategy (e.g., `tokenScan`, `indentScan`).

Note: Logs the change in scan mode for debugging purposes.

The documentation for this class was generated from the following file:

/home/runner/work/byeol/byeol/module/core/worker/parser.hpp

Public Member Functions

Protected Member Functions

Friends

Additional Inherited Members

Detailed Description

Similar Structure to Stela Parser

Scanner - Bison - Parser Structure

lowparser.y Lifecycle Management

Indentation Rule

tokenDispatcher

tokenScan

Member Function Documentation

◆ _onEndWork()

◆ _onWork()

◆ _prepare()

◆ getOpenStrToken()

◆ getSubPod()

◆ onDedent()

◆ onEndFunc()

◆ onIgnoreIndent()

◆ onIndent()

◆ onParseErr()

◆ onScan() [1/3]

◆ onScan() [2/3]

◆ onScan() [3/3]

◆ onScanUnexpected()

◆ onSrcArea()

◆ onTokenBeginStr()

◆ onTokenColon()

◆ onTokenComma()

◆ onTokenEndOfBraces()

◆ onTokenEndOfFile()

◆ onTokenLParan()

◆ onTokenNewLine()

◆ onTokenRParan()

◆ onTokenStartOfStrTemplateBrace()

◆ popState()

◆ pushState()

◆ setScan()