Parser for byeol language source code 더 자세히 ...

#include <parser.hpp>

by::parser에 대한 상속 다이어그램 :

Public 멤버 함수
node *	getSubPod ()
	Retrieves the root node of the parsed sub-pod.

const node *	getSubPod () const BY_CONST_FUNC(getSubPod()) srcSupplies &getSrcSupplies()

const srcSupplies &	getSrcSupplies () const BY_CONST_FUNC(getSrcSupplies()) me &addSupply(const srcSupply &new 1)

me &	addSupply (const tucontainable< srcSupply > &new1)

me &	relSupplies ()

tokenDispatcher &	getDispatcher ()

std::vector< ncnt > &	getIndents ()

nbool	isInit () const

template<typename T >
void	setScan ()
	Sets the scanning mode for the lexer.

void	rel () override

int	pushState (int newState)
	Pushes a new Flex scanner state onto the state stack.

int	popState ()
	Pops a Flex scanner state from the state stack.

nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner, nbool &isBypass) override
	Handles scanning for the lexer, overriding the base `tokenScanable` behavior.

nint	onTokenEndOfFile ()
	Handles the end-of-file token.

nint	onTokenColon (nint tok)
	Handles the colon token.

nint	onTokenNewLine (nint tok)
	Handles the newline token.

nint	onTokenLParan (nint tok)
	Handles the left parenthesis token.

nint	onTokenRParan (nint tok)
	Handles the right parenthesis token.

nint	onTokenComma (nint tok)
	Handles the comma token.

nbool	onTokenEndOfBraces ()
	Handles the end of a braces block (e.g., `}`).

void	onTokenStartOfStrTemplateBrace ()
	Handles the start of a string template brace (e.g., `${`).

void	onTokenBeginStr (nchar token)
	Handles the start of a string literal.

nchar	getOpenStrToken ()
	returns quote character which used to start current string sequence.

nint	onIndent (ncnt col, nint tok)
	Handles an increase in indentation.

nint	onDedent (ncnt col, nint tok)
	Handles a decrease in indentation.

nint	onIgnoreIndent (nint tok)
	Handles ignoring indentation (e.g., for inline blocks).

nchar	onScanUnexpected (const nchar *token)
	Handles an unexpected token encountered during scanning.

void	onSrcArea (const area &area)
	Callback for processing source area information from the lexer.

node *	onGet (const std::string &name)

node *	onGet (const std::string &name, const narr &args)

node *	onGet (node &from, const std::string &name)

node *	onGet (node &from, node &it)

node *	onGet (const std::string &name, const std::string &name2)

node *	onCallAccess (node &it, const narr &args)

FBOExpr *	onUnaryMinus (const node &it)

FBOExpr *	onUnaryNot (const node &it)

FUOExpr *	onUnaryBitwiseNot (const node &it)

node *	onUnaryDoublePlus (node &it)

node *	onUnaryDoubleMinus (node &it)

FUOExpr *	onUnaryPostfixDoublePlus (const node &it)

FUOExpr *	onUnaryPostfixDoubleMinus (const node &it)

FBOExpr *	onAdd (const node &lhs, const node &rhs)

FBOExpr *	onSub (const node &lhs, const node &rhs)

FBOExpr *	onMul (const node &lhs, const node &rhs)

FBOExpr *	onDiv (const node &lhs, const node &rhs)

FBOExpr *	onMod (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseAnd (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseXor (const node &lhs, const node &rhs)

FBOExpr *	onBitwiseOr (const node &lhs, const node &rhs)

FBOExpr *	onLShift (const node &lhs, const node &rhs)

FBOExpr *	onRShift (const node &lhs, const node &rhs)

node *	onAssign (node &lhs, node &rhs)

node *	onAddAssign (node &lhs, node &rhs)

node *	onSubAssign (node &lhs, node &rhs)

node *	onMulAssign (node &lhs, node &rhs)

node *	onDivAssign (node &lhs, node &rhs)

node *	onModAssign (node &lhs, node &rhs)

FBOExpr *	onGt (const node &lhs, const node &rhs)

FBOExpr *	onGe (const node &lhs, const node &rhs)

FBOExpr *	onLt (const node &lhs, const node &rhs)

FBOExpr *	onLe (const node &lhs, const node &rhs)

FBOExpr *	onEq (const node &lhs, const node &rhs)

FBOExpr *	onNe (const node &lhs, const node &rhs)

FBOExpr *	onAnd (const node &lhs, const node &rhs)

FBOExpr *	onOr (const node &lhs, const node &rhs)

node *	onGetGeneric (const std::string &orgName, const args &typeParams)

node *	onGetElem (const node &arr, const node &idx)

node *	onGetArray (node &elemType)

obj *	onPod (const node &path)

obj *	onPod (const std::string &path)

obj *	onPod ()

obj *	onSubPod (obj &subpod)

endExpr *	onEnd (const blockExpr &blk)

blockExpr *	onBlock (const node *stmt)

blockExpr *	onBlock (blockExpr blk, const node stmt)

blockExpr *	onBlock ()

defBlock *	onDefBlock (node *stmt)

defBlock *	onDefBlock (defBlock blk, node stmt)

defBlock *	onDefBlock ()

node *	onFor (const std::string &iterName, const node &expr, const blockExpr &blk)

node *	onWhile (const node &condition, const blockExpr &blk)

node *	onParanthesisAsTuple (narr &tuple)

args *	onTuple ()

args *	onTuple (const node &elem)

args *	onTuple (args &as, const node &elem)

args *	onFuncCallTuple ()

args *	onFuncCallTuple (const node &elem)

args *	onFuncCallTuple (args &as, const node &elem)

args *	onTypeNames (const node &param)

args *	onTypeNames (args &params, const node &param)

template<typename T , typename Arg >
T *	onPrimitive (const Arg &arg)

template<typename T >
T *	onPrimitive ()

node *	onDefProp (const modifier &mod, const std::string &name, const node &rhs)

node *	onDefProp (const std::string &name, const node &rhs)

node *	onDefProp (const node &rhs)

node *	onDefProp (const modifier &mod, const node &rhs)

node *	onDefAssign (const modifier &mod, const std::string &name, const node *rhs)

node *	onDefAssign (const defPropExpr prop, const node rhs)

node *	onDefAssign (const std::string &name, const node *rhs)

obj *	onDefOrigin (const std::string &name, defBlock &blk)

obj *	onDefOrigin (const std::string &name, const narr &args, defBlock &blk)

genericOrigin *	onDefObjGeneric (const std::string &name, const args &typeParams, defBlock &blk)

genericOrigin *	onDefObjGeneric (const std::string &name, const args &typeParams, const narr &args, defBlock &blk)

node *	onDefArray (const narr &items)

node *	onDefSeq (const node &start, const node &end)

void	onCompilationUnit (obj *subpod)

void	onCompilationUnit (obj subpod, defBlock blk)

func *	onFuncSignature (const modifier &mod, const getExpr &access, const node *retType)

func *	onFuncSignature (const getExpr &access, const node *retType)

func *	onFuncSignature (const modifier &mod, node &it, const node *retType)

func *	onFuncSignature (node &it, const node *retType)

func *	onAbstractFunc (func &f)

node *	onFunc (func &func, const blockExpr &blk)

defNestedFuncExpr *	onLambda (const narr &params, const node &retType, const blockExpr &blk)

ctor *	onCtor (const modifier &mod, const narr &args, const blockExpr &blk)

ctor *	onCtor (const narr &args, const blockExpr &blk)

ctor *	onCtor (const modifier &mod, const blockExpr &blk)

ctor *	onCtor (const blockExpr &blk)

narr *	onParams ()

narr *	onParams (const defPropExpr *elem)

narr *	onParams (narr &it, const defPropExpr *elem)

modifier *	onModifier (nbool isPublic, nbool isOverriden)

node *	onDeclStmt (const narr &dotnames)

retExpr *	onRet ()

retExpr *	onRet (node &exp)

breakExpr *	onBreak ()

nextExpr *	onNext ()

evalExpr *	onFillFromOfFuncCall (const node &me, evalExpr &to)

evalExpr *	onEvalExpr (node &type, const narr &a)

evalExpr *	onEvalExpr (node &type, const args &a)

isExpr *	onIs (const node &me, const node &to)

asExpr *	onAs (const node &me, const node &as)

ifExpr *	onIf (const node &condition, const blockExpr &then)

ifExpr *	onIf (const node &condition, const blockExpr &then, const blockExpr &elseBlk)

ifExpr *	onIf (const node &condition, const blockExpr &then, const ifExpr &elseIf)

evalExpr *	onIn (const node &it, const node &container)

void	onEndFunc ()
	Callback invoked at the end of parsing a function definition.

void	onParseErr (const std::string &msg, const nchar *symbolName)
	Reports a parsing error.

nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner)
	Default entry point for token scanning.

virtual nint	onScan (parser &ps, YYSTYPE yylval, YYLTYPE loc, yyscan_t yyscanner, nbool &isBypass)=0
	Pure virtual method for concrete token scanning implementations.

by::tworker< str, pod >(으)로부터 상속된 Public 멤버 함수
	tworker (const errReport &rpt)

errReport &	getReport ()

const errReport &	getReport () const BY_CONST_FUNC(getReport()) me &setReport(errReport &rpt)

me &	setFlag (nint newFlag)

me &	addFlag (nint flag)

me &	delFlag (nint clear)

nbool	isFlag (nint flag) const

nint	getFlag () const

me &	setTask (const pod &root)

me &	setTask (const pod *it) BY_SIDE_FUNC(it

pod *	getTask ()

const pod *	getTask () const BY_CONST_FUNC(getTask()) virtual void rel()

str	work ()

const area &	getArea () const BY_CONST_FUNC(_getArea()) nbool isOk() const

by::tokenScanable(으)로부터 상속된 Public 멤버 함수
nint	onScan (parser &ps, YYSTYPE val, YYLTYPE loc, yyscan_t scanner)
	Default entry point for token scanning.

Protected 멤버 함수
void *	_scanString (const std::string &src, void *scanner)

exprMaker &	_getMaker ()

void	_prepare () override
	Protected virtual method for preparation before starting the work.

void	_onEndWork () override
	Protected virtual method called after the work is completed.

str	_onWork () override

by::tworker< str, pod >(으)로부터 상속된 Protected 멤버 함수
void	_report (baseErr *e)

area &	_getArea ()

void	_onEndErrReport (const errReport &rpt) const

Friends
class	coreInternal

추가로 상속된 멤버들
by::tworker< str, pod >(으)로부터 상속된 Public 타입
enum	logFlag

by::tworker< str, pod >(으)로부터 상속된 Public 속성
me setTask	it

me setTask *	this

상세한 설명

Parser for byeol language source code

Entry point for the byeol parsing component, inheriting from worker. work() returns the parsed result as pod.

Similar Structure to Stela Parser

Since the stela language itself is part of the byeol language, its parser is also based on the byeol language parser. The stela parser is less complex compared to the byeol parser, so it's recommended to examine the stela parser code first before looking at the core module.

Scanner - Bison - Parser Structure

Uses flex and bison, naming flex as lowscanner and bison as lowparser. These low-level scanner and parser only exist inside the parser component and are not exposed externally. When parser::work() executes, it runs lowscanner, which tokenizes and passes tokens to lowparser. Lowparser matches rules and passes those events back to parser. Therefore, parser's functions starting with on are event handling functions that define how to create nodes and construct the AST.

lowparser.y Lifecycle Management

In each rule, calling parser::onXXXX() functions often creates new objects on the heap using the new keyword (not all functions do this). If these arguments are placed directly into tbicontainer, that's fine, but otherwise they must be bound with tstr beforehand to avoid memory leaks.

// - lowparser.y
pod: POD name-access NEWLINE {
    $$ = PS.onPod(*$2); // onpod() returns new pod()
}
 
compilation-unit: pod defblock {
    tstr<obj> pak($1); // Without binding with tstr like this,
 
    PS.onCompilationUnit(pak.get()); // if the pak value has issues inside onCompilationUnit()
                                     // and the operation is cancelled, the pod object
                                     // created on the heap becomes a memory leak.
}

This applies not only to onXXXX() functions but also to string tokens. See the following code:

// -- lowscanner.l
<stateString>\"       { // When string scan ends
    if(!yylval->asStr) yylval->asStr = new std::string(); // Creates string object with new
    ....
    return STRVAL;
}
 
// -- lowparser.y
    ...
   | STRVAL { // When STRVAL comes from scanner
    $$ = PS.onPrimitive<nStr>(*$1); // Pass it first. String is always immutable in this project
                                    // so the object is always copied.
    delete $1; // Since the object is copied, delete the original object.

Indentation Rule

The byeol language applies the offside rule, making it very sensitive to indentation. Unlike typical languages, it must be able to count how many spaces exist immediately after a newline. Once indentation is determined and the code line's scope is confirmed, subsequent spaces should be ignored. Consider the following byeol language example:

def A
    foo(val int) void
        if val > 0
          if val < 5
                  print("0 < val < 5")
        print("end of func") # 1)

After parsing print("0 < val < 5"), when parsing the next line print("end of func"), the parser must count the leading spaces and compare with the space counts for each scope to determine which scope this code line belongs to. Looking at the example, we can intuitively see that print("end of func") has the same spacing as if val > 0, meaning it belongs directly inside the foo() body. This means at this point, both the scope inside if val > 0 and the scope inside if val < 5 are closed. So lowscanner must add 2 DEDENT tokens (signifying scope termination) before lowparser recognizes print("end of func").

tokenDispatcher

Flex uses a separately designated stream called yyin to get characters and define them as tokens. As shown in the example above, adding tokens during parsing means adding specific characters to this stream. Typically unput is used for this, but to flexibly handle multiple unputs or pushing to the front, lowscanner internally uses tokenDispatcher.

tokenScan

As explained earlier, detecting indentation is crucial. Accurate space counting is required, so a strategy pattern is applied, switching between normalScan and indentScan. When a newline is detected, it switches to indentScan to count spaces accurately and determine scope.

멤버 함수 문서화

◆ _onEndWork()

void by::parser::_onEndWork ( )

overrideprotectedvirtual

Protected virtual method called after the work is completed.

This hook allows derived classes to perform any necessary cleanup or finalization after _onWork() has finished.

by::tworker< str, pod >(으)로부터 재구현되었습니다.

◆ _onWork()

str by::parser::_onWork ( )

overrideprotectedvirtual

parse with given srcSupply instances.

매개변수

script is null terminated cstring.

반환값: last parsed sub pod.

by::tworker< str, pod >를 구현.

◆ _prepare()

void by::parser::_prepare ( )

overrideprotectedvirtual

Protected virtual method for preparation before starting the work.

This hook allows derived classes to perform any necessary setup or initialization before _onWork() is invoked.

by::tworker< str, pod >(으)로부터 재구현되었습니다.

◆ getOpenStrToken()

nchar by::parser::getOpenStrToken ( )

returns quote character which used to start current string sequence.

반환값: quote or double quote to be used to open string literal.

◆ getSubPod()

node * by::parser::getSubPod ( )

Retrieves the root node of the parsed sub-pod.

This method provides access to the top-level AST node representing the entire parsed sub-pod.

반환값: A pointer to the root node of the sub-pod.

◆ onDedent()

nint by::parser::onDedent	(	ncnt	col,
		nint	tok )

Handles a decrease in indentation.

매개변수

col	The column count of the new indentation level.
tok	The token ID that triggered the dedentation.

반환값: The token ID for DEDENT.

◆ onEndFunc()

void by::parser::onEndFunc ( )

Callback invoked at the end of parsing a function definition.

This method handles any cleanup or finalization required after a function's body has been parsed.

◆ onIgnoreIndent()

nint by::parser::onIgnoreIndent ( nint tok )

Handles ignoring indentation (e.g., for inline blocks).

매개변수

tok	The token ID.

반환값: The token ID.

◆ onIndent()

nint by::parser::onIndent	(	ncnt	col,
		nint	tok )

Handles an increase in indentation.

매개변수

col	The column count of the new indentation level.
tok	The token ID that triggered the indentation.

반환값: The token ID for INDENT.

◆ onParseErr()

void by::parser::onParseErr	(	const std::string &	msg,
		const nchar *	symbolName )

Reports a parsing error.

매개변수

msg	The error message.
symbolName	The name of the symbol related to the error.

◆ onScan() [1/3]

nint by::tokenScanable::onScan	(	parser &	ps,
		YYSTYPE *	val,
		YYLTYPE *	loc,
		yyscan_t	scanner )

Default entry point for token scanning.

This non-virtual method acts as a wrapper that internally dispatches to the pure virtual onScan method, which must be implemented by concrete scanner strategies.

매개변수

ps	The parser instance.
val	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
scanner	The Flex scanner instance.

반환값: The token ID.

◆ onScan() [2/3]

nint by::parser::onScan	(	parser &	ps,
		YYSTYPE *	val,
		YYLTYPE *	loc,
		yyscan_t	scanner,
		nbool &	isBypass )

overridevirtual

Handles scanning for the lexer, overriding the base tokenScanable behavior.

매개변수

ps	The `parser` instance.
val	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
scanner	The Flex scanner instance.
isBypass	Flag indicating if indentation bypass is active.

반환값: The token ID.

by::tokenScanable를 구현.

◆ onScan() [3/3]

virtual nint by::tokenScanable::onScan	(	parser &	ps,
		YYSTYPE *	yylval,
		YYLTYPE *	loc,
		yyscan_t	yyscanner,
		nbool &	isBypass )

virtual

Pure virtual method for concrete token scanning implementations.

매개변수

ps	The parser instance.
yylval	The YYSTYPE value pointer for the token.
loc	The YYLTYPE location pointer for the token.
yyscanner	The Flex scanner instance.
isBypass	Flag indicating if indentation bypass is active.

반환값: The token ID.

by::tokenScanable를 구현.

◆ onScanUnexpected()

nchar by::parser::onScanUnexpected ( const nchar * token )

Handles an unexpected token encountered during scanning.

매개변수

token The unexpected token string.

반환값: The token ID for an error.

◆ onSrcArea()

void by::parser::onSrcArea ( const area & area )

Callback for processing source area information from the lexer.

매개변수

area	The source area (e.g., line, column) information.

◆ onTokenBeginStr()

void by::parser::onTokenBeginStr ( nchar token )

Handles the start of a string literal.

Stores the opening quote character ('"' or '\'') to distinguish between double-quoted and single-quoted strings during parsing. This enables support for both quote types using the same scanner state.

매개변수

token The opening quote character ('"' or '\'')

◆ onTokenColon()

nint by::parser::onTokenColon ( nint tok )

Handles the colon token.

매개변수

tok	The token ID of the colon.

반환값: The token ID.

◆ onTokenComma()

nint by::parser::onTokenComma ( nint tok )

Handles the comma token.

매개변수

tok	The token ID of the comma.

반환값: The token ID.

◆ onTokenEndOfBraces()

nbool by::parser::onTokenEndOfBraces ( )

Handles the end of a braces block (e.g., }).

반환값: true if the end of braces was handled, false otherwise.

◆ onTokenEndOfFile()

nint by::parser::onTokenEndOfFile ( )

Handles the end-of-file token.

반환값: The token ID for end-of-file.

◆ onTokenLParan()

nint by::parser::onTokenLParan ( nint tok )

Handles the left parenthesis token.

매개변수

tok	The token ID of the left parenthesis.

반환값: The token ID.

◆ onTokenNewLine()

nint by::parser::onTokenNewLine ( nint tok )

Handles the newline token.

매개변수

tok	The token ID of the newline.

반환값: The token ID.

◆ onTokenRParan()

nint by::parser::onTokenRParan ( nint tok )

Handles the right parenthesis token.

매개변수

tok	The token ID of the right parenthesis.

반환값: The token ID.

◆ onTokenStartOfStrTemplateBrace()

void by::parser::onTokenStartOfStrTemplateBrace ( )

Handles the start of a string template brace (e.g., ${).

주의: This method is likely related to parsing interpolated strings.

◆ popState()

int by::parser::popState ( )

Pops a Flex scanner state from the state stack.

반환값: The previous state.

◆ pushState()

int by::parser::pushState ( int newState )

Pushes a new Flex scanner state onto the state stack.

매개변수

newState The new state to push.

반환값: The new state.

◆ setScan()

template<typename T >

void by::parser::setScan ( )

inline

Sets the scanning mode for the lexer.

템플릿 파라메터

T	The type of the scanner strategy (e.g., `tokenScan`, `indentScan`).

주의: Logs the change in scan mode for debugging purposes.

이 클래스에 대한 문서화 페이지는 다음의 파일로부터 생성되었습니다.:

/home/runner/work/byeol/byeol/module/core/worker/parser.hpp

Public 멤버 함수

Protected 멤버 함수

Friends

추가로 상속된 멤버들

상세한 설명

Similar Structure to Stela Parser

Scanner - Bison - Parser Structure

lowparser.y Lifecycle Management

Indentation Rule

tokenDispatcher

tokenScan

멤버 함수 문서화

◆ _onEndWork()

◆ _onWork()

◆ _prepare()

◆ getOpenStrToken()

◆ getSubPod()

◆ onDedent()

◆ onEndFunc()

◆ onIgnoreIndent()

◆ onIndent()

◆ onParseErr()

◆ onScan() [1/3]

◆ onScan() [2/3]

◆ onScan() [3/3]

◆ onScanUnexpected()

◆ onSrcArea()

◆ onTokenBeginStr()

◆ onTokenColon()

◆ onTokenComma()

◆ onTokenEndOfBraces()

◆ onTokenEndOfFile()

◆ onTokenLParan()

◆ onTokenNewLine()

◆ onTokenRParan()

◆ onTokenStartOfStrTemplateBrace()

◆ popState()

◆ pushState()

◆ setScan()