輸入一條SQL語句,PostgreSQL如何解析輸入的SQL,識別SQL型別以及基表/欄位等資訊?接下來的幾節將逐一進行解析.
本節介紹了PostgreSQL的詞法定義檔案(Flex輸入檔案),在檔案src/backend/parser/scan.l中.
如前所述,Flex輸入檔案由四部分組成:
```
%{
Declarations
%}
Definitions
%%
Rules
%%
User subroutines
```
本節介紹第三部分Rules.

### 一、Rules
在Flex的模式檔案中，%%和%%之間的內容被稱為規則（rules），每一行表示一條規則，每條規則由匹配模式（pattern）和 動作（action）組成。其中模式在前面，用正規表示式表示，動作在後面，即C程式碼。每當一個模式被匹配到時，後面的C程式碼將被執行。
Flex會將規則翻譯成名為yylex的函式，該函式掃描輸入檔案（預設標準輸入），當掃描到一個完整的、最長的、可以和某條規則的正規表示式所匹配的輸入時，函式會執行此規則後面的C程式碼。如果程式碼中沒有return語句，則執行完畢後，yylex會繼續執行，開始下一輪的掃描和匹配。注意：當有多條規則的模式被匹配到時， yylex會優先選擇匹配長度最長的那條規則，如果有匹配長度相等的規則，則選擇排在最前面的那條規則。

PG中的規則定義如下：
```

%%

{whitespace}	{
					//--------- 空白字元
					//忽略,不作任何處理
					/* ignore */
				}

{xcstart}		{
					//--------- C風格註釋
					/* Set location in case of syntax error in comment */
					//設定位置,以防註釋中的語法錯誤
					SET_YYLLOC();
					//深度
					yyextra->xcdepth = 0;
					//進入xc狀態
					BEGIN(xc);
					/* Put back any characters past slash-star; see above */
					//把斜槓星後的字元放回去
					// 注意:"/*"是2個字元,從位置2(偏移從0起算)開始把之後的字元放回去
					yyless(2);
				}

<xc>{xcstart}	{
					//遇到下一層的註釋,深度+1
					(yyextra->xcdepth)++;
					/* Put back any characters past slash-star; see above */
					//類似的,把之後的字元放回去
					yyless(2);
				}

<xc>{xcstop}	{
					//層次≤0,回到INITIAL狀態,否則層次減1
					if (yyextra->xcdepth <= 0)
						BEGIN(INITIAL);
					else
						(yyextra->xcdepth)--;
				}

<xc>{xcinside}	{
					//註釋裡面的內容,忽略
					/* ignore */
				}

<xc>{op_chars}	{
					//註釋裡面的內容,忽略
					/* ignore */
				}

<xc>\*+			{
					//註釋裡面的內容,忽略
					/* ignore */
				}

<xc><<eof>>		{ yyerror("unterminated /* comment"); }//遇到結束符,出錯

{xbstart}		{
					/* Binary bit type.
					 * At some point we should simply pass the string
					 * forward to the parser and label it there.
					 * In the meantime, place a leading "b" on the string
					 * to mark it for the input routine as a binary string.
					 */
					//--------- 二進位制位串
					//在某些點上,我們應該簡單的把字串向前傳遞給解析器並標記它
					//在此期間,設定一個打頭的字元"b"以標記該輸入為二進位制串
					SET_YYLLOC();
					BEGIN(xb);
					startlit();
					addlitchar('b', yyscanner);
				}
<xb>{quotestop}	|
<xb>{quotefail} {
					yyless(1);
					BEGIN(INITIAL);
					yylval->str = litbufdup(yyscanner);
					return BCONST;
				}
<xh>{xhinside}	|
<xb>{xbinside}	{
					addlit(yytext, yyleng, yyscanner);
				}
<xh>{quotecontinue}	|
<xb>{quotecontinue}	{
					/* ignore */
				}
<xb><<eof>>		{ yyerror("unterminated bit string literal"); }

{xhstart}		{
					//------------- 十六進位制串
					/* Hexadecimal bit type.
					 * At some point we should simply pass the string
					 * forward to the parser and label it there.
					 * In the meantime, place a leading "x" on the string
					 * to mark it for the input routine as a hex string.
					 */
					SET_YYLLOC();
					BEGIN(xh);
					startlit();
					addlitchar('x', yyscanner);
				}
<xh>{quotestop}	|
<xh>{quotefail} {
					yyless(1);
					BEGIN(INITIAL);
					yylval->str = litbufdup(yyscanner);
					return XCONST;
				}
<xh><<eof>>		{ yyerror("unterminated hexadecimal string literal"); }

{xnstart}		{
					//------------- 國家字元
					/* National character.
					 * We will pass this along as a normal character string,
					 * but preceded with an internally-generated "NCHAR".
					 */
					const ScanKeyword *keyword;

SET_YYLLOC();
					yyless(1);	/* eat only 'n' this time */

keyword = ScanKeywordLookup("nchar",
												yyextra->keywords,
												yyextra->num_keywords);
					if (keyword != NULL)
					{
						yylval->keyword = keyword->name;
						return keyword->value;
					}
					else
					{
						/* If NCHAR isn't a keyword, just return "n" */
						yylval->str = pstrdup("n");
						return IDENT;
					}
				}

{xqstart}		{
					yyextra->warn_on_first_escape = true;
					yyextra->saw_non_ascii = false;
					SET_YYLLOC();
					if (yyextra->standard_conforming_strings)
						BEGIN(xq);
					else
						BEGIN(xe);
					startlit();
				}
{xestart}		{
					yyextra->warn_on_first_escape = false;
					yyextra->saw_non_ascii = false;
					SET_YYLLOC();
					BEGIN(xe);
					startlit();
				}
{xusstart}		{
					SET_YYLLOC();
					if (!yyextra->standard_conforming_strings)
						ereport(ERROR,
								(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
								 errmsg("unsafe use of string constant with Unicode escapes"),
								 errdetail("String constants with Unicode escapes cannot be used when standard_conforming_strings is off."),
								 lexer_errposition()));
					BEGIN(xus);
					startlit();
				}
<xq>{quotestop}	|
<xq>{quotefail} {
					yyless(1);
					BEGIN(INITIAL);
					/*
					 * check that the data remains valid if it might have been
					 * made invalid by unescaping any chars.
					 */
					if (yyextra->saw_non_ascii)
						pg_verifymbstr(yyextra->literalbuf,
									   yyextra->literallen,
									   false);
					yylval->str = litbufdup(yyscanner);
					return SCONST;
				}
<xus>{quotestop} |
<xus>{quotefail} {
					/* throw back all but the quote */
					yyless(1);
					/* xusend state looks for possible UESCAPE */
					BEGIN(xusend);
				}
<xusend>{whitespace} {
					/* stay in xusend state over whitespace */
				}
<xusend><<eof>> |
<xusend>{other} |
<xusend>{xustop1} {
					/* no UESCAPE after the quote, throw back everything */
					yyless(0);
					BEGIN(INITIAL);
					yylval->str = litbuf_udeescape('\\', yyscanner);
					return SCONST;
				}
<xusend>{xustop2} {
					/* found UESCAPE after the end quote */
					BEGIN(INITIAL);
					if (!check_uescapechar(yytext[yyleng - 2]))
					{
						SET_YYLLOC();
						ADVANCE_YYLLOC(yyleng - 2);
						yyerror("invalid Unicode escape character");
					}
					yylval->str = litbuf_udeescape(yytext[yyleng - 2],
												   yyscanner);
					return SCONST;
				}
<xq>{xqdouble} {
					addlitchar('\'', yyscanner);
				}
<xq>{xqinside}  {
					addlit(yytext, yyleng, yyscanner);
				}
<xe>{xeinside}  {
					addlit(yytext, yyleng, yyscanner);
				}
<xe>{xeunicode} {
					pg_wchar	c = strtoul(yytext + 2, NULL, 16);

check_escape_warning(yyscanner);

if (is_utf16_surrogate_first(c))
					{
						yyextra->utf16_first_part = c;
						BEGIN(xeu);
					}
					else if (is_utf16_surrogate_second(c))
						yyerror("invalid Unicode surrogate pair");
					else
						addunicode(c, yyscanner);
				}
<xeu>{xeunicode} {
					pg_wchar	c = strtoul(yytext + 2, NULL, 16);

if (!is_utf16_surrogate_second(c))
						yyerror("invalid Unicode surrogate pair");

c = surrogate_pair_to_codepoint(yyextra->utf16_first_part, c);

addunicode(c, yyscanner);

BEGIN(xe);
				}
<xeu>.			{ yyerror("invalid Unicode surrogate pair"); }
<xeu>\n			{ yyerror("invalid Unicode surrogate pair"); }
<xeu><<eof>>	{ yyerror("invalid Unicode surrogate pair"); }
<xe>{xeunicodefail}	{
					ereport(ERROR,
							(errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE),
							 errmsg("invalid Unicode escape"),
							 errhint("Unicode escapes must be \\uXXXX or \\UXXXXXXXX."),
							 lexer_errposition()));
				}
<xe>{xeescape}  {
					if (yytext[1] == '\'')
					{
						if (yyextra->backslash_quote == BACKSLASH_QUOTE_OFF ||
							(yyextra->backslash_quote == BACKSLASH_QUOTE_SAFE_ENCODING &&
							 PG_ENCODING_IS_CLIENT_ONLY(pg_get_client_encoding())))
							ereport(ERROR,
									(errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
									 errmsg("unsafe use of \\' in a string literal"),
									 errhint("Use '' to write quotes in strings. \\' is insecure in client-only encodings."),
									 lexer_errposition()));
					}
					check_string_escape_warning(yytext[1], yyscanner);
					addlitchar(unescape_single_char(yytext[1], yyscanner),
							   yyscanner);
				}
<xe>{xeoctesc}  {
					unsigned char c = strtoul(yytext + 1, NULL, 8);

check_escape_warning(yyscanner);
					addlitchar(c, yyscanner);
					if (c == '\0' || IS_HIGHBIT_SET(c))
						yyextra->saw_non_ascii = true;
				}
<xe>{xehexesc}  {
					unsigned char c = strtoul(yytext + 2, NULL, 16);

check_escape_warning(yyscanner);
					addlitchar(c, yyscanner);
					if (c == '\0' || IS_HIGHBIT_SET(c))
						yyextra->saw_non_ascii = true;
				}
<xq>{quotecontinue} {
					/* ignore */
				}
<xe>.			{
					/* This is only needed for \ just before EOF */
					addlitchar(yytext[0], yyscanner);
				}
<xq><<eof>>		{ yyerror("unterminated quoted string"); }

{dolqdelim}		{
					SET_YYLLOC();
					yyextra->dolqstart = pstrdup(yytext);
					BEGIN(xdolq);
					startlit();
				}
{dolqfailed}	{
					SET_YYLLOC();
					/* throw back all but the initial "$" */
					yyless(1);
					/* and treat it as {other} */
					return yytext[0];
				}
<xdolq>{dolqdelim} {
					if (strcmp(yytext, yyextra->dolqstart) == 0)
					{
						pfree(yyextra->dolqstart);
						yyextra->dolqstart = NULL;
						BEGIN(INITIAL);
						yylval->str = litbufdup(yyscanner);
						return SCONST;
					}
					else
					{
						/*
						 * When we fail to match $...$ to dolqstart, transfer
						 * the $... part to the output, but put back the final
						 * $ for rescanning.  Consider $delim$...$junk$delim$
						 */
						addlit(yytext, yyleng - 1, yyscanner);
						yyless(yyleng - 1);
					}
				}
<xdolq>{dolqinside} {
					addlit(yytext, yyleng, yyscanner);
				}
<xdolq>{dolqfailed} {
					addlit(yytext, yyleng, yyscanner);
				}
<xdolq>.		{
					/* This is only needed for $ inside the quoted text */
					addlitchar(yytext[0], yyscanner);
				}
<xdolq><<eof>>	{ yyerror("unterminated dollar-quoted string"); }

{xdstart}		{
					SET_YYLLOC();
					BEGIN(xd);
					startlit();
				}
{xuistart}		{
					SET_YYLLOC();
					BEGIN(xui);
					startlit();
				}
<xd>{xdstop}	{
					char	   *ident;

BEGIN(INITIAL);
					if (yyextra->literallen == 0)
						yyerror("zero-length delimited identifier");
					ident = litbufdup(yyscanner);
					if (yyextra->literallen >= NAMEDATALEN)
						truncate_identifier(ident, yyextra->literallen, true);
					yylval->str = ident;
					return IDENT;
				}
<xui>{dquote} {
					yyless(1);
					/* xuiend state looks for possible UESCAPE */
					BEGIN(xuiend);
				}
<xuiend>{whitespace} {
					/* stay in xuiend state over whitespace */
				}
<xuiend><<eof>> |
<xuiend>{other} |
<xuiend>{xustop1} {
					/* no UESCAPE after the quote, throw back everything */
					char	   *ident;
					int			identlen;

yyless(0);

BEGIN(INITIAL);
					if (yyextra->literallen == 0)
						yyerror("zero-length delimited identifier");
					ident = litbuf_udeescape('\\', yyscanner);
					identlen = strlen(ident);
					if (identlen >= NAMEDATALEN)
						truncate_identifier(ident, identlen, true);
					yylval->str = ident;
					return IDENT;
				}
<xuiend>{xustop2}	{
					/* found UESCAPE after the end quote */
					char	   *ident;
					int			identlen;

BEGIN(INITIAL);
					if (yyextra->literallen == 0)
						yyerror("zero-length delimited identifier");
					if (!check_uescapechar(yytext[yyleng - 2]))
					{
						SET_YYLLOC();
						ADVANCE_YYLLOC(yyleng - 2);
						yyerror("invalid Unicode escape character");
					}
					ident = litbuf_udeescape(yytext[yyleng - 2], yyscanner);
					identlen = strlen(ident);
					if (identlen >= NAMEDATALEN)
						truncate_identifier(ident, identlen, true);
					yylval->str = ident;
					return IDENT;
				}
<xd>{xddouble}	{
					addlitchar('"', yyscanner);
				}
<xd>{xdinside}	{
					addlit(yytext, yyleng, yyscanner);
				}
<xd><<eof>>		{ yyerror("unterminated quoted identifier"); }

{xufailed}	{
					char	   *ident;

SET_YYLLOC();
					/* throw back all but the initial u/U */
					yyless(1);
					/* and treat it as {identifier} */
					ident = downcase_truncate_identifier(yytext, yyleng, true);
					yylval->str = ident;
					return IDENT;
				}

{typecast}		{
					SET_YYLLOC();
					return TYPECAST;
				}

{dot_dot}		{
					SET_YYLLOC();
					return DOT_DOT;
				}

{colon_equals}	{
					SET_YYLLOC();
					return COLON_EQUALS;
				}

{equals_greater} {
					SET_YYLLOC();
					return EQUALS_GREATER;
				}

{less_equals}	{
					SET_YYLLOC();
					return LESS_EQUALS;
				}

{greater_equals} {
					SET_YYLLOC();
					return GREATER_EQUALS;
				}

{less_greater}	{
					/* We accept both "<>" and "!=" as meaning NOT_EQUALS */
					SET_YYLLOC();
					return NOT_EQUALS;
				}

{not_equals}	{
					/* We accept both "<>" and "!=" as meaning NOT_EQUALS */
					SET_YYLLOC();
					return NOT_EQUALS;
				}

{self}			{
					SET_YYLLOC();
					return yytext[0];
				}

{operator}		{
					/*
					 * Check for embedded slash-star or dash-dash; those
					 * are comment starts, so operator must stop there.
					 * Note that slash-star or dash-dash at the first
					 * character will match a prior rule, not this one.
					 */
					int			nchars = yyleng;
					char	   *slashstar = strstr(yytext, "/*");
					char	   *dashdash = strstr(yytext, "--");

if (slashstar && dashdash)
					{
						/* if both appear, take the first one */
						if (slashstar > dashdash)
							slashstar = dashdash;
					}
					else if (!slashstar)
						slashstar = dashdash;
					if (slashstar)
						nchars = slashstar - yytext;

/*
					 * For SQL compatibility, '+' and '-' cannot be the
					 * last char of a multi-char operator unless the operator
					 * contains chars that are not in SQL operators.
					 * The idea is to lex '=-' as two operators, but not
					 * to forbid operator names like '?-' that could not be
					 * sequences of SQL operators.
					 */
					if (nchars > 1 &&
						(yytext[nchars - 1] == '+' ||
						 yytext[nchars - 1] == '-'))
					{
						int			ic;

for (ic = nchars - 2; ic >= 0; ic--)
						{
							char c = yytext[ic];
							if (c == '~' || c == '!' || c == '@' ||
								c == '#' || c == '^' || c == '&' ||
								c == '|' || c == '`' || c == '?' ||
								c == '%')
								break;
						}
						if (ic < 0)
						{
							/*
							 * didn't find a qualifying character, so remove
							 * all trailing [+-]
							 */
							do {
								nchars--;
							} while (nchars > 1 &&
								 (yytext[nchars - 1] == '+' ||
								  yytext[nchars - 1] == '-'));
						}
					}

SET_YYLLOC();

if (nchars < yyleng)
					{
						/* Strip the unwanted chars from the token */
						yyless(nchars);
						/*
						 * If what we have left is only one char, and it's
						 * one of the characters matching "self", then
						 * return it as a character token the same way
						 * that the "self" rule would have.
						 */
						if (nchars == 1 &&
							strchr(",()[].;:+-*/%^<>=", yytext[0]))
							return yytext[0];
						/*
						 * Likewise, if what we have left is two chars, and
						 * those match the tokens ">=", "<=", "=>", "<>" or
						 * "!=", then we must return the appropriate token
						 * rather than the generic Op.
						 */
						if (nchars == 2)
						{
							if (yytext[0] == '=' && yytext[1] == '>')
								return EQUALS_GREATER;
							if (yytext[0] == '>' && yytext[1] == '=')
								return GREATER_EQUALS;
							if (yytext[0] == '')
								return NOT_EQUALS;
							if (yytext[0] == '!' && yytext[1] == '=')
								return NOT_EQUALS;
						}
					}

/*
					 * Complain if operator is too long.  Unlike the case
					 * for identifiers, we make this an error not a notice-
					 * and-truncate, because the odds are we are looking at
					 * a syntactic mistake anyway.
					 */
					if (nchars >= NAMEDATALEN)
						yyerror("operator too long");

yylval->str = pstrdup(yytext);
					return Op;
				}

{param}			{
					SET_YYLLOC();
					yylval->ival = atol(yytext + 1);
					return PARAM;
				}

{integer}		{
					SET_YYLLOC();
					return process_integer_literal(yytext, yylval);
				}
{decimal}		{
					SET_YYLLOC();
					yylval->str = pstrdup(yytext);
					return FCONST;
				}
{decimalfail}	{
					/* throw back the .., and treat as integer */
					yyless(yyleng - 2);
					SET_YYLLOC();
					return process_integer_literal(yytext, yylval);
				}
{real}			{
					SET_YYLLOC();
					yylval->str = pstrdup(yytext);
					return FCONST;
				}
{realfail1}		{
					/*
					 * throw back the [Ee], and treat as {decimal}.  Note
					 * that it is possible the input is actually {integer},
					 * but since this case will almost certainly lead to a
					 * syntax error anyway, we don't bother to distinguish.
					 */
					yyless(yyleng - 1);
					SET_YYLLOC();
					yylval->str = pstrdup(yytext);
					return FCONST;
				}
{realfail2}		{
					/* throw back the [Ee][+-], and proceed as above */
					yyless(yyleng - 2);
					SET_YYLLOC();
					yylval->str = pstrdup(yytext);
					return FCONST;
				}

{identifier}	{
					//---------- 識別符號
					const ScanKeyword *keyword;
					char	   *ident;

SET_YYLLOC();

/* Is it a keyword? */
					//是否關鍵字?
					keyword = ScanKeywordLookup(yytext,
												yyextra->keywords,
												yyextra->num_keywords);
					if (keyword != NULL)
					{
						//是,則返回關鍵字值
						yylval->keyword = keyword->name;
						return keyword->value;
					}

/*
					 * No.  Convert the identifier to lower case, and truncate
					 * if necessary.
					 */
					//如果不是關鍵字,則設定為小寫字母,如需要則截斷
					ident = downcase_truncate_identifier(yytext, yyleng, true);
					yylval->str = ident;
					return IDENT;
				}

{other}			{
					SET_YYLLOC();
					return yytext[0];
				}

<<eof>>			{
					SET_YYLLOC();
					yyterminate();
				}

%%
```

### 二、參考資料
[Flex&Bison](http://web.iitd.ac.in/~sumeet/flex__bison.pdf)</eof></eof></xd></xd></xd></xuiend></xuiend></xuiend></eof></xuiend></xuiend></xui></xd></eof></xdolq></xdolq></xdolq></xdolq></xdolq></eof></xq></xe></xq></xe></xe></xe></xe></eof></xeu></xeu></xeu></xeu></xe></xe></xq></xq></xusend></xusend></xusend></eof></xusend></xusend></xus></xus></xq></xq></eof></xh></xh></xh></eof></xb></xb></xh></xb></xh></xb></xb></eof></xc></xc></xc></xc></xc></xc>

PostgreSQL 原始碼解讀（170）- 查詢#90(PG中的詞法定義：scanner.l)#3

相關文章