輸入一條SQL語句,PostgreSQL如何解析輸入的SQL,識別SQL型別以及基表/欄位等資訊?接下來的幾節將逐一進行解析.
本節介紹了PostgreSQL的詞法定義檔案(Flex輸入檔案),在檔案src/backend/parser/scan.l中.
如前所述,Flex輸入檔案由四部分組成:
```
%{
Declarations
%}
Definitions
%%
Rules
%%
User subroutines
```
本節介紹第四部分User subroutines.

### 一、User subroutines
在規則之後是自定義例程,在scan.l中定義的例程主要是對輸入的SQL語句進行解析以及執行初始化和事後清理工作等.

```

/* LCOV_EXCL_STOP */

/*
 * Arrange access to yyextra for subroutines of the main yylex() function.
 * We expect each subroutine to have a yyscanner parameter.  Rather than
 * use the yyget_xxx functions, which might or might not get inlined by the
 * compiler, we cheat just a bit and cast yyscanner to the right type.
 * 為主yylex()函式提供yyextra的訪問.
 * 我們期望每一個子例程都有引數:yyscanner.
 * 相對於使用yyget_xxx函式(可能或不可能被編譯器內聯),我們強制yyscanner為正確的型別.
 */
#undef yyextra
#define yyextra  (((struct yyguts_t *) yyscanner)->yyextra_r)

/* Likewise for a couple of other things we need. */
//定義其他需要的東西:yylloc/yyleng
#undef yylloc
#define yylloc	(((struct yyguts_t *) yyscanner)->yylloc_r)
#undef yyleng
#define yyleng	(((struct yyguts_t *) yyscanner)->yyleng_r)

/*
 * scanner_errposition
 *		Report a lexer or grammar error cursor position, if possible.
 * scanner_errposition : 如可以,報告詞法或語法錯誤位置
 *
 * This is expected to be used within an ereport() call.  The return value
 * is a dummy (always 0, in fact).
 * 該例程在ereport()呼叫中使用.返回值是偽列(實際上通常為0).
 *
 * Note that this can only be used for messages emitted during raw parsing
 * (essentially, scan.l and gram.y), since it requires the yyscanner struct
 * to still be available.
 * 注意,這隻能用於在原始解析期間產生的訊息(scan.l & gram.y),
 *   因為需要yyscanner結構體仍然可用才行.
 */
int
scanner_errposition(int location, core_yyscan_t yyscanner)
{
	int			pos;

if (location < 0)
		return 0;				/* no-op if location is unknown */

/* Convert byte offset to character number */
	pos = pg_mbstrlen_with_len(yyextra->scanbuf, location) + 1;
	/* And pass it to the ereport mechanism */
	return errposition(pos);
}

/*
 * scanner_yyerror
 *		Report a lexer or grammar error.
 * 		報告詞法或語法錯誤.
 *
 * The message's cursor position is whatever YYLLOC was last set to,
 * ie, the start of the current token if called within yylex(), or the
 * most recently lexed token if called from the grammar.
 * This is OK for syntax error messages from the Bison parser, because Bison
 * parsers report error as soon as the first unparsable token is reached.
 * Beware of using yyerror for other purposes, as the cursor position might
 * be misleading!
 * 該訊息遊標的位置在於YYLLOC最後設定的地方,比如如果在yylex()中則是當前的token開始位置,
 *   或者如果是grammer呼叫則為最近一次的詞法token.
 * 在Bison解析器中丟擲語法錯誤是沒有問題的,因為Bison及誒吸氣在遇到第一個無法解析的token時就會報錯.
 * 注意:如果處於其他目的使用yyerror,這時候遊標的位置可能會出現誤導.
 */
void
scanner_yyerror(const char *message, core_yyscan_t yyscanner)
{
	const char *loc = yyextra->scanbuf + *yylloc;

if (*loc == YY_END_OF_BUFFER_CHAR)
	{
		ereport(ERROR,
				(errcode(ERRCODE_SYNTAX_ERROR),
		/* translator: %s is typically the translation of "syntax error" */
				 errmsg("%s at end of input", _(message)),
				 lexer_errposition()));
	}
	else
	{
		ereport(ERROR,
				(errcode(ERRCODE_SYNTAX_ERROR),
		/* translator: first %s is typically the translation of "syntax error" */
				 errmsg("%s at or near \"%s\"", _(message), loc),
				 lexer_errposition()));
	}
}

/*
 * Called before any actual parsing is done
 * 初始化掃描器,在實際解析完成前呼叫
 */
core_yyscan_t
scanner_init(const char *str,
			 core_yy_extra_type *yyext,
			 const ScanKeyword *keywords,
			 int num_keywords)
{
	Size		slen = strlen(str);
	yyscan_t	scanner;

if (yylex_init(&scanner) != 0)
		elog(ERROR, "yylex_init() failed: %m");

core_yyset_extra(yyext, scanner);

yyext->keywords = keywords;
	yyext->num_keywords = num_keywords;

yyext->backslash_quote = backslash_quote;
	yyext->escape_string_warning = escape_string_warning;
	yyext->standard_conforming_strings = standard_conforming_strings;

/*
	 * Make a scan buffer with special termination needed by flex.
	 */
	yyext->scanbuf = (char *) palloc(slen + 2);
	yyext->scanbuflen = slen;
	memcpy(yyext->scanbuf, str, slen);
	yyext->scanbuf[slen] = yyext->scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
	yy_scan_buffer(yyext->scanbuf, slen + 2, scanner);

/* initialize literal buffer to a reasonable but expansible size */
	yyext->literalalloc = 1024;
	yyext->literalbuf = (char *) palloc(yyext->literalalloc);
	yyext->literallen = 0;

return scanner;
}

/*
 * Called after parsing is done to clean up after scanner_init()
 * 在解析完成後呼叫,用於在scanner_init()之後進行清理.
 */
void
scanner_finish(core_yyscan_t yyscanner)
{
	/*
	 * We don't bother to call yylex_destroy(), because all it would do is
	 * pfree a small amount of control storage.  It's cheaper to leak the
	 * storage until the parsing context is destroyed.  The amount of space
	 * involved is usually negligible compared to the output parse tree
	 * anyway.
	 * 不需要呼叫yylex_destroy(),因為所有需要做的事情只是釋放一小塊控制記憶體而已.
	 * 在解析上下文被銷燬前,保留這部分記憶體成本會更低.
	 * 無論如何,與輸出解析樹相比,所涉及到的空間大小通常可以忽略不計.
	 *
	 * We do bother to pfree the scanbuf and literal buffer, but only if they
	 * represent a nontrivial amount of space.  The 8K cutoff is arbitrary.
	 * 需要使用pfree釋放掃描快取和字面值快取,但前提是它們代表了一個不小的空間才需要.
	 * 8K這個數值其實是很隨意的.
	 */
	if (yyextra->scanbuflen >= 8192)
		pfree(yyextra->scanbuf);
	if (yyextra->literalalloc >= 8192)
		pfree(yyextra->literalbuf);
}

static void
addlit(char *ytext, int yleng, core_yyscan_t yyscanner)
{
	/* enlarge buffer if needed */
	//增大快取
	if ((yyextra->literallen + yleng) >= yyextra->literalalloc)
	{
		do
		{
			yyextra->literalalloc *= 2;
		} while ((yyextra->literallen + yleng) >= yyextra->literalalloc);
		yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
												yyextra->literalalloc);
	}
	/* append new data */
	//追加新資料
	memcpy(yyextra->literalbuf + yyextra->literallen, ytext, yleng);
	yyextra->literallen += yleng;
}

static void
addlitchar(unsigned char ychar, core_yyscan_t yyscanner)
{
	/* enlarge buffer if needed */
	if ((yyextra->literallen + 1) >= yyextra->literalalloc)
	{
		yyextra->literalalloc *= 2;
		yyextra->literalbuf = (char *) repalloc(yyextra->literalbuf,
												yyextra->literalalloc);
	}
	/* append new data */
	yyextra->literalbuf[yyextra->literallen] = ychar;
	yyextra->literallen += 1;
}

/*
 * Create a palloc'd copy of literalbuf, adding a trailing null.
 * 建立字面值快取的複製,在末尾增加null.
 */
static char *
litbufdup(core_yyscan_t yyscanner)
{
	int			llen = yyextra->literallen;
	char	   *new;

new = palloc(llen + 1);
	memcpy(new, yyextra->literalbuf, llen);
	new[llen] = '\0';
	return new;
}

static int
process_integer_literal(const char *token, YYSTYPE *lval)
{
	//處理整型字面值
	int			val;
	char	   *endptr;

errno = 0;
	val = strtoint(token, &endptr, 10);
	if (*endptr != '\0' || errno == ERANGE)
	{
		/* integer too large, treat it as a float */
		lval->str = pstrdup(token);
		return FCONST;
	}
	lval->ival = val;
	return ICONST;
}

static unsigned int
hexval(unsigned char c)
{
	//十六進位制
	if (c >= '0' && c <= '9')
		return c - '0';
	if (c >= 'a' && c <= 'f')
		return c - 'a' + 0xA;
	if (c >= 'A' && c <= 'F')
		return c - 'A' + 0xA;
	elog(ERROR, "invalid hexadecimal digit");
	return 0;					/* not reached */
}

static void
check_unicode_value(pg_wchar c, char *loc, core_yyscan_t yyscanner)
{
	if (GetDatabaseEncoding() == PG_UTF8)
		return;

if (c > 0x7F)
	{
		ADVANCE_YYLLOC(loc - yyextra->literalbuf + 3);	/* 3 for U&" */
		yyerror("Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8");
	}
}

static bool
is_utf16_surrogate_first(pg_wchar c)
{
	return (c >= 0xD800 && c <= 0xDBFF);
}

static bool
is_utf16_surrogate_second(pg_wchar c)
{
	return (c >= 0xDC00 && c <= 0xDFFF);
}

static pg_wchar
surrogate_pair_to_codepoint(pg_wchar first, pg_wchar second)
{
	return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}

static void
addunicode(pg_wchar c, core_yyscan_t yyscanner)
{
	char		buf[8];

if (c == 0 || c > 0x10FFFF)
		yyerror("invalid Unicode escape value");
	if (c > 0x7F)
	{
		if (GetDatabaseEncoding() != PG_UTF8)
			yyerror("Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8");
		yyextra->saw_non_ascii = true;
	}
	unicode_to_utf8(c, (unsigned char *) buf);
	addlit(buf, pg_mblen(buf), yyscanner);
}

/* is 'escape' acceptable as Unicode escape character (UESCAPE syntax) ? */
static bool
check_uescapechar(unsigned char escape)
{
	if (isxdigit(escape)
		|| escape == '+'
		|| escape == '\''
		|| escape == '"'
		|| scanner_isspace(escape))
	{
		return false;
	}
	else
		return true;
}

/* like litbufdup, but handle unicode escapes */
static char *
litbuf_udeescape(unsigned char escape, core_yyscan_t yyscanner)
{
	char	   *new;
	char	   *litbuf,
			   *in,
			   *out;
	pg_wchar	pair_first = 0;

/* Make literalbuf null-terminated to simplify the scanning loop */
	litbuf = yyextra->literalbuf;
	litbuf[yyextra->literallen] = '\0';

/*
	 * This relies on the subtle assumption that a UTF-8 expansion cannot be
	 * longer than its escaped representation.
	 */
	new = palloc(yyextra->literallen + 1);

in = litbuf;
	out = new;
	while (*in)
	{
		if (in[0] == escape)
		{
			if (in[1] == escape)
			{
				if (pair_first)
				{
					ADVANCE_YYLLOC(in - litbuf + 3);	/* 3 for U&" */
					yyerror("invalid Unicode surrogate pair");
				}
				*out++ = escape;
				in += 2;
			}
			else if (isxdigit((unsigned char) in[1]) &&
					 isxdigit((unsigned char) in[2]) &&
					 isxdigit((unsigned char) in[3]) &&
					 isxdigit((unsigned char) in[4]))
			{
				pg_wchar	unicode;

unicode = (hexval(in[1]) << 12) +
					(hexval(in[2]) << 8) +
					(hexval(in[3]) << 4) +
					hexval(in[4]);
				check_unicode_value(unicode, in, yyscanner);
				if (pair_first)
				{
					if (is_utf16_surrogate_second(unicode))
					{
						unicode = surrogate_pair_to_codepoint(pair_first, unicode);
						pair_first = 0;
					}
					else
					{
						ADVANCE_YYLLOC(in - litbuf + 3);		/* 3 for U&" */
						yyerror("invalid Unicode surrogate pair");
					}
				}
				else if (is_utf16_surrogate_second(unicode))
					yyerror("invalid Unicode surrogate pair");

if (is_utf16_surrogate_first(unicode))
					pair_first = unicode;
				else
				{
					unicode_to_utf8(unicode, (unsigned char *) out);
					out += pg_mblen(out);
				}
				in += 5;
			}
			else if (in[1] == '+' &&
					 isxdigit((unsigned char) in[2]) &&
					 isxdigit((unsigned char) in[3]) &&
					 isxdigit((unsigned char) in[4]) &&
					 isxdigit((unsigned char) in[5]) &&
					 isxdigit((unsigned char) in[6]) &&
					 isxdigit((unsigned char) in[7]))
			{
				pg_wchar	unicode;

unicode = (hexval(in[2]) << 20) +
					(hexval(in[3]) << 16) +
					(hexval(in[4]) << 12) +
					(hexval(in[5]) << 8) +
					(hexval(in[6]) << 4) +
					hexval(in[7]);
				check_unicode_value(unicode, in, yyscanner);
				if (pair_first)
				{
					if (is_utf16_surrogate_second(unicode))
					{
						unicode = surrogate_pair_to_codepoint(pair_first, unicode);
						pair_first = 0;
					}
					else
					{
						ADVANCE_YYLLOC(in - litbuf + 3);		/* 3 for U&" */
						yyerror("invalid Unicode surrogate pair");
					}
				}
				else if (is_utf16_surrogate_second(unicode))
					yyerror("invalid Unicode surrogate pair");

if (is_utf16_surrogate_first(unicode))
					pair_first = unicode;
				else
				{
					unicode_to_utf8(unicode, (unsigned char *) out);
					out += pg_mblen(out);
				}
				in += 8;
			}
			else
			{
				ADVANCE_YYLLOC(in - litbuf + 3);		/* 3 for U&" */
				yyerror("invalid Unicode escape value");
			}
		}
		else
		{
			if (pair_first)
			{
				ADVANCE_YYLLOC(in - litbuf + 3);		/* 3 for U&" */
				yyerror("invalid Unicode surrogate pair");
			}
			*out++ = *in++;
		}
	}

/* unfinished surrogate pair? */
	if (pair_first)
	{
		ADVANCE_YYLLOC(in - litbuf + 3);				/* 3 for U&" */
		yyerror("invalid Unicode surrogate pair");
	}

*out = '\0';

/*
	 * We could skip pg_verifymbstr if we didn't process any non-7-bit-ASCII
	 * codes; but it's probably not worth the trouble, since this isn't likely
	 * to be a performance-critical path.
	 */
	pg_verifymbstr(new, out - new, false);
	return new;
}

static unsigned char
unescape_single_char(unsigned char c, core_yyscan_t yyscanner)
{
	switch (c)
	{
		case 'b':
			return '\b';
		case 'f':
			return '\f';
		case 'n':
			return '\n';
		case 'r':
			return '\r';
		case 't':
			return '\t';
		default:
			/* check for backslash followed by non-7-bit-ASCII */
			if (c == '\0' || IS_HIGHBIT_SET(c))
				yyextra->saw_non_ascii = true;

return c;
	}
}

static void
check_string_escape_warning(unsigned char ychar, core_yyscan_t yyscanner)
{
	if (ychar == '\'')
	{
		if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
			ereport(WARNING,
					(errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
					 errmsg("nonstandard use of \\' in a string literal"),
					 errhint("Use '' to write quotes in strings, or use the escape string syntax (E'...')."),
					 lexer_errposition()));
		yyextra->warn_on_first_escape = false;	/* warn only once per string */
	}
	else if (ychar == '\\')
	{
		if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
			ereport(WARNING,
					(errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
					 errmsg("nonstandard use of \\\\ in a string literal"),
					 errhint("Use the escape string syntax for backslashes, e.g., E'\\\\'."),
					 lexer_errposition()));
		yyextra->warn_on_first_escape = false;	/* warn only once per string */
	}
	else
		check_escape_warning(yyscanner);
}

static void
check_escape_warning(core_yyscan_t yyscanner)
{
	if (yyextra->warn_on_first_escape && yyextra->escape_string_warning)
		ereport(WARNING,
				(errcode(ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER),
				 errmsg("nonstandard use of escape in a string literal"),
		errhint("Use the escape string syntax for escapes, e.g., E'\\r\\n'."),
				 lexer_errposition()));
	yyextra->warn_on_first_escape = false;		/* warn only once per string */
}

/*
 * Interface functions to make flex use palloc() instead of malloc().
 * It'd be better to make these static, but flex insists otherwise.
 */

void *
core_yyalloc(yy_size_t bytes, core_yyscan_t yyscanner)
{
	return palloc(bytes);
}

void *
core_yyrealloc(void *ptr, yy_size_t bytes, core_yyscan_t yyscanner)
{
	if (ptr)
		return repalloc(ptr, bytes);
	else
		return palloc(bytes);
}

void
core_yyfree(void *ptr, core_yyscan_t yyscanner)
{
	if (ptr)
		pfree(ptr);
}

```

### 二、參考資料
[Flex&Bison](http://web.iitd.ac.in/~sumeet/flex__bison.pdf)

PostgreSQL 原始碼解讀（171）- 查詢#91(PG中的詞法定義：scanner.l)#4

相關文章