diff --git a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java index 45580cb5e..b4e63851d 100644 --- a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java +++ b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java @@ -77,6 +77,14 @@ public P withBackslashEscapeCharacter(boolean allowBackslashEscapeCharacter) { return withFeature(Feature.allowBackslashEscapeCharacter, allowBackslashEscapeCharacter); } + public P withHashLineComments() { + return withFeature(Feature.allowHashLineComments, true); + } + + public P withHashLineComments(boolean allowHashLineComments) { + return withFeature(Feature.allowHashLineComments, allowHashLineComments); + } + public P withUnparenthesizedSubSelects() { return withFeature(Feature.allowUnparenthesizedSubSelects, true); } diff --git a/src/main/java/net/sf/jsqlparser/parser/SimpleCharStream.java b/src/main/java/net/sf/jsqlparser/parser/SimpleCharStream.java index 0e633ebbb..ca81a8dd1 100644 --- a/src/main/java/net/sf/jsqlparser/parser/SimpleCharStream.java +++ b/src/main/java/net/sf/jsqlparser/parser/SimpleCharStream.java @@ -7,9 +7,18 @@ * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 * #L% */ -/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 8.0.0 */ +/* Generated By:JavaCC: Do not edit this line by hand. SimpleCharStream.java Version 8.0.0 */ +/* + * Warning: this file is maintained by hand on top of the JavaCC template. Among other manual + * adjustments it carries the in-buffer '#' rewrite in BeginToken(), which + * Feature.allowHashLineComments depends on. Never overwrite it with a freshly generated + * SimpleCharStream, that would silently drop the rewrite and break allowHashLineComments. + */ package net.sf.jsqlparser.parser; +import net.sf.jsqlparser.parser.feature.Feature; +import net.sf.jsqlparser.parser.feature.FeatureConfiguration; + /** * An implementation of interface CharStream, where the stream is assumed to contain only ASCII * characters (without unicode processing). @@ -42,6 +51,17 @@ public class SimpleCharStream { int available; int tokenBegin; + // MySQL `#` line comments (#2499): under the flag a token-start `#` is + // rewritten in the buffer to HASH_SUBSTITUTION, a character no other + // lexical rule starts with, so the dedicated HASH_LINE_COMMENT production + // wins the match for every `#` form (`# c`, `#c`, `#>`, ...). Rewriting + // the buffer itself (instead of synthesizing reads) keeps the matcher's + // backup / re-read arithmetic intact; the production's action restores + // the `#` in the token image. Wired (with the token manager's + // configuration) before parsing, null keeps this inert. + static final char HASH_SUBSTITUTION = '\u0001'; + FeatureConfiguration featureConfiguration; + /** * Constructor. */ @@ -159,6 +179,11 @@ public final char BeginToken() throws java.io.IOException { absoluteTokenBegin = totalCharsRead; + if (c == '#' && featureConfiguration != null + && featureConfiguration.getAsBoolean(Feature.allowHashLineComments)) { + buffer[bufpos] = HASH_SUBSTITUTION; + c = HASH_SUBSTITUTION; + } return c; } @@ -325,6 +350,23 @@ public void ReInit(Provider dstream) { * Get token literal value. */ public String GetImage() { + // restore the `#` rewritten to HASH_SUBSTITUTION: only the comment + // token's own window starts with the sentinel (token-start rewrites + // only), so this is exact and covers every consumer of GetImage. + // The wiring check keeps parses that never opted in on the original + // cost (the stream is only wired through the feature consumers / + // withConfiguration) + if (featureConfiguration == null) { + return doGetImage(); + } + String image = doGetImage(); + if (!image.isEmpty() && image.charAt(0) == HASH_SUBSTITUTION) { + image = "#" + image.substring(1); + } + return image; + } + + private String doGetImage() { if (bufpos >= tokenBegin) { return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); } else { diff --git a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java index 4c1b6dd51..5daa74eca 100644 --- a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java +++ b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java @@ -795,6 +795,12 @@ public enum Feature { */ allowBackslashEscapeCharacter(false), + /** + * allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary + * operator (#2507: PostgreSQL bitwise XOR / geometric intersection) + */ + allowHashLineComments(false), + /** * allows sub selects without parentheses, e.g. `select * from dual where 1 = select 1` */ diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index a4f155764..9ae702d48 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -91,10 +91,17 @@ public class CCJSqlParser extends AbstractJSqlParser { public CCJSqlParser withConfiguration(FeatureConfiguration configuration) { token_source.configuration = configuration; + jj_input_stream.featureConfiguration = configuration; return this; } public FeatureConfiguration getConfiguration() { + // wire the stream before parsing starts: every feature mutation goes + // through here (the consumers run pre-parse), default-off parsing + // never wires the stream and stays on the zero-cost path + if (jj_input_stream.featureConfiguration != token_source.configuration) { + jj_input_stream.featureConfiguration = token_source.configuration; + } return token_source.configuration; } @@ -1883,6 +1890,19 @@ SPECIAL_TOKEN: < LINE_COMMENT: ("--" | "//") (~["\r","\n"])*> } +// MySQL `#` line comment (#2499), flag-gated. Under +// Feature.allowHashLineComments the stream rewrites a token-start `#` to +// SimpleCharStream.HASH_SUBSTITUTION, a character no other rule starts with, +// so this production wins the match for every `#` form (`# c`, `#c`, `#>`, +// ...) without touching the identifier and JSON-operator lexing of the +// default mode. The action restores the `#` in the token image; unquoted +// identifiers end at their first `#` via the S_IDENTIFIER action, which +// re-lexes the remainder as this comment. +SPECIAL_TOKEN: +{ + < HASH_LINE_COMMENT: "\u0001" (~["\r","\n"])*> +} + // Nested block comments: /* ... /* ... */ ... */ // // Uses a nesting counter (commentNesting in TOKEN_MGR_DECLS) and @@ -1940,12 +1960,46 @@ TOKEN: | ()*) | "$" | ("$" ()*)> +{ + // MySQL `#` line comments (#2499): under the flag an unquoted identifier + // ends at its first `#`, the rest of the line becomes a comment via the + // stream-level substitution (real MySQL reads `42#24` as `42` plus + // comment too). Quoted identifiers and strings keep their `#`. + // the wiring check keeps the flag lookup off the hot path of parses + // that never opted in (the stream is only wired through the feature + // consumers / withConfiguration); getValue avoids the String-based + // getAsBoolean roundtrip + if (input_stream.featureConfiguration != null + && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) { + int hashIndex = matchedToken.image.indexOf('#'); + if (hashIndex > 0) { + input_stream.backup(matchedToken.image.length() - hashIndex); + matchedToken.image = matchedToken.image.substring(0, hashIndex); + } + } +} | <#LETTER: | | [ "#", "_" ] // Not SQL:2016 compliant! > | <#PART_LETTER_NO_DOLLAR: | | [ "#", "_" , "@" ] > | <#PART_LETTER: | | [ "$" , "#", "_" , "@" ] > | ()? > +{ + // same truncation as S_IDENTIFIER: `@@#name` ends at the `#` under + // allowHashLineComments, the rest of the line becomes the comment + // the wiring check keeps the flag lookup off the hot path of parses + // that never opted in (the stream is only wired through the feature + // consumers / withConfiguration); getValue avoids the String-based + // getAsBoolean roundtrip + if (input_stream.featureConfiguration != null + && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) { + int hashIndex = matchedToken.image.indexOf('#'); + if (hashIndex > 0) { + input_stream.backup(matchedToken.image.length() - hashIndex); + matchedToken.image = matchedToken.image.substring(0, hashIndex); + } + } +} // Unicode characters and categories are defined here: https://www.unicode.org/Public/UNIDATA/UnicodeData.txt // SQL:2016 states: diff --git a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java index dd9b55732..4a7258707 100644 --- a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java +++ b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java @@ -514,4 +514,54 @@ void testSingleStatementWithEmptyLines() throws JSQLParserException { + "def'\n" + "where id=?", true); } + + @Test + public void testHashLineCommentsFeature() throws Exception { + // with the flag: MySQL line comments, `#` to end of line, + // unconditional like MySQL itself (no blank needed, `42#24` is a + // comment there too); the statement continues on the next line + assertEquals("SELECT 42", CCJSqlParserUtil + .parse("SELECT 42 # 24", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("SELECT 1 # comment, 2", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("SELECT 1 #comment", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("SELECT 1 #!bang", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 42", CCJSqlParserUtil + .parse("SELECT 42#24", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT a", CCJSqlParserUtil + .parse("SELECT a#b FROM t", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1 FROM t", CCJSqlParserUtil + .parse("SELECT 1 # c\nFROM t", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("# leading\nSELECT 1", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("SELECT 1 #", p -> p.withHashLineComments(true)).toString()); + assertEquals("SELECT 1", CCJSqlParserUtil + .parse("SELECT 1 #\n", p -> p.withHashLineComments(true)).toString()); + // the `#>` family has no meaning in MySQL and comments out like any + // other `#`; quoted forms keep their `#` + assertEquals("SELECT data", CCJSqlParserUtil + .parse("SELECT data #> '{a}' FROM t", p -> p.withHashLineComments(true)) + .toString()); + TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT '#' FROM t", true, + p -> p.withHashLineComments(true)); + TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT \"a#b\" FROM t", true, + p -> p.withHashLineComments(true)); + // default (flag off): the #2507 binary operator, unchanged + assertEquals("SELECT 42 # 24", CCJSqlParserUtil.parse("SELECT 42 # 24").toString()); + assertEquals("SELECT 42#24", CCJSqlParserUtil.parse("SELECT 42#24").toString()); + } + + @Test + public void testHashLineCommentsMySQLStatementSemantics() throws Exception { + // `#temp` is a comment under the flag, so `SELECT #temp FROM t` + // loses the rest of the line and fails, exactly like MySQL; with the + // flag off the same input parses as the #2507 operator expression + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil + .parse("SELECT #temp FROM t", p -> p.withHashLineComments(true))); + assertEquals("SELECT 1 # comment", + CCJSqlParserUtil.parse("SELECT 1 # comment").toString()); + } } diff --git a/src/test/java/net/sf/jsqlparser/statement/select/SelectASTTest.java b/src/test/java/net/sf/jsqlparser/statement/select/SelectASTTest.java index 8c2a36261..173248762 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/SelectASTTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/SelectASTTest.java @@ -14,6 +14,7 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserDefaultVisitor; import net.sf.jsqlparser.parser.CCJSqlParserTreeConstants; +import net.sf.jsqlparser.parser.CCJSqlParser; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.parser.Node; import net.sf.jsqlparser.parser.Token; @@ -212,4 +213,21 @@ public void testSelectASTExtractWithCommentsIssue1580_2() throws JSQLParserExcep assertThat(root.jjtGetFirstToken().specialToken.image) .isEqualTo("/* I want this comment */\n"); } + + @Test + public void testSelectASTHashLineCommentImage() throws Exception { + // a `#` comment under allowHashLineComments is a normal special + // token carrying the original `# ...` text + CCJSqlParser parser = CCJSqlParserUtil.newParser("SELECT 1 # note\nFROM t"); + parser.withHashLineComments(true); + parser.Statement(); + List comments = new ArrayList<>(); + for (Token t = parser.getASTRoot().jjtGetFirstToken(); t.next != null; t = t.next) { + for (Token sp = t.specialToken; sp != null; sp = sp.specialToken) { + comments.add(sp); + } + } + + assertThat(comments).extracting(token -> token.image).containsExactly("# note"); + } }