diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 95bc436c..cd2ef210 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -76,8 +76,8 @@ def _get_insert_token(token): tidx, token = get_next_comment(idx=tidx) def process(self, stmt): - [self.process(sgroup) for sgroup in stmt.get_sublists()] StripCommentsFilter._process(stmt) + [self.process(sgroup) for sgroup in stmt.get_sublists()] return stmt diff --git a/sqlparse/filters/output.py b/sqlparse/filters/output.py index d4e20f29..db062fac 100644 --- a/sqlparse/filters/output.py +++ b/sqlparse/filters/output.py @@ -65,7 +65,7 @@ def _process(self, stream, varname, has_nl): # Escape backslashes before quotes so a backslash preceding a # quote cannot break out of the generated string literal # (GHSA-3496-9g83-7v6x). - else: + elif not token.is_group: token.value = token.value.replace('\\', '\\\\').replace("'", "\\'") # Put the token @@ -116,7 +116,7 @@ def _process(self, stream, varname, has_nl): # Escape backslashes before quotes so a backslash preceding a # quote cannot break out of the generated string literal # (GHSA-3496-9g83-7v6x). - else: + elif not token.is_group: token.value = token.value.replace('\\', '\\\\').replace('"', '\\"') # Put the token diff --git a/sqlparse/sql.py b/sqlparse/sql.py index ec44a6da..ce05ab5c 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -39,35 +39,17 @@ def get_alias(self): return self._get_first_name(reverse=True) -class Token: - """Base class for all other classes in this module. +class TokenBase: + """Base class for ``Token`` and ``TokenList``. - It represents a single token and has two instance attributes: - ``value`` is the unchanged value of the token and ``ttype`` is - the type of the token. + It has a single instance attribute, ``parent``, which if not ``None`` + represents the ``TokenList`` that contains this token. """ - __slots__ = ( - 'is_group', - 'is_keyword', - 'is_newline', - 'is_whitespace', - 'normalized', - 'parent', - 'ttype', - 'value', - ) + __slots__ = 'parent' - def __init__(self, ttype, value): - value = str(value) - self.value = value - self.ttype = ttype + def __init__(self): self.parent = None - self.is_group = False - self.is_keyword = ttype in T.Keyword - self.is_whitespace = self.ttype in T.Whitespace - self.is_newline = self.ttype in T.Newline - self.normalized = value.upper() if self.is_keyword else value def __str__(self): return self.value @@ -84,19 +66,12 @@ def __repr__(self): return "<{cls} {q}{value}{q} at 0x{id:2X}>".format( id=id(self), **locals()) - def _get_repr_name(self): - return str(self.ttype).split('.')[-1] - def _get_repr_value(self): raw = str(self) if len(raw) > 7: raw = raw[:6] + '...' return re.sub(r'\s+', ' ', raw) - def flatten(self): - """Resolve subgroups.""" - yield self - def match(self, ttype, values, regex=False): """Checks whether the token matches the given arguments. @@ -155,24 +130,76 @@ def has_ancestor(self, other): return False -class TokenList(Token): +class Token(TokenBase): + """"A single token. + + It has several additional instance attributes: + ``value`` is the unchanged value of the token + ``ttype`` is the type of the token + ``normalized`` is the value of the token, converted to uppercase if it + is a keyword + ``is_keyword`` is a boolean indicating if the token is a keyword + ``is_whitespace`` is a boolean indicating if the token is whitespace + ``is_newline`` is a boolean indicating if the token is a newline + character + """ + __slots__ = ( + 'is_keyword', + 'is_newline', + 'is_whitespace', + 'normalized', + 'ttype', + 'value', + ) + + is_group = False + + def __init__(self, ttype, value): + super().__init__() + value = str(value) + self.value = value + self.ttype = ttype + self.is_keyword = ttype in T.Keyword + self.is_whitespace = ttype in T.Whitespace + self.is_newline = self.ttype in T.Newline + self.normalized = value.upper() if self.is_keyword else value + + def _get_repr_name(self): + return str(self.ttype).split('.')[-1] + + def flatten(self): + """Resolve subgroups.""" + yield self + + +class TokenList(TokenBase): """A group of tokens. - It has an additional instance attribute ``tokens`` which holds a + It has an additional instance attribute, ``tokens``, which holds a list of child-tokens. """ __slots__ = 'tokens' + is_group = True + ttype = None + is_keyword = False + is_whitespace = False + is_newline = False + def __init__(self, tokens=None): + super().__init__() self.tokens = tokens or [] [setattr(token, 'parent', self) for token in self.tokens] - super().__init__(None, ''.join(token.value for token in self.tokens)) - self.is_group = True - def __str__(self): + @property + def value(self): return ''.join(token.value for token in self.flatten()) + @property + def normalized(self): + return self.value + # weird bug # def __len__(self): # return len(self.tokens) @@ -330,7 +357,6 @@ def group_tokens(self, grp_cls, start, end, include_end=True, grp = start grp.tokens.extend(subtokens) del self.tokens[start_idx + 1:end_idx] - grp.value += ''.join(token.value for token in subtokens) else: subtokens = self.tokens[start_idx:end_idx] grp = grp_cls(subtokens)