Commit 018c61b4 authored by Sven Brauch's avatar Sven Brauch

Work around new behaviour in Python 3.4.3.

Python 3.4.3 effectively reverted most of the changes
to the AST range handling introduced into 3.4 by me.
Thus, this commit introduces a small lexer-like tool
which manually updates the ranges of all affected nodes.

All tests pass, but there will certainly be a few issues left.
parent acf8dbfb
......@@ -256,7 +256,15 @@ void PyDUChainTest::testCrashes_data() {
QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass";
QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)";
QTest::newRow("return_outside_function") << "return 3";
QTest::newRow("attrib") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))";
QTest::newRow("attrib2") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}";
QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()";
QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n"
".format(type(meth))) from None";
QTest::newRow("newline_attrib") << "some_instance \\\n"
". attr1 \\\n"
".funcfunc(argarg, arg2arg) \\\n"
".foo";
QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n"
"in ambiguous_nucleotide_values.iteritems() \\\n"
"if set([codon[0] for codon in codons]).issuperset(set(meanings)))";
......@@ -692,6 +700,8 @@ void PyDUChainTest::testRanges_data()
QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" );
QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" );
QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" );
QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" <<
"25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr");
QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" );
QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" );
......
......@@ -89,9 +89,8 @@ ClassDefinitionAst::ClassDefinitionAst(Ast* parent): StatementAst(parent, Ast::C
}
CodeAst::CodeAst() : name(0)
CodeAst::CodeAst() : Ast(nullptr, Ast::CodeAstType), name(0)
{
astType = Ast::CodeAstType;
}
CodeAst::~CodeAst()
......
......@@ -73,6 +73,7 @@ class KDEVPYTHONPARSER_EXPORT Ast
public:
enum AstType
{
StatementAstType,
FunctionDefinitionAstType,
AssignmentAstType,
PassAstType,
......@@ -97,8 +98,7 @@ public:
ContinueAstType,
AssertionAstType,
AugmentedAssignmentAstType,
CodeAstType,
StatementAstType,
LastStatementType,
ExpressionAstType, // everything below is an expression
NameAstType,
NameConstantAstType,
......@@ -133,6 +133,7 @@ public:
IndexAstType,
LastExpressionType, // keep this at the end of the expr ast list
CodeAstType,
ExceptionHandlerAstType,
AliasAstType, // for imports
IdentifierAstType,
......@@ -185,7 +186,7 @@ public:
Ast(Ast* parent, AstType type);
Ast();
Ast* parent;
Ast* parent = nullptr;
AstType astType;
bool isExpression() const {
......@@ -201,11 +202,19 @@ public:
bool appearsBefore(const Ast* other) {
return startLine < other->startLine || ( startLine == other->startLine && startCol < other->startCol );
};
}
const KDevelop::SimpleRange range() const {
return KDevelop::SimpleRange(startLine, startCol, endLine, endCol);
};
const KTextEditor::Cursor start() const {
return {startLine, startCol};
}
const KTextEditor::Cursor end() const {
return {endLine, endCol};
}
int startCol;
int startLine;
......
......@@ -88,6 +88,109 @@ public:
cutDefinitionPreamble(node->name, "class");
AstDefaultVisitor::visitClassDefinition(node);
};
virtual void visitAttribute(AttributeAst* node) {
/**
* Work around the weird way to count columns in Python's AST module.
* This takes an expression and finds the dots (attribute access tokens),
* then associates the right ranges obtained by their positions with the
* corresponding AST nodes (through recursive calls of this function).
*/
// an attribute access was already found in a previous call, associate it with this node?
if ( ! dots.isEmpty() && node->start() == attributeStart ) {
node->startLine = dots.last().line();
node->startCol = dots.last().column();
node->endLine = node->startLine;
node->endCol = node->startCol + node->attribute->value.length() - 1;
node->attribute->copyRange(node);
dots.pop_back();
visitNode(node->value);
return;
}
// find where the statement starts
Ast* parent = node;
while ( parent->astType != Ast::CodeAstType && parent->parent->astType > Ast::LastStatementType ) {
parent = parent->parent;
}
Ast* statement = parent->parent ? parent->parent : parent;
const QVector<QChar> opening = {'(', '[', '{', '"', '\''};
const QVector<QChar> closing = {')', ']', '}', '"', '\''};
QStack<QChar> blocks;
auto lineno = statement->startLine;
auto line = lines.at(lineno);
int offset = qMax(0, statement->startCol);
bool atDot = false;
bool atContinuationCharacter = false;
int inRightBlock = (lineno < node->startLine || (lineno == node->startLine && offset <= node->startCol)) ? -1 : 0;
// while the statement lasts, step through it char by char to find attribute access tokens
while ( offset < line.size() ) {
// for an expression like [foo.bar, a.b].c(xy.z), are we in the right block?
if ( lineno == node->startLine && offset == node->startCol ) {
inRightBlock = blocks.size();
}
const auto& c = line.at(offset);
// continue to next line if applicable
// TODO escaping
if ( c == '\\' ) {
atContinuationCharacter = true;
}
else if ( ! c.isSpace() ) {
atContinuationCharacter = false;
}
// does this token appear after the node's start position? if not, ignore it
bool later = (lineno > node->startLine) || (lineno == node->startLine && offset >= node->startCol);
if ( atDot && ! c.isSpace() && later ) {
dots.append({lineno, offset});
atDot = false;
}
// handle parentheses
if ( opening.contains(c) ) {
blocks.push(closing.at(opening.indexOf(c)));
}
if ( blocks.isEmpty() && closing.contains(c) && ! dots.isEmpty() ) {
break;
}
if ( ! blocks.isEmpty() && blocks.top() == c ) {
blocks.pop();
}
// if this is a dot and we're in the right block, save it
// saving is deferred until the next non-space character to obtain the correct
// start offset
if ( c == '.' && blocks.size() == inRightBlock ) {
atDot = true;
}
offset++;
// if
// * we're in a block (parentheses, etc)
// * we're not yet at the end of the statement
// * there's a continuation character
// and the rest of the line is only spaces, continue to the next non-empty line
if ( line.mid(offset).trimmed().isEmpty() && (atContinuationCharacter || ! blocks.isEmpty() || lineno < statement->endLine ) ) {
do {
lineno++;
if ( lines.size() <= lineno ) {
qDebug() << "huh?";
// probably not even possible to reach here
return;
}
line = lines.at(lineno);
} while ( line.isEmpty() );
offset = 0;
atContinuationCharacter = false;
}
if ( blocks.size() < inRightBlock ) {
// once out of the right block, can't get back in
inRightBlock = -1;
}
}
attributeStart = node->start();
visitNode(node);
visitNode(node->attribute);
};
// alias for imports (import foo as bar, baz as bang)
// no strings, brackets, or whatever are allowed here, so the "parser"
// can be very straightforward.
......@@ -113,6 +216,8 @@ public:
}
private:
const QStringList lines;
QVector<KTextEditor::Cursor> dots;
KTextEditor::Cursor attributeStart;
// skip the decorators and the "def" at the beginning
......@@ -133,14 +238,16 @@ private:
// cut away decorators
while ( currentLine < lines.size() ) {
if ( ! lines.at(currentLine).trimmed().startsWith('@') ) {
if ( lines.at(currentLine).trimmed().startsWith(defKeyword) ) {
// it's not a decorator, so stop skipping lines.
break;
}
currentLine += 1;
}
qDebug() << "FIX:" << fixNode->range();
fixNode->startLine = currentLine;
fixNode->endLine = currentLine;
qDebug() << "FIXED:" << fixNode->range() << fixNode->astType;
// cut away the "def" / "class"
int currentColumn = -1;
......
......@@ -202,6 +202,7 @@ void PyAstTest::testExpressions_data()
QTest::newRow("name") << "foo";
QTest::newRow("call") << "foo()";
QTest::newRow("attribute") << "foo.bar";
QTest::newRow("attribute_nontoplevel") << "while True: foo.bar";
QTest::newRow("subscript") << "foo[3]";
QTest::newRow("starred") << "*[1, 2, 3 ,4]";
QTest::newRow("list") << "[]";
......
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment