Commit 05e95cac authored by Francis Herne's avatar Francis Herne
Browse files

Initial support for CPython 3.9

This reformats our AST to match that introduced upstream by
 https://bugs.python.org/issue34822 and adjusts the conversion from
 older Python versions' AST to match.

ExtendedSliceAst is removed, in favour of a TupleAst containing
 SliceAsts and/or other expressions.

IndexAst is removed, instead non-slice index expressions are direct
 children of SubscriptAst.

SliceAst and EllipsisAst are now subclasses of ExpressionAst. In the
 latter case I don't understand how it wasn't broken before.

---

The stdlib docfiles are not updated.
 There aren't many changes to builtins in 3.9, but we do need a system to
 re-generate these - perhaps based on Typeshed.

In CPython 3.8 and 3.9 there are several changes to the parser that
 could be useful to us -- particularly end_lineno and end_col_offset.
 This patch doesn't attempt to take advantage of them.

BUG: 419290
parent 898810e1
......@@ -31,7 +31,7 @@ add_definitions( -DTRANSLATION_DOMAIN=\"kdevpython\" )
# CMake looks for exactly the specified version first and ignores newer versions.
# To avoid that, start looking for the newest supported version and work down.
set(Python_ADDITIONAL_VERSIONS 3.8 3.7 3.6 3.5 3.4)
set(Python_ADDITIONAL_VERSIONS 3.9 3.8 3.7 3.6 3.5 3.4)
foreach(_PYTHON_V ${Python_ADDITIONAL_VERSIONS})
find_package(PythonInterp ${_PYTHON_V})
if ( PYTHONINTERP_FOUND )
......@@ -49,7 +49,7 @@ if ( PYTHONINTERP_FOUND AND PYTHON_VERSION_STRING VERSION_GREATER "3.4" )
endif()
if ( NOT PYTHONLIBS_FOUND OR PYTHONLIBS_VERSION_STRING VERSION_LESS "3.4.3" )
message(FATAL_ERROR "Python >= 3.4.3 but < 3.9 with --enable-shared is required to build kdev-python")
message(FATAL_ERROR "Python >= 3.4.3 but < 3.10 with --enable-shared is required to build kdev-python")
endif()
configure_file(kdevpythonversion.h.cmake "${CMAKE_CURRENT_BINARY_DIR}/kdevpythonversion.h" @ONLY)
......
......@@ -1139,9 +1139,9 @@ void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const Declar
}
auto map = MapType::Ptr::dynamicCast(list);
if ( map ) {
if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) {
if ( subscript->slice && subscript->slice->astType != Ast::SliceAstType) {
ExpressionVisitor keyVisitor(currentContext());
keyVisitor.visitNode(static_cast<IndexAst*>(subscript->slice)->value);
keyVisitor.visitNode(subscript->slice);
AbstractType::Ptr key = keyVisitor.lastType();
if ( key ) {
map->addKeyType<Python::UnsureType>(key);
......
......@@ -365,7 +365,7 @@ void ExpressionVisitor::visitSubscript(SubscriptAst* node)
continue;
}
}
if ( (node->slice->astType != Ast::IndexAstType) && type.cast<ListType>() ) {
if ( (node->slice->astType == Ast::SliceAstType) && type.cast<ListType>() ) {
if ( type.cast<MapType>() ) {
continue; // Can't slice dicts.
}
......@@ -373,7 +373,7 @@ void ExpressionVisitor::visitSubscript(SubscriptAst* node)
result = Helper::mergeTypes(result, type);
}
else if ( const auto& indexed = type.cast<IndexedContainer>() ) {
long sliceIndex = integerValue(static_cast<IndexAst*>(node->slice)->value, indexed->typesCount());
long sliceIndex = integerValue(node->slice, indexed->typesCount());
if ( 0 <= sliceIndex && sliceIndex < indexed->typesCount() ) {
result = Helper::mergeTypes(result, indexed->typeAt(sliceIndex).abstractType());
continue;
......
......@@ -128,12 +128,7 @@ DictAst::DictAst(Ast* parent): ExpressionAst(parent, Ast::DictAstType)
}
IndexAst::IndexAst(Ast* parent): SliceAstBase(parent, Ast::IndexAstType), value(nullptr)
{
}
SliceAst::SliceAst(Ast* parent): SliceAstBase(parent, Ast::SliceAstType), lower(nullptr), upper(nullptr), step(nullptr)
SliceAst::SliceAst(Ast* parent): ExpressionAst(parent, Ast::SliceAstType), lower(nullptr), upper(nullptr), step(nullptr)
{
}
......@@ -143,7 +138,7 @@ DictionaryComprehensionAst::DictionaryComprehensionAst(Ast* parent): ExpressionA
}
EllipsisAst::EllipsisAst(Ast* parent): SliceAstBase(parent, Ast::EllipsisAstType)
EllipsisAst::EllipsisAst(Ast* parent): ExpressionAst(parent, Ast::EllipsisAstType)
{
}
......@@ -173,11 +168,6 @@ YieldFromAst::YieldFromAst(Ast* parent) : ExpressionAst(parent, Ast::YieldFromAs
}
ExtendedSliceAst::ExtendedSliceAst(Ast* parent): SliceAstBase(parent, Ast::ExtendedSliceAstType)
{
}
ForAst::ForAst(Ast* parent): StatementAst(parent, Ast::ForAstType), target(nullptr), iterator(nullptr)
{
......@@ -288,11 +278,6 @@ SetComprehensionAst::SetComprehensionAst(Ast* parent): ExpressionAst(parent, Ast
}
SliceAstBase::SliceAstBase(Ast* parent, AstType type): Ast(parent, type)
{
}
StatementAst::StatementAst(Ast* parent, AstType type): Ast(parent, type)
{
......
......@@ -55,7 +55,6 @@ namespace Python {
class ExceptionHandlerAst;
class AliasAst;
class ComprehensionAst;
class SliceAstBase;
class SliceAst;
}
......@@ -102,7 +101,6 @@ public:
NameConstantAstType,
CallAstType,
AttributeAstType,
ExtendedSliceAstType,
DictionaryComprehensionAstType,
BooleanOperationAstType,
BinaryOperationAstType,
......@@ -127,10 +125,8 @@ public:
TupleAstType,
YieldFromAstType,
ComprehensionAstType,
SliceAstType,
EllipsisAstType,
IndexAstType,
AssignmentExpressionAstType,
LastExpressionType, // keep this at the end of the expr ast list
......@@ -671,7 +667,7 @@ class KDEVPYTHONPARSER_EXPORT SubscriptAst : public ExpressionAst {
public:
SubscriptAst(Ast* parent);
ExpressionAst* value;
SliceAstBase* slice;
ExpressionAst* slice;
ExpressionAst::Context context;
};
......@@ -696,18 +692,12 @@ public:
ExpressionAst::Context context;
};
/** Slice classes **/
class KDEVPYTHONPARSER_EXPORT SliceAstBase : public Ast {
public:
SliceAstBase(Ast* parent, AstType type);
};
class KDEVPYTHONPARSER_EXPORT EllipsisAst : public SliceAstBase {
class KDEVPYTHONPARSER_EXPORT EllipsisAst : public ExpressionAst {
public:
EllipsisAst(Ast* parent);
};
class KDEVPYTHONPARSER_EXPORT SliceAst : public SliceAstBase {
class KDEVPYTHONPARSER_EXPORT SliceAst : public ExpressionAst {
public:
SliceAst(Ast* parent);
ExpressionAst* lower;
......@@ -715,17 +705,6 @@ public:
ExpressionAst* step;
};
class KDEVPYTHONPARSER_EXPORT ExtendedSliceAst : public SliceAstBase {
public:
ExtendedSliceAst(Ast* parent);
QList<SliceAst*> dims;
};
class KDEVPYTHONPARSER_EXPORT IndexAst : public SliceAstBase {
public:
IndexAst(Ast* parent);
ExpressionAst* value;
};
/** Independent classes **/
......
......@@ -113,11 +113,6 @@ void AstDefaultVisitor::visitDelete(DeleteAst* node)
visitNodeList(node->targets);
}
void AstDefaultVisitor::visitExtendedSlice(ExtendedSliceAst* node)
{
visitNodeList(node->dims);
}
void AstDefaultVisitor::visitFor(ForAst* node)
{
visitNode(node->target);
......@@ -157,11 +152,6 @@ void AstDefaultVisitor::visitImportFrom(ImportFromAst* node)
visitIdentifier(node->module);
}
void AstDefaultVisitor::visitIndex(IndexAst* node)
{
visitNode(node->value);
}
void AstDefaultVisitor::visitLambda(LambdaAst* node)
{
visitNode(node->arguments);
......
......@@ -93,8 +93,6 @@ public:
void visitTuple(TupleAst* node) override;
void visitEllipsis(EllipsisAst* node) override;
void visitSlice(SliceAst* node) override;
void visitExtendedSlice(ExtendedSliceAst* node) override;
void visitIndex(IndexAst* node) override;
void visitArguments(ArgumentsAst* node) override;
void visitArg(ArgAst* node) override;
void visitKeyword(KeywordAst* node) override;
......@@ -167,8 +165,6 @@ public:
void visitTuple(TupleAst* node) override { AstDefaultVisitor::visitTuple(node); delete node; }
void visitEllipsis(EllipsisAst* node) override { AstDefaultVisitor::visitEllipsis(node); delete node; }
void visitSlice(SliceAst* node) override { AstDefaultVisitor::visitSlice(node); delete node; }
void visitExtendedSlice(ExtendedSliceAst* node) override { AstDefaultVisitor::visitExtendedSlice(node); delete node; }
void visitIndex(IndexAst* node) override { AstDefaultVisitor::visitIndex(node); delete node; }
void visitArguments(ArgumentsAst* node) override { AstDefaultVisitor::visitArguments(node); delete node; }
void visitArg(ArgAst* node) override { AstDefaultVisitor::visitArg(node); delete node; }
void visitKeyword(KeywordAst* node) override { AstDefaultVisitor::visitKeyword(node); delete node; }
......
......@@ -90,8 +90,6 @@ void AstVisitor::visitNode(Ast* node)
case Ast::TupleAstType: this->visitTuple(static_cast<TupleAst*>(node)); break;
case Ast::EllipsisAstType: this->visitEllipsis(static_cast<EllipsisAst*>(node)); break;
case Ast::SliceAstType: this->visitSlice(static_cast<SliceAst*>(node)); break;
case Ast::ExtendedSliceAstType: this->visitExtendedSlice(static_cast<ExtendedSliceAst*>(node)); break;
case Ast::IndexAstType: this->visitIndex(static_cast<IndexAst*>(node)); break;
case Ast::ArgumentsAstType: this->visitArguments(static_cast<ArgumentsAst*>(node)); break;
case Ast::KeywordAstType: this->visitKeyword(static_cast<KeywordAst*>(node)); break;
case Ast::ArgAstType: this->visitArg(static_cast<ArgAst*>(node)); break;
......
......@@ -106,8 +106,6 @@ public:
virtual void visitTuple(TupleAst* node) { Q_UNUSED(node); };
virtual void visitEllipsis(EllipsisAst* node) { Q_UNUSED(node); };
virtual void visitSlice(SliceAst* node) { Q_UNUSED(node); };
virtual void visitExtendedSlice(ExtendedSliceAst* node) { Q_UNUSED(node); };
virtual void visitIndex(IndexAst* node) { Q_UNUSED(node); };
virtual void visitArguments(ArgumentsAst* node) { Q_UNUSED(node); };
virtual void visitKeyword(KeywordAst* node) { Q_UNUSED(node); };
virtual void visitArg(ArgAst* node) { Q_UNUSED(node); };
......
......@@ -313,6 +313,8 @@ for index, lines in sorted(results.items()):
func = func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines).replace('%{APPENDIX}', appendix)
else:
func = simple_func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines)
if index == '_slice':
func = "#if PYTHON_VERSION < QT_VERSION_CHECK(3, 9, 0)\n" + func + "\n#endif\n"
print(func)
print('''};
......
......@@ -426,6 +426,16 @@ PyObject *value = node->v.Constant.value;if (value == Py_None) { NameConstant
result = v;
break;
}
#endif
#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 9, 0)
case Slice_kind: {
SliceAst* v = new SliceAst(parent());
nodeStack.push(v); v->lower = static_cast<ExpressionAst*>(visitNode(node->v.Slice.lower)); nodeStack.pop();
nodeStack.push(v); v->upper = static_cast<ExpressionAst*>(visitNode(node->v.Slice.upper)); nodeStack.pop();
nodeStack.push(v); v->step = static_cast<ExpressionAst*>(visitNode(node->v.Slice.step)); nodeStack.pop();
result = v;
break;
}
#endif
default:
qWarning() << "Unsupported _expr AST type: " << node->kind;
......@@ -480,12 +490,14 @@ PyObject *value = node->v.Constant.value;if (value == Py_None) { NameConstant
return v;
}
#if PYTHON_VERSION < QT_VERSION_CHECK(3, 9, 0)
Ast* visitNode(_slice* node) {
if ( ! node ) return nullptr;
bool ranges_copied = false; Q_UNUSED(ranges_copied);
Ast* result = nullptr;
switch ( node->kind ) {
#if PYTHON_VERSION < QT_VERSION_CHECK(3, 9, 0)
case Slice_kind: {
SliceAst* v = new SliceAst(parent());
nodeStack.push(v); v->lower = static_cast<ExpressionAst*>(visitNode(node->v.Slice.lower)); nodeStack.pop();
......@@ -494,18 +506,21 @@ PyObject *value = node->v.Constant.value;if (value == Py_None) { NameConstant
result = v;
break;
}
#endif
#if PYTHON_VERSION < QT_VERSION_CHECK(3, 9, 0)
case ExtSlice_kind: {
ExtendedSliceAst* v = new ExtendedSliceAst(parent());
nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop();
TupleAst* v = new TupleAst(parent());
nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.ExtSlice.dims); nodeStack.pop();
result = v;
break;
}
#endif
#if PYTHON_VERSION < QT_VERSION_CHECK(3, 9, 0)
case Index_kind: {
IndexAst* v = new IndexAst(parent());
nodeStack.push(v); v->value = static_cast<ExpressionAst*>(visitNode(node->v.Index.value)); nodeStack.pop();
result = v;
return visitNode(node->v.Index.value);
break;
}
#endif
default:
qWarning() << "Unsupported _slice AST type: " << node->kind;
Q_ASSERT(false);
......@@ -537,6 +552,8 @@ PyObject *value = node->v.Constant.value;if (value == Py_None) { NameConstant
return result;
}
#endif
Ast* visitNode(_stmt* node) {
if ( ! node ) return nullptr;
......
......@@ -129,10 +129,11 @@ else {
Q_ASSERT(false);
};;
RULE_FOR _expr;KIND NamedExpr_kind;ACTIONS create|AssignmentExpressionAst set|target->ExpressionAst,target set|value->ExpressionAst,value;SINCE 3.8;;
RULE_FOR _expr;KIND Slice_kind;ACTIONS create|SliceAst set|lower->ExpressionAst,lower set|upper->ExpressionAst,upper set|step->ExpressionAst,step;SINCE 3.9;;
RULE_FOR _slice;KIND Slice_kind;ACTIONS create|SliceAst set|lower->ExpressionAst,lower set|upper->ExpressionAst,upper set|step->ExpressionAst,step;;
RULE_FOR _slice;KIND ExtSlice_kind;ACTIONS create|ExtendedSliceAst set|dims=>SliceAst,dims;;
RULE_FOR _slice;KIND Index_kind;ACTIONS create|IndexAst set|value->ExpressionAst,value;;
RULE_FOR _slice;KIND Slice_kind;ACTIONS create|SliceAst set|lower->ExpressionAst,lower set|upper->ExpressionAst,upper set|step->ExpressionAst,step;BEFORE 3.9;;
RULE_FOR _slice;KIND ExtSlice_kind;ACTIONS create|TupleAst set|elements=>ExpressionAst,dims;BEFORE 3.9;;
RULE_FOR _slice;KIND Index_kind;ACTIONS;BEFORE 3.9;CODE return visitNode(node->v.Index.value);;
RULE_FOR _comprehension;KIND any;ACTIONS create|ComprehensionAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|conditions=>ExpressionAst,ifs;;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment