Commit 983b111a authored by Nikita Sirgienko's avatar Nikita Sirgienko
Browse files

[Python] Massive refactoring

  - Remove communication via DBus, replaced by KProcess
  - Supports text result for a python expression with plot image result
  - Show numpy arrays in full form (because we had solved problem with showing big strings in the variable model)
  - Use Session::setVariableModel instead of handling variable model by self
  - Better interrupt
  - Use expression queue, model updating and expression finishing from Session
  - Remove unused PythonSession members
  - Some tests improvments

Closes T6113, T6114
Differential Revision: https://phabricator.kde.org/D21212
parent 1e144559
......@@ -22,7 +22,6 @@ target_link_libraries(cantor_pythonbackend
KF5::ConfigCore
KF5::ConfigGui
KF5::SyntaxHighlighting
Qt5::DBus
)
install(TARGETS cantor_pythonbackend DESTINATION ${KDE_INSTALL_TARGETS_DEFAULT_ARGS})
......
......@@ -67,9 +67,7 @@ void PythonCompletionObject::fetchCompletions()
"print('|'.join(rlcompleter.Completer(__dict__).global_matches('%1')+rlcompleter.Completer(__dict__).attr_matches('%1')))"
).arg(command());
m_expression = session()->evaluateExpression(expr, Cantor::Expression::FinishingBehavior::DoNotDelete, true);
// TODO: Python exec the expression before connect, so manualy run handler. Uncomment the connection after removing DBus
// connect(m_expression, &Cantor::Expression::statusChanged, this, &PythonCompletionObject::extractCompletions);
extractCompletions(m_expression->status());
connect(m_expression, &Cantor::Expression::statusChanged, this, &PythonCompletionObject::extractCompletions);
}
}
......@@ -97,9 +95,7 @@ void PythonCompletionObject::fetchIdentifierType()
const QString& expr = QString::fromLatin1("callable(%1)").arg(identifier());
m_expression = session()->evaluateExpression(expr, Cantor::Expression::FinishingBehavior::DoNotDelete, true);
// TODO: Python exec the expression before connect, so manualy run handler. Uncomment the connection after removing DBus
// connect(m_expression, &Cantor::Expression::statusChanged, this, &PythonCompletionObject::extractIdentifierType);
extractIdentifierType(m_expression->status());
connect(m_expression, &Cantor::Expression::statusChanged, this, &PythonCompletionObject::extractIdentifierType);
}
}
......
......@@ -52,9 +52,7 @@ void PythonExpression::evaluate()
m_tempFile = nullptr;
}
PythonSession* pythonSession = static_cast<PythonSession*>(session());
pythonSession->runExpression(this);
session()->enqueueExpression(this);
}
QString PythonExpression::internalCommand()
......@@ -104,21 +102,22 @@ QString PythonExpression::internalCommand()
void PythonExpression::parseOutput(QString output)
{
qDebug() << "output: " << output;
qDebug() << "expression output: " << output;
if(command().simplified().startsWith(QLatin1String("help("))){
setResult(new Cantor::HelpResult(output.remove(output.lastIndexOf(QLatin1String("None")), 4)));
} else {
if (!output.isEmpty())
setResult(new Cantor::TextResult(output));
addResult(new Cantor::TextResult(output));
}
setStatus(Cantor::Expression::Done);
if (m_tempFile == nullptr || result() != nullptr) // not plot expression
setStatus(Cantor::Expression::Done);
}
void PythonExpression::parseError(QString error)
{
qDebug() << "error: " << error;
qDebug() << "expression error: " << error;
setErrorMessage(error.replace(QLatin1String("\n"), QLatin1String("<br>")));
setStatus(Cantor::Expression::Error);
......@@ -126,7 +125,24 @@ void PythonExpression::parseError(QString error)
void PythonExpression::imageChanged()
{
addResult(new Cantor::ImageResult(QUrl::fromLocalFile(m_tempFile->fileName())));
if(m_tempFile->size() <= 0)
return;
Cantor::ImageResult* newResult = new Cantor::ImageResult(QUrl::fromLocalFile(m_tempFile->fileName()));
if (result() == nullptr)
setResult(newResult);
else
{
bool found = false;
for (int i = 0; i < results().size(); i++)
if (results()[i]->type() == newResult->type())
{
replaceResult(i, newResult);
found = true;
}
if (!found)
addResult(newResult);
}
setStatus(Done);
}
......
......@@ -146,24 +146,14 @@ void PythonServer::setFilePath(const QString& path)
QString PythonServer::variables(bool parseValue) const
{
// FIXME: This code allows get full form of numpy array, but for big arrays it's could cause performonce problems
// especially for displaying in variables panel
// So, uncomment this, when fix this problem
/*
PyRun_SimpleString(
"try: \n"
" import numpy \n"
" __cantor_numpy_internal__ = numpy.get_printoptions()['threshold'] \n"
" numpy.set_printoptions(threshold=100000000) \n"
"except ModuleNotFoundError: \n"
" pass \n"
"try: \n"
" import numpy \n"
" numpy.set_printoptions(threshold=__cantor_numpy_internal__) \n"
" del __cantor_numpy_internal__ \n"
"except ModuleNotFoundError: \n"
" pass \n"
*/
);
PyRun_SimpleString("__tmp_globals__ = globals()");
PyObject* globals = PyObject_GetAttrString(m_pModule,"__tmp_globals__");
......@@ -171,7 +161,6 @@ QString PythonServer::variables(bool parseValue) const
Py_ssize_t pos = 0;
QStringList vars;
const QChar sep(30); // INFORMATION SEPARATOR TWO
while (PyDict_Next(globals, &pos, &key, &value)) {
const QString& keyString = pyObjectToQString(key);
if (keyString.startsWith(QLatin1String("__")))
......@@ -196,10 +185,19 @@ QString PythonServer::variables(bool parseValue) const
valueString = pyObjectToQString(PyObject_Repr(value));
vars.append(keyString + QChar(31) + valueString);
vars.append(keyString + QChar(17) + valueString);
}
return vars.join(sep);
PyRun_SimpleString(
"try: \n"
" import numpy \n"
" numpy.set_printoptions(threshold=__cantor_numpy_internal__) \n"
" del __cantor_numpy_internal__ \n"
"except ModuleNotFoundError: \n"
" pass \n"
);
return vars.join(QChar(18))+QChar(18);
}
/*
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.
---
Copyright (C) 2015 Minh Ngo <minh@fedoraproject.org>
*/
#include <iostream>
#include <QApplication>
#include <QTimer>
#include <QChar>
#include <QByteArray>
#include "pythonserver.h"
const QChar recordSep(30);
const QChar unitSep(31);
const char messageEnd = 29;
PythonServer server;
QTimer inputTimer;
QMetaObject::Connection connection;
QString inputBuffer;
QLatin1String LOGIN("login");
QLatin1String EXIT("exit");
QLatin1String CODE("code");
QLatin1String FILEPATH("setFilePath");
QLatin1String MODEL("model");
void routeInput() {
QByteArray bytes;
char c;
while (std::cin.get(c))
{
if (messageEnd == c)
break;
else
bytes.append(c);
}
inputBuffer.append(QString::fromLocal8Bit(bytes));
if (inputBuffer.isEmpty())
return;
const QStringList& records = inputBuffer.split(recordSep);
inputBuffer.clear();
if (records.size() == 2)
{
if (records[0] == EXIT)
{
QObject::disconnect(connection);
QObject::connect(&inputTimer, &QTimer::timeout, QCoreApplication::instance(), &QCoreApplication::quit);
}
else if (records[0] == LOGIN)
{
server.login();
const QByteArray bytes = QString::fromLatin1("login done").toLocal8Bit();
//std::cout << bytes.data();
}
else if (records[0] == CODE)
{
server.runPythonCommand(records[1]);
const QString& result =
server.getOutput()
+ unitSep
+ server.getError()
+ QLatin1Char(messageEnd);
const QByteArray bytes = result.toLocal8Bit();
std::cout << bytes.data();
}
else if (records[0] == FILEPATH)
{
server.setFilePath(records[1]);
}
else if (records[0] == MODEL)
{
bool ok;
bool val = records[1].toInt(&ok);
QString result;
if (ok)
result = server.variables(val) + unitSep;
else
result = unitSep + QLatin1String("Invalid argument %1 for 'model' command", val);
result += QLatin1Char(messageEnd);
const QByteArray bytes = result.toLocal8Bit();
std::cout << bytes.data();
}
std::cout.flush();
}
}
int main(int argc, char *argv[])
{
QCoreApplication app(argc, argv);
connection = QObject::connect(&inputTimer, &QTimer::timeout, routeInput);
inputTimer.setInterval(100);
inputTimer.start();
std::cout << "ready" << std::endl;
return app.exec();
}
......@@ -19,6 +19,7 @@
Copyright (C) 2015 Minh Ngo <minh@fedoraproject.org>
*/
#include <defaultvariablemodel.h>
#include "pythonsession.h"
#include "pythonexpression.h"
#include "pythonvariablemodel.h"
......@@ -29,30 +30,36 @@
#include <QDebug>
#include <QDir>
#include <QStandardPaths>
#include <QDBusConnection>
#include <QDBusInterface>
#include <QDBusReply>
#include <KProcess>
#include <QProcess>
#include <KDirWatch>
#include <defaultvariablemodel.h>
#ifndef Q_OS_WIN
#include <signal.h>
#endif
PythonSession::PythonSession(Cantor::Backend* backend, int pythonVersion, const QString serverName, const QString DbusChannelName)
const QChar recordSep(30);
const QChar unitSep(31);
const QChar messageEnd = 29;
PythonSession::PythonSession(Cantor::Backend* backend, int pythonVersion, const QString serverName)
: Session(backend)
, m_variableModel(new PythonVariableModel(this))
, m_currentExpression(nullptr)
, m_pIface(nullptr)
, m_pProcess(nullptr)
, m_process(nullptr)
, serverName(serverName)
, DbusChannelName(DbusChannelName)
, m_pythonVersion(pythonVersion)
, m_needUpdate(false)
{
setVariableModel(new PythonVariableModel(this));
}
PythonSession::~PythonSession()
{
if (m_process) {
m_process->kill();
m_process->deleteLater();
}
}
void PythonSession::login()
......@@ -60,72 +67,56 @@ void PythonSession::login()
qDebug()<<"login";
emit loginStarted();
// TODO: T6113, T6114
if (m_pProcess)
m_pProcess->deleteLater();
m_pProcess = new KProcess(this);
m_pProcess->setOutputChannelMode(KProcess::SeparateChannels);
if (m_process)
m_process->deleteLater();
(*m_pProcess) << QStandardPaths::findExecutable(serverName);
m_process = new QProcess(this);
m_process->setProcessChannelMode(QProcess::ForwardedErrorChannel);
m_pProcess->start();
m_process->start(QStandardPaths::findExecutable(serverName));
m_pProcess->waitForStarted();
m_pProcess->waitForReadyRead();
QTextStream stream(m_pProcess->readAllStandardOutput());
m_process->waitForStarted();
m_process->waitForReadyRead();
QTextStream stream(m_process->readAllStandardOutput());
const QString& readyStatus = QString::fromLatin1("ready");
while (m_pProcess->state() == QProcess::Running)
while (m_process->state() == QProcess::Running)
{
const QString& rl = stream.readLine();
if (rl == readyStatus)
break;
}
if (!QDBusConnection::sessionBus().isConnected())
{
qWarning() << "Can't connect to the D-Bus session bus.\n"
"To start it, run: eval `dbus-launch --auto-syntax`";
return;
}
const QString& serviceName = DbusChannelName + QString::fromLatin1("-%1").arg(m_pProcess->pid());
connect(m_process, &QProcess::readyReadStandardOutput, this, &PythonSession::readOutput);
m_pIface = new QDBusInterface(serviceName,
QString::fromLatin1("/"), QString(), QDBusConnection::sessionBus());
if (!m_pIface->isValid())
{
qWarning() << QDBusConnection::sessionBus().lastError().message();
return;
}
m_variableModel->setPythonServer(m_pIface);
m_pIface->call(QString::fromLatin1("login"));
m_pIface->call(QString::fromLatin1("setFilePath"), worksheetPath);
sendCommand(QLatin1String("login"));
sendCommand(QLatin1String("setFilePath"), QStringList(worksheetPath));
const QStringList& scripts = autorunScripts();
if(!scripts.isEmpty()){
QString autorunScripts = scripts.join(QLatin1String("\n"));
getPythonCommandOutput(autorunScripts);
m_variableModel->update();
evaluateExpression(autorunScripts, Cantor::Expression::DeleteOnFinish, true);
variableModel()->update();
}
const QString& importerFile = QLatin1String(":py/import_default_modules.py");
evaluateExpression(fromSource(importerFile), Cantor::Expression::DeleteOnFinish, true);
changeStatus(Session::Done);
emit loginDone();
}
void PythonSession::logout()
{
// TODO: T6113, T6114
m_pProcess->terminate();
if (!m_process)
return;
sendCommand(QLatin1String("exit"));
m_process = nullptr;
m_variableModel->clearVariables();
variableModel()->clearVariables();
qDebug()<<"logout";
changeStatus(Status::Disable);
......@@ -133,16 +124,31 @@ void PythonSession::logout()
void PythonSession::interrupt()
{
// TODO: T6113, T6114
if (m_pProcess->pid())
m_pProcess->kill();
qDebug()<<"interrupt";
foreach(Cantor::Expression* e, m_runningExpressions)
e->interrupt();
if(!expressionQueue().isEmpty())
{
qDebug()<<"interrupting " << expressionQueue().first()->command();
if(m_process->state() != QProcess::NotRunning)
{
#ifndef Q_OS_WIN
const int pid=m_process->pid();
kill(pid, SIGINT);
#else
; //TODO: interrupt the process on windows
#endif
}
for (Cantor::Expression* expression : expressionQueue())
expression->setStatus(Cantor::Expression::Interrupted);
expressionQueue().clear();
// Cleanup inner state and call octave prompt printing
// If we move this code for interruption to Session, we need add function for
// cleaning before setting Done status
m_output.clear();
m_process->write("\n");
qDebug()<<"done interrupting";
}
m_runningExpressions.clear();
changeStatus(Cantor::Session::Done);
}
......@@ -160,199 +166,71 @@ Cantor::Expression* PythonSession::evaluateExpression(const QString& cmd, Cantor
return expr;
}
void PythonSession::runExpression(PythonExpression* expr)
{
qDebug() << "run expression";
m_currentExpression = expr;
const QString& command = expr->internalCommand();
readExpressionOutput(command);
}
// Is called asynchronously in the Python3 plugin
void PythonSession::readExpressionOutput(const QString& commandProcessing)
{
readOutput(commandProcessing);
}
void PythonSession::getPythonCommandOutput(const QString& commandProcessing)
{
runPythonCommand(commandProcessing);
m_output = getOutput();
m_error = getError();
}
bool PythonSession::identifyKeywords(const QString& command)
{
QString verifyErrorImport;
QString listKeywords;
QString keywordsString;
QString moduleImported;
QString moduleVariable;
getPythonCommandOutput(command);
qDebug() << "verifyErrorImport: ";
if(!m_error.isEmpty()){
qDebug() << "returned false";
return false;
}
moduleImported += identifyPythonModule(command);
moduleVariable += identifyVariableModule(command);
if((moduleVariable.isEmpty()) && (!command.endsWith(QLatin1String("*")))){
keywordsString = command.section(QLatin1String(" "), 3).remove(QLatin1String(" "));
}
if(moduleVariable.isEmpty() && (command.endsWith(QLatin1String("*")))){
listKeywords += QString::fromLatin1("import %1\n" \
"print(dir(%1))\n" \
"del %1\n").arg(moduleImported);
}
if(!moduleVariable.isEmpty()){
listKeywords += QLatin1String("print(dir(") + moduleVariable + QLatin1String("))\n");
}
if(!listKeywords.isEmpty()){
getPythonCommandOutput(listKeywords);
keywordsString = m_output;
keywordsString.remove(QLatin1String("'"));
keywordsString.remove(QLatin1String(" "));
keywordsString.remove(QLatin1String("["));
keywordsString.remove(QLatin1String("]"));
}
QStringList keywordsList = keywordsString.split(QLatin1String(","));
PythonKeywords::instance()->loadFromModule(moduleVariable, keywordsList);
qDebug() << "Module imported" << moduleImported;
return true;
}
QString PythonSession::identifyPythonModule(const QString& command) const
{
QString module;
if(command.contains(QLatin1String("import "))){
module = command.section(QLatin1String(" "), 1, 1);
}
qDebug() << "module identified" << module;
return module;
}
QString PythonSession::identifyVariableModule(const QString& command) const
QSyntaxHighlighter* PythonSession::syntaxHighlighter(QObject* parent)
{
QString variable;
if(command.contains(QLatin1String("import "))){
variable = command.section(QLatin1String(" "), 1, 1);
}
if((command.contains(QLatin1String("import "))) && (command.contains(QLatin1String(" as ")))){
variable = command.section(QLatin1String(" "), 3, 3);
}
if(command.contains(QLatin1String("from "))){
variable = QLatin1String("");
}
qDebug() << "variable identified" << variable;
return variable;
return new PythonHighlighter(parent, this, m_pythonVersion);
}
void PythonSession::expressionFinished()
Cantor::CompletionObject* PythonSession::completionFor(const QString& command, int index)
{
qDebug()<< "finished";
PythonExpression* expression = qobject_cast<PythonExpression*>(sender());
m_runningExpressions.removeAll(expression);
qDebug() << "size: " << m_runningExpressions.size();
return new PythonCompletionObject(command, index, this);
}
void PythonSession::updateOutput()
void PythonSession::runFirstExpression()
{
m_needUpdate |= !m_currentExpression->isInternal();
if(m_error.isEmpty()){
m_currentExpression->parseOutput(m_output);
qDebug() << "output: " << m_output;
} else {