[GSoC 2020] Fixing problem with wrong R highlighting and additional highlighting improvments

src/backends/R/rhighlighter.cpp

@@ -22,6 +22,7 @@
 #include "rhighlighter.h"
 #include "rkeywords.h"
 #include "rsession.h"
+#include "rvariablemodel.h"
 
 #include <QTextEdit>
 #include <QDebug>
@@ -35,49 +36,44 @@ const QStringList RHighlighter::specials_list=QStringList()
 
 RHighlighter::RHighlighter(QObject* parent, RSession* session) : Cantor::DefaultHighlighter(parent, session)
 {
+    Cantor::DefaultVariableModel* model = session->variableModel();
+    if (model)
+    {
+        RVariableModel* RModel = static_cast<RVariableModel*>(model);
+        connect(RModel, &RVariableModel::constantsAdded, this, &RHighlighter::addVariables);
+        connect(RModel, &RVariableModel::constantsRemoved, this, &RHighlighter::removeRules);
+    }
+
     addKeywords(RKeywords::instance()->keywords());
 
     foreach (const QString& s, operators_list)
-        operators.append(QRegExp(s));
+        addRule(QRegularExpression(s), operatorFormat());
     foreach (const QString& s, specials_list)
-        specials.append(QRegExp(QLatin1String("\\b")+s+QLatin1String("\\b")));
-}
+        addRule(QRegularExpression(QLatin1String("\\b")+s+QLatin1String("\\b")), commentFormat());
 
-// FIXME: due to lack of lookbehinds in QRegExp here we use a flag showing if we need to shift the boundary of formatting
-// to make up for the accidentally matched character
-void RHighlighter::formatRule(const QRegExp &p, const QTextCharFormat &fmt, const QString& text,bool shift)
-{
-    int index = p.indexIn(text);
-    while (index >= 0) {
-        int length = p.matchedLength();
-        setFormat(index+(shift?1:0),  length-(shift?1:0),  fmt);
-       index = p.indexIn(text,  index + length);
-    }
+    addRule(QRegularExpression(QStringLiteral("\"[^\"]*\"")), stringFormat());
+    addRule(QRegularExpression(QStringLiteral("'[^']*'")), stringFormat());
+
+    addRule(QRegularExpression(QStringLiteral("#[^\n]*")), commentFormat());
 }
 
-void RHighlighter::massFormat(const QVector<QRegExp> &p, const QTextCharFormat &fmt, const QString& text,bool shift)
+QStringList RHighlighter::parseBlockTextToWords(const QString& originalText)
 {
-    foreach (const QRegExp &rule, p)
-        formatRule(rule,fmt,text,shift);
-}
+    QString text = originalText;
 
+    static const QString replacer1 = QLatin1String("___CANTOR_R_REPLACER_1___");
+    static const QString replacer2 = QLatin1String("___CANTOR_R_REPLACER_2___");
 
-void RHighlighter::highlightBlock(const QString& text)
-{
-    if(text.isEmpty())
-        return;
-
-    //Do some backend independent highlighting (brackets etc.)
-    DefaultHighlighter::highlightBlock(text);
-
-    //Let's mark every functionlike call as an error, then paint right ones in their respective format
-    // TODO: find more elegant solution not involving double formatting
-    formatRule(QRegExp(QLatin1String("\\b[A-Za-z0-9_]+(?=\\()")),errorFormat(),text);
-
-    //formatRule(QRegExp("[^A-Za-z_]-?([0-9]+)?(((e|i)?-?)|\\.)[0-9]*L?"),numberFormat(),text,true); // TODO: erroneous number formats, refine
-    massFormat(operators,operatorFormat(),text);
-    massFormat(specials,commentFormat(),text); // FIXME must be distinct
-    massFormat(functions,functionFormat(),text);
-    massFormat(variables,variableFormat(),text);
-    formatRule(QRegExp(QLatin1String("\"[^\"]+\"")),stringFormat(),text); // WARNING a bit redundant
+    text.replace(QLatin1String("-"), replacer1);
+    text.replace(QLatin1String("."), replacer2);
+
+    QStringList words = text.split(QRegularExpression(QStringLiteral("\\b")), QString::SkipEmptyParts);
+
+    for (int i = 0; i < words.size(); i++)
+    {
+        words[i].replace(replacer1, QLatin1String("-"));
+        words[i].replace(replacer2, QLatin1String("."));
+    }
+
+    return words;
 }

src/backends/R/rhighlighter.h

@@ -34,7 +34,7 @@ class RHighlighter : public Cantor::DefaultHighlighter
     ~RHighlighter() override = default;
 
   protected:
-    void highlightBlock(const QString &text) override;
+    QStringList parseBlockTextToWords(const QString& text) override;
 
   private:
     inline void formatRule(const QRegExp &p, const QTextCharFormat &fmt, const QString& text,bool shift=false);

src/backends/R/rserver/rserver.cpp

@@ -448,7 +448,7 @@ void RServer::listSymbols()
 {
     setStatus(RServer::Busy);
 
-    QStringList vars, values, funcs;
+    QStringList vars, values, funcs, constants;
     int errorOccurred; // TODO: error checks
 
     /* Obtaining a list of user namespace objects */
@@ -485,23 +485,39 @@ void RServer::listSymbols()
 
         if (!m_parsedNamespaces.contains(packageName))
         {
-            QStringList foundedFunctions;
+            CachedParsedNamespace cache;
 
-            char pos[32];
-            sprintf(pos,"%d",i+1);
+            //char pos[32];
+            //sprintf(pos,"%d",i+1);
             SEXP f=PROTECT(R_tryEval(lang2(install("ls"),ScalarInteger(i+1)),nullptr,&errorOccurred));
             for (int j=0;j<length(f);j++)
-                foundedFunctions<<QString::fromUtf8(translateCharUTF8(STRING_ELT(f,j)));
+            {
+                SEXP object = STRING_ELT(f,j);
+                const QString& name = QString::fromUtf8(translateCharUTF8(object));
+                SEXP value = installChar(object);
+                int errorOccurred2 = 2;
+                //TODO error handling
+                //FIXME without this unused typeof evaling - server crash on certain symbols
+                SEXP test = PROTECT(R_tryEval(lang2(install("typeof"), value),nullptr,&errorOccurred2));
+                Q_UNUSED(test);
+
+                SEXP resultIs = PROTECT(R_tryEval(lang2(install("is.function"), value),nullptr, &errorOccurred2));
+                if (QString::fromUtf8(translateCharUTF8(asChar(resultIs))) == QLatin1String("TRUE"))
+                    cache.functions << name;
+                else
+                    cache.constants << name;
+            }
             UNPROTECT(1);
 
-            m_parsedNamespaces[packageName] = foundedFunctions;
+            m_parsedNamespaces[packageName] = cache;
         }
 
-        funcs += m_parsedNamespaces[packageName];
+        funcs += m_parsedNamespaces[packageName].functions;
+        constants += m_parsedNamespaces[packageName].constants;
     }
     UNPROTECT(1);
 
-    const QString output = vars.join(recordSep) + unitSep + values.join(recordSep) + unitSep + funcs.join(recordSep);
+    const QString output = vars.join(recordSep) + unitSep + values.join(recordSep) + unitSep + funcs.join(recordSep) + unitSep + constants.join(recordSep);
     emit expressionFinished(RServer::SuccessCode, output, QStringList());
     setStatus(Idle);
 }

src/backends/R/rserver/rserver.h

@@ -67,6 +67,12 @@ class RServer : public QObject
     void runCommand(const QString& cmd, bool internal=false);
     void answerRequest(const QString& answer);
 
+  private:
+    struct CachedParsedNamespace {
+        QStringList functions;
+        QStringList constants;
+    };
+
   private:
     void setStatus(Status status);
     void newPlotDevice();
@@ -76,6 +82,7 @@ class RServer : public QObject
   private:
     const static QChar recordSep;
     const static QChar unitSep;
+
   private:
     bool m_isInitialized;
     bool m_isCompletionAvailable;
@@ -84,7 +91,7 @@ class RServer : public QObject
     QString m_tmpDir;
     QString m_curPlotFile;
     QStringList m_expressionFiles;
-    QMap<QString, QStringList> m_parsedNamespaces;
+    QMap<QString, CachedParsedNamespace> m_parsedNamespaces;
 };
 
 #endif /* _RSERVER_H */

src/backends/R/rvariablemodel.cpp

@@ -19,6 +19,7 @@
 */
 
 #include "rvariablemodel.h"
+#include "rkeywords.h"
 #include "rsession.h"
 
 #include <result.h>
@@ -60,9 +61,10 @@ void RVariableModel::parseResult(Cantor::Expression::Status status)
 
             const QString output = m_expression->result()->data().toString();
 
-            const QStringList names = output.section(unitSep, 0, 0).split(recordSep, QString::SkipEmptyParts);
-            const QStringList values = output.section(unitSep, 1, 1).split(recordSep, QString::SkipEmptyParts);
-            const QStringList funcs = output.section(unitSep, 2, 2).split(recordSep, QString::SkipEmptyParts);
+            const QStringList& names = output.section(unitSep, 0, 0).split(recordSep, QString::SkipEmptyParts);
+            const QStringList& values = output.section(unitSep, 1, 1).split(recordSep, QString::SkipEmptyParts);
+            QStringList funcs = output.section(unitSep, 2, 2).split(recordSep, QString::SkipEmptyParts);
+            const QStringList& constants = output.section(unitSep, 3, 3).split(recordSep, QString::SkipEmptyParts);
 
             QList<Variable> vars;
             if (!values.isEmpty()) // Variables management disabled
@@ -74,7 +76,16 @@ void RVariableModel::parseResult(Cantor::Expression::Status status)
 
             setVariables(vars);
 
+            // Remove primitive function "(" because it not function for user calling (i guess)
+            // And the function with name like this make highlighting worse actually
+            funcs.removeOne(QLatin1String("("));
+
+            // Also removes syntax keywords from functions list, like "function"
+            for (const QString& keyword: RKeywords::instance()->keywords())
+                funcs.removeOne(keyword);
+
             setFunctions(funcs);
+            setConstants(constants);
             break;
         }
         case Expression::Status::Error:
@@ -91,3 +102,43 @@ void RVariableModel::parseResult(Cantor::Expression::Status status)
     m_expression->deleteLater();
     m_expression = nullptr;
 }
+
+void RVariableModel::setConstants(QStringList newConstants)
+{
+    QStringList addedConstants;
+    QStringList removedConstants;
+
+    //remove the old variables
+    int i = 0;
+    while (i < m_constants.size())
+    {
+        //check if this var is present in the new variables
+        bool found = false;
+        for (const QString& constant : newConstants)
+            if(m_constants[i] == constant)
+            {
+                found=true;
+                break;
+            }
+
+        if(!found)
+        {
+            removedConstants << m_constants[i];
+            m_constants.removeAt(i);
+        }
+        else
+            i++;
+    }
+
+    for (const QString& constant : newConstants)
+    {
+        if (!m_constants.contains(constant))
+        {
+            addedConstants << constant;
+            m_constants.append(constant);
+        }
+    }
+
+    emit constantsAdded(addedConstants);
+    emit constantsRemoved(removedConstants);
+}

src/backends/R/rvariablemodel.h

@@ -32,13 +32,23 @@ class RVariableModel : public Cantor::DefaultVariableModel
     RVariableModel( RSession* session);
     ~RVariableModel() override;
 
+    // List of virables from other R namespaces (packages), which can be treted as constants, like "pi"
+    //QStringList constants() const;
+
     void update() override;
 
+  Q_SIGNALS:
+    void constantsAdded(QStringList);
+    void constantsRemoved(QStringList);
+
   public Q_SLOTS:
     void parseResult(Cantor::Expression::Status status);
 
   private:
-    QStringList m_functions;
+    void setConstants(QStringList constants);
+
+  private:
+    QStringList m_constants;
     Cantor::Expression* m_expression;
 };
 

src/lib/defaulthighlighter.cpp

@@ -212,11 +212,16 @@ void DefaultHighlighter::highlightPairs(const QString& text)
 
 }
 
+QStringList Cantor::DefaultHighlighter::parseBlockTextToWords(const QString& text)
+{
+    return text.split(QRegularExpression(QStringLiteral("\\b")), QString::SkipEmptyParts);
+}
+
 void DefaultHighlighter::highlightWords(const QString& text)
 {
     //qDebug() << "DefaultHighlighter::highlightWords";
 
-    const QStringList& words = text.split(QRegularExpression(QStringLiteral("\\b")), QString::SkipEmptyParts);
+    const QStringList& words = parseBlockTextToWords(text);
     int count;
     int pos = 0;
 

src/lib/defaulthighlighter.h

@@ -92,6 +92,8 @@ class CANTOR_EXPORT DefaultHighlighter : public QSyntaxHighlighter
      */
     void highlightBlock(const QString& text) override;
 
+    virtual QStringList parseBlockTextToWords(const QString& text);
+
     bool skipHighlighting(const QString& text);
 
     QTextCharFormat functionFormat() const;