Members of the KDE Community are recommended to subscribe to the kde-community mailing list at https://mail.kde.org/mailman/listinfo/kde-community to allow them to participate in important discussions and receive other important announcements

Commit 9e9c2055 authored by Alex Richardson's avatar Alex Richardson

Add a RAII class for clang_tokenize

REVIEW: 125301
parent 7a300854
......@@ -327,11 +327,8 @@ bool isInsideComment(CXTranslationUnit unit, CXFile file, const KTextEditor::Cur
// tokenize the whole range from the start until 'position'
// if we detect a comment token at this position, return true
CXToken* tokens = nullptr;
unsigned int nTokens = 0;
clang_tokenize(unit, range, &tokens, &nTokens);
for (unsigned int i = 0; i < nTokens; ++i) {
CXToken token = tokens[i];
const ClangTokens tokens(unit, range);
for (CXToken token : tokens) {
CXTokenKind tokenKind = clang_getTokenKind(token);
if (tokenKind != CXToken_Comment) {
continue;
......
......@@ -132,11 +132,9 @@ ClangProblem::ClangProblem(CXDiagnostic diagnostic, CXTranslationUnit unit)
// a token we can use for building the range
auto nextLocation = clang_getLocation(unit, file, line, column + 100);
auto rangeToTokenize = clang_getRange(location, nextLocation);
CXToken *tokens = nullptr;
unsigned numTokens = 0;
clang_tokenize(unit, rangeToTokenize, &tokens, &numTokens);
if (numTokens) {
docRange.setRange(ClangRange(clang_getTokenExtent(unit, tokens[0])).toRange());
const ClangTokens tokens(unit, rangeToTokenize);
if (tokens.size()) {
docRange.setRange(ClangRange(clang_getTokenExtent(unit, tokens.at(0))).toRange());
}
}
......
......@@ -173,11 +173,8 @@ void TodoExtractor::extractTodos()
return;
}
CXToken* tokens = nullptr;
unsigned int nTokens = 0;
clang_tokenize(m_unit, range, &tokens, &nTokens);
for (unsigned int i = 0; i < nTokens; ++i) {
CXToken token = tokens[i];
const ClangTokens tokens(m_unit, range);
for (CXToken token : tokens) {
CXTokenKind tokenKind = clang_getTokenKind(token);
if (tokenKind != CXToken_Comment) {
continue;
......@@ -206,7 +203,6 @@ void TodoExtractor::extractTodos()
m_problems << problem;
}
}
clang_disposeTokens(m_unit, tokens, nTokens);
}
QList< ProblemPointer > TodoExtractor::problems() const
......
......@@ -77,16 +77,12 @@ private:
CXTranslationUnit TU = m_session.unit();
auto cursor = clang_getTranslationUnitCursor(TU);
CXSourceRange range = clang_getCursorExtent(cursor);
CXToken *tokens = 0;
unsigned int nTokens = 0;
clang_tokenize(TU, range, &tokens, &nTokens);
for (unsigned int i = 0; i < nTokens; i++)
{
CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
const ClangTokens tokens(TU, range);
for (CXToken token : tokens) {
CXString spelling = clang_getTokenSpelling(TU, token);
qout << "token= " << clang_getCString(spelling) << endl;
clang_disposeString(spelling);
}
clang_disposeTokens(TU, tokens, nTokens);
}
if (!m_session.unit()) {
......
......@@ -109,9 +109,6 @@ void TestCodeCompletion::initTestCase()
TestCore::initialize();
ClangSettingsManager::self()->m_enableTesting = true;
auto languages = ICore::self()->languageController()->languagesForUrl(QUrl::fromLocalFile("/tmp/foo.cpp"));
QCOMPARE(languages.size(), 1);
QCOMPARE(languages[0]->name(), QStringLiteral("clang"));
}
void TestCodeCompletion::cleanupTestCase()
......
......@@ -163,3 +163,37 @@ RangeInRevision ClangRange::toRangeInRevision() const
ClangRange::~ClangRange()
{
}
ClangTokens::ClangTokens(CXTranslationUnit unit, CXSourceRange range)
: m_unit(unit)
{
clang_tokenize(m_unit, range, &m_tokens, &m_numTokens);
}
ClangTokens::~ClangTokens()
{
clang_disposeTokens(m_unit, m_tokens, m_numTokens);
}
CXToken* ClangTokens::begin() const
{
return m_tokens;
}
CXToken* ClangTokens::end() const
{
return m_tokens + m_numTokens;
}
std::reverse_iterator<CXToken*> ClangTokens::rbegin() const
{
return std::reverse_iterator<CXToken*>(end());
}
std::reverse_iterator<CXToken*> ClangTokens::rend() const
{
return std::reverse_iterator<CXToken*>(begin());
}
uint ClangTokens::size() const
{
return m_numTokens;
}
CXToken ClangTokens::at(uint index) const {
Q_ASSERT(index < m_numTokens);
return m_tokens[index];
}
......@@ -27,6 +27,8 @@
#include <util/path.h>
#include <iterator>
class QTextStream;
namespace KTextEditor {
......@@ -108,4 +110,21 @@ private:
CXSourceRange m_range;
};
class ClangTokens
{
public:
ClangTokens(CXTranslationUnit unit, CXSourceRange range);
~ClangTokens();
CXToken* begin() const;
CXToken* end() const;
std::reverse_iterator<CXToken*> rbegin() const;
std::reverse_iterator<CXToken*> rend() const;
uint size() const;
CXToken at(uint index) const;
private:
CXTranslationUnit m_unit;
CXToken* m_tokens;
uint m_numTokens;
};
#endif // CLANGTYPES_H
......@@ -70,8 +70,6 @@ CXChildVisitResult paramVisitor(CXCursor cursor, CXCursor /*parent*/, CXClientDa
}
FunctionInfo *info = static_cast<FunctionInfo*>(data);
CXToken *tokens;
unsigned int numTokens;
ClangRange range(clang_getCursorExtent(cursor));
CXFile file;
......@@ -87,12 +85,10 @@ CXChildVisitResult paramVisitor(CXCursor cursor, CXCursor /*parent*/, CXClientDa
//the declaration or definition, and the default arguments don't have lexical
//parents. So this range check is the only thing that really works.
if ((info->fileName.isEmpty() || fileName == info->fileName) && info->range.contains(range.toRange())) {
clang_tokenize(info->unit, range.range(), &tokens, &numTokens);
for (unsigned int i = 0; i < numTokens; i++) {
info->stringParts.append(ClangString(clang_getTokenSpelling(info->unit, tokens[i])).toString());
const ClangTokens tokens(info->unit, range.range());
for (CXToken token : tokens) {
info->stringParts.append(ClangString(clang_getTokenSpelling(info->unit, token)).toString());
}
clang_disposeTokens(info->unit, tokens, numTokens);
}
return CXChildVisit_Continue;
}
......@@ -254,23 +250,19 @@ QByteArray ClangUtils::getRawContents(CXTranslationUnit unit, CXSourceRange rang
clang_getFileLocation(rangeEnd, nullptr, nullptr, nullptr, &end);
QByteArray result;
CXToken *tokens = 0;
unsigned int nTokens = 0;
clang_tokenize(unit, range, &tokens, &nTokens);
for (unsigned int i = 0; i < nTokens; i++) {
const auto location = ClangLocation(clang_getTokenLocation(unit, tokens[i]));
const ClangTokens tokens(unit, range);
for (CXToken token : tokens) {
const auto location = ClangLocation(clang_getTokenLocation(unit, token));
unsigned int offset;
clang_getFileLocation(location, nullptr, nullptr, nullptr, &offset);
Q_ASSERT(offset >= start);
const int fillCharacters = offset - start - result.size();
Q_ASSERT(fillCharacters >= 0);
result.append(QByteArray(fillCharacters, ' '));
const auto spelling = clang_getTokenSpelling(unit, tokens[i]);
const auto spelling = clang_getTokenSpelling(unit, token);
result.append(clang_getCString(spelling));
clang_disposeString(spelling);
}
clang_disposeTokens(unit, tokens, nTokens);
// Clang always appends the full range of the last token, even if this exceeds the end of the requested range.
// Fix this.
result.chop((result.size() - 1) - (end - start));
......@@ -282,16 +274,14 @@ bool ClangUtils::isExplicitlyDefaultedOrDeleted(CXCursor cursor)
{
// TODO: expose clang::FunctionDecl::isDeleted() and clang::FunctionDecl::isExplicitlyDefaulted() in libclang
auto declCursor = clang_getCanonicalCursor(cursor);
uint numTokens = 0;
CXToken* tokens = nullptr;
CXTranslationUnit tu = clang_Cursor_getTranslationUnit(declCursor);
clang_tokenize(tu, clang_getCursorExtent(declCursor), &tokens, &numTokens);
ClangTokens tokens(tu, clang_getCursorExtent(declCursor));
int parenDepth = 0; // we want to ignore =delete within parentheses
bool foundFirstClosingParen = false; // a function needs at least one closing parenthesis before the = delete
bool lastTokenWasEquals = false;
bool explicitlyDefaultedOrDeleted = false;
for (uint i = 0; i < numTokens && !explicitlyDefaultedOrDeleted; ++i) {
auto kind = clang_getTokenKind(tokens[i]);
for (uint i = 0; i < tokens.size() && !explicitlyDefaultedOrDeleted; ++i) {
auto kind = clang_getTokenKind(tokens.at(i));
switch (kind) {
case CXToken_Comment:
break;
......@@ -300,7 +290,7 @@ bool ClangUtils::isExplicitlyDefaultedOrDeleted(CXCursor cursor)
lastTokenWasEquals = false;
break;
case CXToken_Punctuation: {
ClangString spelling(clang_getTokenSpelling(tu, tokens[i]));
ClangString spelling(clang_getTokenSpelling(tu, tokens.at(i)));
const char* spellingCStr = spelling.c_str();
if (strcmp(spellingCStr, "(") == 0) {
parenDepth++;
......@@ -316,7 +306,7 @@ bool ClangUtils::isExplicitlyDefaultedOrDeleted(CXCursor cursor)
if (!lastTokenWasEquals || parenDepth > 0) {
break;
}
ClangString spelling(clang_getTokenSpelling(tu, tokens[i]));
ClangString spelling(clang_getTokenSpelling(tu, tokens.at(i)));
const char* spellingCStr = spelling.c_str();
if (strcmp(spellingCStr, "default") == 0 || strcmp(spellingCStr, "delete") == 0) {
explicitlyDefaultedOrDeleted = true; // break loop
......@@ -325,10 +315,7 @@ bool ClangUtils::isExplicitlyDefaultedOrDeleted(CXCursor cursor)
}
break;
}
default:
Q_UNREACHABLE();
}
}
clang_disposeTokens(tu, tokens, numTokens);
return explicitlyDefaultedOrDeleted;
}
......@@ -114,9 +114,6 @@ namespace ClangUtils
* such API in libclang so having one function to check both cases is more efficient (only tokenize once)
*/
bool isExplicitlyDefaultedOrDeleted(CXCursor cursor);
};
#endif // CLANGUTILS_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment