Commit 0b99ea7b authored by Simon Persson's avatar Simon Persson

Make filedigger significantly faster for large directories

By not checking file sizes for all versions until asked for the info by the versionlistmodel.

Also, quite important! Fix wrong "modified" time stamps for files. Bug in generateSubnodes(), was not reading metadata from the metadata stream for all files, needs to be read unconditionally so that later files will match up with the stream.
parent 96bfefd9
......@@ -67,7 +67,7 @@ void MergedNode::getBupUrl(int pVersionIndex, KUrl *pComplete, QString *pRepoPat
pComplete->setProtocol(QLatin1String("bup"));
pComplete->addPath(lRepo->objectName());
pComplete->addPath(lRepo->mBranchName);
pComplete->addPath(vfsTimeToString(mVersions.at(pVersionIndex)->mCommitTime));
pComplete->addPath(vfsTimeToString(mVersionList.at(pVersionIndex)->mCommitTime));
}
if(pRepoPath) {
*pRepoPath = lRepo->objectName();
......@@ -76,7 +76,7 @@ void MergedNode::getBupUrl(int pVersionIndex, KUrl *pComplete, QString *pRepoPat
*pBranchName = lRepo->mBranchName;
}
if(pCommitTime) {
*pCommitTime = mVersions.at(pVersionIndex)->mCommitTime;
*pCommitTime = mVersionList.at(pVersionIndex)->mCommitTime;
}
if(pPathInRepo) {
pPathInRepo->clear();
......@@ -105,11 +105,9 @@ MergedNodeList &MergedNode::subNodes() {
void MergedNode::generateSubNodes() {
NameMap lSubNodeMap;
VersionMapIterator lVersionIter(mVersionMap);
while(lVersionIter.hasNext()) {
VersionData *lCurrentVersion = lVersionIter.next().value();
foreach(VersionData *lCurrentVersion, mVersionList) {
git_tree *lTree;
if(0 != git_tree_lookup(&lTree, mRepository, &lVersionIter.key())) {
if(0 != git_tree_lookup(&lTree, mRepository, &lCurrentVersion->mOid)) {
continue;
}
git_blob *lMetadataBlob = NULL;
......@@ -153,34 +151,31 @@ void MergedNode::generateSubNodes() {
mSubNodes->append(lSubNode);
}
}
if(!lSubNode->mVersionMap.contains(*lOid)) {
quint64 lSize;
bool lAlreadySeen = false;
foreach(VersionData *lVersion, lSubNode->mVersionList) {
if(lVersion->mOid == *lOid) {
lAlreadySeen = true;
break;
}
}
if(S_ISDIR(lMode)) {
if(!lAlreadySeen) {
lSubNode->mVersionList.append(new VersionData(lOid, lCurrentVersion->mCommitTime,
lCurrentVersion->mModifiedDate, 0));
}
} else {
quint64 lModifiedDate;
if(S_ISDIR(lMode)) {
lSize = 0;
lModifiedDate = lCurrentVersion->mModifiedDate;
Metadata lMetadata;
if(lMetadataStream != NULL && 0 == readMetadata(*lMetadataStream, lMetadata)) {
lModifiedDate = lMetadata.mMtime;
} else {
Metadata lMetadata;
if(lMetadataStream != NULL && 0 == readMetadata(*lMetadataStream, lMetadata)) {
lModifiedDate = lMetadata.mMtime;
} else {
lModifiedDate = lCurrentVersion->mModifiedDate;
}
if(lChunked) {
lSize = calculateChunkFileSize(lOid, mRepository);
} else {
git_blob *lBlob;
if(0 == git_blob_lookup(&lBlob, mRepository, lOid)) {
lSize = git_blob_rawsize(lBlob);
git_blob_free(lBlob);
} else {
lSize = 0;
}
}
lModifiedDate = lCurrentVersion->mModifiedDate;
}
if(!lAlreadySeen) {
lSubNode->mVersionList.append(new VersionData(lChunked, lOid,
lCurrentVersion->mCommitTime,
lModifiedDate));
}
VersionData *lVersionData = new VersionData(lCurrentVersion->mCommitTime, lModifiedDate, lSize);
lSubNode->mVersionMap.insert(*lOid, lVersionData);
lSubNode->mVersions.append(lVersionData);
}
}
if(lMetadataStream != NULL) {
......@@ -191,7 +186,7 @@ void MergedNode::generateSubNodes() {
}
qSort(mSubNodes->begin(), mSubNodes->end(), mergedNodeLessThan);
foreach(MergedNode *lNode, *mSubNodes) {
qSort(lNode->mVersions.begin(), lNode->mVersions.end(), versionGreaterThan);
qSort(lNode->mVersionList.begin(), lNode->mVersionList.end(), versionGreaterThan);
}
}
......@@ -224,9 +219,7 @@ MergedRepository::MergedRepository(QObject *pParent, const QString &pRepositoryP
continue;
}
git_time_t lTime = git_commit_time(lCommit);
VersionData *lVersionData = new VersionData(lTime, lTime, 0);
mVersionMap.insert(*git_commit_tree_id(lCommit), lVersionData);
mVersions.append(lVersionData);
mVersionList.append(new VersionData(git_commit_tree_id(lCommit), lTime, lTime, 0));
git_commit_free(lCommit);
}
}
......@@ -250,3 +243,23 @@ bool operator ==(const git_oid &pOidA, const git_oid &pOidB) {
QByteArray b = QByteArray::fromRawData((char *)pOidB.id, GIT_OID_RAWSZ);
return a == b;
}
quint64 VersionData::size() {
if(mSizeIsValid) {
return mSize;
}
if(mChunkedFile) {
mSize = calculateChunkFileSize(&mOid, MergedNode::mRepository);
} else {
git_blob *lBlob;
if(0 == git_blob_lookup(&lBlob, MergedNode::mRepository, &mOid)) {
mSize = git_blob_rawsize(lBlob);
git_blob_free(lBlob);
} else {
mSize = 0;
}
}
mSizeIsValid = true;
return mSize;
}
......@@ -32,26 +32,38 @@ bool operator ==(const git_oid &pOidA, const git_oid &pOidB);
#include <sys/stat.h>
struct VersionData {
VersionData(quint64 pCommitTime, quint64 pModifiedDate, quint64 pSize)
: mCommitTime(pCommitTime), mModifiedDate(pModifiedDate), mSize(pSize)
{}
VersionData(bool pChunkedFile, const git_oid *pOid, quint64 pCommitTime, quint64 pModifiedDate)
:mChunkedFile(pChunkedFile), mOid(*pOid), mCommitTime(pCommitTime), mModifiedDate(pModifiedDate)
{
mSizeIsValid = false;
}
VersionData(const git_oid *pOid, quint64 pCommitTime, quint64 pModifiedDate, quint64 pSize)
:mOid(*pOid), mCommitTime(pCommitTime), mModifiedDate(pModifiedDate), mSize(pSize)
{
mSizeIsValid = true;
}
quint64 size();
bool mSizeIsValid;
bool mChunkedFile;
git_oid mOid;
quint64 mCommitTime;
quint64 mModifiedDate;
protected:
quint64 mSize;
};
class MergedNode;
typedef QList<MergedNode*> MergedNodeList;
typedef QListIterator<MergedNode*> MergedNodeListIterator;
typedef QHash<git_oid, VersionData *> VersionMap;
typedef QHashIterator<git_oid, VersionData *> VersionMapIterator;
typedef QList<VersionData *> VersionList;
typedef QListIterator<VersionData *> VersionListIterator;
class MergedNode: public QObject {
Q_OBJECT
friend class VersionData;
public:
MergedNode(QObject *pParent, const QString &pName, uint pMode);
virtual ~MergedNode() {
......@@ -63,7 +75,7 @@ public:
void getBupUrl(int pVersionIndex, KUrl *pComplete, QString *pRepoPath = NULL, QString *pBranchName = NULL,
quint64 *pCommitTime = NULL, QString *pPathInRepo = NULL) const;
virtual MergedNodeList &subNodes();
const VersionList *versionList() const { return &mVersions; }
const VersionList *versionList() const { return &mVersionList; }
uint mode() const { return mMode; }
protected:
......@@ -72,8 +84,7 @@ protected:
static git_revwalk *mRevisionWalker;
static git_repository *mRepository;
uint mMode;
VersionMap mVersionMap;
VersionList mVersions;
VersionList mVersionList;
MergedNodeList *mSubNodes;
};
......
......@@ -63,13 +63,13 @@ QVariant VersionListModel::data(const QModelIndex &pIndex, int pRole) const {
case VersionMimeTypeRole:
return KMimeType::findByUrl(mNode->objectName(), mNode->mode())->name();
case VersionSizeRole:
return lData->mSize;
return lData->size();
case VersionSourceInfoRole: {
BupSourceInfo lSourceInfo;
mNode->getBupUrl(pIndex.row(), &lSourceInfo.mBupKioPath, &lSourceInfo.mRepoPath, &lSourceInfo.mBranchName,
&lSourceInfo.mCommitTime, &lSourceInfo.mPathInRepo);
lSourceInfo.mIsDirectory = mNode->isDirectory();
lSourceInfo.mSize = lData->mSize;
lSourceInfo.mSize = lData->size();
return QVariant::fromValue<BupSourceInfo>(lSourceInfo);
}
default:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment