filterpmail.cpp 13.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/***************************************************************************
                          FilterPMail.cxx  -  Pegasus-Mail import
                             -------------------
    begin                : Sat Jan 6 2001
    copyright            : (C) 2001 by Holger Schurig <holgerschurig@gmx.de>
                           (C) 2005 by Danny Kukawka <danny.kukawka@web.de>
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

Allen Winter's avatar
Allen Winter committed
18
#include "filterpmail.h"
19 20
#include "mailimporter_debug.h"

Allen Winter's avatar
Allen Winter committed
21 22 23 24 25
#include <KLocalizedString>

#include <QFileDialog>
#include <QRegularExpression>
#include <QTemporaryFile>
26 27 28

using namespace MailImporter;

Laurent Montel's avatar
Laurent Montel committed
29 30 31 32 33 34 35 36 37
FilterPMail::FilterPMail()
    : Filter(i18n("Import Folders From Pegasus-Mail"),
             i18n("Holger Schurig <br>( rewritten by Danny Kukawka )"),
             i18n("<p>Select the Pegasus-Mail directory on your system (containing *.CNM, *.PMM and *.MBX files). "
                  "On many systems this is stored in C:\\pmail\\mail or C:\\pmail\\mail\\admin</p>"
                  "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders "
                  "will be stored under: \"PegasusMail-Import\".</p>"))
{
}
38 39 40 41 42 43 44 45

FilterPMail::~FilterPMail()
{
}

void FilterPMail::import()
{
    // Select directory from where I have to import files
Laurent Montel's avatar
Laurent Montel committed
46
    const QString maildir = QFileDialog::getExistingDirectory(nullptr, QString(), QDir::homePath());
47
    importMails(maildir);
48 49
}

Laurent Montel's avatar
Laurent Montel committed
50
void FilterPMail::importMails(const QString &chosenDir)
51
{
52
    if (chosenDir.isEmpty()) {
53 54 55
        filterInfo()->alert(i18n("No directory selected."));
        return;
    }
56
    setMailDir(chosenDir);
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82

    // Count total number of files to be processed
    filterInfo()->addInfoLogEntry(i18n("Counting files..."));
    dir.setPath(mailDir());
    const QStringList files = dir.entryList(QStringList() << QStringLiteral("*.[cC][nN][mM]") << QStringLiteral("*.[pP][mM][mM]") << QStringLiteral("*.[mM][bB][xX]"), QDir::Files, QDir::Name);
    totalFiles = files.count();
    currentFile = 0;
    qCDebug(MAILIMPORTER_LOG) << "Count is" << totalFiles;

    if (!(folderParsed = parseFolderMatrix(mailDir()))) {
        filterInfo()->addErrorLogEntry(i18n("Cannot parse the folder structure; continuing import without subfolder support."));
    }

    filterInfo()->addInfoLogEntry(i18n("Importing new mail files ('.cnm')..."));
    processFiles(QStringLiteral("*.[cC][nN][mM]"), &FilterPMail::importNewMessage);
    filterInfo()->addInfoLogEntry(i18n("Importing mail folders ('.pmm')..."));
    processFiles(QStringLiteral("*.[pP][mM][mM]"), &FilterPMail::importMailFolder);
    filterInfo()->addInfoLogEntry(i18n("Importing 'UNIX' mail folders ('.mbx')..."));
    processFiles(QStringLiteral("*.[mM][bB][xX]"), &FilterPMail::importUnixMailFolder);

    filterInfo()->addInfoLogEntry(i18n("Finished importing emails from %1", mailDir()));
    filterInfo()->setCurrent(100);
    filterInfo()->setOverall(100);
}

/** this looks for all files with the filemask 'mask' and calls the 'workFunc' on each of them */
Laurent Montel's avatar
Laurent Montel committed
83
void FilterPMail::processFiles(const QString &mask, void (FilterPMail::*workFunc)(const QString &))
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
{
    if (filterInfo()->shouldTerminate()) {
        return;
    }

    const QStringList files = dir.entryList(QStringList(mask), QDir::Files, QDir::Name);
    //qCDebug(MAILIMPORTER_LOG) <<"Mask is" << mask <<" count is" << files.count();
    QStringList::ConstIterator end = files.constEnd();
    for (QStringList::ConstIterator mailFile = files.constBegin(); mailFile != end; ++mailFile) {
        // Notify current file
        QFileInfo mailfilem_filterInfoo(*mailFile);
        filterInfo()->setFrom(mailfilem_filterInfoo.fileName());

        // Clear the other fields
        filterInfo()->setTo(QString());
        filterInfo()->setCurrent(QString());
        filterInfo()->setCurrent(-1);

        // call worker function, increase progressbar
        (this->*workFunc)(dir.filePath(*mailFile));
        ++currentFile;
Laurent Montel's avatar
Laurent Montel committed
105
        filterInfo()->setOverall((int)((float)currentFile / totalFiles * 100));
106 107 108 109 110 111 112 113 114 115 116 117 118
        filterInfo()->setCurrent(100);
        if (filterInfo()->shouldTerminate()) {
            return;
        }
    }
}

/** this function imports one *.CNM message */
void FilterPMail::importNewMessage(const QString &file)
{
    QString destFolder(QStringLiteral("PegasusMail-Import/New Messages"));
    filterInfo()->setTo(destFolder);

119 120 121
    if (!importMessage(destFolder, file, filterInfo()->removeDupMessage())) {
        filterInfo()->addErrorLogEntry(i18n("Could not import %1", file));
    }
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
}

/** this function imports one mail folder file (*.PMM) */
void FilterPMail::importMailFolder(const QString &file)
{
    // Format of a PMM file:
    // First comes a header with 128 bytes. At the beginning is the name of
    // the folder. Then there are some unknown bytes (strings). At offset 128
    // the first message starts.
    //
    // Each message is terminated by a 0x1A byte. The next message follows
    // immediately.
    //
    // The last message is followed by a 0x1A, too.
    //
    // 000000 6d 61 69 6c 73 65 72 76 65 72 2d 70 72 6f 6a 65    mailserver-proje
    // 000010 63 74 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ct..............
    // 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
    // 000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
    // 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
    // 000050 00 00 00 00 00 00 36 30 34 37 35 37 32 45 3a 36    ......6047572E:6
    // 000060 46 34 39 3a 46 4f 4c 30 31 33 35 35 00 00 00 00    F49:FOL01355....
    // 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
    // 000080 52 65 74 75 72 6e 2d 50 61 74 68 3a 20 3c 75 72    Return-Path: <ur
    // ...
    // 000cb0 2d 2d 2d 2d 2d 2d 2d 2d 2d 2b 0d 0a 1a 52 65 74    ---------+...Ret
    // 000cc0 75 72 6e 2d 50 61 74 68 3a 20 3c 62 6f 75 6e 63    urn-Path: <bounc
    // ...
    // 04dc50 46 30 33 38 44 2e 36 31 35 44 37 34 44 30 2d 2d    F038D.615D74D0--
    // 04dc60 0d 0a 1a

    struct {
        char folder[86];
        char id[42];
    } pmm_head;

    QFile f(file);
    if (!f.open(QIODevice::ReadOnly)) {
        filterInfo()->alert(i18n("Unable to open %1, skipping", file));
    } else {
        // Get folder name
Laurent Montel's avatar
Laurent Montel committed
163
        long l = f.read((char *)&pmm_head, sizeof(pmm_head));
164 165 166 167 168 169 170
        QString folder(i18nc("define folder name when we will import pegasus mail", "PegasusMail-Import") + QLatin1Char('/'));
        if (folderParsed) {
            folder.append(getFolderName(QString::fromLatin1(pmm_head.id)));
        } else {
            folder.append(QString::fromLatin1(pmm_head.folder));
        }
        filterInfo()->setTo(folder);
Laurent Montel's avatar
Laurent Montel committed
171
        filterInfo()->addInfoLogEntry(i18n("Importing %1", QStringLiteral("../") + QString::fromLatin1(pmm_head.folder)));
172 173 174 175 176 177 178

        QByteArray input(MAX_LINE, '\0');
        bool first_msg = true;

        while (!f.atEnd()) {
            QTemporaryFile tempfile;
            tempfile.open();
Laurent Montel's avatar
Laurent Montel committed
179
            filterInfo()->setCurrent((int)(((float)f.pos() / f.size()) * 100));
180 181 182 183 184 185 186

            if (!first_msg) {
                // set the filepos back to last line minus the separate char (0x1a)
                f.seek(f.pos() - l + 1);
            }

            // no problem to loose the last line in file. This only contains a separate char
Laurent Montel's avatar
Laurent Montel committed
187
            while (!f.atEnd() && (l = f.readLine(input.data(), MAX_LINE))) {
188 189 190 191 192 193
                if (filterInfo()->shouldTerminate()) {
                    return;
                }
                if (input.at(0) == 0x1a) {
                    break;
                } else {
194
                    tempfile.write(input.constData(), l);
195 196 197
                }
            }
            tempfile.flush();
198 199 200
            if (!importMessage(folder, tempfile.fileName(), filterInfo()->removeDupMessage())) {
                filterInfo()->addErrorLogEntry(i18n("Could not import %1", tempfile.fileName()));
            }
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224

            first_msg = false;
        }
    }
    f.close();
}

/** imports a 'unix' format mail folder (*.MBX) */
void FilterPMail::importUnixMailFolder(const QString &file)
{
    struct {
        char folder[58];
        char id[31];
    } pmg_head;

    QFile f;
    QString folder(QStringLiteral("PegasusMail-Import/")), s(file), separate;
    QByteArray line(MAX_LINE, '\0');
    int n = 0, l = 0;

    /** Get the folder name */
    s.replace(QRegularExpression(QStringLiteral("mbx$")), QStringLiteral("pmg"));
    s.replace(QRegularExpression(QStringLiteral("MBX$")), QStringLiteral("PMG"));
    f.setFileName(s);
Laurent Montel's avatar
Laurent Montel committed
225
    if (!f.open(QIODevice::ReadOnly)) {
226 227 228
        filterInfo()->alert(i18n("Unable to open %1, skipping", s));
        return;
    } else {
Laurent Montel's avatar
Laurent Montel committed
229
        f.read((char *)&pmg_head, sizeof(pmg_head));
230 231 232 233 234 235 236 237 238 239 240 241 242 243
        f.close();

        if (folderParsed) {
            folder.append(getFolderName(QString::fromLatin1(pmg_head.id)));
        } else {
            folder.append(QString::fromLatin1(pmg_head.folder));
        }

        filterInfo()->setTo(folder);
        filterInfo()->setTo(folder);
    }

    /** Read in the mbox */
    f.setFileName(file);
Laurent Montel's avatar
Laurent Montel committed
244
    if (!f.open(QIODevice::ReadOnly)) {
245 246 247 248
        filterInfo()->alert(i18n("Unable to open %1, skipping", s));
    } else {
        filterInfo()->addInfoLogEntry(i18n("Importing %1", QStringLiteral("../") + QString::fromLatin1(pmg_head.folder)));
        l = f.readLine(line.data(), MAX_LINE); // read the first line which is unneeded
Laurent Montel's avatar
Laurent Montel committed
249
        while (!f.atEnd()) {
250 251 252 253 254 255
            QTemporaryFile tempfile;
            tempfile.open();

            // we lost the last line, which is the first line of the new message in
            // this lopp, but this is ok, because this is the separate line with
            // "From ???@???" and we can forget them
Laurent Montel's avatar
Laurent Montel committed
256
            while (!f.atEnd() && (l = f.readLine(line.data(), MAX_LINE)) && ((separate = QString::fromLatin1(line.data())).left(5) != QLatin1String("From "))) {
257 258 259 260 261 262
                tempfile.write(line.data(), l);
                if (filterInfo()->shouldTerminate()) {
                    return;
                }
            }
            tempfile.flush();
263 264 265
            if (!importMessage(folder, tempfile.fileName(), filterInfo()->removeDupMessage())) {
                filterInfo()->addErrorLogEntry(i18n("Could not import %1", tempfile.fileName()));
            }
266 267 268

            n++;
            filterInfo()->setCurrent(i18n("Message %1", n));
Laurent Montel's avatar
Laurent Montel committed
269
            filterInfo()->setCurrent((int)(((float)f.pos() / f.size()) * 100));
270 271 272 273 274 275
        }
    }
    f.close();
}

/** Parse the m_filterInfoormation about folderstructure to folderMatrix */
Laurent Montel's avatar
Laurent Montel committed
276
bool FilterPMail::parseFolderMatrix(const QString &chosendir)
277 278 279 280 281
{
    qCDebug(MAILIMPORTER_LOG) << "Start parsing the foldermatrix.";
    filterInfo()->addInfoLogEntry(i18n("Parsing the folder structure..."));

    QFile hierarch(chosendir + QLatin1String("/hierarch.pm"));
Laurent Montel's avatar
Laurent Montel committed
282
    if (!hierarch.open(QIODevice::ReadOnly)) {
283 284 285 286 287 288 289 290 291 292 293
        filterInfo()->alert(i18n("Unable to open %1, skipping", chosendir + QLatin1String("hierarch.pm")));
        return false;
    } else {
        QByteArray tmpRead;
        while (!hierarch.atEnd()) {
            tmpRead = hierarch.readLine();
            if (tmpRead.isEmpty()) {
                break;
            }
            QString tmpArray[5];
            tmpRead.remove(tmpRead.length() - 2, 2);
294
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
295
            QStringList tmpList = QString::fromLatin1(tmpRead).split(QLatin1Char(','), QString::SkipEmptyParts);
296 297 298
#else
            QStringList tmpList = QString::fromLatin1(tmpRead).split(QLatin1Char(','), Qt::SkipEmptyParts);
#endif
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
            int i = 0;
            QStringList::ConstIterator end(tmpList.constEnd());
            for (QStringList::ConstIterator it = tmpList.constBegin(); it != end; ++it, ++i) {
                QString _tmp = *it;
                if (i < 5) {
                    tmpArray[i] = _tmp.remove(QLatin1Char('\"'));
                } else {
                    hierarch.close();
                    return false;
                }
            }
            folderMatrix.append(FolderStructure(tmpArray));
        }
    }
    hierarch.close();
    return true;
}

/** get the foldername for a given file ID from folderMatrix */
QString FilterPMail::getFolderName(const QString &ID)
{
    bool found = false;
    QString folder;
    QString search = ID;

    while (!found) {
        FolderStructureIterator end(folderMatrix.end());
        for (FolderStructureIterator it = folderMatrix.begin(); it != end; ++it) {
            FolderStructure tmp = *it;

            QString _ID = tmp[2];
            if (_ID == search) {
                QString _type = tmp[0] + tmp[1];
                if ((_type == QLatin1String("21"))) {
                    found = true;
                    break;
                } else {
                    folder.prepend((tmp[4] + QLatin1Char('/')));
                    search = tmp[3];
                }
            }
        }
    }
    return folder;
}