Commit 05924776 authored by Enrico Ros's avatar Enrico Ros

The long awaited (by me) memory manager is in place. It unloads pixmaps

not on screen starting from the oldest generated one. Rules (aka memory
profiles) are simple by now, but they work as expected.
Plus: added functions for getting Linux total memory and free memory by
querying the /proc filesystem. Generator creation by mimetype. -Fixes.
BROKEN: thumbnail, presentation (need to reimpl a virtual method)
TODO1: notify generators when cancelling jobs.
TODO2: add forward caching.
TODO3: rationalize code, clean it up and update README.png
TODO4: periodically check for free memory and unload pixmaps if needed.
TODO5: wisely choose default values for memory profiles.

svn path=/branches/kpdf_experiments/kdegraphics/kpdf/; revision=372514
parent 70ea81ca
......@@ -35,7 +35,7 @@
//NOTE: XPDF/Splash *implementation dependant* code is marked with '###'
//BEGIN KPDFOutputDev
KPDFOutputDev::KPDFOutputDev( GeneratorPDF * parent, SplashColor paperColor )
KPDFOutputDev::KPDFOutputDev( PDFGenerator * parent, SplashColor paperColor )
: SplashOutputDev( splashModeRGB8, false, paperColor ), m_pixmap( 0 ),
m_generator( parent ), m_text( 0 )
{
......
......@@ -25,7 +25,7 @@
class QPixmap;
class TextPage;
class GeneratorPDF;
class PDFGenerator;
class KPDFLink;
class KPDFPageRect;
......@@ -42,7 +42,7 @@ class KPDFPageRect;
class KPDFOutputDev : public SplashOutputDev
{
public:
KPDFOutputDev( GeneratorPDF * parent, SplashColor paperColor );
KPDFOutputDev( PDFGenerator * parent, SplashColor paperColor );
virtual ~KPDFOutputDev();
// to be called before PDFDoc->displayPage( thisclass, .. )
......@@ -84,7 +84,7 @@ class KPDFOutputDev : public SplashOutputDev
int m_pixmapWidth;
int m_pixmapHeight;
QPixmap * m_pixmap;
GeneratorPDF * m_generator;
PDFGenerator * m_generator;
// text page generated on demand
TextPage * m_text;
......
......@@ -7,16 +7,17 @@ Legend:
(*) - Some parts of this item are already done
In progress on the branch (first item comes first):
-> create a memory manager in Document with different profiles [80%]
-> FIX: viewport changes the right way when clicking links and TOC items (also
suggested by Mikolaj Machowski). Create a great viewport definition and merge
it inside the synopsis too. [70% done]
-> memory manager with different profiles (mem/cpu tradeoff: {memory saving, normal, memory aggressive}) [20%]
Things to do in order to merge in HEAD (first item has highest priority):
-> take care of naming on merge, too differences (remove some kpdf_* prefixes
and rename internals too document->kpdfdocument, page->kpdfpage, etc..)
Higher priority after merge:
-> move toolbar view actions in the PageView instead of the part.. maybe.. or not...
-> link thumbnails view with document [first, the 'Viewport' must be defined]
-> usability: layout 2PPV [1 2,3 4,5 6] -> [1,2 3,4 5]. add option for 'ebook' style alignment. (by Mikolaj)
-> usability: trigger redraw on 'filter text' on current page (by Mikolaj)
......
......@@ -31,7 +31,7 @@
#include "generator_pdf.h" // PDF generator
//#include "generator_ps.H" // PS generator
// structure used internally by KPDFDocument for local variables storage
// structures used internally by KPDFDocument for local variables storage
class KPDFDocumentPrivate
{
public:
......@@ -47,13 +47,23 @@ class KPDFDocumentPrivate
int currentPage;
// observers related (note: won't delete oservers)
QMap< int, KPDFDocumentObserver* > observers;
QMap< int, class ObserverData* > observers;
};
struct ObserverData
{
// public data fields
KPDFDocumentObserver * observer;
QMap< int, int > pageMemory;
int totalMemory;
// public constructor: initialize data
ObserverData( KPDFDocumentObserver * obs ) : observer( obs ), totalMemory( 0 ) {};
};
#define foreachObserver( cmd ) {\
QMap<int,KPDFDocumentObserver*>::iterator it = d->observers.begin();\
QMap<int,KPDFDocumentObserver*>::iterator end = d->observers.end();\
for ( ; it != end ; ++ it ) { (*it)-> cmd ; } }
QMap< int, ObserverData * >::iterator it = d->observers.begin(), end = d->observers.end();\
for ( ; it != end ; ++ it ) { (*it)->observer-> cmd ; } }
KPDFDocument::KPDFDocument()
: generator( 0 ), d( new KPDFDocumentPrivate )
......@@ -80,9 +90,19 @@ bool KPDFDocument::openDocument( const QString & docFile )
// reset internal status and frees memory
closeDocument();
// create the generator
// TODO: switch on mimetype for generator selection
generator = new GeneratorPDF();
// create the generator based on the file's mimetype
KMimeType::Ptr mime = KMimeType::findByPath( docFile );
QString mimeName = mime->name();
if ( mimeName == "application/pdf" )
generator = new PDFGenerator();
// else if ( mimeName == "application/postscript" )
// generator = new PSGenerator();
else
{
kdWarning() << "Unknown mimetype '" << mimeName << "'." << endl;
return false;
}
documentFileName = docFile;
bool openOk = generator->loadDocument( docFile, pages_vector );
if ( !openOk )
......@@ -127,7 +147,7 @@ void KPDFDocument::closeDocument()
void KPDFDocument::addObserver( KPDFDocumentObserver * pObserver )
{
// keep the pointer to the observer in a map
d->observers[ pObserver->observerId() ] = pObserver;
d->observers[ pObserver->observerId() ] = new ObserverData( pObserver );
// if the observer is added while a document is already opened, tell it
if ( !pages_vector.isEmpty() )
......@@ -137,7 +157,17 @@ void KPDFDocument::addObserver( KPDFDocumentObserver * pObserver )
void KPDFDocument::removeObserver( KPDFDocumentObserver * pObserver )
{
// remove observer from the map. it won't receive notifications anymore
d->observers.remove( pObserver->observerId() );
if ( d->observers.contains( pObserver->observerId() ) )
{
// free observer data
int observerId = pObserver->observerId();
QValueVector<KPDFPage*>::iterator it = pages_vector.begin(), end = pages_vector.end();
for ( ; it != end; ++it )
(*it)->deletePixmap( observerId );
// delete observer
delete d->observers[ observerId ];
d->observers.remove( observerId );
}
}
void KPDFDocument::reparseConfig()
......@@ -185,15 +215,65 @@ bool KPDFDocument::okToPrint() const
}
void KPDFDocument::requestPixmap( int id, uint page, int width, int height, bool syn )
void KPDFDocument::requestPixmap( int id, int page, int width, int height, bool syn )
{
KPDFPage * kp = pages_vector[ page ];
if ( !generator || !kp || kp->width() < 1 || kp->height() < 1 )
return;
// 1. Update statistics (pageMemory / totalMemory) adding this pixmap
ObserverData * obs = d->observers[ id ];
if ( obs->pageMemory.contains( page ) )
obs->totalMemory -= obs->pageMemory[ page ];
int pixmapMemory = 4 * width * height / 1024;
obs->pageMemory[ page ] = pixmapMemory;
obs->totalMemory += pixmapMemory;
//
int memoryToFree;
switch ( Settings::memoryLevel() )
{
case Settings::EnumMemoryLevel::Low:
memoryToFree = obs->totalMemory;
break;
case Settings::EnumMemoryLevel::Normal:
memoryToFree = obs->totalMemory - mTotalMemory()/4;
printf("%d\n",memoryToFree);
break;
case Settings::EnumMemoryLevel::Aggressive:
memoryToFree = 0;
break;
}
// 2. FREE Memory Loop. remove older data first.
int freed = 0;
if ( memoryToFree > 0 )
{
QMap< int, int >::iterator it = obs->pageMemory.begin(), end = obs->pageMemory.end();
while ( (it != end) && (memoryToFree > 0) )
{
int freeNumber = it.key();
if ( page != freeNumber && obs->observer->canUnloadPixmap( freeNumber ) )
{
// update mem stats
memoryToFree -= it.data();
obs->totalMemory -= it.data();
obs->pageMemory.remove( it );
// delete pixmap
pages_vector[ freeNumber ]->deletePixmap( id );
freed++;
}
++it;
}
}
kdWarning() << "[" << obs->totalMemory << "] Removed " << freed << " pages. " << obs->pageMemory.count() << " pages kept in memory." << endl;
// 3. Request next pixmap to generator
bool pixChanged = generator->requestPixmap( id, kp, width, height, syn );
if ( pixChanged )
d->observers[id]->notifyPixmapChanged( page );
d->observers[id]->observer->notifyPixmapChanged( page );
}
void KPDFDocument::requestTextPage( uint page )
......@@ -202,6 +282,8 @@ void KPDFDocument::requestTextPage( uint page )
if ( !generator || !kp )
return;
// Memory management for TextPages
generator->requestTextPage( kp );
}
......@@ -424,6 +506,58 @@ bool KPDFDocument::print( KPrinter &printer )
}
int KPDFDocument::mTotalMemory()
{
#ifdef __linux__
// if /proc/meminfo doesn't exist, return 128MB
QFile memFile( "/proc/meminfo" );
if ( !memFile.open( IO_ReadOnly ) )
return 131072;
// read /proc/meminfo and sum up the contents of 'MemFree', 'Buffers'
// and 'Cached' fields. consider swapped memory as used memory.
QTextStream readStream( &memFile );
while ( !readStream.atEnd() )
{
QString entry = readStream.readLine();
if ( entry.startsWith( "MemTotal:" ) )
return entry.section( ' ', -2, -2 ).toInt();
}
#endif
return 131072;
}
int KPDFDocument::mFreeMemory()
{
#ifdef __linux__
// if /proc/meminfo doesn't exist, return 128MB
QFile memFile( "/proc/meminfo" );
if ( !memFile.open( IO_ReadOnly ) )
return 131072;
// read /proc/meminfo and sum up the contents of 'MemFree', 'Buffers'
// and 'Cached' fields. consider swapped memory as used memory.
int memoryFree = 0;
QString entry;
QTextStream readStream( &memFile );
while ( !readStream.atEnd() )
{
entry = readStream.readLine();
if ( entry.startsWith( "MemFree:" ) ||
entry.startsWith( "Buffers:" ) ||
entry.startsWith( "Cached:" ) ||
entry.startsWith( "SwapFree:" ) )
memoryFree += entry.section( ' ', -2, -2 ).toInt();
if ( entry.startsWith( "SwapTotal:" ) )
memoryFree -= entry.section( ' ', -2, -2 ).toInt();
}
memFile.close();
return memoryFree;
#else
return 131072;
#endif
}
QString KPDFDocument::giveAbsolutePath( const QString & fileName )
{
if ( documentFileName.isEmpty() )
......
......@@ -34,13 +34,16 @@ class KPDFDocumentObserver
// you must give each observer a unique ID (used for notifications)
virtual uint observerId() const = 0;
// monitor changes in pixmaps (generation thread complete)
virtual void notifyPixmapChanged( int /*pageNumber*/ ) {};
virtual void notifyPixmapsCleared() {};
// commands from the Document to all observers
virtual void pageSetup( const QValueVector<KPDFPage*> & /*pages*/, bool /*documentChanged*/ ) {};
virtual void pageSetCurrent( int /*pageNumber*/, const QRect & /*viewport*/ = QRect() ) {};
// queries to observers
virtual bool canUnloadPixmap( int /*pageNum*/ ) { return true; }
// monitor changes in pixmaps (generation thread complete)
virtual void notifyPixmapChanged( int /*pageNumber*/ ) {};
virtual void notifyPixmapsCleared() {};
};
#define PRESENTATION_ID 1
......@@ -90,7 +93,8 @@ class KPDFDocument
bool okToPrint() const;
// perform actions on document / pages
void requestPixmap( int id, uint page, int width, int height, bool syncronous = false );
//void requestPixmaps( int id, const QValueList<int> & pages, int width, int height, bool syncronous = false );
void requestPixmap( int id, int pageNum, int width, int height, bool syncronous = false );
void requestTextPage( uint page );
void setCurrentPage( int page, const QRect & viewport = QRect() );
void findText( const QString & text = "", bool caseSensitive = false );
......@@ -100,6 +104,10 @@ class KPDFDocument
bool print( KPrinter &printer );
private:
// memory management related functions
int mTotalMemory();
int mFreeMemory();
// more private functions
QString giveAbsolutePath( const QString & fileName );
bool openRelativeFile( const QString & fileName );
void processPageList( bool documentChanged );
......@@ -129,6 +137,7 @@ struct DocumentInfo
producer,
subject,
title,
mimeType,
format,
formatVersion,
encryption,
......
......@@ -10,6 +10,7 @@
#ifndef _KPDF_GENERATOR_H_
#define _KPDF_GENERATOR_H_
#include <qobject.h>
#include <qvaluevector.h>
#include <qstring.h>
class KPrinter;
......@@ -19,6 +20,17 @@ class KPDFDocument;
class DocumentSynopsis;
class DocumentInfo;
/* Note: on contents generation and asyncronous queries.
* Many observers may want to request data syncronously or asyncronously.
* - Sync requests. These should be done in-place. Syncronous events in the
* queue have precedence on all the asyncronous ones.
* - Async request must be done in real background. That usually means a
* thread, such as QThread derived classes.
* Once contents are available, they must be immediately stored in the
* KPDFPage they refer to, and a signal is emitted as soon as storing
* (even for sync or async queries) has been done.
*/
/**
* @short [Abstract Class] The information generator.
*
......@@ -29,8 +41,9 @@ class DocumentInfo;
* class stores the resulting data into 'KPDFPage's. The data will then be
* displayed by the GUI components (pageView, thumbnailList, etc..).
*/
class Generator
class Generator : public QObject
{
Q_OBJECT
public:
// load a document and fill up the pagesVector
virtual bool loadDocument( const QString & fileName, QValueVector< KPDFPage* > & pagesVector ) = 0;
......@@ -43,13 +56,16 @@ class Generator
enum Permissions { Modify = 1, Copy = 2, Print = 4, AddNotes = 8 };
virtual bool allowed( int /*permisisons*/ ) { return true; }
// perform actions (/request content generation)
virtual bool print( KPrinter& printer ) = 0;
// generator core
virtual bool print( KPrinter& printer ) { return false; }
virtual bool requestPixmap( int id, KPDFPage * page, int width, int height, bool syncronous = false ) = 0;
virtual void requestTextPage( KPDFPage * page ) = 0;
// check configuration and return if something changed
// check configuration and return true if something changed
virtual bool reparseConfig() { return false; }
signals:
void contentsChanged( const KPDFPage * page );
};
#endif
......@@ -35,7 +35,7 @@
#include "QOutputDev.h"
GeneratorPDF::GeneratorPDF()
PDFGenerator::PDFGenerator()
: pdfdoc( 0 ), kpdfOutputDev( 0 ),
docInfoDirty( true ), docSynopsisDirty( true )
{
......@@ -43,7 +43,7 @@ GeneratorPDF::GeneratorPDF()
reparseConfig();
}
GeneratorPDF::~GeneratorPDF()
PDFGenerator::~PDFGenerator()
{
docLock.lock();
delete kpdfOutputDev;
......@@ -52,7 +52,7 @@ GeneratorPDF::~GeneratorPDF()
}
bool GeneratorPDF::loadDocument( const QString & fileName, QValueVector<KPDFPage*> & pagesVector )
bool PDFGenerator::loadDocument( const QString & fileName, QValueVector<KPDFPage*> & pagesVector )
{
// create PDFDoc for the given file
GString *filename = new GString( QFile::encodeName( fileName ) );
......@@ -122,7 +122,7 @@ bool GeneratorPDF::loadDocument( const QString & fileName, QValueVector<KPDFPage
}
const DocumentInfo * GeneratorPDF::documentInfo()
const DocumentInfo * PDFGenerator::documentInfo()
{
if ( docInfoDirty )
{
......@@ -135,6 +135,7 @@ const DocumentInfo * GeneratorPDF::documentInfo()
docInfo.producer = getDocumentInfo("Producer");
docInfo.subject = getDocumentInfo("Subject");
docInfo.title = getDocumentInfo("Title");
docInfo.mimeType = "application/pdf";
docInfo.format = "PDF";
if ( pdfdoc )
{
......@@ -156,7 +157,7 @@ const DocumentInfo * GeneratorPDF::documentInfo()
return &docInfo;
}
const DocumentSynopsis * GeneratorPDF::documentSynopsis()
const DocumentSynopsis * PDFGenerator::documentSynopsis()
{
if ( !docSynopsisDirty )
return &docSyn;
......@@ -180,7 +181,7 @@ const DocumentSynopsis * GeneratorPDF::documentSynopsis()
return &docSyn;
}
void GeneratorPDF::addSynopsisChildren( QDomNode * parent, GList * items )
void PDFGenerator::addSynopsisChildren( QDomNode * parent, GList * items )
{
int numItems = items->getLength();
for ( int i = 0; i < numItems; ++i )
......@@ -220,7 +221,7 @@ void GeneratorPDF::addSynopsisChildren( QDomNode * parent, GList * items )
}
bool GeneratorPDF::print( KPrinter& printer )
bool PDFGenerator::print( KPrinter& printer )
{
KTempFile tf( QString::null, ".ps" );
PSOutputDev *psOut = new PSOutputDev(tf.name().latin1(), pdfdoc->getXRef(), pdfdoc->getCatalog(), 1, pdfdoc->getNumPages(), psModePS);
......@@ -258,7 +259,7 @@ bool GeneratorPDF::print( KPrinter& printer )
}
}
bool GeneratorPDF::requestPixmap( int id, KPDFPage * page, int width, int height, bool syncronous )
bool PDFGenerator::requestPixmap( int id, KPDFPage * page, int width, int height, bool syncronous )
{
//kdDebug() << "id: " << id << " is requesting pixmap for page " << page->number() << " [" << width << " x " << height << "]." << endl;
if ( syncronous )
......@@ -301,7 +302,7 @@ bool GeneratorPDF::requestPixmap( int id, KPDFPage * page, int width, int height
return false;
}
void GeneratorPDF::requestTextPage( KPDFPage * page )
void PDFGenerator::requestTextPage( KPDFPage * page )
{
// build a TextPage using the lightweight KPDFTextDev generator..
KPDFTextDev td;
......@@ -312,7 +313,7 @@ void GeneratorPDF::requestTextPage( KPDFPage * page )
page->setSearchPage( td.takeTextPage() );
}
bool GeneratorPDF::reparseConfig()
bool PDFGenerator::reparseConfig()
{
// load paper color from Settings or use the white default color
QColor color = ( (Settings::renderMode() == Settings::EnumRenderMode::Paper ) &&
......@@ -337,7 +338,7 @@ bool GeneratorPDF::reparseConfig()
return false;
}
KPDFLinkGoto::Viewport GeneratorPDF::decodeLinkViewport( GString * namedDest, LinkDest * dest )
KPDFLinkGoto::Viewport PDFGenerator::decodeLinkViewport( GString * namedDest, LinkDest * dest )
// note: this function is called when processing a page, when the MUTEX is already LOCKED
{
KPDFLinkGoto::Viewport vp;
......@@ -398,7 +399,7 @@ KPDFLinkGoto::Viewport GeneratorPDF::decodeLinkViewport( GString * namedDest, Li
}
QString GeneratorPDF::getDocumentInfo( const QString & data ) const
QString PDFGenerator::getDocumentInfo( const QString & data ) const
{
// [Albert] Code adapted from pdfinfo.cc on xpdf
Object info;
......@@ -457,7 +458,7 @@ QString GeneratorPDF::getDocumentInfo( const QString & data ) const
return i18n( "Unknown" );
}
QString GeneratorPDF::getDocumentDate( const QString & data ) const
QString PDFGenerator::getDocumentDate( const QString & data ) const
{
// [Albert] Code adapted from pdfinfo.cc on xpdf
Object info;
......
......@@ -14,6 +14,7 @@
#include <qmutex.h>
#include <qcolor.h>
#include <qstring.h>
#include <qthread.h>
#include "generator.h"
#include "document.h"
#include "link.h"
......@@ -25,13 +26,22 @@ class KPDFOutputDev;
/**
* @short A generator that builds contents from a PDF document.
*
* ...
* All Generator features are supported and implented by this one.
* Internally this holds a reference to xpdf's core objects and provides
* contents generation using the PDFDoc object and a couple of OutputDevices
* called KPDFOutputDev and KPDFTextDev (both defined in QOutputDev.h).
*
* For generating page contents we tell PDFDoc to render a page and grab
* contents from out OutputDevs when rendering finishes.
*
* Background asyncronous contents providing is done via a QThread inherited
* class defined at the bottom of the file.
*/
class GeneratorPDF : public Generator
class PDFGenerator : public Generator
{
public:
GeneratorPDF();
virtual ~GeneratorPDF();
PDFGenerator();
virtual ~PDFGenerator();
// [INHERITED] load a document and fill up the pagesVector
bool loadDocument( const QString & fileName, QValueVector<KPDFPage*> & pagesVector );
......@@ -69,33 +79,29 @@ class GeneratorPDF : public Generator
DocumentSynopsis docSyn;
};
/*
#ifndef THUMBNAILGENERATOR_H
#define THUMBNAILGENERATOR_H
#include <qthread.h>
class QMutex;
class ThumbnailGenerator : public QThread
/**
* @short A thread that builds contents for PDFGenerator in the background.
*
*
*/
class PDFGeneratorThread : public QThread
{
/*
public:
ThumbnailGenerator(PDFDoc *doc, QMutex *docMutex, int page, double ppp, QObject *o);
PDFGeneratorThread(PDFDoc *doc, QMutex *docMutex, int page, double ppp, QObject *o);
int getPage() const;
protected:
void run();
private:
PDFDoc *m_doc;
QMutex *m_docMutex;
int m_page;
QObject *m_o;
double m_ppp;
};
#endif
*/
};
#endif
......@@ -53,6 +53,8 @@ bool KPDFPage::hasPixmap( int id, int width, int height ) const
{
if ( !m_pixmaps.contains( id ) )
return false;
if ( width == -1 || height == -1 )
return true;
QPixmap * p = m_pixmaps[ id ];
return p ? ( p->width() == width && p->height() == height ) : false;
}
......@@ -136,6 +138,15 @@ void KPDFPage::setRects( const QValueList< KPDFPageRect * > rects )
m_rects = rects;
}
void KPDFPage::deletePixmap( int id )
{
if ( m_pixmaps.contains( id ) )
{
delete m_pixmaps[ id ];
m_pixmaps.remove( id );
}
}
void KPDFPage::deletePixmapsAndRects()
{
// delete all stored pixmaps
......
......@@ -45,7 +45,7 @@ class KPDFPage
inline float height() const { return m_height; }
inline float ratio() const { return m_height / m_width; }
bool hasPixmap( int id, int width, int height ) const;
bool hasPixmap( int id, int width = -1, int height = -1 ) const;
bool hasSearchPage() const;
bool hasRect( int mouseX, int mouseY ) const;
const KPDFPageRect * getRect( int mouseX, int mouseY ) const;
......@@ -57,10 +57,11 @@ class KPDFPage
inline void toggleAttribute( int att ) { m_attributes ^= att; }
bool hasText( const QString & text, bool strictCase, bool fromTop );
// set contents (by KPDFDocument)
// set/delete contents (by KPDFDocument)
void setPixmap( int id, QPixmap * pixmap );
void setSearchPage( TextPage * text );
void setRects( const QValueList< KPDFPageRect * > rects );
void deletePixmap( int id );
void deletePixmapsAndRects();
private:
......
......@@ -53,6 +53,7 @@ public:
PageViewItem * activeItem; //equal to items[vectorIndex]
QValueVector< PageViewItem * > items;
int vectorIndex;
QValueList< PageViewItem * > visibleItems;