Greatly reduce memory usage required by Konsole's scrollback buffer and...

Greatly reduce memory usage required by Konsole's scrollback buffer and improve releasing of scrollback memory to OS when no longer needed.

* Reduce amount of memory required for representing characters in scrollback buffer by not storing formatting data for each character.  Instead store UTF-16 characters and formatting ranges
* Allocate memory for scrollback buffer use a custom mmap-based memory pool - this allows the memory to be released to the OS sooner when freed.

There is a small performance cost when printing a large number of lines - see bug report for figures.

Patch by Michael Meier

REVIEW: http://reviewboard.kde.org/r/802/
BUG:176974


svn path=/trunk/KDE/kdebase/apps/konsole/; revision=987102

src/Character.h

@@ -104,6 +104,11 @@ public:
    */
   ColorEntry::FontWeight fontWeight(const ColorEntry* base) const;
   
+  /** 
+   * returns true if the format (color, rendition flag) of the compared characters is equal
+   */
+  bool equalsFormat(const Character &other) const;
+
   /** 
    * Compares two characters and returns true if they have the same unicode character value,
    * rendition and colors.
@@ -140,6 +145,14 @@ inline bool Character::isTransparent(const ColorEntry* base) const
           base[backgroundColor._u+2+(backgroundColor._v?BASE_COLORS:0)].transparent);
 }
 
+inline bool Character::equalsFormat(const Character& other) const
+{
+  return 
+    backgroundColor==other.backgroundColor &&
+    foregroundColor==other.foregroundColor &&
+    rendition==other.rendition;
+}	
+
 inline ColorEntry::FontWeight Character::fontWeight(const ColorEntry* base) const
 {
     if (backgroundColor._colorSpace == COLOR_SPACE_DEFAULT)

src/History.cpp

@@ -528,6 +528,257 @@ void HistoryScrollBlockArray::addLine(bool)
 {
 }
 
+////////////////////////////////////////////////////////////////
+// Compact History Scroll //////////////////////////////////////
+////////////////////////////////////////////////////////////////
+void* CompactHistoryBlock::allocate ( size_t length )
+{
+ Q_ASSERT ( length > 0 );
+  if ( tail-blockStart+length > blockLength )
+    return NULL;
+
+  void* block = tail;
+  tail += length;
+  //kDebug() << "allocated " << length << " bytes at address " << block;
+  allocCount++;
+  return block;
+}
+
+void CompactHistoryBlock::deallocate ( )
+{
+  allocCount--;
+  Q_ASSERT ( allocCount >= 0 );
+}
+
+void* CompactHistoryBlockList::allocate(size_t size)
+{
+  CompactHistoryBlock* block;
+  if ( list.isEmpty() || list.last()->remaining() < size)
+  {
+    block = new CompactHistoryBlock();
+    list.append ( block );
+    //kDebug() << "new block created, remaining " << block->remaining() << "number of blocks=" << list.size();
+  }
+  else
+  {
+    block = list.last();
+    //kDebug() << "old block used, remaining " << block->remaining();
+  }
+  return block->allocate(size);
+}
+
+void CompactHistoryBlockList::deallocate(void* ptr)
+{
+  Q_ASSERT( !list.isEmpty());
+  
+  int i=0;  
+  CompactHistoryBlock *block = list.at(i);
+  while ( i<list.size() && !block->contains(ptr) )
+  { 
+    i++;
+    block=list.at(i);
+  }
+
+  Q_ASSERT( i<list.size() );
+  
+  block->deallocate();
+
+  if (!block->isInUse())
+  {
+    list.removeAt(i);
+    delete block;
+    //kDebug() << "block deleted, new size = " << list.size();
+  }
+}
+
+CompactHistoryBlockList::~CompactHistoryBlockList()
+{
+  qDeleteAll ( list.begin(), list.end() );
+  list.clear();
+}
+
+void* CompactHistoryLine::operator new (size_t size, CompactHistoryBlockList& blockList)
+{
+  return blockList.allocate(size);
+}
+
+CompactHistoryLine::CompactHistoryLine ( const TextLine& line, CompactHistoryBlockList& bList ) 
+  : formatLength(0),
+    blockList(bList)
+{
+  length=line.size();
+      
+  if (line.size() > 0) {
+    formatLength=1;
+    int k=1;
+  
+    // count number of different formats in this text line
+    Character c = line[0];
+    while ( k<length )
+    {
+      if ( !(line[k].equalsFormat(c)))
+      {
+        formatLength++; // format change detected
+        c=line[k];
+      }
+      k++;
+    }
+  
+    //kDebug() << "number of different formats in string: " << formatLength;
+    formatArray = (CharacterFormat*) blockList.allocate(sizeof(CharacterFormat)*formatLength);
+    Q_ASSERT (formatArray!=NULL);
+    text = (quint16*) blockList.allocate(sizeof(quint16)*line.size());
+    Q_ASSERT (text!=NULL);
+  
+    length=line.size();
+    formatLength=formatLength;
+    wrapped=false;
+  
+    // record formats and their positions in the format array
+    c=line[0];
+    formatArray[0].setFormat ( c );
+    formatArray[0].startPos=0;                        // there's always at least 1 format (for the entire line, unless a change happens)
+  
+    k=1;                                              // look for possible format changes
+    int j=1;
+    while ( k<length && j<formatLength )
+    {
+      if (!(line[k].equalsFormat(c)))
+      {
+        c=line[k];
+        formatArray[j].setFormat(c);
+        formatArray[j].startPos=k;
+        //kDebug() << "format entry " << j << " at pos " << formatArray[j].startPos << " " << &(formatArray[j].startPos) ;
+        j++;
+      }
+      k++;
+    }
+  
+    // copy character values
+    for ( int i=0; i<line.size(); i++ )
+    {
+      text[i]=line[i].character;
+      //kDebug() << "char " << i << " at mem " << &(text[i]);
+    }
+  }
+  //kDebug() << "line created, length " << length << " at " << &(length);
+}
+
+CompactHistoryLine::~CompactHistoryLine()
+{
+  //kDebug() << "~CHL";
+  if (length>0) {
+    blockList.deallocate(text);
+    blockList.deallocate(formatArray);
+  }
+  blockList.deallocate(this);  
+}
+
+void CompactHistoryLine::getCharacter ( int index, Character &r )
+{
+  Q_ASSERT ( index < length );
+  int formatPos=0;
+  while ( ( formatPos+1 ) < formatLength && index >= formatArray[formatPos+1].startPos )
+    formatPos++;
+
+  r.character=text[index];
+  r.rendition = formatArray[formatPos].rendition;
+  r.foregroundColor = formatArray[formatPos].fgColor;
+  r.backgroundColor = formatArray[formatPos].bgColor;
+}
+
+void CompactHistoryLine::getCharacters ( Character* array, int length, int startColumn )
+{
+  Q_ASSERT ( startColumn >= 0 && length >= 0 );
+  Q_ASSERT ( startColumn+length <= ( int ) getLength() );
+
+  for ( int i=startColumn; i<length+startColumn; i++ )
+  {
+    getCharacter ( i, array[i-startColumn] );
+  }
+}
+
+CompactHistoryScroll::CompactHistoryScroll ( unsigned int maxLineCount )
+    : HistoryScroll ( new CompactHistoryType ( maxLineCount ) )
+    ,lines()
+    ,blockList()
+{
+  //kDebug() << "scroll of length " << maxLineCount << " created";
+  setMaxNbLines ( maxLineCount );
+}
+
+CompactHistoryScroll::~CompactHistoryScroll()
+{
+  qDeleteAll ( lines.begin(), lines.end() );
+  lines.clear();
+}
+
+void CompactHistoryScroll::addCellsVector ( const TextLine& cells )
+{
+  CompactHistoryLine *line;
+  line = new(blockList) CompactHistoryLine ( cells, blockList );
+
+  if ( lines.size() > ( int ) _maxLineCount )
+  {
+    delete lines.takeAt ( 0 );
+  }
+  lines.append ( line );
+}
+
+void CompactHistoryScroll::addCells ( const Character a[], int count )
+{
+  TextLine newLine ( count );
+  qCopy ( a,a+count,newLine.begin() );
+  addCellsVector ( newLine );
+}
+
+void CompactHistoryScroll::addLine ( bool previousWrapped )
+{
+  CompactHistoryLine *line = lines.last();
+  //kDebug() << "last line at address " << line;
+  line->setWrapped(previousWrapped);
+}
+
+int CompactHistoryScroll::getLines()
+{
+  return lines.size();
+}
+
+int CompactHistoryScroll::getLineLen ( int lineNumber )
+{
+  Q_ASSERT ( lineNumber >= 0 && lineNumber < lines.size() );
+  CompactHistoryLine* line = lines[lineNumber];
+  //kDebug() << "request for line at address " << line;
+  return line->getLength();
+}
+
+
+void CompactHistoryScroll::getCells ( int lineNumber, int startColumn, int count, Character* buffer )
+{
+  if ( count == 0 ) return;
+  Q_ASSERT ( lineNumber < lines.size() );
+  CompactHistoryLine* line = lines[lineNumber];
+  Q_ASSERT ( startColumn <= line->getLength() - count );
+  line->getCharacters ( buffer, count, startColumn );
+}
+
+void CompactHistoryScroll::setMaxNbLines ( unsigned int lineCount )
+{
+  _maxLineCount = lineCount;
+
+  while (lines.size() > (int) lineCount) {
+    delete lines.takeAt(0);
+  }
+  //kDebug() << "set max lines to: " << _maxLineCount;
+}
+
+bool CompactHistoryScroll::isWrappedLine ( int lineNumber )
+{
+  Q_ASSERT ( lineNumber < lines.size() );
+  return lines[lineNumber]->isWrapped();
+}
+
+
 //////////////////////////////////////////////////////////////////////
 // History Types
 //////////////////////////////////////////////////////////////////////
@@ -698,3 +949,35 @@ int HistoryTypeFile::maximumLineCount() const
 {
   return 0;
 }
+
+//////////////////////////////
+
+CompactHistoryType::CompactHistoryType ( unsigned int nbLines )
+    : m_nbLines ( nbLines )
+{
+}
+
+bool CompactHistoryType::isEnabled() const
+{
+  return true;
+}
+
+int CompactHistoryType::maximumLineCount() const
+{
+  return m_nbLines;
+}
+
+HistoryScroll* CompactHistoryType::scroll ( HistoryScroll *old ) const
+{
+  if ( old )
+  {
+    CompactHistoryScroll *oldBuffer = dynamic_cast<CompactHistoryScroll*> ( old );
+    if ( oldBuffer )
+    {
+      oldBuffer->setMaxNbLines ( m_nbLines );
+      return oldBuffer;
+    }
+    delete old;
+  }
+  return new CompactHistoryScroll ( m_nbLines );
+}

src/History.h

@@ -33,6 +33,9 @@
 #include "BlockArray.h"
 #include "Character.h"
 
+// map
+#include <sys/mman.h>
+
 namespace Konsole
 {
 
@@ -254,6 +257,133 @@ protected:
   QHash<int,size_t> m_lineLengths;
 };
 
+//////////////////////////////////////////////////////////////////////
+// History using compact storage
+// This implementation uses a list of fixed-sized blocks
+// where history lines are allocated in (avoids heap fragmentation)
+//////////////////////////////////////////////////////////////////////
+typedef QVector<Character> TextLine;
+
+class CharacterFormat
+{
+public:
+  bool equalsFormat(const CharacterFormat &other) const {
+    return other.rendition==rendition && other.fgColor==fgColor && other.bgColor==bgColor;
+  }
+
+  bool equalsFormat(const Character &c) const {
+    return c.rendition==rendition && c.foregroundColor==fgColor && c.backgroundColor==bgColor;
+  }
+
+  void setFormat(const Character& c) {
+    rendition=c.rendition;
+    fgColor=c.foregroundColor;
+    bgColor=c.backgroundColor;
+  }
+
+  CharacterColor fgColor, bgColor;
+  quint16 startPos;
+  quint8 rendition;
+};
+
+class CompactHistoryBlock
+{
+public:
+  
+  CompactHistoryBlock(){
+    blockLength = 4096*64; // 256kb
+    head = (quint8*) mmap(0, blockLength, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
+    //head = (quint8*) malloc(blockLength);
+    Q_ASSERT(head != MAP_FAILED);
+    tail = blockStart = head;
+    allocCount=0;
+  }
+  
+  virtual ~CompactHistoryBlock(){
+    //free(blockStart);
+    munmap(blockStart, blockLength);
+  }
+  
+  virtual unsigned int remaining(){ return blockStart+blockLength-tail;}
+  virtual unsigned  length() { return blockLength; }
+  virtual void* allocate(size_t length);
+  virtual bool contains(void *addr) {return addr>=blockStart && addr<(blockStart+blockLength);}
+  virtual void deallocate();
+  virtual bool isInUse(){ return allocCount!=0; } ;
+
+private:
+  size_t blockLength;
+  quint8* head;
+  quint8* tail;
+  quint8* blockStart;
+  int allocCount;
+};
+
+class CompactHistoryBlockList {
+public:
+  CompactHistoryBlockList() {};
+  ~CompactHistoryBlockList();
+
+  void *allocate( size_t size );
+  void deallocate(void *);
+  int length() {return list.size();}
+private:
+  QList<CompactHistoryBlock*> list;
+};
+
+class CompactHistoryLine
+{
+public:
+  CompactHistoryLine(const TextLine&, CompactHistoryBlockList& blockList);
+  virtual ~CompactHistoryLine();
+
+  // custom new operator to allocate memory from custom pool instead of heap
+  static void *operator new( size_t size, CompactHistoryBlockList& blockList);
+  static void operator delete( void *) { /* do nothing, deallocation from pool is done in destructor*/ } ;
+
+  virtual void getCharacters(Character* array, int length, int startColumn) ;
+  virtual void getCharacter(int index, Character &r) ;
+  virtual bool isWrapped() const {return wrapped;};
+  virtual void setWrapped(bool isWrapped) { wrapped=isWrapped;};
+  virtual unsigned int getLength() const {return length;};
+
+protected:
+  CompactHistoryBlockList& blockList;
+  CharacterFormat* formatArray;
+  quint16 length;
+  quint16* text;
+  quint16 formatLength;
+  bool wrapped;
+};
+
+class CompactHistoryScroll : public HistoryScroll
+{
+  typedef QList<CompactHistoryLine*> HistoryArray;
+
+public:
+  CompactHistoryScroll(unsigned int maxNbLines = 1000);
+  virtual ~CompactHistoryScroll();
+
+  virtual int  getLines();
+  virtual int  getLineLen(int lineno);
+  virtual void getCells(int lineno, int colno, int count, Character res[]);
+  virtual bool isWrappedLine(int lineno);
+
+  virtual void addCells(const Character a[], int count);
+  virtual void addCellsVector(const TextLine& cells);
+  virtual void addLine(bool previousWrapped=false);
+
+  void setMaxNbLines(unsigned int nbLines);
+  unsigned int maxNbLines() const { return _maxLineCount; }
+
+private:
+  bool hasDifferentColors(const TextLine& line) const;
+  HistoryArray lines;
+  CompactHistoryBlockList blockList;
+  
+  unsigned int _maxLineCount;
+};
+
 //////////////////////////////////////////////////////////////////////
 // History type
 //////////////////////////////////////////////////////////////////////
@@ -340,6 +470,21 @@ protected:
   unsigned int m_nbLines;
 };
 
+class CompactHistoryType : public HistoryType
+{
+public:
+  CompactHistoryType(unsigned int size);
+  
+  virtual bool isEnabled() const;
+  virtual int maximumLineCount() const;
+
+  virtual HistoryScroll* scroll(HistoryScroll *) const;
+
+protected:
+  unsigned int m_nbLines;
+};
+
+
 #endif
 
 }

src/SessionController.cpp

@@ -863,7 +863,7 @@ void SessionController::scrollBackOptionsChanged(int mode, int lines, bool saveT
             _session->setHistoryType( HistoryTypeNone() );
             break;
          case HistorySizeDialog::FixedSizeHistory:
-            _session->setHistoryType( HistoryTypeBuffer(lines) );
+            _session->setHistoryType( CompactHistoryType(lines) );
             break;
          case HistorySizeDialog::UnlimitedHistory:
              _session->setHistoryType( HistoryTypeFile() );

src/SessionManager.cpp

@@ -515,7 +515,7 @@ void SessionManager::applyProfile(Session* session, const Profile::Ptr info , bo
             case Profile::FixedSizeHistory:
                 {
                     int lines = info->property<int>(Profile::HistorySize);
-                    session->setHistoryType( HistoryTypeBuffer(lines) );
+                    session->setHistoryType( CompactHistoryType(lines) );
                 }
                 break;
             case Profile::UnlimitedHistory: