summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas Bruederli <thomas@roundcube.net>2014-02-06 16:25:16 (GMT)
committerThomas Bruederli <thomas@roundcube.net>2014-02-06 16:25:16 (GMT)
commit2ade247a0cef2cce7177cf6d68b87d1c3f19cedd (patch)
tree64ff8000a1fc0e65b9acd65b01f1a872a8ae36a7
parent7d39a1808a041143d3829c8e38d3d3376948f384 (diff)
downloadroundcubemail-plugins-kolab-2ade247a0cef2cce7177cf6d68b87d1c3f19cedd.tar.gz
Return a kolab_storage_dataset itertor object from kolab_storage_cache::select()
to manage memory usage for large result sets (#2828). Attention! Do not cast the return value of kolab_storage_folder::select() calls into an array anymore.
-rw-r--r--plugins/libkolab/lib/kolab_storage_cache.php20
-rw-r--r--plugins/libkolab/lib/kolab_storage_dataset.php129
2 files changed, 142 insertions, 7 deletions
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index efd61ef..f261220 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -239,14 +239,14 @@ class kolab_storage_cache
);
if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
- $this->objects[$msguid] = $this->_unserialize($sql_arr);
+ $this->objects = array($msguid => $this->_unserialize($sql_arr)); // store only this object in memory (#2827)
}
}
// fetch from IMAP if not present in cache
if (empty($this->objects[$msguid])) {
$result = $this->_fetch(array($msguid), $type, $foldername);
- $this->objects[$msguid] = $result[0];
+ $this->objects = array($msguid => $result[0]); // store only this object in memory (#2827)
}
}
@@ -438,14 +438,16 @@ class kolab_storage_cache
*/
public function select($query = array(), $uids = false)
{
- $result = array();
+ $result = $uids ? array() : new kolab_storage_dataset($this);
// read from local cache DB (assume it to be synchronized)
if ($this->ready) {
$this->_read_folder_data();
+ // fetch full object data on one query if a small result set is expected
+ $fetchall = !$uids && $this->count($query) < 500;
$sql_result = $this->db->query(
- "SELECT " . ($uids ? 'msguid, uid' : '*') . " FROM $this->cache_table ".
+ "SELECT " . (!$fetchall ? 'msguid, msguid AS _msguid, uid' : '*') . " FROM $this->cache_table ".
"WHERE folder_id=? " . $this->_sql_where($query),
$this->folder_id
);
@@ -459,9 +461,13 @@ class kolab_storage_cache
$this->uid2msg[$sql_arr['uid']] = $sql_arr['msguid'];
$result[] = $sql_arr['uid'];
}
- else if ($object = $this->_unserialize($sql_arr)) {
+ else if ($fetchall && ($object = $this->_unserialize($sql_arr))) {
$result[] = $object;
}
+ else {
+ // only add msguid to dataset index
+ $result[] = $sql_arr;
+ }
}
}
// use IMAP
@@ -491,7 +497,7 @@ class kolab_storage_cache
if (!$uids && count($result) == 1) {
if ($msguid = $result[0]['_msguid']) {
$this->uid2msg[$result[0]['uid']] = $msguid;
- $this->objects[$msguid] = $result[0];
+ $this->objects = array($msguid => $result[0]);
}
}
@@ -618,7 +624,7 @@ class kolab_storage_cache
*/
protected function _fetch($index, $type = null, $folder = null)
{
- $results = array();
+ $results = new kolab_storage_dataset($this);
foreach ((array)$index as $msguid) {
if ($object = $this->folder->read_object($msguid, $type, $folder)) {
$results[] = $object;
diff --git a/plugins/libkolab/lib/kolab_storage_dataset.php b/plugins/libkolab/lib/kolab_storage_dataset.php
new file mode 100644
index 0000000..17e66de
--- /dev/null
+++ b/plugins/libkolab/lib/kolab_storage_dataset.php
@@ -0,0 +1,129 @@
+<?php
+
+/**
+ * Dataset class providing the results of a select operation on a kolab_storage_folder.
+ *
+ * Can be used as a normal array as well as an iterator in foreach() loops.
+ *
+ * @version @package_version@
+ * @author Thomas Bruederli <bruederli@kolabsys.com>
+ *
+ * Copyright (C) 2014, Kolab Systems AG <contact@kolabsys.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+class kolab_storage_dataset implements Iterator, ArrayAccess
+{
+ private $cache; // kolab_storage_cache instance to use for fetching data
+ private $memlimit = 0;
+ private $buffer = false;
+ private $index = array();
+ private $data = array();
+ private $iteratorkey = -1;
+
+ /**
+ * Default constructor
+ *
+ * @param object kolab_storage_cache instance to be used for fetching objects upon access
+ */
+ public function __construct($cache)
+ {
+ $this->cache = $cache;
+
+ // enable in-memory buffering up until 1/5 of the available memory
+ if (function_exists('memory_get_usage')) {
+ $this->memlimit = parse_bytes(ini_get('memory_limit')) / 5;
+ $this->buffer = true;
+ }
+ }
+
+
+ /*** Implement PHP ArrayAccess interface ***/
+
+ public function offsetSet($offset, $value)
+ {
+ $uid = $value['_msguid'];
+
+ if (is_null($offset)) {
+ $offset = count($this->index);
+ $this->index[] = $uid;
+ }
+ else {
+ $this->index[$offset] = $uid;
+ }
+
+ // keep full payload data in memory if possible
+ if ($this->memlimit && $this->buffer && isset($value['_mailbox'])) {
+ $this->data[$offset] = $value;
+
+ // check memory usage and stop buffering
+ if ($offset % 10 == 0) {
+ $this->buffer = memory_get_usage() < $this->memlimit;
+ }
+ }
+ }
+
+ public function offsetExists($offset)
+ {
+ return isset($this->index[$offset]);
+ }
+
+ public function offsetUnset($offset)
+ {
+ unset($this->index[$offset]);
+ }
+
+ public function offsetGet($offset)
+ {
+ if (isset($this->data[$offset])) {
+ return $this->data[$offset];
+ }
+ else if ($msguid = $this->index[$offset]) {
+ return $this->cache->get($msguid);
+ }
+
+ return null;
+ }
+
+
+ /*** Implement PHP Iterator interface ***/
+
+ public function current()
+ {
+ return $this->offsetGet($this->iteratorkey);
+ }
+
+ public function key()
+ {
+ return $this->iteratorkey;
+ }
+
+ public function next()
+ {
+ $this->iteratorkey++;
+ return $this->valid();
+ }
+
+ public function rewind()
+ {
+ $this->iteratorkey = 0;
+ }
+
+ public function valid()
+ {
+ return !empty($this->index[$this->iteratorkey]);
+ }
+
+}