Moved readers to iterators

Instead of the hasNext() / next() syntax, readers now implements the PHP iterator pattern.
It allows readers to be used with a foreach() loop.

All readers now share the same structure (CSV is treated as having exactly one sheet):
- one concrete Reader
- one SheetIterator, exposed by the Reader
- one or more Sheets, returned at every iteration
- one RowIterator, exposed by the Sheet

Introducing the concept of sheets for CSV may be kind of confusing but it makes Spout way more consistent.
Also, this confusion may be resolved by creating a wrapper around the readers if needed.

-- This commit does not delete the old files, not change the folder structure for Writers. This will be done in another commit.

src/Spout/Reader/XLSX/Helper/SharedStringsCaching/CachingStrategyFactory.php

@@ -0,0 +1,154 @@
+<?php
+
+namespace Box\Spout\Reader\XLSX\Helper\SharedStringsCaching;
+
+/**
+ * Class CachingStrategyFactory
+ *
+ * @package Box\Spout\Reader\XLSX\Helper\SharedStringsCaching
+ */
+class CachingStrategyFactory
+{
+    /**
+     * The memory amount needed to store a string was obtained empirically from this data:
+     *
+     *        ------------------------------------
+     *        | Number of chars⁺ | Memory needed |
+     *        ------------------------------------
+     *        |           3,000  |         1 MB  |
+     *        |          15,000  |         2 MB  |
+     *        |          30,000  |         5 MB  |
+     *        |          75,000  |        11 MB  |
+     *        |         150,000  |        21 MB  |
+     *        |         300,000  |        43 MB  |
+     *        |         750,000  |       105 MB  |
+     *        |       1,500,000  |       210 MB  |
+     *        |       2,250,000  |       315 MB  |
+     *        |       3,000,000  |       420 MB  |
+     *        |       4,500,000  |       630 MB  |
+     *        ------------------------------------
+     *
+     *        ⁺ All characters were 1 byte long
+     *
+     * This gives a linear graph where each 1-byte character requires about 150 bytes to be stored.
+     * Given that some characters can take up to 4 bytes, we need 600 bytes per character to be safe.
+     * Also, there is on average about 20 characters per cell (this is entirely empirical data...).
+     *
+     * This means that in order to store one shared string in memory, the memory amount needed is:
+     *   => 20 * 600 ≈ 12KB
+     */
+    const AMOUNT_MEMORY_NEEDED_PER_STRING_IN_KB = 12;
+
+    /**
+     * To avoid running out of memory when extracting a huge number of shared strings, they can be saved to temporary files
+     * instead of in memory. Then, when accessing a string, the corresponding file contents will be loaded in memory
+     * and the string will be quickly retrieved.
+     * The performance bottleneck is not when creating these temporary files, but rather when loading their content.
+     * Because the contents of the last loaded file stays in memory until another file needs to be loaded, it works
+     * best when the indexes of the shared strings are sorted in the sheet data.
+     * 10,000 was chosen because it creates small files that are fast to be loaded in memory.
+     */
+    const MAX_NUM_STRINGS_PER_TEMP_FILE = 10000;
+
+    /** @var CachingStrategyFactory|null Singleton instance */
+    protected static $instance = null;
+
+    /**
+     * Private constructor for singleton
+     */
+    private function __construct()
+    {
+    }
+
+    /**
+     * Returns the singleton instance of the factory
+     *
+     * @return CachingStrategyFactory
+     */
+    public static function getInstance()
+    {
+        if (self::$instance === null) {
+            self::$instance = new CachingStrategyFactory();
+        }
+
+        return self::$instance;
+    }
+
+    /**
+     * Returns the best caching strategy, given the number of unique shared strings
+     * and the amount of memory available.
+     *
+     * @param int $sharedStringsUniqueCount Number of unique shared strings
+     * @param string|void $tempFolder Temporary folder where the temporary files to store shared strings will be stored
+     * @return CachingStrategyInterface The best caching strategy
+     */
+    public function getBestCachingStrategy($sharedStringsUniqueCount, $tempFolder = null)
+    {
+        if ($this->isInMemoryStrategyUsageSafe($sharedStringsUniqueCount)) {
+            return new InMemoryStrategy($sharedStringsUniqueCount);
+        } else {
+            return new FileBasedStrategy($tempFolder, self::MAX_NUM_STRINGS_PER_TEMP_FILE);
+        }
+    }
+
+    /**
+     * Returns whether it is safe to use in-memory caching, given the number of unique shared strings
+     * and the amount of memory available.
+     *
+     * @param int $sharedStringsUniqueCount Number of unique shared strings
+     * @return bool
+     */
+    protected function isInMemoryStrategyUsageSafe($sharedStringsUniqueCount)
+    {
+        $memoryAvailable = $this->getMemoryLimitInKB();
+
+        if ($memoryAvailable === -1) {
+            // if cannot get memory limit or if memory limit set as unlimited, don't trust and play safe
+            return ($sharedStringsUniqueCount < self::MAX_NUM_STRINGS_PER_TEMP_FILE);
+        } else {
+            $memoryNeeded = $sharedStringsUniqueCount * self::AMOUNT_MEMORY_NEEDED_PER_STRING_IN_KB;
+            return ($memoryAvailable > $memoryNeeded);
+        }
+    }
+
+    /**
+     * Returns the PHP "memory_limit" in Kilobytes
+     *
+     * @return float
+     */
+    protected function getMemoryLimitInKB()
+    {
+        $memoryLimitFormatted = $this->getMemoryLimitFromIni();
+        $memoryLimitFormatted = strtolower(trim($memoryLimitFormatted));
+
+        // No memory limit
+        if ($memoryLimitFormatted === '-1') {
+            return -1;
+        }
+
+        if (preg_match('/(\d+)([bkmgt])b?/', $memoryLimitFormatted, $matches)) {
+            $amount = intval($matches[1]);
+            $unit = $matches[2];
+
+            switch ($unit) {
+                case 'b': return ($amount / 1024);
+                case 'k': return $amount;
+                case 'm': return ($amount * 1024);
+                case 'g': return ($amount * 1024 * 1024);
+                case 't': return ($amount * 1024 * 1024 * 1024);
+            }
+        }
+
+        return -1;
+    }
+
+    /**
+     * Returns the formatted "memory_limit" value
+     *
+     * @return string
+     */
+    protected function getMemoryLimitFromIni()
+    {
+        return ini_get('memory_limit');
+    }
+}

src/Spout/Reader/XLSX/Helper/SharedStringsCaching/CachingStrategyInterface.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Box\Spout\Reader\XLSX\Helper\SharedStringsCaching;
+
+/**
+ * Interface CachingStrategyInterface
+ *
+ * @package Box\Spout\Reader\XLSX\Helper\SharedStringsCaching
+ */
+interface CachingStrategyInterface
+{
+    /**
+     * Adds the given string to the cache.
+     *
+     * @param string $sharedString The string to be added to the cache
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return void
+     */
+    public function addStringForIndex($sharedString, $sharedStringIndex);
+
+    /**
+     * Closes the cache after the last shared string was added.
+     * This prevents any additional string from being added to the cache.
+     *
+     * @return void
+     */
+    public function closeCache();
+
+    /**
+     * Returns the string located at the given index from the cache.
+     *
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return string The shared string at the given index
+     * @throws \Box\Spout\Reader\Exception\SharedStringNotFoundException If no shared string found for the given index
+     */
+    public function getStringAtIndex($sharedStringIndex);
+
+    /**
+     * Destroys the cache, freeing memory and removing any created artifacts
+     *
+     * @return void
+     */
+    public function clearCache();
+}

src/Spout/Reader/XLSX/Helper/SharedStringsCaching/FileBasedStrategy.php

@@ -0,0 +1,188 @@
+<?php
+
+namespace Box\Spout\Reader\XLSX\Helper\SharedStringsCaching;
+
+use Box\Spout\Common\Helper\FileSystemHelper;
+use Box\Spout\Common\Helper\GlobalFunctionsHelper;
+use Box\Spout\Reader\Exception\SharedStringNotFoundException;
+
+/**
+ * Class FileBasedStrategy
+ *
+ * This class implements the file-based caching strategy for shared strings.
+ * Shared strings are stored in small files (with a max number of strings per file).
+ * This strategy is slower than an in-memory strategy but is used to avoid out of memory crashes.
+ *
+ * @package Box\Spout\Reader\XLSX\Helper\SharedStringsCaching
+ */
+class FileBasedStrategy implements CachingStrategyInterface
+{
+    /** Value to use to escape the line feed character ("\n") */
+    const ESCAPED_LINE_FEED_CHARACTER = '_x000A_';
+
+    /** @var \Box\Spout\Common\Helper\GlobalFunctionsHelper Helper to work with global functions */
+    protected $globalFunctionsHelper;
+
+    /** @var \Box\Spout\Common\Helper\FileSystemHelper Helper to perform file system operations */
+    protected $fileSystemHelper;
+
+    /**
+     * @var int Maximum number of strings that can be stored in one temp file
+     * @see CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE
+     */
+    protected $maxNumStringsPerTempFile;
+
+    /** @var resource Pointer to the last temp file a shared string was written to */
+    protected $tempFilePointer;
+
+    /**
+     * @var string Path of the temporary file whose contents is currently stored in memory
+     * @see CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE
+     */
+    protected $inMemoryTempFilePath;
+
+    /**
+     * @var string Contents of the temporary file that was last read
+     * @see CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE
+     */
+    protected $inMemoryTempFileContents;
+
+    /**
+     * @param string|null $tempFolder Temporary folder where the temporary files to store shared strings will be stored
+     * @param int $maxNumStringsPerTempFile Maximum number of strings that can be stored in one temp file
+     */
+    public function __construct($tempFolder, $maxNumStringsPerTempFile)
+    {
+        $rootTempFolder = ($tempFolder) ?: sys_get_temp_dir();
+        $this->fileSystemHelper = new FileSystemHelper($rootTempFolder);
+        $this->tempFolder = $this->fileSystemHelper->createFolder($rootTempFolder, uniqid('sharedstrings'));
+
+        $this->maxNumStringsPerTempFile = $maxNumStringsPerTempFile;
+
+        $this->globalFunctionsHelper = new GlobalFunctionsHelper();
+        $this->tempFilePointer = null;
+    }
+
+    /**
+     * Adds the given string to the cache.
+     *
+     * @param string $sharedString The string to be added to the cache
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return void
+     */
+    public function addStringForIndex($sharedString, $sharedStringIndex)
+    {
+        $tempFilePath = $this->getSharedStringTempFilePath($sharedStringIndex);
+
+        if (!$this->globalFunctionsHelper->file_exists($tempFilePath)) {
+            if ($this->tempFilePointer) {
+                $this->globalFunctionsHelper->fclose($this->tempFilePointer);
+            }
+            $this->tempFilePointer = $this->globalFunctionsHelper->fopen($tempFilePath, 'w');
+        }
+
+        // The shared string retrieval logic expects each cell data to be on one line only
+        // Encoding the line feed character allows to preserve this assumption
+        $lineFeedEncodedSharedString = $this->escapeLineFeed($sharedString);
+
+        $this->globalFunctionsHelper->fwrite($this->tempFilePointer, $lineFeedEncodedSharedString . PHP_EOL);
+    }
+
+    /**
+     * Returns the path for the temp file that should contain the string for the given index
+     *
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return string The temp file path for the given index
+     */
+    protected function getSharedStringTempFilePath($sharedStringIndex)
+    {
+        $numTempFile = intval($sharedStringIndex / $this->maxNumStringsPerTempFile);
+        return $this->tempFolder . '/sharedstrings' . $numTempFile;
+    }
+
+    /**
+     * Closes the cache after the last shared string was added.
+     * This prevents any additional string from being added to the cache.
+     *
+     * @return void
+     */
+    public function closeCache()
+    {
+        // close pointer to the last temp file that was written
+        if ($this->tempFilePointer) {
+            $this->globalFunctionsHelper->fclose($this->tempFilePointer);
+        }
+    }
+
+
+    /**
+     * Returns the string located at the given index from the cache.
+     *
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return string The shared string at the given index
+     * @throws \Box\Spout\Reader\Exception\SharedStringNotFoundException If no shared string found for the given index
+     */
+    public function getStringAtIndex($sharedStringIndex)
+    {
+        $tempFilePath = $this->getSharedStringTempFilePath($sharedStringIndex);
+        $indexInFile = $sharedStringIndex % $this->maxNumStringsPerTempFile;
+
+        if (!$this->globalFunctionsHelper->file_exists($tempFilePath)) {
+            throw new SharedStringNotFoundException("Shared string temp file not found: $tempFilePath ; for index: $sharedStringIndex");
+        }
+
+        if ($this->inMemoryTempFilePath !== $tempFilePath) {
+            // free memory
+            unset($this->inMemoryTempFileContents);
+
+            $this->inMemoryTempFileContents = explode(PHP_EOL, $this->globalFunctionsHelper->file_get_contents($tempFilePath));
+            $this->inMemoryTempFilePath = $tempFilePath;
+        }
+
+        $sharedString = null;
+        if (array_key_exists($indexInFile, $this->inMemoryTempFileContents)) {
+            $escapedSharedString = $this->inMemoryTempFileContents[$indexInFile];
+            $sharedString = $this->unescapeLineFeed($escapedSharedString);
+        }
+
+        if ($sharedString === null) {
+            throw new SharedStringNotFoundException("Shared string not found for index: $sharedStringIndex");
+        }
+
+        return rtrim($sharedString, PHP_EOL);
+    }
+
+    /**
+     * Escapes the line feed characters (\n)
+     *
+     * @param string $unescapedString
+     * @return string
+     */
+    private function escapeLineFeed($unescapedString)
+    {
+        return str_replace("\n", self::ESCAPED_LINE_FEED_CHARACTER, $unescapedString);
+    }
+
+    /**
+     * Unescapes the line feed characters (\n)
+     *
+     * @param string $escapedString
+     * @return string
+     */
+    private function unescapeLineFeed($escapedString)
+    {
+        return str_replace(self::ESCAPED_LINE_FEED_CHARACTER, "\n", $escapedString);
+    }
+
+    /**
+     * Destroys the cache, freeing memory and removing any created artifacts
+     *
+     * @return void
+     */
+    public function clearCache()
+    {
+        if ($this->tempFolder) {
+            $this->fileSystemHelper->deleteFolderRecursively($this->tempFolder);
+        }
+    }
+}

src/Spout/Reader/XLSX/Helper/SharedStringsCaching/InMemoryStrategy.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace Box\Spout\Reader\XLSX\Helper\SharedStringsCaching;
+
+use Box\Spout\Reader\Exception\SharedStringNotFoundException;
+
+/**
+ * Class InMemoryStrategy
+ *
+ * This class implements the in-memory caching strategy for shared strings.
+ * This strategy is used when the number of unique strings is low, compared to the memory available.
+ *
+ * @package Box\Spout\Reader\XLSX\Helper\SharedStringsCaching
+ */
+class InMemoryStrategy implements CachingStrategyInterface
+{
+    /** @var \SplFixedArray Array used to cache the shared strings */
+    protected $inMemoryCache;
+
+    /** @var bool Whether the cache has been closed */
+    protected $isCacheClosed;
+
+    /**
+     * @param int $sharedStringsUniqueCount Number of unique shared strings
+     */
+    public function __construct($sharedStringsUniqueCount)
+    {
+        $this->inMemoryCache = new \SplFixedArray($sharedStringsUniqueCount);
+        $this->isCacheClosed = false;
+    }
+
+    /**
+     * Adds the given string to the cache.
+     *
+     * @param string $sharedString The string to be added to the cache
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return void
+     */
+    public function addStringForIndex($sharedString, $sharedStringIndex)
+    {
+        if (!$this->isCacheClosed) {
+            $this->inMemoryCache->offsetSet($sharedStringIndex, $sharedString);
+        }
+    }
+
+    /**
+     * Closes the cache after the last shared string was added.
+     * This prevents any additional string from being added to the cache.
+     *
+     * @return void
+     */
+    public function closeCache()
+    {
+        $this->isCacheClosed = true;
+    }
+
+    /**
+     * Returns the string located at the given index from the cache.
+     *
+     * @param int $sharedStringIndex Index of the shared string in the sharedStrings.xml file
+     * @return string The shared string at the given index
+     * @throws \Box\Spout\Reader\Exception\SharedStringNotFoundException If no shared string found for the given index
+     */
+    public function getStringAtIndex($sharedStringIndex)
+    {
+        try {
+            return $this->inMemoryCache->offsetGet($sharedStringIndex);
+        } catch (\RuntimeException $e) {
+            throw new SharedStringNotFoundException("Shared string not found for index: $sharedStringIndex");
+        }
+    }
+
+    /**
+     * Destroys the cache, freeing memory and removing any created artifacts
+     *
+     * @return void
+     */
+    public function clearCache()
+    {
+        unset($this->inMemoryCache);
+        $this->isCacheClosed = false;
+    }
+}