Merge pull request #67 from box/caching_strategies

Caching strategies

tests/Spout/Reader/Helper/XLSX/SharedStringsCaching/CachingStrategyFactoryTest.php

@@ -0,0 +1,99 @@
+<?php
+
+namespace Box\Spout\Reader\Helper\XLSX\SharedStringsCaching;
+
+/**
+ * Class CachingStrategyFactoryTest
+ *
+ * @package Box\Spout\Reader\Helper\XLSX\SharedStringsCaching
+ */
+class CachingStrategyFactoryTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * @return array
+     */
+    public function dataProviderForTestGetBestCachingStrategy()
+    {
+        return [
+            [CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE, -1, 'FileBasedStrategy'],
+            [CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE + 10, -1, 'FileBasedStrategy'],
+            [CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE - 10, -1, 'InMemoryStrategy'],
+            [10 , CachingStrategyFactory::AMOUNT_MEMORY_NEEDED_PER_STRING_IN_KB * 10, 'FileBasedStrategy'],
+            [15, CachingStrategyFactory::AMOUNT_MEMORY_NEEDED_PER_STRING_IN_KB * 10, 'FileBasedStrategy'],
+            [5 , CachingStrategyFactory::AMOUNT_MEMORY_NEEDED_PER_STRING_IN_KB * 10, 'InMemoryStrategy'],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestGetBestCachingStrategy
+     *
+     * @param int $sharedStringsUniqueCount
+     * @param int $memoryLimitInKB
+     * @param string $expectedStrategyClassName
+     * @return void
+     */
+    public function testGetBestCachingStrategy($sharedStringsUniqueCount, $memoryLimitInKB, $expectedStrategyClassName)
+    {
+        /** @var CachingStrategyFactory|\PHPUnit_Framework_MockObject_MockObject $factoryStub */
+        $factoryStub = $this
+            ->getMockBuilder('\Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\CachingStrategyFactory')
+            ->disableOriginalConstructor()
+            ->setMethods(['getMemoryLimitInKB'])
+            ->getMock();
+
+        $factoryStub->method('getMemoryLimitInKB')->willReturn($memoryLimitInKB);
+
+        \ReflectionHelper::setStaticValue('\Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\CachingStrategyFactory', 'instance', $factoryStub);
+
+        $strategy = $factoryStub->getBestCachingStrategy($sharedStringsUniqueCount, null);
+
+        $fullExpectedStrategyClassName = 'Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\\' . $expectedStrategyClassName;
+        $this->assertEquals($fullExpectedStrategyClassName, get_class($strategy));
+
+        $strategy->clearCache();
+        \ReflectionHelper::reset();
+    }
+
+    /**
+     * @return array
+     */
+    public function dataProviderForTestGetMemoryLimitInKB()
+    {
+        return [
+            ['-1', -1],
+            ['invalid', -1],
+            ['1024B', 1],
+            ['128K', 128],
+            ['256KB', 256],
+            ['512M', 512 * 1024],
+            ['2MB', 2 * 1024],
+            ['1G', 1 * 1024 * 1024],
+            ['10GB', 10 * 1024 * 1024],
+            ['2T', 2 * 1024 * 1024 * 1024],
+            ['5TB', 5 * 1024 * 1024 * 1024],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestGetMemoryLimitInKB
+     *
+     * @param string $memoryLimitFormatted
+     * @param float $expectedMemoryLimitInKB
+     * @return void
+     */
+    public function testGetMemoryLimitInKB($memoryLimitFormatted, $expectedMemoryLimitInKB)
+    {
+        /** @var CachingStrategyFactory|\PHPUnit_Framework_MockObject_MockObject $factoryStub */
+        $factoryStub = $this
+            ->getMockBuilder('\Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\CachingStrategyFactory')
+            ->disableOriginalConstructor()
+            ->setMethods(['getMemoryLimitFromIni'])
+            ->getMock();
+
+        $factoryStub->method('getMemoryLimitFromIni')->willReturn($memoryLimitFormatted);
+
+        $memoryLimitInKB = \ReflectionHelper::callMethodOnObject($factoryStub, 'getMemoryLimitInKB');
+
+        $this->assertEquals($expectedMemoryLimitInKB, $memoryLimitInKB);
+    }
+}

tests/Spout/Reader/Helper/XLSX/SharedStringsHelperTest.php

@@ -2,6 +2,9 @@
 
 namespace Box\Spout\Reader\Helper\XLSX;
 
+use Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\CachingStrategyFactory;
+use Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\FileBasedStrategy;
+use Box\Spout\Reader\Helper\XLSX\SharedStringsCaching\InMemoryStrategy;
 use Box\Spout\TestUsingResource;
 
 /**
@@ -33,46 +36,6 @@ class SharedStringsHelperTest extends \PHPUnit_Framework_TestCase
         $this->sharedStringsHelper->cleanup();
     }
 
-    /**
-     * @return void
-     */
-    public function testExtractSharedStringsShouldCreateTempFileWithSharedStrings()
-    {
-        $this->sharedStringsHelper->extractSharedStrings();
-
-        $tempFolder = \ReflectionHelper::getValueOnObject($this->sharedStringsHelper, 'tempFolder');
-
-        $filesInTempFolder = $this->getFilesInFolder($tempFolder);
-        $this->assertEquals(1, count($filesInTempFolder), 'One temp file should have been created in the temp folder.');
-
-        $tempFileContents = file_get_contents($filesInTempFolder[0]);
-        $tempFileContentsPerLine = explode(PHP_EOL, $tempFileContents);
-
-        $this->assertEquals('s1--A1', $tempFileContentsPerLine[0]);
-        $this->assertEquals('s1--E5', $tempFileContentsPerLine[24]);
-    }
-
-    /**
-     * Returns all files that are in the given folder.
-     * It does not include "." and ".." and is not recursive.
-     *
-     * @param string $folderPath
-     * @return array
-     */
-    private function getFilesInFolder($folderPath)
-    {
-        $files = [];
-        $directoryIterator = new \DirectoryIterator($folderPath);
-
-        foreach ($directoryIterator as $fileInfo) {
-            if ($fileInfo->isFile()) {
-                $files[] = $fileInfo->getPathname();
-            }
-        }
-
-        return $files;
-    }
-
     /**
      * @expectedException \Box\Spout\Reader\Exception\SharedStringNotFoundException
      * @return void
@@ -95,6 +58,9 @@ class SharedStringsHelperTest extends \PHPUnit_Framework_TestCase
 
         $sharedString = $this->sharedStringsHelper->getStringAtIndex(24);
         $this->assertEquals('s1--E5', $sharedString);
+
+        $usedCachingStrategy = \ReflectionHelper::getValueOnObject($this->sharedStringsHelper, 'cachingStrategy');
+        $this->assertTrue($usedCachingStrategy instanceof InMemoryStrategy);
     }
 
     /**
@@ -115,4 +81,32 @@ class SharedStringsHelperTest extends \PHPUnit_Framework_TestCase
 
         $sharedStringsHelper->cleanup();
     }
+
+    /**
+     * @return void
+     */
+    public function testGetStringAtIndexWithFileBasedStrategy()
+    {
+        // force the file-based strategy by setting no memory limit
+        $originalMemoryLimit = ini_get('memory_limit');
+        ini_set('memory_limit', '-1');
+
+        $resourcePath = $this->getResourcePath('sheet_with_lots_of_shared_strings.xlsx');
+        $sharedStringsHelper = new SharedStringsHelper($resourcePath);
+
+        $sharedStringsHelper->extractSharedStrings();
+
+        $sharedString = $sharedStringsHelper->getStringAtIndex(0);
+        $this->assertEquals('str', $sharedString);
+
+        $sharedString = $sharedStringsHelper->getStringAtIndex(CachingStrategyFactory::MAX_NUM_STRINGS_PER_TEMP_FILE + 1);
+        $this->assertEquals('str', $sharedString);
+
+        $usedCachingStrategy = \ReflectionHelper::getValueOnObject($sharedStringsHelper, 'cachingStrategy');
+        $this->assertTrue($usedCachingStrategy instanceof FileBasedStrategy);
+
+        $sharedStringsHelper->cleanup();
+
+        ini_set('memory_limit', $originalMemoryLimit);
+    }
 }

tests/Spout/Reader/XLSXTest.php

@@ -2,6 +2,7 @@
 
 namespace Box\Spout\Reader;
 
+use Box\Spout\Common\Exception\IOException;
 use Box\Spout\Common\Type;
 use Box\Spout\TestUsingResource;
 
@@ -245,18 +246,39 @@ class XLSXTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals($expectedRow, $allRows[0], 'Pronunciation data should be removed.');
     }
 
+
     /**
+     * @return array
+     */
+    public function dataProviderForTestReadShouldBeProtectedAgainstAttacks()
+    {
+        return [
+            ['attack_billion_laughs.xlsx'],
+            ['attack_quadratic_blowup.xlsx'],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestReadShouldBeProtectedAgainstAttacks
+     * @NOTE: The LIBXML_NOENT is used to ACTUALLY substitute entities (and should therefore not be used)
+     *
+     * @param string $fileName
      * @return void
      */
-    public function testReadShouldBeProtectedAgainstBillionLaughsAttack()
+    public function testReadShouldBeProtectedAgainstAttacks($fileName)
     {
-        $allRows = $this->getAllRowsForFile('billion_laughs_test_file.xlsx');
+        $startTime = microtime(true);
 
-        $expectedMaxMemoryUsage = 30 * 1024 * 1024; // 30MB
-        $this->assertLessThan($expectedMaxMemoryUsage, memory_get_peak_usage(true), 'Entities should not be expanded and therefore consume all the memory.');
+        try {
+            $this->getAllRowsForFile($fileName);
+            $this->fail('An exception should have been thrown');
+        } catch (IOException $exception) {
+            $duration = microtime(true) - $startTime;
+            $this->assertLessThan(10, $duration, 'Entities should not be expanded and therefore take more than 10 seconds to be parsed.');
 
-        $expectedFirstRow = ['s1--A1', 's1--B1', 's1--C1', 's1--D1', 's1--E1'];
-        $this->assertEquals($expectedFirstRow, $allRows[0], 'Entities should be ignored when reading XML files.');
+            $expectedMaxMemoryUsage = 30 * 1024 * 1024; // 30MB
+            $this->assertLessThan($expectedMaxMemoryUsage, memory_get_peak_usage(true), 'Entities should not be expanded and therefore consume all the memory.');
+        }
     }
 
     /**

tests/Spout/ReflectionHelper.php

@@ -89,4 +89,27 @@ class ReflectionHelper
 
         return $value;
     }
+
+    /**
+     * Invoke a the given public or protected method on the given object.
+     *
+     * @param object $object
+     * @param string $methodName
+     * @param *mixed|null $params
+     *
+     * @return mixed|null
+     */
+    public static function callMethodOnObject($object, $methodName)
+    {
+        $params = func_get_args();
+        array_shift($params); // object
+        array_shift($params); // methodName
+
+        $className = get_class($object);
+        $class = new ReflectionClass($className);
+        $method = $class->getMethod($methodName);
+        $method->setAccessible(true);
+
+        return $method->invokeArgs($object, $params);
+    }
 }