Make max read bytes per line configurable

- Remove max read bytes constant
- Update reader, reader options, row iterator adding max read bytes properties
- Add test data
- Update tests
This commit is contained in:
Alexandr Jeliuc 2016-12-15 04:38:58 +02:00
parent 521f799366
commit fcb163fb8c
5 changed files with 68 additions and 6 deletions

View File

@ -87,6 +87,17 @@ class Reader extends AbstractReader
return $this;
}
/**
* Set maximum bytes to read per line
* @param $maxReadBytesPerLine
* @return Reader
*/
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
{
$this->getOptions()->setMaxReadBytesPerLine($maxReadBytesPerLine);
return $this;
}
/**
* Returns whether stream wrappers are supported
*

View File

@ -24,6 +24,12 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
/** @var string Defines the End of line */
protected $endOfLineCharacter = "\n";
/**
* Alignment with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
* @var int Number of bytes to read
*/
protected $maxReadBytesPerLine = 32768;
/**
* @return string
*/
@ -107,4 +113,26 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
$this->endOfLineCharacter = $endOfLineCharacter;
return $this;
}
/**
* Sets maximum bytes to read in line
*
* @param int $maxReadBytesPerLine
* @return ReaderOptions
*/
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
{
$this->maxReadBytesPerLine = $maxReadBytesPerLine;
return $this;
}
/**
* Gets maximum bytes to read in line
*
* @return int
*/
public function getMaxReadBytesPerLine()
{
return $this->maxReadBytesPerLine;
}
}

View File

@ -14,10 +14,9 @@ use Box\Spout\Common\Helper\EncodingHelper;
class RowIterator implements IteratorInterface
{
/**
* If no value is given to fgetcsv(), it defaults to 8192 (which may be too low).
* Alignement with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
* @var int Number of bytes to read
*/
const MAX_READ_BYTES_PER_LINE = 32768;
protected $maxReadBytesPerLine;
/** @var resource Pointer to the CSV file to read */
protected $filePointer;
@ -67,6 +66,7 @@ class RowIterator implements IteratorInterface
$this->fieldEnclosure = $options->getFieldEnclosure();
$this->encoding = $options->getEncoding();
$this->inputEOLDelimiter = $options->getEndOfLineCharacter();
$this->maxReadBytesPerLine = $options->getMaxReadBytesPerLine();
$this->shouldPreserveEmptyRows = $options->shouldPreserveEmptyRows();
$this->globalFunctionsHelper = $globalFunctionsHelper;
@ -177,7 +177,7 @@ class RowIterator implements IteratorInterface
*/
protected function getNextUTF8EncodedRow()
{
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, self::MAX_READ_BYTES_PER_LINE, $this->fieldDelimiter, $this->fieldEnclosure);
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, $this->maxReadBytesPerLine, $this->fieldDelimiter, $this->fieldEnclosure);
if ($encodedRowData === false) {
return false;
}

View File

@ -428,6 +428,26 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
$this->assertEquals($expectedRows, $allRows, 'Cell values should not be trimmed');
}
/**
* maxReadBytesPerLine should be configurable
* @return void
*/
public function testShouldReadLinesOfAnySize()
{
$allColumns = $this->getAllRowsForFile(
'csv_with_long_lines.csv',
';',
'"',
"\n",
EncodingHelper::ENCODING_UTF8,
false,
(1.5 * 1024 * 1024)
)[0];
$expectedNumOfColumns = 4;
$this->assertEquals($expectedNumOfColumns, count($allColumns));
}
/**
* @param string $fileName
* @param string|void $fieldDelimiter
@ -443,8 +463,9 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
$fieldEnclosure = '"',
$endOfLineCharacter = "\n",
$encoding = EncodingHelper::ENCODING_UTF8,
$shouldPreserveEmptyRows = false)
{
$shouldPreserveEmptyRows = false,
$maxReadBytesPerLine = 32768
) {
$allRows = [];
$resourcePath = $this->getResourcePath($fileName);
@ -456,6 +477,7 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
->setEndOfLineCharacter($endOfLineCharacter)
->setEncoding($encoding)
->setShouldPreserveEmptyRows($shouldPreserveEmptyRows)
->setMaxReadBytesPerLine($maxReadBytesPerLine)
->open($resourcePath);
foreach ($reader->getSheetIterator() as $sheetIndex => $sheet) {

File diff suppressed because one or more lines are too long