Make max read bytes per line configurable
- Remove max read bytes constant - Update reader, reader options, row iterator adding max read bytes properties - Add test data - Update tests
This commit is contained in:
parent
521f799366
commit
fcb163fb8c
@ -87,6 +87,17 @@ class Reader extends AbstractReader
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set maximum bytes to read per line
|
||||
* @param $maxReadBytesPerLine
|
||||
* @return Reader
|
||||
*/
|
||||
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||
{
|
||||
$this->getOptions()->setMaxReadBytesPerLine($maxReadBytesPerLine);
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether stream wrappers are supported
|
||||
*
|
||||
|
@ -24,6 +24,12 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
|
||||
/** @var string Defines the End of line */
|
||||
protected $endOfLineCharacter = "\n";
|
||||
|
||||
/**
|
||||
* Alignment with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
|
||||
* @var int Number of bytes to read
|
||||
*/
|
||||
protected $maxReadBytesPerLine = 32768;
|
||||
|
||||
/**
|
||||
* @return string
|
||||
*/
|
||||
@ -107,4 +113,26 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
|
||||
$this->endOfLineCharacter = $endOfLineCharacter;
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets maximum bytes to read in line
|
||||
*
|
||||
* @param int $maxReadBytesPerLine
|
||||
* @return ReaderOptions
|
||||
*/
|
||||
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||
{
|
||||
$this->maxReadBytesPerLine = $maxReadBytesPerLine;
|
||||
return $this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets maximum bytes to read in line
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function getMaxReadBytesPerLine()
|
||||
{
|
||||
return $this->maxReadBytesPerLine;
|
||||
}
|
||||
}
|
||||
|
@ -14,10 +14,9 @@ use Box\Spout\Common\Helper\EncodingHelper;
|
||||
class RowIterator implements IteratorInterface
|
||||
{
|
||||
/**
|
||||
* If no value is given to fgetcsv(), it defaults to 8192 (which may be too low).
|
||||
* Alignement with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
|
||||
* @var int Number of bytes to read
|
||||
*/
|
||||
const MAX_READ_BYTES_PER_LINE = 32768;
|
||||
protected $maxReadBytesPerLine;
|
||||
|
||||
/** @var resource Pointer to the CSV file to read */
|
||||
protected $filePointer;
|
||||
@ -67,6 +66,7 @@ class RowIterator implements IteratorInterface
|
||||
$this->fieldEnclosure = $options->getFieldEnclosure();
|
||||
$this->encoding = $options->getEncoding();
|
||||
$this->inputEOLDelimiter = $options->getEndOfLineCharacter();
|
||||
$this->maxReadBytesPerLine = $options->getMaxReadBytesPerLine();
|
||||
$this->shouldPreserveEmptyRows = $options->shouldPreserveEmptyRows();
|
||||
$this->globalFunctionsHelper = $globalFunctionsHelper;
|
||||
|
||||
@ -177,7 +177,7 @@ class RowIterator implements IteratorInterface
|
||||
*/
|
||||
protected function getNextUTF8EncodedRow()
|
||||
{
|
||||
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, self::MAX_READ_BYTES_PER_LINE, $this->fieldDelimiter, $this->fieldEnclosure);
|
||||
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, $this->maxReadBytesPerLine, $this->fieldDelimiter, $this->fieldEnclosure);
|
||||
if ($encodedRowData === false) {
|
||||
return false;
|
||||
}
|
||||
|
@ -428,6 +428,26 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
||||
$this->assertEquals($expectedRows, $allRows, 'Cell values should not be trimmed');
|
||||
}
|
||||
|
||||
/**
|
||||
* maxReadBytesPerLine should be configurable
|
||||
* @return void
|
||||
*/
|
||||
public function testShouldReadLinesOfAnySize()
|
||||
{
|
||||
$allColumns = $this->getAllRowsForFile(
|
||||
'csv_with_long_lines.csv',
|
||||
';',
|
||||
'"',
|
||||
"\n",
|
||||
EncodingHelper::ENCODING_UTF8,
|
||||
false,
|
||||
(1.5 * 1024 * 1024)
|
||||
)[0];
|
||||
|
||||
$expectedNumOfColumns = 4;
|
||||
$this->assertEquals($expectedNumOfColumns, count($allColumns));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $fileName
|
||||
* @param string|void $fieldDelimiter
|
||||
@ -443,8 +463,9 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
||||
$fieldEnclosure = '"',
|
||||
$endOfLineCharacter = "\n",
|
||||
$encoding = EncodingHelper::ENCODING_UTF8,
|
||||
$shouldPreserveEmptyRows = false)
|
||||
{
|
||||
$shouldPreserveEmptyRows = false,
|
||||
$maxReadBytesPerLine = 32768
|
||||
) {
|
||||
$allRows = [];
|
||||
$resourcePath = $this->getResourcePath($fileName);
|
||||
|
||||
@ -456,6 +477,7 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
||||
->setEndOfLineCharacter($endOfLineCharacter)
|
||||
->setEncoding($encoding)
|
||||
->setShouldPreserveEmptyRows($shouldPreserveEmptyRows)
|
||||
->setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||
->open($resourcePath);
|
||||
|
||||
foreach ($reader->getSheetIterator() as $sheetIndex => $sheet) {
|
||||
|
1
tests/resources/csv/csv_with_long_lines.csv
Normal file
1
tests/resources/csv/csv_with_long_lines.csv
Normal file
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user