Make max read bytes per line configurable
- Remove max read bytes constant - Update reader, reader options, row iterator adding max read bytes properties - Add test data - Update tests
This commit is contained in:
parent
521f799366
commit
fcb163fb8c
@ -87,6 +87,17 @@ class Reader extends AbstractReader
|
|||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set maximum bytes to read per line
|
||||||
|
* @param $maxReadBytesPerLine
|
||||||
|
* @return Reader
|
||||||
|
*/
|
||||||
|
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||||
|
{
|
||||||
|
$this->getOptions()->setMaxReadBytesPerLine($maxReadBytesPerLine);
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns whether stream wrappers are supported
|
* Returns whether stream wrappers are supported
|
||||||
*
|
*
|
||||||
|
@ -24,6 +24,12 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
|
|||||||
/** @var string Defines the End of line */
|
/** @var string Defines the End of line */
|
||||||
protected $endOfLineCharacter = "\n";
|
protected $endOfLineCharacter = "\n";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Alignment with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
|
||||||
|
* @var int Number of bytes to read
|
||||||
|
*/
|
||||||
|
protected $maxReadBytesPerLine = 32768;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return string
|
* @return string
|
||||||
*/
|
*/
|
||||||
@ -107,4 +113,26 @@ class ReaderOptions extends \Box\Spout\Reader\Common\ReaderOptions
|
|||||||
$this->endOfLineCharacter = $endOfLineCharacter;
|
$this->endOfLineCharacter = $endOfLineCharacter;
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets maximum bytes to read in line
|
||||||
|
*
|
||||||
|
* @param int $maxReadBytesPerLine
|
||||||
|
* @return ReaderOptions
|
||||||
|
*/
|
||||||
|
public function setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||||
|
{
|
||||||
|
$this->maxReadBytesPerLine = $maxReadBytesPerLine;
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets maximum bytes to read in line
|
||||||
|
*
|
||||||
|
* @return int
|
||||||
|
*/
|
||||||
|
public function getMaxReadBytesPerLine()
|
||||||
|
{
|
||||||
|
return $this->maxReadBytesPerLine;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,10 +14,9 @@ use Box\Spout\Common\Helper\EncodingHelper;
|
|||||||
class RowIterator implements IteratorInterface
|
class RowIterator implements IteratorInterface
|
||||||
{
|
{
|
||||||
/**
|
/**
|
||||||
* If no value is given to fgetcsv(), it defaults to 8192 (which may be too low).
|
* @var int Number of bytes to read
|
||||||
* Alignement with other functions like fgets() is discussed here: https://bugs.php.net/bug.php?id=48421
|
|
||||||
*/
|
*/
|
||||||
const MAX_READ_BYTES_PER_LINE = 32768;
|
protected $maxReadBytesPerLine;
|
||||||
|
|
||||||
/** @var resource Pointer to the CSV file to read */
|
/** @var resource Pointer to the CSV file to read */
|
||||||
protected $filePointer;
|
protected $filePointer;
|
||||||
@ -67,6 +66,7 @@ class RowIterator implements IteratorInterface
|
|||||||
$this->fieldEnclosure = $options->getFieldEnclosure();
|
$this->fieldEnclosure = $options->getFieldEnclosure();
|
||||||
$this->encoding = $options->getEncoding();
|
$this->encoding = $options->getEncoding();
|
||||||
$this->inputEOLDelimiter = $options->getEndOfLineCharacter();
|
$this->inputEOLDelimiter = $options->getEndOfLineCharacter();
|
||||||
|
$this->maxReadBytesPerLine = $options->getMaxReadBytesPerLine();
|
||||||
$this->shouldPreserveEmptyRows = $options->shouldPreserveEmptyRows();
|
$this->shouldPreserveEmptyRows = $options->shouldPreserveEmptyRows();
|
||||||
$this->globalFunctionsHelper = $globalFunctionsHelper;
|
$this->globalFunctionsHelper = $globalFunctionsHelper;
|
||||||
|
|
||||||
@ -177,7 +177,7 @@ class RowIterator implements IteratorInterface
|
|||||||
*/
|
*/
|
||||||
protected function getNextUTF8EncodedRow()
|
protected function getNextUTF8EncodedRow()
|
||||||
{
|
{
|
||||||
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, self::MAX_READ_BYTES_PER_LINE, $this->fieldDelimiter, $this->fieldEnclosure);
|
$encodedRowData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, $this->maxReadBytesPerLine, $this->fieldDelimiter, $this->fieldEnclosure);
|
||||||
if ($encodedRowData === false) {
|
if ($encodedRowData === false) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -428,6 +428,26 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
|||||||
$this->assertEquals($expectedRows, $allRows, 'Cell values should not be trimmed');
|
$this->assertEquals($expectedRows, $allRows, 'Cell values should not be trimmed');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* maxReadBytesPerLine should be configurable
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function testShouldReadLinesOfAnySize()
|
||||||
|
{
|
||||||
|
$allColumns = $this->getAllRowsForFile(
|
||||||
|
'csv_with_long_lines.csv',
|
||||||
|
';',
|
||||||
|
'"',
|
||||||
|
"\n",
|
||||||
|
EncodingHelper::ENCODING_UTF8,
|
||||||
|
false,
|
||||||
|
(1.5 * 1024 * 1024)
|
||||||
|
)[0];
|
||||||
|
|
||||||
|
$expectedNumOfColumns = 4;
|
||||||
|
$this->assertEquals($expectedNumOfColumns, count($allColumns));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $fileName
|
* @param string $fileName
|
||||||
* @param string|void $fieldDelimiter
|
* @param string|void $fieldDelimiter
|
||||||
@ -443,8 +463,9 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
|||||||
$fieldEnclosure = '"',
|
$fieldEnclosure = '"',
|
||||||
$endOfLineCharacter = "\n",
|
$endOfLineCharacter = "\n",
|
||||||
$encoding = EncodingHelper::ENCODING_UTF8,
|
$encoding = EncodingHelper::ENCODING_UTF8,
|
||||||
$shouldPreserveEmptyRows = false)
|
$shouldPreserveEmptyRows = false,
|
||||||
{
|
$maxReadBytesPerLine = 32768
|
||||||
|
) {
|
||||||
$allRows = [];
|
$allRows = [];
|
||||||
$resourcePath = $this->getResourcePath($fileName);
|
$resourcePath = $this->getResourcePath($fileName);
|
||||||
|
|
||||||
@ -456,6 +477,7 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
|
|||||||
->setEndOfLineCharacter($endOfLineCharacter)
|
->setEndOfLineCharacter($endOfLineCharacter)
|
||||||
->setEncoding($encoding)
|
->setEncoding($encoding)
|
||||||
->setShouldPreserveEmptyRows($shouldPreserveEmptyRows)
|
->setShouldPreserveEmptyRows($shouldPreserveEmptyRows)
|
||||||
|
->setMaxReadBytesPerLine($maxReadBytesPerLine)
|
||||||
->open($resourcePath);
|
->open($resourcePath);
|
||||||
|
|
||||||
foreach ($reader->getSheetIterator() as $sheetIndex => $sheet) {
|
foreach ($reader->getSheetIterator() as $sheetIndex => $sheet) {
|
||||||
|
1
tests/resources/csv/csv_with_long_lines.csv
Normal file
1
tests/resources/csv/csv_with_long_lines.csv
Normal file
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user