Attempt to convert the non UTF-8 strings to UTF-8

src/Spout/Reader/CSV/Helper/EncodingHelper.php

@@ -1,92 +0,0 @@
-<?php
-
-namespace Box\Spout\Reader\CSV\Helper;
-
-/**
- * Class EncodingHelper
- * This class provides helper functions to work with encodings.
- *
- * @package Box\Spout\Reader\CSV\Helper
- */
-class EncodingHelper
-{
-    /** Definition of the encodings that can have a BOM */
-    const ENCODING_UTF8     = 'UTF-8';
-    const ENCODING_UTF16_LE = 'UTF-16LE';
-    const ENCODING_UTF16_BE = 'UTF-16BE';
-    const ENCODING_UTF32_LE = 'UTF-32LE';
-    const ENCODING_UTF32_BE = 'UTF-32BE';
-
-    /** Definition of the BOMs for the different encodings */
-    const BOM_UTF8     = "\xEF\xBB\xBF";
-    const BOM_UTF16_LE = "\xFF\xFE";
-    const BOM_UTF16_BE = "\xFE\xFF";
-    const BOM_UTF32_LE = "\xFF\xFE\x00\x00";
-    const BOM_UTF32_BE = "\x00\x00\xFE\xFF";
-
-    /** @var \Box\Spout\Common\Helper\GlobalFunctionsHelper Helper to work with global functions */
-    protected $globalFunctionsHelper;
-
-    /** @var array Map representing the encodings supporting BOMs (key) and their associated BOM (value) */
-    protected $supportedEncodingsWithBom;
-
-    /**
-     * @param \Box\Spout\Common\Helper\GlobalFunctionsHelper $globalFunctionsHelper
-     */
-    public function __construct($globalFunctionsHelper)
-    {
-        $this->globalFunctionsHelper = $globalFunctionsHelper;
-
-        $this->supportedEncodingsWithBom = [
-            self::ENCODING_UTF8     => self::BOM_UTF8,
-            self::ENCODING_UTF16_LE => self::BOM_UTF16_LE,
-            self::ENCODING_UTF16_BE => self::BOM_UTF16_BE,
-            self::ENCODING_UTF32_LE => self::BOM_UTF32_LE,
-            self::ENCODING_UTF32_BE => self::BOM_UTF32_BE,
-        ];
-    }
-
-    /**
-     * Returns the number of bytes to use as offset in order to skip the BOM.
-     *
-     * @param resource $filePointer Pointer to the file to check
-     * @param string $encoding Encoding of the file to check
-     * @return int Bytes offset to apply to skip the BOM (0 means no BOM)
-     */
-    public function getBytesOffsetToSkipBOM($filePointer, $encoding)
-    {
-        $byteOffsetToSkipBom = 0;
-
-        if ($this->hasBom($filePointer, $encoding)) {
-            $bomUsed = $this->supportedEncodingsWithBom[$encoding];
-
-            // we skip the N first bytes
-            $byteOffsetToSkipBom = strlen($bomUsed);
-        }
-
-        return $byteOffsetToSkipBom;
-    }
-
-    /**
-     * Returns whether the file identified by the given pointer has a BOM.
-     *
-     * @param resource $filePointer Pointer to the file to check
-     * @param string $encoding Encoding of the file to check
-     * @return bool TRUE if the file has a BOM, FALSE otherwise
-     */
-    protected function hasBOM($filePointer, $encoding)
-    {
-        $hasBOM = false;
-
-        $this->globalFunctionsHelper->rewind($filePointer);
-
-        if (array_key_exists($encoding, $this->supportedEncodingsWithBom)) {
-            $potentialBom = $this->supportedEncodingsWithBom[$encoding];
-            $numBytesInBom = strlen($potentialBom);
-
-            $hasBOM = ($this->globalFunctionsHelper->fgets($filePointer, $numBytesInBom + 1) === $potentialBom);
-        }
-
-        return $hasBOM;
-    }
-}

src/Spout/Reader/CSV/Reader.php

@@ -4,6 +4,7 @@ namespace Box\Spout\Reader\CSV;
 
 use Box\Spout\Reader\AbstractReader;
 use Box\Spout\Common\Exception\IOException;
+use Box\Spout\Common\Helper\EncodingHelper;
 
 /**
  * Class Reader
@@ -26,7 +27,7 @@ class Reader extends AbstractReader
     protected $fieldEnclosure = '"';
 
     /** @var string Encoding of the CSV file to be read */
-    protected $encoding = 'UTF-8';
+    protected $encoding = EncodingHelper::ENCODING_UTF8;
 
     /**
      * Sets the field delimiter for the CSV.
@@ -69,6 +70,7 @@ class Reader extends AbstractReader
 
     /**
      * Opens the file at the given path to make it ready to be read.
+     * If setEncoding() was not called, it assumes that the file is encoded in UTF-8.
      *
      * @param  string $filePath Path of the CSV file to be read
      * @return void

src/Spout/Reader/CSV/RowIterator.php

@@ -2,8 +2,8 @@
 
 namespace Box\Spout\Reader\CSV;
 
-use Box\Spout\Reader\CSV\Helper\EncodingHelper;
 use Box\Spout\Reader\IteratorInterface;
+use Box\Spout\Common\Helper\EncodingHelper;
 
 /**
  * Class RowIterator
@@ -37,6 +37,9 @@ class RowIterator implements IteratorInterface
     /** @var \Box\Spout\Common\Helper\GlobalFunctionsHelper Helper to work with global functions */
     protected $globalFunctionsHelper;
 
+    /** @var \Box\Spout\Common\Helper\EncodingHelper Helper to work with different encodings */
+    protected $encodingHelper;
+
     /**
      * @param resource $filePointer Pointer to the CSV file to read
      * @param string $fieldDelimiter Character that delimits fields
@@ -101,6 +104,7 @@ class RowIterator implements IteratorInterface
      * @link http://php.net/manual/en/iterator.next.php
      *
      * @return void
+     * @throws \Box\Spout\Common\Exception\EncodingConversionException If unable to convert data to UTF-8
      */
     public function next()
     {
@@ -109,7 +113,8 @@ class RowIterator implements IteratorInterface
 
         if (!$this->hasReachedEndOfFile) {
             do {
-               $lineData = $this->globalFunctionsHelper->fgetcsv($this->filePointer, 0, $this->fieldDelimiter, $this->fieldEnclosure);
+                $utf8EncodedLineData = $this->getNextUTF8EncodedLine();
+                $lineData = $this->globalFunctionsHelper->str_getcsv($utf8EncodedLineData, $this->fieldDelimiter, $this->fieldEnclosure);
            } while ($lineData === false || ($lineData !== null && $this->isEmptyLine($lineData)));
 
             if ($lineData !== false && $lineData !== null) {
@@ -119,6 +124,25 @@ class RowIterator implements IteratorInterface
         }
     }
 
+    /**
+     * Returns the next line, converted if necessary to UTF-8.
+     * Neither fgets nor fgetcsv don't work with non UTF-8 data... so we need to do some things manually.
+     *
+     * @return string The next line for the current file pointer, encoded in UTF-8
+     * @throws \Box\Spout\Common\Exception\EncodingConversionException If unable to convert data to UTF-8
+     */
+    protected function getNextUTF8EncodedLine()
+    {
+        // Read until the EOL delimiter or EOF is reached. The delimiter's encoding needs to match the CSV's encoding.
+        $encodedEOLDelimiter = $this->encodingHelper->attemptConversionFromUTF8("\n", $this->encoding);
+        $encodedLineData = $this->globalFunctionsHelper->stream_get_line($this->filePointer, 0, $encodedEOLDelimiter);
+
+        // Once the line has been read, it can be converted to UTF-8
+        $utf8EncodedLineData = $this->encodingHelper->attemptConversionToUTF8($encodedLineData, $this->encoding);
+
+        return $utf8EncodedLineData;
+    }
+
     /**
      * @param array $lineData Array containing the cells value for the line
      * @return bool Whether the given line is empty