Attempt to convert the non UTF-8 strings to UTF-8

tests/Spout/Common/Helper/EncodingHelperTest.php

@@ -0,0 +1,223 @@
+<?php
+
+namespace Box\Spout\Common\Helper;
+
+use Box\Spout\TestUsingResource;
+
+/**
+ * Class EncodingHelperTest
+ *
+ * @package Box\Spout\Common\Helper
+ */
+class EncodingHelperTest extends \PHPUnit_Framework_TestCase
+{
+    use TestUsingResource;
+
+    /**
+     * @return array
+     */
+    public function dataProviderForTestGetBytesOffsetToSkipBOM()
+    {
+        return [
+            ['csv_with_utf8_bom.csv', EncodingHelper::ENCODING_UTF8, 3],
+            ['csv_with_utf16be_bom.csv', EncodingHelper::ENCODING_UTF16_BE, 2],
+            ['csv_with_utf32le_bom.csv', EncodingHelper::ENCODING_UTF32_LE, 4],
+            ['csv_with_encoding_utf16le_no_bom.csv', EncodingHelper::ENCODING_UTF16_LE, 0],
+            ['csv_standard.csv', EncodingHelper::ENCODING_UTF8, 0],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestGetBytesOffsetToSkipBOM
+     *
+     * @param string $fileName
+     * @param string $encoding
+     * @param int $expectedBytesOffset
+     * @return void
+     */
+    public function testGetBytesOffsetToSkipBOM($fileName, $encoding, $expectedBytesOffset)
+    {
+        $resourcePath = $this->getResourcePath($fileName);
+        $filePointer = fopen($resourcePath, 'r');
+
+        $encodingHelper = new EncodingHelper(new GlobalFunctionsHelper());
+        $bytesOffset = $encodingHelper->getBytesOffsetToSkipBOM($filePointer, $encoding);
+
+        $this->assertEquals($expectedBytesOffset, $bytesOffset);
+    }
+
+    /**
+     * @return array
+     */
+    public function dataProviderForIconvOrMbstringUsage()
+    {
+        return [
+            [$shouldUseIconv = true],
+            [$shouldNotUseIconv = false],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForIconvOrMbstringUsage
+     * @expectedException \Box\Spout\Common\Exception\EncodingConversionException
+     *
+     * @param bool $shouldUseIconv
+     * @return void
+     */
+    public function testAttemptConversionToUTF8ShouldThrowIfConversionFailed($shouldUseIconv)
+    {
+        $helperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\GlobalFunctionsHelper')
+                        ->setMethods(['iconv', 'mb_convert_encoding'])
+                        ->getMock();
+        $helperStub->method('iconv')->willReturn(false);
+        $helperStub->method('mb_convert_encoding')->willReturn(false);
+
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->setConstructorArgs([$helperStub])
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn($shouldUseIconv);
+        $encodingHelperStub->method('canUseMbString')->willReturn(true);
+
+        $encodingHelperStub->attemptConversionToUTF8('input', EncodingHelper::ENCODING_UTF16_LE);
+    }
+
+    /**
+     * @expectedException \Box\Spout\Common\Exception\EncodingConversionException
+     *
+     * @return void
+     */
+    public function testAttemptConversionToUTF8ShouldThrowIfConversionNotSupported()
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->disableOriginalConstructor()
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn(false);
+        $encodingHelperStub->method('canUseMbString')->willReturn(false);
+
+        $encodingHelperStub->attemptConversionToUTF8('input', EncodingHelper::ENCODING_UTF16_LE);
+    }
+
+    /**
+     * @dataProvider dataProviderForIconvOrMbstringUsage
+     *
+     * @param bool $shouldUseIconv
+     * @return void
+     */
+    public function testAttemptConversionToUTF8ShouldReturnReencodedString($shouldUseIconv)
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->setConstructorArgs([new GlobalFunctionsHelper()])
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn($shouldUseIconv);
+        $encodingHelperStub->method('canUseMbString')->willReturn(true);
+
+        $encodedString = iconv(EncodingHelper::ENCODING_UTF8, EncodingHelper::ENCODING_UTF16_LE, 'input');
+        $decodedString = $encodingHelperStub->attemptConversionToUTF8($encodedString, EncodingHelper::ENCODING_UTF16_LE);
+
+        $this->assertEquals('input', $decodedString);
+    }
+
+    /**
+     * @return void
+     */
+    public function testAttemptConversionToUTF8ShouldBeNoopWhenTargetIsUTF8()
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->disableOriginalConstructor()
+                        ->setMethods(['canUseIconv'])
+                        ->getMock();
+        $encodingHelperStub->expects($this->never())->method('canUseIconv');
+
+        $decodedString = $encodingHelperStub->attemptConversionToUTF8('input', EncodingHelper::ENCODING_UTF8);
+        $this->assertEquals('input', $decodedString);
+    }
+
+    /**
+     * @dataProvider dataProviderForIconvOrMbstringUsage
+     * @expectedException \Box\Spout\Common\Exception\EncodingConversionException
+     *
+     * @param bool $shouldUseIconv
+     * @return void
+     */
+    public function testAttemptConversionFromUTF8ShouldThrowIfConversionFailed($shouldUseIconv)
+    {
+        $helperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\GlobalFunctionsHelper')
+                        ->setMethods(['iconv', 'mb_convert_encoding'])
+                        ->getMock();
+        $helperStub->method('iconv')->willReturn(false);
+        $helperStub->method('mb_convert_encoding')->willReturn(false);
+
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->setConstructorArgs([$helperStub])
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn($shouldUseIconv);
+        $encodingHelperStub->method('canUseMbString')->willReturn(true);
+
+        $encodingHelperStub->attemptConversionFromUTF8('input', EncodingHelper::ENCODING_UTF16_LE);
+    }
+
+    /**
+     * @expectedException \Box\Spout\Common\Exception\EncodingConversionException
+     *
+     * @return void
+     */
+    public function testAttemptConversionFromUTF8ShouldThrowIfConversionNotSupported()
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->disableOriginalConstructor()
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn(false);
+        $encodingHelperStub->method('canUseMbString')->willReturn(false);
+
+        $encodingHelperStub->attemptConversionFromUTF8('input', EncodingHelper::ENCODING_UTF16_LE);
+    }
+
+    /**
+     * @dataProvider dataProviderForIconvOrMbstringUsage
+     *
+     * @param bool $shouldUseIconv
+     * @return void
+     */
+    public function testAttemptConversionFromUTF8ShouldReturnReencodedString($shouldUseIconv)
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->setConstructorArgs([new GlobalFunctionsHelper()])
+                        ->setMethods(['canUseIconv', 'canUseMbString'])
+                        ->getMock();
+        $encodingHelperStub->method('canUseIconv')->willReturn($shouldUseIconv);
+        $encodingHelperStub->method('canUseMbString')->willReturn(true);
+
+        $encodedString = $encodingHelperStub->attemptConversionFromUTF8('input', EncodingHelper::ENCODING_UTF16_LE);
+        $encodedStringWithIconv = iconv(EncodingHelper::ENCODING_UTF8, EncodingHelper::ENCODING_UTF16_LE, 'input');
+
+        $this->assertEquals($encodedStringWithIconv, $encodedString);
+    }
+
+    /**
+     * @return void
+     */
+    public function testAttemptConversionFromUTF8ShouldBeNoopWhenTargetIsUTF8()
+    {
+        /** @var EncodingHelper $encodingHelperStub */
+        $encodingHelperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\EncodingHelper')
+                        ->disableOriginalConstructor()
+                        ->setMethods(['canUseIconv'])
+                        ->getMock();
+        $encodingHelperStub->expects($this->never())->method('canUseIconv');
+
+        $encodedString = $encodingHelperStub->attemptConversionFromUTF8('input', EncodingHelper::ENCODING_UTF8);
+        $this->assertEquals('input', $encodedString);
+    }
+}

tests/Spout/Reader/CSV/ReaderTest.php

@@ -2,8 +2,9 @@
 
 namespace Box\Spout\Reader\CSV;
 
-use Box\Spout\Common\Type;
 use Box\Spout\Reader\ReaderFactory;
+use Box\Spout\Common\Type;
+use Box\Spout\Common\Helper\EncodingHelper;
 use Box\Spout\TestUsingResource;
 
 /**
@@ -167,15 +168,96 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
     }
 
     /**
+     * @return array
+     */
+    public function dataProviderForTestReadShouldSkipBom()
+    {
+        return [
+            ['csv_with_utf8_bom.csv', EncodingHelper::ENCODING_UTF8],
+            ['csv_with_utf16le_bom.csv', EncodingHelper::ENCODING_UTF16_LE],
+            ['csv_with_utf16be_bom.csv', EncodingHelper::ENCODING_UTF16_BE],
+            ['csv_with_utf32le_bom.csv', EncodingHelper::ENCODING_UTF32_LE],
+            ['csv_with_utf32be_bom.csv', EncodingHelper::ENCODING_UTF32_BE],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestReadShouldSkipBom
+     *
+     * @param string $fileName
+     * @param string $fileEncoding
      * @return void
      */
-    public function testReadShouldSkipUtf8Bom()
+    public function testReadShouldSkipBom($fileName, $fileEncoding)
     {
-        $allRows = $this->getAllRowsForFile('csv_with_utf8_bom.csv');
+        $allRows = $this->getAllRowsForFile($fileName, ',', '"', $fileEncoding);
 
         $expectedRows = [
             ['csv--11', 'csv--12', 'csv--13'],
             ['csv--21', 'csv--22', 'csv--23'],
+            ['csv--31', 'csv--32', 'csv--33'],
+        ];
+        $this->assertEquals($expectedRows, $allRows);
+    }
+
+    /**
+     * @return array
+     */
+    public function dataProviderForTestReadShouldSupportNonUTF8FilesWithoutBOMs()
+    {
+        $shouldUseIconv = true;
+        $shouldNotUseIconv = false;
+
+        return [
+            ['csv_with_encoding_utf16le_no_bom.csv', EncodingHelper::ENCODING_UTF16_LE, $shouldUseIconv],
+            ['csv_with_encoding_utf16le_no_bom.csv', EncodingHelper::ENCODING_UTF16_LE, $shouldNotUseIconv],
+            ['csv_with_encoding_cp1252.csv', 'CP1252', $shouldUseIconv],
+            ['csv_with_encoding_cp1252.csv', 'CP1252', $shouldNotUseIconv],
+        ];
+    }
+
+    /**
+     * @dataProvider dataProviderForTestReadShouldSupportNonUTF8FilesWithoutBOMs
+     *
+     * @param string $fileName
+     * @param string $fileEncoding
+     * @param bool $shouldUseIconv
+     * @return void
+     */
+    public function testReadShouldSupportNonUTF8FilesWithoutBOMs($fileName, $fileEncoding, $shouldUseIconv)
+    {
+        $allRows = [];
+        $resourcePath = $this->getResourcePath($fileName);
+
+        $helperStub = $this->getMockBuilder('\Box\Spout\Common\Helper\GlobalFunctionsHelper')
+                        ->setMethods(['function_exists'])
+                        ->getMock();
+
+        $returnValueMap = [
+            ['iconv', $shouldUseIconv],
+            ['mb_convert_encoding', true],
+        ];
+        $helperStub->method('function_exists')->will($this->returnValueMap($returnValueMap));
+
+        /** @var \Box\Spout\Reader\CSV\Reader $reader */
+        $reader = ReaderFactory::create(Type::CSV);
+        $reader
+            ->setGlobalFunctionsHelper($helperStub)
+            ->setEncoding($fileEncoding)
+            ->open($resourcePath);
+
+        foreach ($reader->getSheetIterator() as $sheet) {
+            foreach ($sheet->getRowIterator() as $row) {
+                $allRows[] = $row;
+            }
+        }
+
+        $reader->close();
+
+        $expectedRows = [
+            ['csv--11', 'csv--12', 'csv--13'],
+            ['csv--21', 'csv--22', 'csv--23'],
+            ['csv--31', 'csv--32', 'csv--33'],
         ];
         $this->assertEquals($expectedRows, $allRows);
     }
@@ -228,18 +310,25 @@ class ReaderTest extends \PHPUnit_Framework_TestCase
      * @param string $fileName
      * @param string|void $fieldDelimiter
      * @param string|void $fieldEnclosure
+     * @param string|void $encoding
      * @return array All the read rows the given file
      */
-    private function getAllRowsForFile($fileName, $fieldDelimiter = ",", $fieldEnclosure = '"')
+    private function getAllRowsForFile(
+        $fileName,
+        $fieldDelimiter = ',',
+        $fieldEnclosure = '"',
+        $encoding = EncodingHelper::ENCODING_UTF8)
     {
         $allRows = [];
         $resourcePath = $this->getResourcePath($fileName);
 
+        /** @var \Box\Spout\Reader\CSV\Reader $reader */
         $reader = ReaderFactory::create(Type::CSV);
-        $reader->setFieldDelimiter($fieldDelimiter);
-        $reader->setFieldEnclosure($fieldEnclosure);
-
-        $reader->open($resourcePath);
+        $reader
+            ->setFieldDelimiter($fieldDelimiter)
+            ->setFieldEnclosure($fieldEnclosure)
+            ->setEncoding($encoding)
+            ->open($resourcePath);
 
         foreach ($reader->getSheetIterator() as $sheetIndex => $sheet) {
             foreach ($sheet->getRowIterator() as $rowIndex => $row) {

tests/Spout/Writer/CSV/WriterTest.php

@@ -2,8 +2,9 @@
 
 namespace Box\Spout\Writer\CSV;
 
-use Box\Spout\Common\Type;
 use Box\Spout\TestUsingResource;
+use Box\Spout\Common\Type;
+use Box\Spout\Common\Helper\EncodingHelper;
 use Box\Spout\Writer\WriterFactory;
 
 /**
@@ -70,7 +71,7 @@ class WriterTest extends \PHPUnit_Framework_TestCase
         ];
         $writtenContent = $this->writeToCsvFileAndReturnWrittenContent($allRows, 'csv_with_utf8_bom.csv');
 
-        $this->assertContains(Writer::BOM_UTF8, $writtenContent, 'The CSV file should contain a UTF-8 BOM');
+        $this->assertContains(EncodingHelper::BOM_UTF8, $writtenContent, 'The CSV file should contain a UTF-8 BOM');
     }
 
     /**
@@ -162,6 +163,6 @@ class WriterTest extends \PHPUnit_Framework_TestCase
     private function trimWrittenContent($writtenContent)
     {
         // remove line feeds and UTF-8 BOM
-        return trim($writtenContent, PHP_EOL . Writer::BOM_UTF8);
+        return trim($writtenContent, PHP_EOL . EncodingHelper::BOM_UTF8);
     }
 }

tests/resources/csv/csv_with_encoding_cp1252.csv

@@ -0,0 +1,3 @@
+csv--11,csv--12,csv--13
+csv--21,csv--22,csv--23
+csv--31,csv--32,csv--33

tests/resources/csv/csv_with_utf8_bom.csv

@@ -1,2 +1,3 @@
 csv--11,csv--12,csv--13
-csv--21,csv--22,csv--23
+csv--21,csv--22,csv--23
+csv--31,csv--32,csv--33