Merge pull request #72 from alexander-nitsche/feature-handle-utf8-in-html

vanderlee · web-flow · commit d47fe93afd3b · 2023-05-14T23:48:11.000+02:00
FEATURE: Handle UTF-8 in HTML
diff --git a/README.md b/README.md
@@ -76,8 +76,10 @@ $syllable->setHyphen(new Hyphen\Dash());
 // By default, all words are hyphenated.
 $syllable->setMinWordLength(5);
 
-// Output hyphenated text.
+// Output hyphenated text ..
 echo $syllable->hyphenateText('Provide your own paragraphs...');
+// .. or hyphenated HTML.
+echo $syllable->hyphenateHtmlText('<b>... with highlighted text.</b>');
 ```
 
 See the [demo.php](demo.php) file for a working example.
@@ -137,7 +139,7 @@ Words need to contain at least this many character to be hyphenated.
 #### public setLibxmlOptions(int $libxmlOptions)
 
 Options to use for HTML parsing by libxml.
-See https://www.php.net/manual/de/libxml.constants.php.
+**See:** https://www.php.net/manual/de/libxml.constants.php.
 
 #### public excludeAll()
 
@@ -194,6 +196,13 @@ Hyphenate all words in the plain text.
 
 Hyphenate all readable text in the HTML, excluding HTML tags and
 attributes.
+**Deprecated:** Use the UTF-8 capable hyphenateHtmlText() instead. This method is kept only for backward compatibility and will be removed in the next major version 2.0.
+
+#### public hyphenateHtmlText(string $html): string
+
+Hyphenate all readable text in the HTML, excluding HTML tags and
+attributes.
+This method is UTF-8 capable and should be preferred over hyphenateHtml().
 
 #### public histogramText(string $text): array
 
diff --git a/build/classes/Reflection.php b/build/classes/Reflection.php
@@ -114,8 +114,11 @@ protected function parse($class)
                             $annotation = explode(' ', $line);
                             $returnType = $annotation[1] !== 'void' ? $annotation[1] : '';
                         } elseif (strpos($line, '@see') === 0) {
-                            $annotation = explode(' ', $line);
-                            $commentLines[] = 'See '.$annotation[1].'.';
+                            $annotation = explode(' ', $line, 2);
+                            $commentLines[] = '**See:** '.rtrim($annotation[1], '.').'.';
+                        } elseif (strpos($line, '@deprecated') === 0) {
+                            $annotation = explode(' ', $line, 2);
+                            $commentLines[] = '**Deprecated:** '.rtrim($annotation[1], '.').'.';
                         } elseif (!empty($line)) {
                             $commentLines[] = $line;
                         }
diff --git a/composer.json b/composer.json
@@ -24,6 +24,7 @@
   "require": {
     "php": ">=5.6",
     "ext-json": "*",
+    "ext-libxml": "*",
     "ext-mbstring": "*",
     "ext-dom": "*"
   },
diff --git a/src/Syllable.php b/src/Syllable.php
@@ -500,6 +500,8 @@ public function hyphenateText($text)
      * Hyphenate all readable text in the HTML, excluding HTML tags and
      * attributes.
      *
+     * @deprecated Use the UTF-8 capable hyphenateHtmlText() instead. This method is kept only for backward compatibility and will be removed in the next major version 2.0.
+     *
      * @param string $html
      *
      * @return string
@@ -520,6 +522,72 @@ public function hyphenateHtml($html)
         return $dom->saveHTML();
     }
 
+    /**
+     * Hyphenate all readable text in the HTML, excluding HTML tags and
+     * attributes.
+     *
+     * This method is UTF-8 capable and should be preferred over hyphenateHtml().
+     *
+     * @param string $html
+     *
+     * @return string
+     */
+    public function hyphenateHtmlText($html)
+    {
+        $charset = mb_detect_encoding($html);
+        list($bodyContent, $beforeBodyContent, $afterBodyContent) = $this->parseHtmlText($html);
+        $html = "<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.0 Transitional//EN' 'http://www.w3.org/TR/REC-html40/loose.dtd'>".
+            '<html>'.
+                '<head>'.
+                    "<meta http-equiv='content-type' content='text/html; charset=$charset'>".
+                '</head>'.
+                "<body>$bodyContent</body>".
+            '</html>';
+
+        $dom = new DOMDocument();
+        $dom->resolveExternals = true;
+        $dom->loadHTML($html, $this->libxmlOptions);
+
+        // filter excludes
+        $xpath = new DOMXPath($dom);
+        $excludedNodes = $this->excludes ? $xpath->query(join('|', $this->excludes)) : null;
+        $includedNodes = $this->includes ? $xpath->query(join('|', $this->includes)) : null;
+
+        $this->hyphenateHtmlDom($dom, $excludedNodes, $includedNodes);
+
+        $hyphenatedBodyContent = $dom->saveHTML($dom->getElementsByTagName('body')->item(0));
+        $hyphenatedBodyContent = mb_substr($hyphenatedBodyContent, mb_strlen('<body>'), -mb_strlen('</body>'));
+        $hyphenatedHtml = $beforeBodyContent.$hyphenatedBodyContent.$afterBodyContent;
+
+        return $hyphenatedHtml;
+    }
+
+    /**
+     * @param string $html
+     *
+     * @return array
+     */
+    private function parseHtmlText($html)
+    {
+        if (($bodyContentEnd = mb_strrpos($html, '</body>')) !== false) {
+            $bodyContentStart = mb_strpos($html, '<body');
+            $bodyContentStart = $bodyContentStart + strcspn($html, '>', $bodyContentStart) + 1;
+            $beforeBodyContent = mb_substr($html, 0, $bodyContentStart);
+            $afterBodyContent = mb_substr($html, $bodyContentEnd);
+            $bodyContent = mb_substr($html, $bodyContentStart, -mb_strlen($afterBodyContent));
+        } else {
+            $beforeBodyContent = '';
+            $afterBodyContent = '';
+            $bodyContent = $html;
+        }
+
+        return [
+            $bodyContent,
+            $beforeBodyContent,
+            $afterBodyContent,
+        ];
+    }
+
     /**
      * Add hyphenation to the DOM nodes.
      *
diff --git a/tests/build/DocumentationManagerTest.php b/tests/build/DocumentationManagerTest.php
@@ -64,10 +64,15 @@ public function delegateSucceeds()
 you will not use any other functions. Browse the code under src/ for all 
 available functions.
 
+#### public setMethodsDeprecated(array $methods = [])
+
+The deprecated public setter method.
+**Deprecated:** Use setMethods() instead.
+
 #### public setMethods(array $methods = [])
 
 The public setter method.
-See https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.
+**See:** https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.
 
 #### public getMethods(): array
 
diff --git a/tests/build/ReflectionFixture.php b/tests/build/ReflectionFixture.php
@@ -14,6 +14,20 @@ class ReflectionFixture
      */
     protected static $parameters;
 
+    /**
+     * The deprecated public setter method.
+     *
+     * @param array $methods
+     *
+     * @return void
+     *
+     * @deprecated Use setMethods() instead.
+     */
+    public function setMethodsDeprecated($methods = [])
+    {
+        $this->methods = $methods;
+    }
+
     /**
      * The public setter method.
      *
diff --git a/tests/build/ReflectionTest.php b/tests/build/ReflectionTest.php
@@ -32,9 +32,13 @@ public function getPublicMethodsWithSignatureAndComment()
         $class = ReflectionFixture::class;
 
         $expected = [
+            [
+                'signature' => 'public setMethodsDeprecated(array $methods = [])',
+                'comment'   => "The deprecated public setter method.\n**Deprecated:** Use setMethods() instead.",
+            ],
             [
                 'signature' => 'public setMethods(array $methods = [])',
-                'comment'   => "The public setter method.\nSee https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.",
+                'comment'   => "The public setter method.\n**See:** https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.",
             ],
             [
                 'signature' => 'public getMethods(): array',
diff --git a/tests/src/SyllableTest.php b/tests/src/SyllableTest.php
@@ -573,6 +573,64 @@ public function testHyphenateHtml()
             ."\n", $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext'));
     }
 
+    /**
+     * @return array[]
+     */
+    public function dataHyphenateHtmlText()
+    {
+        return [
+            [
+                'Ridiculously <b class="unsplittable">complicated</b> metatext — with dash entity.',
+                'Ridicu-lous-ly <b class="unsplittable">com-pli-cat-ed</b> meta-text — with dash en-ti-ty.',
+            ],
+            [
+                '<html>'.
+                    '<body>'.
+                        'Ridiculously <b class="unsplittable">complicated</b> metatext — with dash entity.'.
+                    '</body>'.
+                '</html>',
+                '<html>'.
+                    '<body>'.
+                        'Ridicu-lous-ly <b class="unsplittable">com-pli-cat-ed</b> meta-text — with dash en-ti-ty.'.
+                    '</body>'.
+                '</html>',
+            ],
+            [
+                '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'.
+                '<html>'.
+                    '<body class="body-class">'.
+                        'Ridiculously <b class="unsplittable">complicated</b> metatext — with dash entity.'.
+                    '</body>'.
+                '</html>',
+                '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'.
+                '<html>'.
+                    '<body class="body-class">'.
+                        'Ridicu-lous-ly <b class="unsplittable">com-pli-cat-ed</b> meta-text — with dash en-ti-ty.'.
+                    '</body>'.
+                '</html>',
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider dataHyphenateHtmlText
+     *
+     * @return void
+     */
+    public function testHyphenateHtmlText($html, $expected)
+    {
+        $this->object->setHyphen('-');
+
+        // Test that incoming content is never wrapped with the implicit doctype or
+        // html and body tag of DOMDocument. It always behaves as if LIBXML_HTML_NOIMPLIED
+        // and LIBXML_HTML_NODEFDTD are set.
+        $this->object->setLibxmlOptions(0);
+        $this->assertEquals($expected, $this->object->hyphenateHtmlText($html));
+
+        $this->object->setLibxmlOptions(LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
+        $this->assertEquals($expected, $this->object->hyphenateHtmlText($html));
+    }
+
     /**
      * @return void
      */
@@ -647,6 +705,13 @@ public function testExcludeElement()
             ."\n",
             $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>')
         );
+        $this->assertEquals(
+            'Ridicu-lous-ly <b class="unsplittable">complicated</b> meta-text <i>ex-trav-a-gan-za</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>'))
+        );
     }
 
     /**
@@ -664,6 +729,13 @@ public function testExcludeElements()
             ."\n",
             $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>')
         );
+        $this->assertEquals(
+            'Ridicu-lous-ly <b class="unsplittable">complicated</b> meta-text <i>extravaganza</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>'))
+        );
     }
 
     /**
@@ -682,6 +754,13 @@ public function testExcludeAllAndInclude()
             ."\n",
             $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>')
         );
+        $this->assertEquals(
+            'Ridiculously <b class="unsplittable">com-pli-cat-ed</b> metatext <i>extravaganza</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>'))
+        );
     }
 
     /**
@@ -696,9 +775,16 @@ public function testExcludeAndInclude()
         // Do not Hypenate content within <b>
         $this->assertEquals(
             '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'
-            ."\n".'<html><body><p>Ridicu-lous-ly <b class="unsplittable">complicated <i>ex-trav-a-gan-za</i></b> meta-text</p></body></html>'
+            ."\n".'<html><body><p>Ridicu-lous-ly <b class="unsplittable">complicated </b> meta-text <i>ex-trav-a-gan-za</i></p></body></html>'
             ."\n",
-            $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated <i>extravaganza</i></b> metatext')
+            $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated </b> metatext <i>extravaganza</i>')
+        );
+        $this->assertEquals(
+            'Ridicu-lous-ly <b class="unsplittable">complicated </b> meta-text <i>ex-trav-a-gan-za</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated </b> metatext <i>extravaganza</i>'))
         );
     }
 
@@ -717,6 +803,13 @@ public function testExcludeAttribute()
             ."\n",
             $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>')
         );
+        $this->assertEquals(
+            'Ridicu-lous-ly <b class="unsplittable">complicated</b> meta-text <i>ex-trav-a-gan-za</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated</b> metatext <i>extravaganza</i>'))
+        );
     }
 
     /**
@@ -734,5 +827,40 @@ public function testExcludeAttributeValue()
             ."\n",
             $this->object->hyphenateHtml('Ridiculously <b class="unsplittable">complicated</b> metatext <i class="go right ahead">extravaganza</i>')
         );
+        $this->assertEquals(
+            'Ridicu-lous-ly <b class="unsplittable">complicated</b> meta-text <i class="go right ahead">ex-trav-a-gan-za</i>',
+            // Old libxml versions of PHP < 7.4 occasionally added a line break in the output of
+            // \DOMDocument::saveHTML(). These line breaks are not yet handled in the Syllable
+            // implementation, but only quick and dirty in these tests by removing the trailing line breaks.
+            rtrim($this->object->hyphenateHtmlText('Ridiculously <b class="unsplittable">complicated</b> metatext <i class="go right ahead">extravaganza</i>'))
+        );
+    }
+
+    /**
+     * @return void
+     */
+    public function testUtf8Characters()
+    {
+        $this->object->setHyphen('-');
+
+        $this->object->setLanguage('de');
+        $this->assertEquals(
+            'Äu-ßerst kom-pli-zier-ter Me-ta-text.',
+            $this->object->hyphenateText('Äußerst komplizierter Metatext.')
+        );
+        $this->assertEquals(
+            'Äu-ßerst <b class="unsplittable">kom-pli-zier-ter</b> Me-ta-text.',
+            $this->object->hyphenateHtmlText('Äußerst <b class="unsplittable">komplizierter</b> Metatext.')
+        );
+
+        $this->object->setLanguage('uk');
+        $this->assertEquals(
+            'Над-зви-чайно скла-дний ме-та-те-кст.',
+            $this->object->hyphenateText('Надзвичайно складний метатекст.')
+        );
+        $this->assertEquals(
+            'Над-зви-чайно <b class="unsplittable">скла-дний</b> ме-та-те-кст.',
+            $this->object->hyphenateHtmlText('Надзвичайно <b class="unsplittable">складний</b> метатекст.')
+        );
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -32,9 +32,13 @@ public function getPublicMethodsWithSignatureAndComment()`
`32`	`32`	`$class = ReflectionFixture::class;`
`33`	`33`
`34`	`34`	`$expected = [`
	`35`	`+ [`
	`36`	`+ 'signature' => 'public setMethodsDeprecated(array $methods = [])',`
	`37`	`+ 'comment' => "The deprecated public setter method.\nDeprecated: Use setMethods() instead.",`
	`38`	`+ ],`
`35`	`39`	`[`
`36`	`40`	`'signature' => 'public setMethods(array $methods = [])',`
`37`		`- 'comment' => "The public setter method.\nSee https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.",`
	`41`	`+ 'comment' => "The public setter method.\nSee: https://github.com/vanderlee/phpSyllable/blob/master/tests/build/ReflectionFixture.php.",`
`38`	`42`	`],`
`39`	`43`	`[`
`40`	`44`	`'signature' => 'public getMethods(): array',`