Fix encoding support for ruby-1.9.x

Author: svenyurgensson

README.markdown

@@ -8,18 +8,18 @@
 
 ### Manual
 
-    git clone https://github.com/ssoper/summarize.git
+    git clone https://github.com/svenyurgensson/summarize.git
     cd summarize
     rake build
     gem build summarize.gemspec
-    gem install summarize-1.0.3.gem
+    gem install summarize-1.0.4.gem
 
 ## Usage
 
 The summarize method is added to File which you can use to summarize the contents of any plain text file
 
     File.open('path/to/file').summarize
-    
+
 Or use the String method
 
     "text to summarize".summarize
@@ -43,13 +43,15 @@ Topics can also be returned
 ## Dependencies
 
 You must have glib-2.0 and libxml-2.0 installed and properly configured.
-    
+
 ## Author
 
 Gem written by Sean Soper ([@ssoper](http://twitter.com/ssoper))
 
 The Open Text Summarizer library was written by Nadav Rotem and can be found at <http://libots.sourceforge.net/>
 
+Gem fixed to handle UTF-8 encoding by Yury Batenko ([@svenyurgensson](http://twitter.com/svenyurgensson))
+
 ## License
 
 Copyright (C) 2010 Sean Soper <[email protected]>

ext/summarize/dictionary.c

@@ -2,7 +2,7 @@
  *  dictionary.c
  *
  *  Copyright (C) 2003 Nadav Rotem <[email protected]>
- * 
+ *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; either version 2 of the License, or
@@ -40,28 +40,28 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   xmlNodePtr stem=NULL;
   xmlNodePtr pre=NULL;
   xmlNodePtr post=NULL;
-  xmlNodePtr syno=NULL;	    		/* synonyms     */
-  xmlNodePtr manual=NULL;	   	/* manual  */
-  xmlNodePtr step1_pre=NULL;		/* step1  */
-  xmlNodePtr step1_post=NULL;		/* step1  */
-  
-  xmlNodePtr parse=NULL;	   	/* parser rules */
-  xmlNodePtr pbreak=NULL;	      
-  xmlNodePtr pdbreak=NULL;		
-  
-  xmlNodePtr tc_words=NULL;	   	/* term count dictionary   */
-  xmlNodePtr tf_words=NULL;	   	/* term frequency dictionary   */
-  
+  xmlNodePtr syno=NULL;         /* synonyms     */
+  xmlNodePtr manual=NULL;     /* manual  */
+  xmlNodePtr step1_pre=NULL;    /* step1  */
+  xmlNodePtr step1_post=NULL;   /* step1  */
+
+  xmlNodePtr parse=NULL;      /* parser rules */
+  xmlNodePtr pbreak=NULL;
+  xmlNodePtr pdbreak=NULL;
+
+  xmlNodePtr tc_words=NULL;     /* term count dictionary   */
+  xmlNodePtr tf_words=NULL;     /* term frequency dictionary   */
+
 
   OtsStemRule * rule=Doc->stem;
-	
+
   char *local_dict_name;
-  
+
   local_dict_name = g_strdup_printf ("%s.xml", name);
 
 
-	if (g_file_test(local_dict_name,G_FILE_TEST_EXISTS))
-		  doc = xmlParseFile (local_dict_name); /* it warns to the screen so we cant use it; enable for web services only */
+  if (g_file_test(local_dict_name,G_FILE_TEST_EXISTS))
+      doc = xmlParseFile (local_dict_name); /* it warns to the screen so we cant use it; enable for web services only */
   if (doc == NULL) return (FALSE);
 
   head = xmlDocGetRootElement (doc);
@@ -82,23 +82,23 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   if (head != NULL)
     stem = head->xmlChildrenNode;
   while ((stem != NULL)
-	 && (xmlStrcmp (stem->name, (const xmlChar *) "stemmer")))
+   && (xmlStrcmp (stem->name, (const xmlChar *) "stemmer")))
     {
       stem = stem->next;
     }
 
   if (head != NULL)
     parse = head->xmlChildrenNode;
   while ((parse != NULL)
-	 && (xmlStrcmp (parse->name, (const xmlChar *) "parser")))
+   && (xmlStrcmp (parse->name, (const xmlChar *) "parser")))
     {
       parse = parse->next;
     }
 
   if (head != NULL)
     tc_words = head->xmlChildrenNode;
   while ((tc_words != NULL)
-	 && (xmlStrcmp (tc_words->name, (const xmlChar *) "grader-tc")))
+   && (xmlStrcmp (tc_words->name, (const xmlChar *) "grader-tc")))
     {
       tc_words = tc_words->next;
     }
@@ -107,12 +107,12 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   if (head != NULL)
     tf_words = head->xmlChildrenNode;
   while ((tf_words != NULL)
-	 && (xmlStrcmp (tf_words->name, (const xmlChar *) "grader-tf")))
+   && (xmlStrcmp (tf_words->name, (const xmlChar *) "grader-tf")))
     {
       tf_words = tf_words->next;
     }
-    
-    
+
+
 
   if (stem != NULL)
     pre = stem->xmlChildrenNode;
@@ -132,15 +132,15 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   if (stem != NULL)
     syno = stem->xmlChildrenNode;
   while ((syno != NULL)
-	 && (xmlStrcmp (syno->name, (const xmlChar *) "synonyms")))
+   && (xmlStrcmp (syno->name, (const xmlChar *) "synonyms")))
     {
       syno = syno->next;
     }
 
   if (stem != NULL)
     manual = stem->xmlChildrenNode;
   while ((manual != NULL)
-	 && (xmlStrcmp (manual->name, (const xmlChar *) "manual")))
+   && (xmlStrcmp (manual->name, (const xmlChar *) "manual")))
     {
       manual = manual->next;
     }
@@ -149,31 +149,31 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   if (stem != NULL)
     step1_pre = stem->xmlChildrenNode;
   while ((step1_pre != NULL)
-	 && (xmlStrcmp (step1_pre->name, (const xmlChar *) "step1_pre")))
+   && (xmlStrcmp (step1_pre->name, (const xmlChar *) "step1_pre")))
     {
       step1_pre = step1_pre->next;
     }
-    
-    
-    
+
+
+
   if (stem != NULL)
     step1_post = stem->xmlChildrenNode;
   while ((step1_post != NULL)
-	 && (xmlStrcmp (step1_post->name, (const xmlChar *) "step1_post")))
+   && (xmlStrcmp (step1_post->name, (const xmlChar *) "step1_post")))
     {
       step1_post = step1_post->next;
     }
 
 
   if (pre != NULL)
-    pre = pre->xmlChildrenNode;	/*point to first word */
+    pre = pre->xmlChildrenNode; /*point to first word */
   while (pre != NULL)
     {
       if (0 == xmlStrcmp (pre->name, (const xmlChar *) "rule"))
-	rule->RemovePre =
-	  g_list_append (rule->RemovePre,
-			 (xmlNodeListGetString
-				 (doc, pre->xmlChildrenNode, 1)));
+  rule->RemovePre =
+    g_list_append (rule->RemovePre,
+       (xmlNodeListGetString
+         (doc, pre->xmlChildrenNode, 1)));
       pre = pre->next;
     }
 
@@ -183,10 +183,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (post != NULL)
     {
       if (0 == xmlStrcmp (post->name, (const xmlChar *) "rule"))
-	rule->RemovePost =
-	  g_list_append (rule->RemovePost,
-			       (xmlNodeListGetString
-				 (doc, post->xmlChildrenNode, 1)));
+  rule->RemovePost =
+    g_list_append (rule->RemovePost,
+             (xmlNodeListGetString
+         (doc, post->xmlChildrenNode, 1)));
       post = post->next;
     }
 
@@ -195,10 +195,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (syno != NULL)
     {
       if (0 == xmlStrcmp (syno->name, (const xmlChar *) "rule"))
-	rule->synonyms =
-	  g_list_append (rule->synonyms,
-			     (xmlNodeListGetString
-				 (doc, syno->xmlChildrenNode, 1)));
+  rule->synonyms =
+    g_list_append (rule->synonyms,
+           (xmlNodeListGetString
+         (doc, syno->xmlChildrenNode, 1)));
       syno = syno->next;
     }
 
@@ -207,10 +207,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (manual != NULL)
     {
       if (0 == xmlStrcmp (manual->name, (const xmlChar *) "rule"))
-	rule->manual =
-	  g_list_append (rule->manual,
-			      (xmlNodeListGetString
-				 (doc, manual->xmlChildrenNode, 1)));
+  rule->manual =
+    g_list_append (rule->manual,
+            (xmlNodeListGetString
+         (doc, manual->xmlChildrenNode, 1)));
       manual = manual->next;
     }
 
@@ -222,10 +222,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (step1_pre != NULL)
     {
       if (0 == xmlStrcmp (step1_pre->name, (const xmlChar *) "rule"))
-	rule->step1_pre =
-	  g_list_append (rule->step1_pre,
-			      (xmlNodeListGetString
-				 (doc, step1_pre->xmlChildrenNode, 1)));
+  rule->step1_pre =
+    g_list_append (rule->step1_pre,
+            (xmlNodeListGetString
+         (doc, step1_pre->xmlChildrenNode, 1)));
       step1_pre = step1_pre->next;
     }
 
@@ -236,10 +236,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (step1_post != NULL)
     {
       if (0 == xmlStrcmp (step1_post->name, (const xmlChar *) "rule"))
-	rule->step1_post =
-	  g_list_append (rule->step1_post,
-			        (xmlNodeListGetString
-				 (doc, step1_post->xmlChildrenNode, 1)));
+  rule->step1_post =
+    g_list_append (rule->step1_post,
+              (xmlNodeListGetString
+         (doc, step1_post->xmlChildrenNode, 1)));
       step1_post = step1_post->next;
     }
 
@@ -258,18 +258,18 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
     {
       pdbreak = pdbreak->next;
     }
-    
-    
+
+
   /*Parser break*/
   if (pbreak != NULL)
     pbreak = pbreak->xmlChildrenNode;
   while (pbreak != NULL)
     {
       if (0 == xmlStrcmp (pbreak->name, (const xmlChar *) "rule"))
-	rule->ParserBreak =
-	  g_list_append (rule->ParserBreak,
-			        (xmlNodeListGetString
-				 (doc, pbreak->xmlChildrenNode, 1)));
+  rule->ParserBreak =
+    g_list_append (rule->ParserBreak,
+              (xmlNodeListGetString
+         (doc, pbreak->xmlChildrenNode, 1)));
       pbreak = pbreak->next;
     }
 
@@ -279,10 +279,10 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (pdbreak != NULL)
     {
       if (0 == xmlStrcmp (pdbreak->name, (const xmlChar *) "rule"))
-	rule->ParserDontBreak =
-	  g_list_append (rule->ParserDontBreak,
-			        (xmlNodeListGetString
-				 (doc, pdbreak->xmlChildrenNode, 1)));
+  rule->ParserDontBreak =
+    g_list_append (rule->ParserDontBreak,
+              (xmlNodeListGetString
+         (doc, pdbreak->xmlChildrenNode, 1)));
       pdbreak = pdbreak->next;
     }
 
@@ -293,37 +293,37 @@ ots_load_xml_dictionary (OtsArticle * Doc,unsigned const char *name)
   while (tc_words != NULL)
     {
       if (0 == xmlStrcmp (tc_words->name, (const xmlChar *) "word"))
-	{
-		xmlChar *key;
-		key=xmlNodeListGetString(doc, tc_words->xmlChildrenNode,1);
-	   Doc->dict = g_list_append (Doc->dict,(gpointer)ots_new_wordEntery(key));
+  {
+    xmlChar *key;
+    key=xmlNodeListGetString(doc, tc_words->xmlChildrenNode,1);
+     Doc->dict = g_list_append (Doc->dict,(gpointer)ots_new_wordEntery(key));
       xmlFree(key);
    }
       tc_words = tc_words->next;
     }
-      
-  
+
+
   /*Term Frequency load dict*/
-  
+
   if (tf_words != NULL)
     tf_words = tf_words->xmlChildrenNode;
   while (tf_words != NULL)
     {
       if (0 == xmlStrcmp (tf_words->name, (const xmlChar *) "word"))
-	{
-		xmlChar *key;
-		xmlChar *idf_key;
-		key=xmlNodeListGetString(doc, tf_words->xmlChildrenNode,1);
-	   
-	   idf_key=xmlGetProp(tf_words,"idf");
-	   Doc->tf_terms = g_list_append (Doc->tf_terms,ots_new_OtsWordTF(key,atof(idf_key)));
+  {
+    xmlChar *key;
+    xmlChar *idf_key;
+    key=xmlNodeListGetString(doc, tf_words->xmlChildrenNode,1);
+
+     idf_key=xmlGetProp(tf_words,"idf");
+     Doc->tf_terms = g_list_append (Doc->tf_terms,ots_new_OtsWordTF(key,atof(idf_key)));
       xmlFree(key);
       xmlFree(idf_key);
    }
       tf_words = tf_words->next;
     }
-    
-    
+
+
   xmlFreeDoc(doc);
   xmlCleanupParser ();
   g_free(local_dict_name);

ext/summarize/summarize.c

@@ -6,6 +6,10 @@
 #include <glib-object.h>
 #include <ruby.h>
 
+#ifdef HAVE_RUBY_ENCODING_H
+  #include <ruby/encoding.h>
+#endif
+
 #include "libots.h"
 #include "summarize.h"
 
@@ -16,6 +20,9 @@ void Init_summarize() {
 }
 
 static VALUE summarize(const VALUE self, volatile VALUE rb_str, volatile VALUE rb_dict_file, const VALUE rb_ratio, const VALUE rb_topics) {
+  #ifdef HAVE_RUBY_ENCODING_H
+    int enc = rb_enc_find_index("UTF-8");
+  #endif
   long int length = RSTRING_LEN(rb_str);
   char *text = StringValuePtr(rb_str);
   char *dictionary_file = StringValuePtr(rb_dict_file);
@@ -41,6 +48,11 @@ static VALUE summarize(const VALUE self, volatile VALUE rb_str, volatile VALUE r
   summary = rb_str_new2(ots_get_doc_text(doc, &result_len));
   topics = rb_str_new2((const char *)doc->title);
 
+  #ifdef HAVE_RUBY_ENCODING_H
+    rb_enc_associate_index(summary, enc);
+    rb_enc_associate_index(summary, enc);
+  #endif
+
   ots_free_article(doc);
 
   if (rb_topics == Qtrue) {