indexer improvements & fix for underscores

darcs-hash:20050816032408-50fdc-6e41585c9b97d70a218877b8ad169df9117d9965.gz

inc/indexer.php

@@ -37,19 +37,20 @@ function idx_getPageWords($page){
     
     $words = array();
     foreach ($tokens as $word => $count) {
-        $word = utf8_strtolower($word);
 
         // simple filter to restrict use of utf8_stripspecials 
-        if (preg_match('/\W/', $word)) {
+        if (preg_match('/[^0-9A-Za-z]/u', $word)) {
             $arr = explode(' ', utf8_stripspecials($word,' ','._\-:'));
             $arr = array_count_values($arr);
             
             foreach ($arr as $w => $c) {
                 if (!is_numeric($w) && strlen($w) < 3) continue;
+    		    $w = utf8_strtolower($w);
                 $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0);
             }
         } else {
             if (!is_numeric($w) && strlen($w) < 3) continue;
+	        $word = strtolower($word);
             $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0);
         }
     }

inc/lang/en/stopwords.txt

@@ -12,6 +12,7 @@ their
 com
 for
 from
+into
 how
 that
 the