[Scala] 集合知プログラミング generatefeedvector.py を scalaで書いてみる

集合知プログラミングの第３章で紹介されているgeneratefeedvector.pyをscalaでかいてみる。
指摘大歓迎です！！
※結構時間がかかってしまったうえに、ＰＧ長い。。。
[追記]
id:kmizushima さんから指摘を頂いたので、修正してみた。

package jp.shohu.auto_gene

import scala.io.Source

object autogene {
  import scala.io.Source
  import scala.xml._
  import scala.collection.mutable.{Queue,HashMap}
  import scala.util.matching.Regex
  import java.io.FileOutputStream
  import scala.collection.jcl.ArrayList
  import java.util.Date
  import DisposableFile._

  def main(args: Array[String]) = {
    var wordcounts =  HashMap[String, Any]()
    var apcount =  HashMap[String, Int]()
    // URL一覧読込み
    val urls = Source.fromFile( "feedlist.txt" ).getLines.toList
    val size = urls.size.toFloat
    var cnt = 1
    for (url <- urls) {
      try {
          println(new Date() + "counter = " + cnt + "/" + size);
          var (title, wc) = getwordcounts(url);
	      wordcounts(title) = wc
	      for (word <- wc.keySet){
	        if(word.length() > 2){
	            if(apcount.contains(word)) apcount(word) = apcount.get(word).get + 1
	            else apcount(word) = 0
	        }
	      }
	       }catch{
	         case e: Exception => 
	           e.printStackTrace
	           println("Failed to parse feed " + url)
	       }
       cnt += 1;
    }
    var wordlist = List[String]();
    // 余分な単語は無視する
    for ((w, f) <- apcount){
      val frac = f.toFloat / size;
      if (frac > 0.05 && frac < 0.9) wordlist = w :: wordlist;
    }
    
    // TODO getBytesで書き込むのをけしたい
    openOut("blogdata.txt") {
      write("Blog".getBytes)
      for(word <- wordlist) write(("\t" + word).getBytes)
      write("\n".getBytes)
	  for ((blog, map) <- wordcounts){
	    val wc = map.asInstanceOf[HashMap[String, Int]]
	    write(blog.getBytes)
        for (word <- wordlist){
          if (wc.contains(word)) write(("\t" + wc.get(word).get).getBytes)
          else write("\t0".getBytes)
        }
        write("\n".getBytes)
      }
    }
  }
  
  def getwordcounts(url:String):(String,HashMap[String, Int]) = {
      // RSS読込んでく
      val feed = common.getFeedElem(url);
      val items = feed\"item";
      var h = new HashMap[String, Int]()
      for(item <- items){
        var summary = (item\"summary").text;
        if (summary == "") summary = (item\"description").text
        val words = getwords((item\"title").text + ' ' + summary);
        for (word <- words){
          // 3文字以上の文字のみ取得
          if (word.length() > 2){
            if(h.contains(word)) h(word) = h.get(word).get + 1
            else h(word) = 0
          }
        }
      }
      // 複数の値を返したいけどコンテナとするクラスを定義するのがメンドイ、というときにタプルが使える。
      return ((feed\"channel"\"title").text,h)
  }
  
  // 単語をカウント
  val re = new Regex( "<.*?>" )
  def getwords(html:String):Seq[String] = {
    // HTMLタグを取り除く
    val txt = re.replaceAllIn(html, "");
    return for (s <- splitter.split(txt) if s != "") yield s
  }
  
}

// 共通
object common {
  import java.net.{URLConnection, URL}
  import java.io.FileOutputStream
  import scala.xml._

  //Java の API を利用
  def getFeedElem(feed:String):Elem = {
    val url = new URL(feed)
    val conn = url.openConnection
    XML.load(conn.getInputStream)
  }
}

// ファイル出力用
import java.io._
import scala.util.DynamicVariable
object DisposableFile extends OutputStream {
  val out = new DynamicVariable[FileOutputStream](null)
  def openOut(path: String)(block: => Unit) {
    val out_ = new FileOutputStream(path)
    try {
      out.withValue(out_) { block }
    } finally {
      out_.close()
      // Console.println("closed")
    }
  }
  def write(b: Int) = out.value.write(b)
  override def write(b: Array[Byte]) = out.value.write(b)
  override def write(b: Array[Byte], off: Int, len: Int) = out.value.write(b, off, len)
  override def flush = out.value.flush
}

// 形態素解析
object splitter {
  import java.net.URLEncoder

  val appid="XXXXXXX"
  val pageurl="http://api.jlp.yahoo.co.jp/MAService/V1/parse"

  def split(sentence:String):Seq[String] = {
    // URL文字列に変換
    val statement = URLEncoder.encode(sentence , "UTF-8");
    val query = pageurl+"?appid="+appid+"&results=ma&uniq_filter=1|2|3|4|5|9|10&sentence="+statement
    val feed = common.getFeedElem(query);
    val wordList = feed\"ma_result"\"word_list"\"word";
    return for(word <- wordList) yield (word\"surface").text;
  }
}