org.annotation.wordfreak.annotator
Class SimpleTokenizer

java.lang.Object
  org.annotation.wordfreak.annotator.SimpleTokenizer

public class SimpleTokenizer

extends java.lang.Object

Constructor Summary

SimpleTokenizer()

Method Summary

boolean	getAlphabeticSplit()
double[]	getTokProbs()
static void	main(java.lang.String[] args)
void	setAlphabeticSplit(boolean alphaSplit)
Span[]	tokenize(java.lang.String s)
java.util.List	tokenizeToList(java.lang.String s)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SimpleTokenizer

public SimpleTokenizer()

Method Detail

tokenize

public Span[] tokenize(java.lang.String s)

getTokProbs

public double[] getTokProbs()

getAlphabeticSplit

public boolean getAlphabeticSplit()

setAlphabeticSplit

public void setAlphabeticSplit(boolean alphaSplit)

tokenizeToList

public java.util.List tokenizeToList(java.lang.String s)

main

public static void main(java.lang.String[] args)
                 throws java.io.UnsupportedEncodingException

Throws:

java.io.UnsupportedEncodingException