ivory.lsh.bitext

Class PreprocessHelper

java.lang.Object

ivory.lsh.bitext.PreprocessHelper

public class PreprocessHelper
extends Object

Constructor Summary

Constructors

Constructor and Description
PreprocessHelper(int minVectorTerms, int minSentenceLength, Configuration conf) Implemented for non-cluster mode, files read directly from local FS
PreprocessHelper(int minVectorTerms, int minSentenceLength, JobConf conf) Implemented for HDFS cluster mode, files read from local cache

Method Summary

Methods

Modifier and Type	Method and Description
HMapSFW	createEDocVector(String sentence)
HMapSFW	createEDocVector(String sentence, HMapSIW term2Tf)
HMapSFW	createFDocVector(String sentence)
HMapSFW	createFDocVector(String sentence, HMapSIW term2Tf)
opennlp.model.MaxentModel	getClassifier()
TTable_monolithic_IFAs	getE2F()
float	getEOOVRate(String eSent)
opennlp.tools.sentdetect.SentenceDetectorME	getESentenceModel()
ArrayListWritable<Text>	getESentences(String text, ArrayListWritable<HMapSFW> vectors, ArrayListOfIntsWritable sentLengths)
Vocab	getESrc()
Tokenizer	getETokenizer()
Vocab	getETrg()
TTable_monolithic_IFAs	getF2E()
float	getFOOVRate(String fSent)
opennlp.tools.sentdetect.SentenceDetectorME	getFSentenceModel()
ArrayListWritable<Text>	getFSentences(String text, ArrayListWritable<HMapSFW> vectors, ArrayListOfIntsWritable sentLengths)
Vocab	getFSrc()
Tokenizer	getFTokenizer()
Vocab	getFTrg()
void	loadModels(JobConf job)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

PreprocessHelper

public PreprocessHelper(int minVectorTerms,
                int minSentenceLength,
                Configuration conf)
                 throws Exception

Implemented for non-cluster mode, files read directly from local FS

Throws:

Exception

PreprocessHelper

public PreprocessHelper(int minVectorTerms,
                int minSentenceLength,
                JobConf conf)
                 throws Exception

Implemented for HDFS cluster mode, files read from local cache

Throws:

Exception

Method Detail

createEDocVector

public HMapSFW createEDocVector(String sentence)

createEDocVector

public HMapSFW createEDocVector(String sentence,
                       HMapSIW term2Tf)

createFDocVector

public HMapSFW createFDocVector(String sentence)

createFDocVector

public HMapSFW createFDocVector(String sentence,
                       HMapSIW term2Tf)

getClassifier

public opennlp.model.MaxentModel getClassifier()

getE2F

public TTable_monolithic_IFAs getE2F()

getEOOVRate

public float getEOOVRate(String eSent)

getESentenceModel

public opennlp.tools.sentdetect.SentenceDetectorME getESentenceModel()

getESentences

public ArrayListWritable<Text> getESentences(String text,
                                    ArrayListWritable<HMapSFW> vectors,
                                    ArrayListOfIntsWritable sentLengths)
                                      throws ClassNotFoundException,
                                             InstantiationException,
                                             IllegalAccessException,
                                             IOException

Throws:

ClassNotFoundException

InstantiationException

IllegalAccessException

IOException

getESrc

public Vocab getESrc()

getETokenizer

public Tokenizer getETokenizer()

getETrg

public Vocab getETrg()

getF2E

public TTable_monolithic_IFAs getF2E()

getFOOVRate

public float getFOOVRate(String fSent)

getFSentenceModel

public opennlp.tools.sentdetect.SentenceDetectorME getFSentenceModel()

getFSentences

public ArrayListWritable<Text> getFSentences(String text,
                                    ArrayListWritable<HMapSFW> vectors,
                                    ArrayListOfIntsWritable sentLengths)
                                      throws ClassNotFoundException,
                                             InstantiationException,
                                             IllegalAccessException,
                                             IOException

Throws:

ClassNotFoundException

InstantiationException

IllegalAccessException

IOException

getFSrc

public Vocab getFSrc()

getFTokenizer

public Tokenizer getFTokenizer()

getFTrg

public Vocab getFTrg()

loadModels

public void loadModels(JobConf job)
                throws Exception

Throws:

Exception