Java生态中的NLP框架详解

InputStream modelIn = new FileInputStream("en-sent.bin");
SentenceModel model = new SentenceModel(modelIn);
SentenceDetectorME sentenceDetector = new SentenceDetectorME(model);
String sentences[] = sentenceDetector.sentDetect("First sentence. Second sentence.");

Properties props = new Properties();
props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner, parse, sentiment");
StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
Annotation document = new Annotation("Stanford CoreNLP is great!");
pipeline.annotate(document);

TokenizerFactory tokenizerFactory = IndoEuropeanTokenizerFactory.INSTANCE;
Tokenizer tokenizer = tokenizerFactory.tokenizer("This is LingPipe.", 0, "This is LingPipe.".length());
for (Token token : tokenizer)
    System.out.println("Token: " + token);

AnalysisEngine engine = AnalysisEngineFactory.createEngine(
    createEngineDescription(
        LanguageToolSegmenter.class,
        LanguageToolLemmatizer.class));
JCas jcas = engine.newJCas();
jcas.setDocumentText("This is DKPro Core.");
engine.process(jcas);

InstanceList instances = new InstanceList(new SerialPipes(pipes));
instances.addThruPipe(new LineIterator("data.txt"));
ParallelTopicModel model = new ParallelTopicModel(5, 1.0, 0.01);
model.addInstances(instances);
model.estimate();

TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
Word2Vec vec = new Word2Vec.Builder()
    .minWordFrequency(5)
    .iterations(1)
    .layerSize(100)
    .seed(42)
    .windowSize(5)
    .iterate(iter)
    .tokenizerFactory(tokenizerFactory)
    .build();
vec.fit();