Solr通过特殊字符分词实现自定义分词器详解

<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
 <analyzer>
  <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 </analyzer>
</fieldType>

<dependency>
  <groupId>org.apache.solr</groupId>
  <artifactId>solr-core</artifactId>
  <version>6.0.0</version>
</dependency>

package com.trainning.project.custom;

import java.util.Arrays;
import java.util.Collection;
import java.util.Map;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.UnicodeWhitespaceTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;

/**
* @author JiangChao
* @date 2017年4月2日下午3:41:13
*/
public class MyVerticalLineTokenizerFactory extends TokenizerFactory{
 public static final String RULE_JAVA = "java";
 public static final String RULE_UNICODE = "unicode";
 private static final Collection<String> RULE_NAMES = Arrays.asList(RULE_JAVA, RULE_UNICODE);

 private final String rule;

 /** Creates a new MyVerticalLineTokenizerFactory */
 public MyVerticalLineTokenizerFactory(Map<String,String> args) {
  super(args);

  rule = get(args, "rule", RULE_NAMES, RULE_JAVA);

  if (!args.isEmpty()) {
  throw new IllegalArgumentException("Unknown parameters: " + args);
  }
 }

 @Override
 public Tokenizer create(AttributeFactory factory) {
  switch (rule) {
  case RULE_JAVA:
   return new MyVerticalLineTokenizer(factory);
  case RULE_UNICODE:
   return new UnicodeWhitespaceTokenizer(factory);
  default:
   throw new AssertionError();
  }
 }
}

package com.trainning.project.custom;

import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;

/**
* @author JiangChao
* @date 2017年4月2日下午9:46:18
*/
public class MyVerticalLineTokenizer extends CharTokenizer {

 public MyVerticalLineTokenizer() {

 }
 public MyVerticalLineTokenizer(AttributeFactory factory) {
  super(factory);
  }

  /** Collects only characters which do not satisfy
  * 参数c指的是term的ASCII值，竖线的值为 124
  */
  @Override
  protected boolean isTokenChar(int c) {
  return !(c == 124);
  }
}

 /** Collects only characters which do not satisfy
  * 参数c指的是term的ASCII值，逗号的值为 44
  */
  @Override
  protected boolean isTokenChar(int c) {
  return !(c == 44);
  }

<fieldType name="vertical_text" class="solr.TextField">
 <analyzer>
  <tokenizer class="com.trainning.project.custom.MyVerticalLineTokenizerFactory"/>
 </analyzer>
 </fieldType>
 <field name="custom" type="vertical_text" indexed="true" stored="false"/>