【二】StandardFilter分词过滤器的使用

在  if (!input.incrementToken())判断中，调用构建TokenScreamComponent时传入的分词器获取token分词
并存入CharTermAttribute中

//A TokenFilter is a TokenStream whose input is another TokenStream.
public class StandardFilter extends TokenFilter {
     /**
        * 去除词语末尾的"'s"   如  it's-> it
        * 去除缩略语中的"."  如U.S.A -> USA
        */
     //词元的内容，如"tearcher"  "[e-mail]"  "1421"用来保存Token字符串；
    private final org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt ;
    private final org.apache.lucene.analysis.tokenattributes.TypeAttribute typeAtt ;
    //Token的大概14多重类型

    private final static String ACRONYM = "";
    private final static String APOSTROPHE = "";

    protected StandardFilter(TokenStream input) {
        super(input);
          termAtt = addAttribute(CharTermAttribute.class);
          typeAtt = addAttribute(TypeAttribute.class);

    }

    @Override
    public boolean incrementToken() throws IOException {

        if (!input.incrementToken())//这里会立即调用你所传入分词器的incrementToken方法
        {
            return false;
        }
        char[] buffer = termAtt.buffer();
        final int bufferLength = termAtt.length();
        final String type = typeAtt.type();
        if (type ==String.valueOf(APOSTROPHE)
                    && bufferLength >= 2
                        && buffer[bufferLength-2] == '\''
                                && (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {

                termAtt.setLength((bufferLength - 2));
         } else if (type == String.valueOf(ACRONYM)) {
                int upto = 0;
                for(int i=0;i<bufferLength;i++) {
                      char c = buffer[i];
                      if (c != '.')
                            buffer[upto++] = c;
                }
                termAtt.setLength(upto);
          }
      return true;
    }

}