/* The following code was generated by JFlex 1.5.1 */
using System;
using System.IO;
using Lucene.Net.Analysis.TokenAttributes;

namespace Lucene.Net.Analysis.Standard
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    /*
	
    WARNING: if you change ClassicTokenizerImpl.jflex and need to regenerate
          the tokenizer, only use the trunk version of JFlex 1.5 at the moment!
	
    */
    /// <summary>
    /// This class implements the classic lucene <see cref="StandardTokenizer"/> up until 3.0 
    /// </summary>

    internal class ClassicTokenizerImpl : IStandardTokenizerInterface
    {
        /// <summary>
        /// This character denotes the end of file </summary>
        public static readonly int YYEOF = -1;

        /// <summary>
        /// initial size of the lookahead buffer </summary>
        private static readonly int ZZ_BUFFERSIZE = 4096;

        /// <summary>
        /// lexical states </summary>
        public const int YYINITIAL = 0;

        /// <summary>
        /// ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
        /// ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
        ///                  at the beginning of a line
        /// l is of the form l = 2*k, k a non negative integer
        /// </summary>
        private static readonly int[] ZZ_LEXSTATE = { 0, 0 };

        /// <summary>
        /// Translates characters to character classes
        /// </summary>
        private const string ZZ_CMAP_PACKED =
            "\x0026\x0000\x0001\x0005\x0001\x0003\x0004\x0000\x0001\x0009\x0001\x0007\x0001\x0004\x0001\x0009\x000A\x0002\x0006\x0000" +
            "\x0001\x0006\x001A\x000A\x0004\x0000\x0001\x0008\x0001\x0000\x001A\x000A\x002F\x0000\x0001\x000A\x000A\x0000\x0001\x000A" +
            "\x0004\x0000\x0001\x000A\x0005\x0000\x0017\x000A\x0001\x0000\x001F\x000A\x0001\x0000\u0128\x000A\x0002\x0000\x0012\x000A" +
            "\x001C\x0000\x005E\x000A\x0002\x0000\x0009\x000A\x0002\x0000\x0007\x000A\x000E\x0000\x0002\x000A\x000E\x0000\x0005\x000A" +
            "\x0009\x0000\x0001\x000A\x008B\x0000\x0001\x000A\x000B\x0000\x0001\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0001\x000A" +
            "\x0001\x0000\x0014\x000A\x0001\x0000\x002C\x000A\x0001\x0000\x0008\x000A\x0002\x0000\x001A\x000A\x000C\x0000\x0082\x000A" +
            "\x000A\x0000\x0039\x000A\x0002\x0000\x0002\x000A\x0002\x0000\x0002\x000A\x0003\x0000\x0026\x000A\x0002\x0000\x0002\x000A" +
            "\x0037\x0000\x0026\x000A\x0002\x0000\x0001\x000A\x0007\x0000\x0027\x000A\x0048\x0000\x001B\x000A\x0005\x0000\x0003\x000A" +
            "\x002E\x0000\x001A\x000A\x0005\x0000\x000B\x000A\x0015\x0000\x000A\x0002\x0007\x0000\x0063\x000A\x0001\x0000\x0001\x000A" +
            "\x000F\x0000\x0002\x000A\x0009\x0000\x000A\x0002\x0003\x000A\x0013\x0000\x0001\x000A\x0001\x0000\x001B\x000A\x0053\x0000" +
            "\x0026\x000A\u015f\x0000\x0035\x000A\x0003\x0000\x0001\x000A\x0012\x0000\x0001\x000A\x0007\x0000\x000A\x000A\x0004\x0000" +
            "\x000A\x0002\x0015\x0000\x0008\x000A\x0002\x0000\x0002\x000A\x0002\x0000\x0016\x000A\x0001\x0000\x0007\x000A\x0001\x0000" +
            "\x0001\x000A\x0003\x0000\x0004\x000A\x0022\x0000\x0002\x000A\x0001\x0000\x0003\x000A\x0004\x0000\x000A\x0002\x0002\x000A" +
            "\x0013\x0000\x0006\x000A\x0004\x0000\x0002\x000A\x0002\x0000\x0016\x000A\x0001\x0000\x0007\x000A\x0001\x0000\x0002\x000A" +
            "\x0001\x0000\x0002\x000A\x0001\x0000\x0002\x000A\x001F\x0000\x0004\x000A\x0001\x0000\x0001\x000A\x0007\x0000\x000A\x0002" +
            "\x0002\x0000\x0003\x000A\x0010\x0000\x0007\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0016\x000A" +
            "\x0001\x0000\x0007\x000A\x0001\x0000\x0002\x000A\x0001\x0000\x0005\x000A\x0003\x0000\x0001\x000A\x0012\x0000\x0001\x000A" +
            "\x000F\x0000\x0001\x000A\x0005\x0000\x000A\x0002\x0015\x0000\x0008\x000A\x0002\x0000\x0002\x000A\x0002\x0000\x0016\x000A" +
            "\x0001\x0000\x0007\x000A\x0001\x0000\x0002\x000A\x0002\x0000\x0004\x000A\x0003\x0000\x0001\x000A\x001E\x0000\x0002\x000A" +
            "\x0001\x0000\x0003\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0006\x000A\x0003\x0000\x0003\x000A\x0001\x0000\x0004\x000A" +
            "\x0003\x0000\x0002\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0002\x000A\x0003\x0000\x0002\x000A\x0003\x0000\x0003\x000A" +
            "\x0003\x0000\x0008\x000A\x0001\x0000\x0003\x000A\x002D\x0000\x0009\x0002\x0015\x0000\x0008\x000A\x0001\x0000\x0003\x000A" +
            "\x0001\x0000\x0017\x000A\x0001\x0000\x000A\x000A\x0001\x0000\x0005\x000A\x0026\x0000\x0002\x000A\x0004\x0000\x000A\x0002" +
            "\x0015\x0000\x0008\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0017\x000A\x0001\x0000\x000A\x000A\x0001\x0000\x0005\x000A" +
            "\x0024\x0000\x0001\x000A\x0001\x0000\x0002\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0008\x000A\x0001\x0000\x0003\x000A" +
            "\x0001\x0000\x0017\x000A\x0001\x0000\x0010\x000A\x0026\x0000\x0002\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0012\x000A" +
            "\x0003\x0000\x0018\x000A\x0001\x0000\x0009\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0007\x000A\x0039\x0000\x0001\x0001" +
            "\x0030\x000A\x0001\x0001\x0002\x000A\x000C\x0001\x0007\x000A\x0009\x0001\x000A\x0002\x0027\x0000\x0002\x000A\x0001\x0000" +
            "\x0001\x000A\x0002\x0000\x0002\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0001\x000A\x0006\x0000\x0004\x000A\x0001\x0000" +
            "\x0007\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0002\x000A\x0001\x0000" +
            "\x0004\x000A\x0001\x0000\x0002\x000A\x0009\x0000\x0001\x000A\x0002\x0000\x0005\x000A\x0001\x0000\x0001\x000A\x0009\x0000" +
            "\x000A\x0002\x0002\x0000\x0002\x000A\x0022\x0000\x0001\x000A\x001F\x0000\x000A\x0002\x0016\x0000\x0008\x000A\x0001\x0000" +
            "\x0022\x000A\x001D\x0000\x0004\x000A\x0074\x0000\x0022\x000A\x0001\x0000\x0005\x000A\x0001\x0000\x0002\x000A\x0015\x0000" +
            "\x000A\x0002\x0006\x0000\x0006\x000A\x004A\x0000\x0026\x000A\x000A\x0000\x0027\x000A\x0009\x0000\x005A\x000A\x0005\x0000" +
            "\x0044\x000A\x0005\x0000\x0052\x000A\x0006\x0000\x0007\x000A\x0001\x0000\x003F\x000A\x0001\x0000\x0001\x000A\x0001\x0000" +
            "\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0027\x000A\x0001\x0000" +
            "\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x001F\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000" +
            "\x0007\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000\x0007\x000A\x0001\x0000" +
            "\x0017\x000A\x0001\x0000\x001F\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000" +
            "\x0027\x000A\x0001\x0000\x0013\x000A\x000E\x0000\x0009\x0002\x002E\x0000\x0055\x000A\x000C\x0000\u026c\x000A\x0002\x0000" +
            "\x0008\x000A\x000A\x0000\x001A\x000A\x0005\x0000\x004B\x000A\x0095\x0000\x0034\x000A\x002C\x0000\x000A\x0002\x0026\x0000" +
            "\x000A\x0002\x0006\x0000\x0058\x000A\x0008\x0000\x0029\x000A\u0557\x0000\x009C\x000A\x0004\x0000\x005A\x000A\x0006\x0000" +
            "\x0016\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0026\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0008\x000A\x0001\x0000" +
            "\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x001F\x000A\x0002\x0000\x0035\x000A\x0001\x0000" +
            "\x0007\x000A\x0001\x0000\x0001\x000A\x0003\x0000\x0003\x000A\x0001\x0000\x0007\x000A\x0003\x0000\x0004\x000A\x0002\x0000" +
            "\x0006\x000A\x0004\x0000\x000D\x000A\x0005\x0000\x0003\x000A\x0001\x0000\x0007\x000A\x0082\x0000\x0001\x000A\x0082\x0000" +
            "\x0001\x000A\x0004\x0000\x0001\x000A\x0002\x0000\x000A\x000A\x0001\x0000\x0001\x000A\x0003\x0000\x0005\x000A\x0006\x0000" +
            "\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0001\x0000\x0003\x000A\x0001\x0000" +
            "\x0007\x000A\u0ecb\x0000\x0002\x000A\x002A\x0000\x0005\x000A\x000A\x0000\x0001\x000B\x0054\x000B\x0008\x000B\x0002\x000B" +
            "\x0002\x000B\x005A\x000B\x0001\x000B\x0003\x000B\x0006\x000B\x0028\x000B\x0003\x000B\x0001\x0000\x005E\x000A\x0011\x0000" +
            "\x0018\x000A\x0038\x0000\x0010\x000B\u0100\x0000\x0080\x000B\x0080\x0000\u19b6\x000B\x000A\x000B\x0040\x0000\u51a6\x000B" +
            "\x005A\x000B\u048d\x000A\u0773\x0000\u2ba4\x000A\u215c\x0000\u012e\x000B\x00D2\x000B\x0007\x000A\x000C\x0000\x0005\x000A" +
            "\x0005\x0000\x0001\x000A\x0001\x0000\x000A\x000A\x0001\x0000\x000D\x000A\x0001\x0000\x0005\x000A\x0001\x0000\x0001\x000A" +
            "\x0001\x0000\x0002\x000A\x0001\x0000\x0002\x000A\x0001\x0000\x006C\x000A\x0021\x0000\u016b\x000A\x0012\x0000\x0040\x000A" +
            "\x0002\x0000\x0036\x000A\x0028\x0000\x000C\x000A\x0074\x0000\x0003\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0087\x000A" +
            "\x0013\x0000\x000A\x0002\x0007\x0000\x001A\x000A\x0006\x0000\x001A\x000A\x000A\x0000\x0001\x000B\x003A\x000B\x001F\x000A" +
            "\x0003\x0000\x0006\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0003\x000A\x0023\x0000";

        /// <summary>
        /// Translates characters to character classes
        /// </summary>
        private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED);

        /// <summary>
        /// Translates DFA states to action switch labels.
        /// </summary>
        private static readonly int[] ZZ_ACTION = ZzUnpackAction();

        private const string ZZ_ACTION_PACKED_0 =
            "\x0001\x0000\x0001\x0001\x0003\x0002\x0001\x0003\x000B\x0000\x0001\x0002\x0003\x0004\x0002\x0000" +
            "\x0001\x0005\x0001\x0000\x0001\x0005\x0003\x0004\x0006\x0005\x0001\x0006\x0001\x0004\x0002\x0007" +
            "\x0001\x0008\x0001\x0000\x0001\x0008\x0003\x0000\x0002\x0008\x0001\x0009\x0001\x000A\x0001\x0004";

        private static int[] ZzUnpackAction()
        {
            int[] result = new int[50];
            int offset = 0;
            offset = ZzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
            return result;
        }

        private static int ZzUnpackAction(string packed, int offset, int[] result)
        {
            int i = 0; // index in packed string
            int j = offset; // index in unpacked array
            int l = packed.Length;
            while (i < l)
            {
                int count = packed[i++];
                int value = packed[i++];
                do
                {
                    result[j++] = value;
                } while (--count > 0);
            }
            return j;
        }


        /// <summary>
        /// Translates a state to a row index in the transition table
        /// </summary>
        private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap();

        private const string ZZ_ROWMAP_PACKED_0 =
            "\x0000\x0000\x0000\x000C\x0000\x0018\x0000\x0024\x0000\x0030\x0000\x000C\x0000\x003C\x0000\x0048" +
            "\x0000\x0054\x0000\x0060\x0000\x006C\x0000\x0078\x0000\x0084\x0000\x0090\x0000\x009C\x0000\x00A8" +
            "\x0000\x00B4\x0000\x00C0\x0000\x00CC\x0000\x00D8\x0000\x00E4\x0000\x00F0\x0000\x00FC\x0000\u0108" +
            "\x0000\u0114\x0000\u0120\x0000\u012c\x0000\u0138\x0000\u0144\x0000\u0150\x0000\u015c\x0000\u0168" +
            "\x0000\u0174\x0000\u0180\x0000\u018c\x0000\u0198\x0000\u01a4\x0000\x00A8\x0000\u01b0\x0000\u01bc" +
            "\x0000\u01c8\x0000\u01d4\x0000\u01e0\x0000\u01ec\x0000\u01f8\x0000\x003C\x0000\x006C\x0000\u0204" +
            "\x0000\u0210\x0000\u021c";

        private static int[] ZzUnpackRowMap()
        {
            int[] result = new int[50];
            int offset = 0;
            offset = ZzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
            return result;
        }

        private static int ZzUnpackRowMap(string packed, int offset, int[] result)
        {
            int i = 0; // index in packed string
            int j = offset; // index in unpacked array
            int l = packed.Length;
            while (i < l)
            {
                int high = packed[i++] << 16;
                result[j++] = high | packed[i++];
            }
            return j;
        }

        /// <summary>
        /// The transition table of the DFA
        /// </summary>
        private static readonly int[] ZZ_TRANS = ZzUnpackTrans();

        private const string ZZ_TRANS_PACKED_0 =
            "\x0001\x0002\x0001\x0003\x0001\x0004\x0007\x0002\x0001\x0005\x0001\x0006\x000D\x0000\x0002\x0003" +
            "\x0001\x0000\x0001\x0007\x0001\x0000\x0001\x0008\x0002\x0009\x0001\x000A\x0001\x0003\x0002\x0000" +
            "\x0001\x0003\x0001\x0004\x0001\x0000\x0001\x000B\x0001\x0000\x0001\x0008\x0002\x000C\x0001\x000D" +
            "\x0001\x0004\x0002\x0000\x0001\x0003\x0001\x0004\x0001\x000E\x0001\x000F\x0001\x0010\x0001\x0011" +
            "\x0002\x0009\x0001\x000A\x0001\x0012\x0002\x0000\x0001\x0013\x0001\x0014\x0007\x0000\x0001\x0015" +
            "\x0002\x0000\x0002\x0016\x0007\x0000\x0001\x0016\x0002\x0000\x0001\x0017\x0001\x0018\x0007\x0000" +
            "\x0001\x0019\x0003\x0000\x0001\x001A\x0007\x0000\x0001\x000A\x0002\x0000\x0001\x001B\x0001\x001C" +
            "\x0007\x0000\x0001\x001D\x0002\x0000\x0001\x001E\x0001\x001F\x0007\x0000\x0001\x0020\x0002\x0000" +
            "\x0001\x0021\x0001\x0022\x0007\x0000\x0001\x0023\x000B\x0000\x0001\x0024\x0002\x0000\x0001\x0013" +
            "\x0001\x0014\x0007\x0000\x0001\x0025\x000B\x0000\x0001\x0026\x0002\x0000\x0002\x0016\x0007\x0000" +
            "\x0001\x0027\x0002\x0000\x0001\x0003\x0001\x0004\x0001\x000E\x0001\x0007\x0001\x0010\x0001\x0011" +
            "\x0002\x0009\x0001\x000A\x0001\x0012\x0002\x0000\x0002\x0013\x0001\x0000\x0001\x0028\x0001\x0000" +
            "\x0001\x0008\x0002\x0029\x0001\x0000\x0001\x0013\x0002\x0000\x0001\x0013\x0001\x0014\x0001\x0000" +
            "\x0001\x002A\x0001\x0000\x0001\x0008\x0002\x002B\x0001\x002C\x0001\x0014\x0002\x0000\x0001\x0013" +
            "\x0001\x0014\x0001\x0000\x0001\x0028\x0001\x0000\x0001\x0008\x0002\x0029\x0001\x0000\x0001\x0015" +
            "\x0002\x0000\x0002\x0016\x0001\x0000\x0001\x002D\x0002\x0000\x0001\x002D\x0002\x0000\x0001\x0016" +
            "\x0002\x0000\x0002\x0017\x0001\x0000\x0001\x0029\x0001\x0000\x0001\x0008\x0002\x0029\x0001\x0000" +
            "\x0001\x0017\x0002\x0000\x0001\x0017\x0001\x0018\x0001\x0000\x0001\x002B\x0001\x0000\x0001\x0008" +
            "\x0002\x002B\x0001\x002C\x0001\x0018\x0002\x0000\x0001\x0017\x0001\x0018\x0001\x0000\x0001\x0029" +
            "\x0001\x0000\x0001\x0008\x0002\x0029\x0001\x0000\x0001\x0019\x0003\x0000\x0001\x001A\x0001\x0000" +
            "\x0001\x002C\x0002\x0000\x0003\x002C\x0001\x001A\x0002\x0000\x0002\x001B\x0001\x0000\x0001\x002E" +
            "\x0001\x0000\x0001\x0008\x0002\x0009\x0001\x000A\x0001\x001B\x0002\x0000\x0001\x001B\x0001\x001C" +
            "\x0001\x0000\x0001\x002F\x0001\x0000\x0001\x0008\x0002\x000C\x0001\x000D\x0001\x001C\x0002\x0000" +
            "\x0001\x001B\x0001\x001C\x0001\x0000\x0001\x002E\x0001\x0000\x0001\x0008\x0002\x0009\x0001\x000A" +
            "\x0001\x001D\x0002\x0000\x0002\x001E\x0001\x0000\x0001\x0009\x0001\x0000\x0001\x0008\x0002\x0009" +
            "\x0001\x000A\x0001\x001E\x0002\x0000\x0001\x001E\x0001\x001F\x0001\x0000\x0001\x000C\x0001\x0000" +
            "\x0001\x0008\x0002\x000C\x0001\x000D\x0001\x001F\x0002\x0000\x0001\x001E\x0001\x001F\x0001\x0000" +
            "\x0001\x0009\x0001\x0000\x0001\x0008\x0002\x0009\x0001\x000A\x0001\x0020\x0002\x0000\x0002\x0021" +
            "\x0001\x0000\x0001\x000A\x0002\x0000\x0003\x000A\x0001\x0021\x0002\x0000\x0001\x0021\x0001\x0022" +
            "\x0001\x0000\x0001\x000D\x0002\x0000\x0003\x000D\x0001\x0022\x0002\x0000\x0001\x0021\x0001\x0022" +
            "\x0001\x0000\x0001\x000A\x0002\x0000\x0003\x000A\x0001\x0023\x0004\x0000\x0001\x000E\x0006\x0000" +
            "\x0001\x0024\x0002\x0000\x0001\x0013\x0001\x0014\x0001\x0000\x0001\x0030\x0001\x0000\x0001\x0008" +
            "\x0002\x0029\x0001\x0000\x0001\x0015\x0002\x0000\x0002\x0016\x0001\x0000\x0001\x002D\x0002\x0000" +
            "\x0001\x002D\x0002\x0000\x0001\x0027\x0002\x0000\x0002\x0013\x0007\x0000\x0001\x0013\x0002\x0000" +
            "\x0002\x0017\x0007\x0000\x0001\x0017\x0002\x0000\x0002\x001B\x0007\x0000\x0001\x001B\x0002\x0000" +
            "\x0002\x001E\x0007\x0000\x0001\x001E\x0002\x0000\x0002\x0021\x0007\x0000\x0001\x0021\x0002\x0000" +
            "\x0002\x0031\x0007\x0000\x0001\x0031\x0002\x0000\x0002\x0013\x0007\x0000\x0001\x0032\x0002\x0000" +
            "\x0002\x0031\x0001\x0000\x0001\x002D\x0002\x0000\x0001\x002D\x0002\x0000\x0001\x0031\x0002\x0000" +
            "\x0002\x0013\x0001\x0000\x0001\x0030\x0001\x0000\x0001\x0008\x0002\x0029\x0001\x0000\x0001\x0013" +
            "\x0001\x0000";

        private static int[] ZzUnpackTrans()
        {
            int[] result = new int[552];
            int offset = 0;
            offset = ZzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
            return result;
        }

        private static int ZzUnpackTrans(string packed, int offset, int[] result)
        {
            int i = 0; // index in packed string
            int j = offset; // index in unpacked array
            int l = packed.Length;
            while (i < l)
            {
                int count = packed[i++];
                int value = packed[i++];
                value--;
                do
                {
                    result[j++] = value;
                } while (--count > 0);
            }
            return j;
        }


        /* error codes */
        private const int ZZ_UNKNOWN_ERROR = 0;
        private const int ZZ_NO_MATCH = 1;
        private const int ZZ_PUSHBACK_2BIG = 2;

        /* error messages for the codes above */
        private static readonly string[] ZZ_ERROR_MSG = {
            "Unkown internal scanner error",
            "Error: could not match input",
            "Error: pushback value was too large"
        };

        /// <summary>
        /// ZZ_ATTRIBUTE[aState] contains the attributes of state <c>aState</c>
        /// </summary>
        private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute();

        private const string ZZ_ATTRIBUTE_PACKED_0 =
            "\x0001\x0000\x0001\x0009\x0003\x0001\x0001\x0009\x000B\x0000\x0004\x0001\x0002\x0000\x0001\x0001" +
            "\x0001\x0000\x000F\x0001\x0001\x0000\x0001\x0001\x0003\x0000\x0005\x0001";

        private static int[] ZzUnpackAttribute()
        {
            int[] result = new int[50];
            int offset = 0;
            offset = ZzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
            return result;
        }

        private static int ZzUnpackAttribute(string packed, int offset, int[] result)
        {
            int i = 0; // index in packed string
            int j = offset; // index in unpacked array
            int l = packed.Length;
            while (i < l)
            {
                int count = packed[i++];
                int value = packed[i++];
                do
                {
                    result[j++] = value;
                } while (--count > 0);
            }
            return j;
        }

        /// <summary>
        /// the input device </summary>
        private TextReader zzReader;

        /// <summary>
        /// the current state of the DFA </summary>
        private int zzState;

        /// <summary>
        /// the current lexical state </summary>
        private int zzLexicalState = YYINITIAL;

        /// <summary>
        /// this buffer contains the current text to be matched and is
        ///    the source of the YyText string 
        /// </summary>
        private char[] zzBuffer = new char[ZZ_BUFFERSIZE];

        /// <summary>
        /// the textposition at the last accepting state </summary>
        private int zzMarkedPos;

        /// <summary>
        /// the current text position in the buffer </summary>
        private int zzCurrentPos;

        /// <summary>
        /// startRead marks the beginning of the YyText string in the buffer </summary>
        private int zzStartRead;

        /// <summary>
        /// endRead marks the last character in the buffer, that has been read
        ///    from input 
        /// </summary>
        private int zzEndRead;

        /// <summary>
        /// number of newlines encountered up to the start of the matched text </summary>
        private int yyline;

        /// <summary>
        /// the number of characters up to the start of the matched text </summary>
        private int yyChar;

#pragma warning disable 169, 414
        /// <summary>
        /// the number of characters from the last newline up to the start of the 
        /// matched text
        /// </summary>
        private int yycolumn;

        /// <summary>
        /// zzAtBOL == true &lt;=&gt; the scanner is currently at the beginning of a line
        /// </summary>
        private bool zzAtBOL = true;

        /// <summary>
        /// zzAtEOF == true &lt;=&gt; the scanner is at the EOF </summary>
        private bool zzAtEOF;

        /// <summary>
        /// denotes if the user-EOF-code has already been executed </summary>
        private bool zzEOFDone;

#pragma warning restore 169, 414

        /* user code: */

        public static readonly int ALPHANUM = StandardTokenizer.ALPHANUM;
#pragma warning disable 612, 618
        public static readonly int APOSTROPHE = StandardTokenizer.APOSTROPHE;
        public static readonly int ACRONYM = StandardTokenizer.ACRONYM;
        public static readonly int COMPANY = StandardTokenizer.COMPANY;
        public static readonly int EMAIL = StandardTokenizer.EMAIL;
        public static readonly int HOST = StandardTokenizer.HOST;
        public static readonly int NUM = StandardTokenizer.NUM;
        public static readonly int CJ = StandardTokenizer.CJ;
        public static readonly int ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
#pragma warning restore 612, 618

        public static readonly string[] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;

        public int YyChar
        {
            get { return yyChar; }
        }

        /// <summary>
        /// Fills ICharTermAttribute with the current token text.
        /// </summary>
        public void GetText(ICharTermAttribute t)
        {
            t.CopyBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
        }



        /// <summary>
        /// Creates a new scanner
        /// </summary>
        /// <param name="in">  the TextReader to read input from. </param>
        internal ClassicTokenizerImpl(TextReader @in)
        {
            this.zzReader = @in;
        }


        /// <summary>
        /// Unpacks the compressed character translation table.
        /// </summary>
        /// <param name="packed">   the packed character translation table </param>
        /// <returns>         the unpacked character translation table </returns>
        private static char[] ZzUnpackCMap(string packed)
        {
            char[] map = new char[0x10000];
            int i = 0; // index in packed string
            int j = 0; // index in unpacked array
            while (i < 1138)
            {
                int count = packed[i++];
                char value = packed[i++];
                do
                {
                    map[j++] = value;
                } while (--count > 0);
            }
            return map;
        }


        /// <summary>
        /// Refills the input buffer.
        /// </summary>
        /// <returns>      <c>false</c>, iff there was new input.
        /// </returns>
        /// <exception cref="IOException">  if any I/O-Error occurs </exception>
        private bool ZzRefill()
        {

            /* first: make room (if you can) */
            if (zzStartRead > 0)
            {
                Array.Copy(zzBuffer, zzStartRead, zzBuffer, 0, zzEndRead - zzStartRead);

                /* translate stored positions */
                zzEndRead -= zzStartRead;
                zzCurrentPos -= zzStartRead;
                zzMarkedPos -= zzStartRead;
                zzStartRead = 0;
            }

            /* is the buffer big enough? */
            if (zzCurrentPos >= zzBuffer.Length)
            {
                /* if not: blow it up */
                char[] newBuffer = new char[zzCurrentPos * 2];
                Array.Copy(zzBuffer, 0, newBuffer, 0, zzBuffer.Length);
                zzBuffer = newBuffer;
            }

            /* finally: fill the buffer with new input */
            int numRead = zzReader.Read(zzBuffer, zzEndRead, zzBuffer.Length - zzEndRead);

            if (numRead > 0)
            {
                zzEndRead += numRead;
                return false;
            }
            // unlikely but not impossible: read 0 characters, but not at end of stream    
            if (numRead == 0)
            {
                int c = zzReader.Read();
                if (c == -1)
                {
                    return true;
                }
                else
                {
                    zzBuffer[zzEndRead++] = (char)c;
                    return false;
                }
            }

            // numRead < 0
            return true;
        }


        /// <summary>
        /// Disposes the input stream.
        /// </summary>
        public void YyClose()
        {
            zzAtEOF = true; // indicate end of file
            zzEndRead = zzStartRead; // invalidate buffer

            if (zzReader != null)
            {
                zzReader.Dispose();
            }
        }


        /// <summary>
        /// Resets the scanner to read from a new input stream.
        /// Does not close the old reader.
        /// <para/>
        /// All internal variables are reset, the old input stream 
        /// <b>cannot</b> be reused (internal buffer is discarded and lost).
        /// Lexical state is set to <see cref="YYINITIAL"/>.
        /// <para/>
        /// Internal scan buffer is resized down to its initial length, if it has grown.
        /// </summary>
        /// <param name="reader">   the new input stream  </param>
        public void YyReset(TextReader reader)
        {
            zzReader = reader;
            zzAtBOL = true;
            zzAtEOF = false;
            zzEOFDone = false;
            zzEndRead = zzStartRead = 0;
            zzCurrentPos = zzMarkedPos = 0;
            yyline = yyChar = yycolumn = 0;
            zzLexicalState = YYINITIAL;
            if (zzBuffer.Length > ZZ_BUFFERSIZE)
            {
                zzBuffer = new char[ZZ_BUFFERSIZE];
            }
        }


        /// <summary>
        /// Returns the current lexical state.
        /// </summary>
        public int YyState
        {
            get { return zzLexicalState; }
        }


        /// <summary>
        /// Enters a new lexical state
        /// </summary>
        /// <param name="newState"> the new lexical state </param>
        public void YyBegin(int newState)
        {
            zzLexicalState = newState;
        }


        /// <summary>
        /// Returns the text matched by the current regular expression.
        /// </summary>
        public string YyText
        {
            get { return new string(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead); }
        }


        /// <summary>
        /// Returns the character at position <tt>pos</tt> from the 
        /// matched text. 
        /// 
        /// It is equivalent to YyText[pos], but faster
        /// </summary>
        /// <param name="pos"> the position of the character to fetch. 
        ///            A value from 0 to YyLength-1.
        /// </param>
        /// <returns> the character at position pos </returns>
        public char YyCharAt(int pos)
        {
            return zzBuffer[zzStartRead + pos];
        }


        /// <summary>
        /// Returns the length of the matched text region.
        /// </summary>
        public int YyLength
        {
            get { return zzMarkedPos - zzStartRead; }
        }


        /// <summary>
        /// Reports an error that occured while scanning.
        /// <para/>
        /// In a wellformed scanner (no or only correct usage of 
        /// YyPushBack(int) and a match-all fallback rule) this method 
        /// will only be called with things that "Can't Possibly Happen".
        /// If this method is called, something is seriously wrong
        /// (e.g. a JFlex bug producing a faulty scanner etc.).
        /// <para/>
        /// Usual syntax/scanner level error handling should be done
        /// in error fallback rules.
        /// </summary>
        /// <param name="errorCode">  the code of the errormessage to display </param>
        private void ZzScanError(int errorCode)
        {
            string message;
            try
            {
                message = ZZ_ERROR_MSG[errorCode];
            }
            catch (System.IndexOutOfRangeException)
            {
                message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
            }

            throw new Exception(message);
        }


        /// <summary>
        /// Pushes the specified amount of characters back into the input stream.
        /// 
        /// They will be read again by then next call of the scanning method
        /// </summary>
        /// <param name="number">  the number of characters to be read again.
        ///                This number must not be greater than YyLength! </param>
        public virtual void YyPushBack(int number)
        {
            if (number > YyLength)
            {
                ZzScanError(ZZ_PUSHBACK_2BIG);
            }

            zzMarkedPos -= number;
        }


        /// <summary>
        /// Resumes scanning until the next regular expression is matched,
        /// the end of input is encountered or an I/O-Error occurs.
        /// </summary>
        /// <returns>      the next token </returns>
        /// <exception cref="IOException">  if any I/O-Error occurs </exception>
        public virtual int GetNextToken()
        {
            int zzInput;
            int zzAction;

            // cached fields:
            int zzCurrentPosL;
            int zzMarkedPosL;
            int zzEndReadL = zzEndRead;
            char[] zzBufferL = zzBuffer;
            char[] zzCMapL = ZZ_CMAP;

            int[] zzTransL = ZZ_TRANS;
            int[] zzRowMapL = ZZ_ROWMAP;
            int[] zzAttrL = ZZ_ATTRIBUTE;

            while (true)
            {
                zzMarkedPosL = zzMarkedPos;

                yyChar += zzMarkedPosL - zzStartRead;

                zzAction = -1;

                zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

                zzState = ZZ_LEXSTATE[zzLexicalState];

                // set up zzAction for empty match case:
                int zzAttributes = zzAttrL[zzState];
                if ((zzAttributes & 1) == 1)
                {
                    zzAction = zzState;
                }


                {
                    while (true)
                    {

                        if (zzCurrentPosL < zzEndReadL)
                        {
                            zzInput = zzBufferL[zzCurrentPosL++];
                        }
                        else if (zzAtEOF)
                        {
                            zzInput = StandardTokenizerInterface.YYEOF;
                            goto zzForActionBreak;
                        }
                        else
                        {
                            // store back cached positions
                            zzCurrentPos = zzCurrentPosL;
                            zzMarkedPos = zzMarkedPosL;
                            bool eof = ZzRefill();
                            // get translated positions and possibly new buffer
                            zzCurrentPosL = zzCurrentPos;
                            zzMarkedPosL = zzMarkedPos;
                            zzBufferL = zzBuffer;
                            zzEndReadL = zzEndRead;
                            if (eof)
                            {
                                zzInput = StandardTokenizerInterface.YYEOF;
                                goto zzForActionBreak;
                            }
                            else
                            {
                                zzInput = zzBufferL[zzCurrentPosL++];
                            }
                        }
                        int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
                        if (zzNext == -1)
                        {
                            goto zzForActionBreak;
                        }
                        zzState = zzNext;

                        zzAttributes = zzAttrL[zzState];
                        if ((zzAttributes & 1) == 1)
                        {
                            zzAction = zzState;
                            zzMarkedPosL = zzCurrentPosL;
                            if ((zzAttributes & 8) == 8)
                            {
                                goto zzForActionBreak;
                            }
                        }

                    }
                }
                zzForActionBreak:

                // store back cached position
                zzMarkedPos = zzMarkedPosL;

                switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction])
                {
                    case 1:
                        { // Break so we don't hit fall-through warning:
                            break; // ignore
                        }
                        //goto case 11; // unreachable
                    case 11:
                        break;
                    case 2:
                        {
                            return ALPHANUM;
                        }
                    case 12:
                        break;
                    case 3:
                        {
                            return CJ;
                        }
                    case 13:
                        break;
                    case 4:
                        {
                            return HOST;
                        }
                    case 14:
                        break;
                    case 5:
                        {
                            return NUM;
                        }
                    case 15:
                        break;
                    case 6:
                        {
                            return APOSTROPHE;
                        }
                    case 16:
                        break;
                    case 7:
                        {
                            return COMPANY;
                        }
                    case 17:
                        break;
                    case 8:
                        {
                            return ACRONYM_DEP;
                        }
                    case 18:
                        break;
                    case 9:
                        {
                            return ACRONYM;
                        }
                    case 19:
                        break;
                    case 10:
                        {
                            return EMAIL;
                        }
                    case 20:
                        break;
                    default:
                        if (zzInput == StandardTokenizerInterface.YYEOF && zzStartRead == zzCurrentPos)
                        {
                            zzAtEOF = true;
                            return StandardTokenizerInterface.YYEOF;
                        }
                        else
                        {
                            ZzScanError(ZZ_NO_MATCH);
                        }
                        break;
                }
            }
        }
    }
}
