using Lucene.Net.Analysis.Core;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Support;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using Console = Lucene.Net.Support.SystemConsole;

namespace Lucene.Net.Analysis.Synonym
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    public class TestSynonymMapFilter : BaseTokenStreamTestCase
    {

        private SynonymMap.Builder b;
        private Tokenizer tokensIn;
        private SynonymFilter tokensOut;
        private ICharTermAttribute termAtt;
        private IPositionIncrementAttribute posIncrAtt;
        private IPositionLengthAttribute posLenAtt;
        private IOffsetAttribute offsetAtt;

        private static Regex space = new Regex(" +", RegexOptions.Compiled);

        private void Add(string input, string output, bool keepOrig)
        {
            if (VERBOSE)
            {
                Console.WriteLine("  add input=" + input + " output=" + output + " keepOrig=" + keepOrig);
            }

            CharsRef inputCharsRef = new CharsRef();
            SynonymMap.Builder.Join(space.Split(input).TrimEnd(), inputCharsRef);

            CharsRef outputCharsRef = new CharsRef();
            SynonymMap.Builder.Join(space.Split(output).TrimEnd(), outputCharsRef);

            b.Add(inputCharsRef, outputCharsRef, keepOrig);
        }

        private void AssertEquals(CharTermAttribute term, string expected)
        {
            assertEquals(expected.Length, term.Length);
            char[] buffer = term.Buffer;
            for (int chIDX = 0; chIDX < expected.Length; chIDX++)
            {
                assertEquals(expected[chIDX], buffer[chIDX]);
            }
        }

        // For the output string: separate positions with a space,
        // and separate multiple tokens at each position with a
        // /.  If a token should have end offset != the input
        // token's end offset then add :X to it:

        // TODO: we should probably refactor this guy to use/take analyzer,
        // the tests are a little messy
        private void Verify(string input, string output)
        {
            if (VERBOSE)
            {
                Console.WriteLine("TEST: verify input=" + input + " expectedOutput=" + output);
            }

            tokensIn.SetReader(new StringReader(input));
            tokensOut.Reset();
            string[] expected = output.Split(' ').TrimEnd();
            int expectedUpto = 0;
            while (tokensOut.IncrementToken())
            {

                if (VERBOSE)
                {
                    Console.WriteLine("  incr token=" + termAtt.ToString() + " posIncr=" + posIncrAtt.PositionIncrement + " startOff=" + offsetAtt.StartOffset + " endOff=" + offsetAtt.EndOffset);
                }

                assertTrue(expectedUpto < expected.Length);
                int startOffset = offsetAtt.StartOffset;
                int endOffset = offsetAtt.EndOffset;

                string[] expectedAtPos = expected[expectedUpto++].Split('/').TrimEnd();
                for (int atPos = 0; atPos < expectedAtPos.Length; atPos++)
                {
                    if (atPos > 0)
                    {
                        assertTrue(tokensOut.IncrementToken());
                        if (VERBOSE)
                        {
                            Console.WriteLine("  incr token=" + termAtt.ToString() + " posIncr=" + posIncrAtt.PositionIncrement + " startOff=" + offsetAtt.StartOffset + " endOff=" + offsetAtt.EndOffset);
                        }
                    }
                    int colonIndex = expectedAtPos[atPos].IndexOf(':');
                    int underbarIndex = expectedAtPos[atPos].IndexOf('_');
                    string expectedToken;
                    int expectedEndOffset;
                    int expectedPosLen;
                    if (colonIndex != -1)
                    {
                        expectedToken = expectedAtPos[atPos].Substring(0, colonIndex - 0);
                        if (underbarIndex != -1)
                        {
                            expectedEndOffset = int.Parse(expectedAtPos[atPos].Substring(1 + colonIndex, underbarIndex - (1 + colonIndex)));
                            expectedPosLen = int.Parse(expectedAtPos[atPos].Substring(1 + underbarIndex));
                        }
                        else
                        {
                            expectedEndOffset = int.Parse(expectedAtPos[atPos].Substring(1 + colonIndex));
                            expectedPosLen = 1;
                        }
                    }
                    else
                    {
                        expectedToken = expectedAtPos[atPos];
                        expectedEndOffset = endOffset;
                        expectedPosLen = 1;
                    }
                    assertEquals(expectedToken, termAtt.ToString());
                    assertEquals(atPos == 0 ? 1 : 0, posIncrAtt.PositionIncrement);
                    // start/end offset of all tokens at same pos should
                    // be the same:
                    assertEquals(startOffset, offsetAtt.StartOffset);
                    assertEquals(expectedEndOffset, offsetAtt.EndOffset);
                    assertEquals(expectedPosLen, posLenAtt.PositionLength);
                }
            }
            tokensOut.End();
            tokensOut.Dispose();
            if (VERBOSE)
            {
                Console.WriteLine("  incr: END");
            }
            assertEquals(expectedUpto, expected.Length);
        }

        [Test]
        public virtual void TestDontKeepOrig()
        {
            b = new SynonymMap.Builder(true);
            Add("a b", "foo", false);

            SynonymMap map = b.Build();
            Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper(this, map);

            AssertAnalyzesTo(analyzer, "a b c", 
                            new string[] { "foo", "c" }, 
                            new int[] { 0, 4 }, 
                            new int[] { 3, 5 }, 
                            null, 
                            new int[] { 1, 1 }, 
                            new int[] { 1, 1 }, 
                            true);
            CheckAnalysisConsistency(Random(), analyzer, false, "a b c");
        }

        private class AnalyzerAnonymousInnerClassHelper : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
            }
        }

        [Test]
        public virtual void TestDoKeepOrig()
        {
            b = new SynonymMap.Builder(true);
            Add("a b", "foo", true);

            SynonymMap map = b.Build();

            Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper2(this, map);

            AssertAnalyzesTo(analyzer, "a b c", 
                            new string[] { "a", "foo", "b", "c" }, 
                            new int[] { 0, 0, 2, 4 }, 
                            new int[] { 1, 3, 3, 5 }, 
                            null, 
                            new int[] { 1, 0, 1, 1 }, 
                            new int[] { 1, 2, 1, 1 }, 
                            true);
            CheckAnalysisConsistency(Random(), analyzer, false, "a b c");
        }

        private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper2(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
            }
        }

        [Test]
        public virtual void TestBasic()
        {
            b = new SynonymMap.Builder(true);
            Add("a", "foo", true);
            Add("a b", "bar fee", true);
            Add("b c", "dog collar", true);
            Add("c d", "dog harness holder extras", true);
            Add("m c e", "dog barks loudly", false);
            Add("i j k", "feep", true);

            Add("e f", "foo bar", false);
            Add("e f", "baz bee", false);

            Add("z", "boo", false);
            Add("y", "bee", true);

            tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
            tokensIn.Reset();
            assertTrue(tokensIn.IncrementToken());
            assertFalse(tokensIn.IncrementToken());
            tokensIn.End();
            tokensIn.Dispose();

            tokensOut = new SynonymFilter(tokensIn, b.Build(), true);
            termAtt = tokensOut.AddAttribute<ICharTermAttribute>();
            posIncrAtt = tokensOut.AddAttribute<IPositionIncrementAttribute>();
            posLenAtt = tokensOut.AddAttribute<IPositionLengthAttribute>();
            offsetAtt = tokensOut.AddAttribute<IOffsetAttribute>();

            Verify("a b c", "a/bar b/fee c");

            // syn output extends beyond input tokens
            Verify("x a b c d", "x a/bar b/fee c/dog d/harness holder extras");

            Verify("a b a", "a/bar b/fee a/foo");

            // outputs that add to one another:
            Verify("c d c d", "c/dog d/harness c/holder/dog d/extras/harness holder extras");

            // two outputs for same input
            Verify("e f", "foo/baz bar/bee");

            // verify multi-word / single-output offsets:
            Verify("g i j k g", "g i/feep:7_3 j k g");

            // mixed keepOrig true/false:
            Verify("a m c e x", "a/foo dog barks loudly x");
            Verify("c d m c e x", "c/dog d/harness holder/dog extras/barks loudly x");
            assertTrue(tokensOut.CaptureCount > 0);

            // no captureStates when no syns matched
            Verify("p q r s t", "p q r s t");
            assertEquals(0, tokensOut.CaptureCount);

            // no captureStates when only single-input syns, w/ no
            // lookahead needed, matched
            Verify("p q z y t", "p q boo y/bee t");
            assertEquals(0, tokensOut.CaptureCount);
        }

        private string GetRandomString(char start, int alphabetSize, int length)
        {
            Debug.Assert(alphabetSize <= 26);
            char[] s = new char[2 * length];
            for (int charIDX = 0; charIDX < length; charIDX++)
            {
                s[2 * charIDX] = (char)(start + Random().Next(alphabetSize));
                s[2 * charIDX + 1] = ' ';
            }
            return new string(s);
        }

        protected class OneSyn
        {
            internal string @in;
            internal IList<string> @out;
            internal bool keepOrig;
        }

        protected virtual string SlowSynMatcher(string doc, IList<OneSyn> syns, int maxOutputLength)
        {
            assertTrue(doc.Length % 2 == 0);
            int numInputs = doc.Length / 2;
            bool[] keepOrigs = new bool[numInputs];
            bool[] hasMatch = new bool[numInputs];
            Arrays.Fill(keepOrigs, false);
            string[] outputs = new string[numInputs + maxOutputLength];
            OneSyn[] matches = new OneSyn[numInputs];
            foreach (OneSyn syn in syns)
            {
                int idx = -1;
                while (true)
                {
                    idx = doc.IndexOf(syn.@in, 1 + idx, StringComparison.Ordinal);
                    if (idx == -1)
                    {
                        break;
                    }
                    assertTrue(idx % 2 == 0);
                    int matchIDX = idx / 2;
                    assertTrue(syn.@in.Length % 2 == 1);
                    if (matches[matchIDX] == null)
                    {
                        matches[matchIDX] = syn;
                    }
                    else if (syn.@in.Length > matches[matchIDX].@in.Length)
                    {
                        // Greedy conflict resolution: longer match wins:
                        matches[matchIDX] = syn;
                    }
                    else
                    {
                        assertTrue(syn.@in.Length < matches[matchIDX].@in.Length);
                    }
                }
            }

            // Greedy conflict resolution: if syn matches a range of inputs,
            // it prevents other syns from matching that range
            for (int inputIDX = 0; inputIDX < numInputs; inputIDX++)
            {
                OneSyn match = matches[inputIDX];
                if (match != null)
                {
                    int synInLength = (1 + match.@in.Length) / 2;
                    for (int nextInputIDX = inputIDX + 1; nextInputIDX < numInputs && nextInputIDX < (inputIDX + synInLength); nextInputIDX++)
                    {
                        matches[nextInputIDX] = null;
                    }
                }
            }

            // Fill overlapping outputs:
            for (int inputIDX = 0; inputIDX < numInputs; inputIDX++)
            {
                OneSyn syn = matches[inputIDX];
                if (syn == null)
                {
                    continue;
                }
                for (int idx = 0; idx < (1 + syn.@in.Length) / 2; idx++)
                {
                    hasMatch[inputIDX + idx] = true;
                    keepOrigs[inputIDX + idx] |= syn.keepOrig;
                }
                foreach (string synOut in syn.@out)
                {
                    string[] synOutputs = synOut.Split(' ').TrimEnd();
                    assertEquals(synOutputs.Length, (1 + synOut.Length) / 2);
                    int matchEnd = inputIDX + synOutputs.Length;
                    int synUpto = 0;
                    for (int matchIDX = inputIDX; matchIDX < matchEnd; matchIDX++)
                    {
                        if (outputs[matchIDX] == null)
                        {
                            outputs[matchIDX] = synOutputs[synUpto++];
                        }
                        else
                        {
                            outputs[matchIDX] = outputs[matchIDX] + "/" + synOutputs[synUpto++];
                        }
                        int endOffset;
                        if (matchIDX < numInputs)
                        {
                            int posLen;
                            if (synOutputs.Length == 1)
                            {
                                // Add full endOffset
                                endOffset = (inputIDX * 2) + syn.@in.Length;
                                posLen = syn.keepOrig ? (1 + syn.@in.Length) / 2 : 1;
                            }
                            else
                            {
                                // Add endOffset matching input token's
                                endOffset = (matchIDX * 2) + 1;
                                posLen = 1;
                            }
                            outputs[matchIDX] = outputs[matchIDX] + ":" + endOffset + "_" + posLen;
                        }
                    }
                }
            }

            StringBuilder sb = new StringBuilder();
            string[] inputTokens = doc.Split(' ').TrimEnd();
            int limit = inputTokens.Length + maxOutputLength;
            for (int inputIDX = 0; inputIDX < limit; inputIDX++)
            {
                bool posHasOutput = false;
                if (inputIDX >= numInputs && outputs[inputIDX] == null)
                {
                    break;
                }
                if (inputIDX < numInputs && (!hasMatch[inputIDX] || keepOrigs[inputIDX]))
                {
                    assertTrue(inputTokens[inputIDX].Length != 0);
                    sb.Append(inputTokens[inputIDX]);
                    posHasOutput = true;
                }

                if (outputs[inputIDX] != null)
                {
                    if (posHasOutput)
                    {
                        sb.Append('/');
                    }
                    sb.Append(outputs[inputIDX]);
                }
                else if (!posHasOutput)
                {
                    continue;
                }
                if (inputIDX < limit - 1)
                {
                    sb.Append(' ');
                }
            }

            return sb.ToString();
        }

        [Test]
        public virtual void TestRandom()
        {

            int alphabetSize = TestUtil.NextInt(Random(), 2, 7);

            int docLen = AtLeast(3000);
            //final int docLen = 50;

            string document = GetRandomString('a', alphabetSize, docLen);

            if (VERBOSE)
            {
                Console.WriteLine("TEST: doc=" + document);
            }

            int numSyn = AtLeast(5);
            //final int numSyn = 2;

            IDictionary<string, OneSyn> synMap = new Dictionary<string, OneSyn>();
            IList<OneSyn> syns = new List<OneSyn>();
            bool dedup = Random().nextBoolean();
            if (VERBOSE)
            {
                Console.WriteLine("  dedup=" + dedup);
            }
            b = new SynonymMap.Builder(dedup);
            for (int synIDX = 0; synIDX < numSyn; synIDX++)
            {
                string synIn = GetRandomString('a', alphabetSize, TestUtil.NextInt(Random(), 1, 5)).Trim();
                OneSyn s = synMap.ContainsKey(synIn) ? synMap[synIn] : null;
                if (s == null)
                {
                    s = new OneSyn();
                    s.@in = synIn;
                    syns.Add(s);
                    s.@out = new List<string>();
                    synMap[synIn] = s;
                    s.keepOrig = Random().nextBoolean();
                }
                string synOut = GetRandomString('0', 10, TestUtil.NextInt(Random(), 1, 5)).Trim();
                s.@out.Add(synOut);
                Add(synIn, synOut, s.keepOrig);
                if (VERBOSE)
                {
                    Console.WriteLine("  syns[" + synIDX + "] = " + s.@in + " -> " + s.@out + " keepOrig=" + s.keepOrig);
                }
            }

            tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
            tokensIn.Reset();
            assertTrue(tokensIn.IncrementToken());
            assertFalse(tokensIn.IncrementToken());
            tokensIn.End();
            tokensIn.Dispose();

            tokensOut = new SynonymFilter(tokensIn, b.Build(), true);
            termAtt = tokensOut.AddAttribute<ICharTermAttribute>();
            posIncrAtt = tokensOut.AddAttribute<IPositionIncrementAttribute>();
            posLenAtt = tokensOut.AddAttribute<IPositionLengthAttribute>();
            offsetAtt = tokensOut.AddAttribute<IOffsetAttribute>();

            if (dedup)
            {
                PruneDups(syns);
            }

            string expected = SlowSynMatcher(document, syns, 5);

            if (VERBOSE)
            {
                Console.WriteLine("TEST: expected=" + expected);
            }

            Verify(document, expected);
        }

        private void PruneDups(IList<OneSyn> syns)
        {
            ISet<string> seen = new HashSet<string>();
            foreach (OneSyn syn in syns)
            {
                int idx = 0;
                while (idx < syn.@out.Count)
                {
                    string @out = syn.@out[idx];
                    if (!seen.Contains(@out))
                    {
                        seen.Add(@out);
                        idx++;
                    }
                    else
                    {
                        syn.@out.RemoveAt(idx);
                    }
                }
                seen.Clear();
            }
        }

        private string RandomNonEmptyString()
        {
            while (true)
            {
                string s = TestUtil.RandomUnicodeString(Random()).Trim();
                if (s.Length != 0 && s.IndexOf('\u0000') == -1)
                {
                    return s;
                }
            }
        }

        /// <summary>
        /// simple random test, doesn't verify correctness.
        ///  does verify it doesnt throw exceptions, or that the stream doesn't misbehave
        /// </summary>
        [Test]
        public virtual void TestRandom2()
        {
            int numIters = AtLeast(3);
            for (int i = 0; i < numIters; i++)
            {
                b = new SynonymMap.Builder(Random().nextBoolean());
                int numEntries = AtLeast(10);
                for (int j = 0; j < numEntries; j++)
                {
                    Add(RandomNonEmptyString(), RandomNonEmptyString(), Random().nextBoolean());
                }
                SynonymMap map = b.Build();
                bool ignoreCase = Random().nextBoolean();

                Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper100(this, map, ignoreCase);

                CheckRandomData(Random(), analyzer, 100);
            }
        }

        private class AnalyzerAnonymousInnerClassHelper100 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;
            private bool ignoreCase;

            public AnalyzerAnonymousInnerClassHelper100(TestSynonymMapFilter outerInstance, SynonymMap map, bool ignoreCase)
            {
                this.outerInstance = outerInstance;
                this.map = map;
                this.ignoreCase = ignoreCase;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
            }
        }

        // NOTE: this is an invalid test... SynFilter today can't
        // properly consume a graph... we can re-enable this once
        // we fix that...
        /*
        // Adds MockGraphTokenFilter before SynFilter:
        public void TestRandom2GraphBefore() throws Exception {
          final int numIters = AtLeast(10);
          Random random = random();
          for (int i = 0; i < numIters; i++) {
            b = new SynonymMap.Builder(random.nextBoolean());
            final int numEntries = AtLeast(10);
            for (int j = 0; j < numEntries; j++) {
              add(randomNonEmptyString(), randomNonEmptyString(), random.nextBoolean());
            }
            final SynonymMap map = b.Build();
            final boolean ignoreCase = random.nextBoolean();

            final Analyzer analyzer = new Analyzer() {
              @Override
              protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(graph, map, ignoreCase));
              }
            };

            checkRandomData(random, analyzer, 1000*RANDOM_MULTIPLIER);
          }
        }
        */

        // Adds MockGraphTokenFilter after SynFilter:
        [Test]
        public virtual void TestRandom2GraphAfter()
        {
            int numIters = AtLeast(3);
            Random random = Random();
            for (int i = 0; i < numIters; i++)
            {
                b = new SynonymMap.Builder(random.nextBoolean());
                int numEntries = AtLeast(10);
                for (int j = 0; j < numEntries; j++)
                {
                    Add(RandomNonEmptyString(), RandomNonEmptyString(), random.nextBoolean());
                }
                SynonymMap map = b.Build();
                bool ignoreCase = random.nextBoolean();

                Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper101(this, map, ignoreCase);

                CheckRandomData(random, analyzer, 100);
            }
        }

        private class AnalyzerAnonymousInnerClassHelper101 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;
            private bool ignoreCase;

            public AnalyzerAnonymousInnerClassHelper101(TestSynonymMapFilter outerInstance, SynonymMap map, bool ignoreCase)
            {
                this.outerInstance = outerInstance;
                this.map = map;
                this.ignoreCase = ignoreCase;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
                TokenStream graph = new MockGraphTokenFilter(Random(), syns);
                return new TokenStreamComponents(tokenizer, graph);
            }
        }

        [Test]
        public virtual void TestEmptyTerm()
        {
            Random random = Random();
            int numIters = AtLeast(10);
            for (int i = 0; i < numIters; i++)
            {
                b = new SynonymMap.Builder(random.nextBoolean());
                int numEntries = AtLeast(10);
                for (int j = 0; j < numEntries; j++)
                {
                    Add(RandomNonEmptyString(), RandomNonEmptyString(), random.nextBoolean());
                }
                SynonymMap map = b.Build();
                bool ignoreCase = random.nextBoolean();

                Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper3(this, map, ignoreCase);

                CheckAnalysisConsistency(random, analyzer, random.nextBoolean(), "");
            }
        }

        private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;
            private bool ignoreCase;

            public AnalyzerAnonymousInnerClassHelper3(TestSynonymMapFilter outerInstance, SynonymMap map, bool ignoreCase)
            {
                this.outerInstance = outerInstance;
                this.map = map;
                this.ignoreCase = ignoreCase;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new KeywordTokenizer(reader);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
            }
        }

        /// <summary>
        /// simple random test like testRandom2, but for larger docs
        /// </summary>
        [Test]
        public virtual void TestRandomHuge()
        {
            Random random = Random();
            int numIters = AtLeast(3);
            for (int i = 0; i < numIters; i++)
            {
                b = new SynonymMap.Builder(random.nextBoolean());
                int numEntries = AtLeast(10);
                if (VERBOSE)
                {
                    Console.WriteLine("TEST: iter=" + i + " numEntries=" + numEntries);
                }
                for (int j = 0; j < numEntries; j++)
                {
                    Add(RandomNonEmptyString(), RandomNonEmptyString(), random.nextBoolean());
                }
                SynonymMap map = b.Build();
                bool ignoreCase = random.nextBoolean();

                Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper4(this, map, ignoreCase);

                CheckRandomData(random, analyzer, 100, 1024);
            }
        }

        private class AnalyzerAnonymousInnerClassHelper4 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;
            private bool ignoreCase;

            public AnalyzerAnonymousInnerClassHelper4(TestSynonymMapFilter outerInstance, SynonymMap map, bool ignoreCase)
            {
                this.outerInstance = outerInstance;
                this.map = map;
                this.ignoreCase = ignoreCase;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
            }
        }

        // LUCENE-3375
        [Test]
        public virtual void TestVanishingTerms()
        {
            string testFile = "aaa => aaaa1 aaaa2 aaaa3\n" + "bbb => bbbb1 bbbb2\n";

            SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(Random()));
            parser.Parse(new StringReader(testFile));
            SynonymMap map = parser.Build();

            Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper5(this, map);

            // where did my pot go?!
            AssertAnalyzesTo(analyzer, "xyzzy bbb pot of gold", new string[] { "xyzzy", "bbbb1", "pot", "bbbb2", "of", "gold" });

            // this one nukes 'pot' and 'of'
            // xyzzy aaa pot of gold -> xyzzy aaaa1 aaaa2 aaaa3 gold
            AssertAnalyzesTo(analyzer, "xyzzy aaa pot of gold", new string[] { "xyzzy", "aaaa1", "pot", "aaaa2", "of", "aaaa3", "gold" });
        }

        private class AnalyzerAnonymousInnerClassHelper5 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper5(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestBasic2()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            Add("aaa", "aaaa1 aaaa2 aaaa3", keepOrig);
            Add("bbb", "bbbb1 bbbb2", keepOrig);
            tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
            tokensIn.Reset();
            assertTrue(tokensIn.IncrementToken());
            assertFalse(tokensIn.IncrementToken());
            tokensIn.End();
            tokensIn.Dispose();

            tokensOut = new SynonymFilter(tokensIn, b.Build(), true);
            termAtt = tokensOut.AddAttribute<ICharTermAttribute>();
            posIncrAtt = tokensOut.AddAttribute<IPositionIncrementAttribute>();
            posLenAtt = tokensOut.AddAttribute<IPositionLengthAttribute>();
            offsetAtt = tokensOut.AddAttribute<IOffsetAttribute>();

#pragma warning disable 162
            if (keepOrig)
            {
                Verify("xyzzy bbb pot of gold", "xyzzy bbb/bbbb1 pot/bbbb2 of gold");
                Verify("xyzzy aaa pot of gold", "xyzzy aaa/aaaa1 pot/aaaa2 of/aaaa3 gold");
            }
            else
            {
                Verify("xyzzy bbb pot of gold", "xyzzy bbbb1 pot/bbbb2 of gold");
                Verify("xyzzy aaa pot of gold", "xyzzy aaaa1 pot/aaaa2 of/aaaa3 gold");
            }
#pragma warning restore 612, 618
        }

        [Test]
        public virtual void TestMatching()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            Add("a b", "ab", keepOrig);
            Add("a c", "ac", keepOrig);
            Add("a", "aa", keepOrig);
            Add("b", "bb", keepOrig);
            Add("z x c v", "zxcv", keepOrig);
            Add("x c", "xc", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper6(this, map);

            CheckOneTerm(a, "$", "$");
            CheckOneTerm(a, "a", "aa");
            CheckOneTerm(a, "b", "bb");

            AssertAnalyzesTo(a, "a $", new string[] { "aa", "$" }, new int[] { 1, 1 });

            AssertAnalyzesTo(a, "$ a", new string[] { "$", "aa" }, new int[] { 1, 1 });

            AssertAnalyzesTo(a, "a a", new string[] { "aa", "aa" }, new int[] { 1, 1 });

            AssertAnalyzesTo(a, "z x c v", new string[] { "zxcv" }, new int[] { 1 });

            AssertAnalyzesTo(a, "z x c $", new string[] { "z", "xc", "$" }, new int[] { 1, 1, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper6 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper6(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRepeatsOff()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            Add("a b", "ab", keepOrig);
            Add("a b", "ab", keepOrig);
            Add("a b", "ab", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper7(this, map);

            AssertAnalyzesTo(a, "a b", new string[] { "ab" }, new int[] { 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper7 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper7(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRepeatsOn()
        {
            b = new SynonymMap.Builder(false);
            const bool keepOrig = false;
            Add("a b", "ab", keepOrig);
            Add("a b", "ab", keepOrig);
            Add("a b", "ab", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper8(this, map);

            AssertAnalyzesTo(a, "a b", new string[] { "ab", "ab", "ab" }, new int[] { 1, 0, 0 });
        }

        private class AnalyzerAnonymousInnerClassHelper8 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper8(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRecursion()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            Add("zoo", "zoo", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper9(this, map);

            AssertAnalyzesTo(a, "zoo zoo $ zoo", new string[] { "zoo", "zoo", "$", "zoo" }, new int[] { 1, 1, 1, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper9 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper9(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRecursion2()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            Add("zoo", "zoo", keepOrig);
            Add("zoo", "zoo zoo", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper10(this, map);

            // verify("zoo zoo $ zoo", "zoo/zoo zoo/zoo/zoo $/zoo zoo/zoo zoo");
            AssertAnalyzesTo(a, "zoo zoo $ zoo", new string[] { "zoo", "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo", "zoo" }, new int[] { 1, 0, 1, 0, 0, 1, 0, 1, 0, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper10 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper10(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestOutputHangsOffEnd()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = false;
            // b hangs off the end (no input token under it):
            Add("a", "a b", keepOrig);
            tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
            tokensIn.Reset();
            assertTrue(tokensIn.IncrementToken());
            assertFalse(tokensIn.IncrementToken());
            tokensIn.End();
            tokensIn.Dispose();

            tokensOut = new SynonymFilter(tokensIn, b.Build(), true);
            termAtt = tokensOut.AddAttribute<ICharTermAttribute>();
            posIncrAtt = tokensOut.AddAttribute<IPositionIncrementAttribute>();
            offsetAtt = tokensOut.AddAttribute<IOffsetAttribute>();
            posLenAtt = tokensOut.AddAttribute<IPositionLengthAttribute>();

            // Make sure endOffset inherits from previous input token:
            Verify("a", "a b:1");
        }

        [Test]
        public virtual void TestIncludeOrig()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = true;
            Add("a b", "ab", keepOrig);
            Add("a c", "ac", keepOrig);
            Add("a", "aa", keepOrig);
            Add("b", "bb", keepOrig);
            Add("z x c v", "zxcv", keepOrig);
            Add("x c", "xc", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper11(this, map);

            AssertAnalyzesTo(a, "$", new string[] { "$" }, new int[] { 1 });
            AssertAnalyzesTo(a, "a", new string[] { "a", "aa" }, new int[] { 1, 0 });
            AssertAnalyzesTo(a, "a", new string[] { "a", "aa" }, new int[] { 1, 0 });
            AssertAnalyzesTo(a, "$ a", new string[] { "$", "a", "aa" }, new int[] { 1, 1, 0 });
            AssertAnalyzesTo(a, "a $", new string[] { "a", "aa", "$" }, new int[] { 1, 0, 1 });
            AssertAnalyzesTo(a, "$ a !", new string[] { "$", "a", "aa", "!" }, new int[] { 1, 1, 0, 1 });
            AssertAnalyzesTo(a, "a a", new string[] { "a", "aa", "a", "aa" }, new int[] { 1, 0, 1, 0 });
            AssertAnalyzesTo(a, "b", new string[] { "b", "bb" }, new int[] { 1, 0 });
            AssertAnalyzesTo(a, "z x c v", new string[] { "z", "zxcv", "x", "c", "v" }, new int[] { 1, 0, 1, 1, 1 });
            AssertAnalyzesTo(a, "z x c $", new string[] { "z", "x", "xc", "c", "$" }, new int[] { 1, 1, 0, 1, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper11 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper11(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRecursion3()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = true;
            Add("zoo zoo", "zoo", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper12(this, map);

            AssertAnalyzesTo(a, "zoo zoo $ zoo", new string[] { "zoo", "zoo", "zoo", "$", "zoo" }, new int[] { 1, 0, 1, 1, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper12 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper12(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestRecursion4()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = true;
            Add("zoo zoo", "zoo", keepOrig);
            Add("zoo", "zoo zoo", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper13(this, map);

            AssertAnalyzesTo(a, "zoo zoo $ zoo", new string[] { "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" }, new int[] { 1, 0, 1, 1, 1, 0, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper13 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper13(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestMultiwordOffsets()
        {
            b = new SynonymMap.Builder(true);
            const bool keepOrig = true;
            Add("national hockey league", "nhl", keepOrig);
            SynonymMap map = b.Build();
            Analyzer a = new AnalyzerAnonymousInnerClassHelper14(this, map);

            AssertAnalyzesTo(a, "national hockey league", new string[] { "national", "nhl", "hockey", "league" }, new int[] { 0, 0, 9, 16 }, new int[] { 8, 22, 15, 22 }, new int[] { 1, 0, 1, 1 });
        }

        private class AnalyzerAnonymousInnerClassHelper14 : Analyzer
        {
            private readonly TestSynonymMapFilter outerInstance;

            private SynonymMap map;

            public AnalyzerAnonymousInnerClassHelper14(TestSynonymMapFilter outerInstance, SynonymMap map)
            {
                this.outerInstance = outerInstance;
                this.map = map;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
            }
        }

        [Test]
        public virtual void TestEmpty()
        {
            Tokenizer tokenizer = new MockTokenizer(new StringReader("aa bb"));
            try
            {
                new SynonymFilter(tokenizer, (new SynonymMap.Builder(true)).Build(), true);
                fail("did not hit expected exception");
            }
            catch (System.ArgumentException iae)
            {
                // expected
                assertEquals("fst must be non-null", iae.Message);
            }
        }
    }
}