#if FEATURE_CONCURRENTMERGESCHEDULER
using System;
using System.Diagnostics;
using System.Threading;
using Lucene.Net.Documents;
using Console = Lucene.Net.Support.SystemConsole;

namespace Lucene.Net.Index
{
    using Lucene.Net.Randomized.Generators;
    using Lucene.Net.Support;
    using NUnit.Framework;
    using System.IO;
    using Directory = Lucene.Net.Store.Directory;
    using Document = Documents.Document;
    using Field = Field;
    using Lucene41PostingsFormat = Lucene.Net.Codecs.Lucene41.Lucene41PostingsFormat;
    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;

    /*
         * Licensed to the Apache Software Foundation (ASF) under one or more
         * contributor license agreements.  See the NOTICE file distributed with
         * this work for additional information regarding copyright ownership.
         * The ASF licenses this file to You under the Apache License, Version 2.0
         * (the "License"); you may not use this file except in compliance with
         * the License.  You may obtain a copy of the License at
         *
         *     http://www.apache.org/licenses/LICENSE-2.0
         *
         * Unless required by applicable law or agreed to in writing, software
         * distributed under the License is distributed on an "AS IS" BASIS,
         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
         * See the License for the specific language governing permissions and
         * limitations under the License.
         */

    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
    using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
    using StringField = StringField;
    using TestUtil = Lucene.Net.Util.TestUtil;
    using TextField = TextField;
    using Attributes;
    using Lucene.Net.Util;

    [TestFixture]
    public class TestConcurrentMergeScheduler : LuceneTestCase
    {
        private class FailOnlyOnFlush : MockDirectoryWrapper.Failure
        {
            private readonly TestConcurrentMergeScheduler OuterInstance;

            public FailOnlyOnFlush(TestConcurrentMergeScheduler outerInstance)
            {
                this.OuterInstance = outerInstance;
            }

            new internal bool DoFail;
            internal bool HitExc;

            public override void SetDoFail()
            {
                this.DoFail = true;
                HitExc = false;
            }

            public override void ClearDoFail()
            {
                this.DoFail = false;
            }

            public override void Eval(MockDirectoryWrapper dir)
            {
                if (DoFail && TestThread())
                {
                    // LUCENENET specific: for these to work in release mode, we have added [MethodImpl(MethodImplOptions.NoInlining)]
                    // to each possible target of the StackTraceHelper. If these change, so must the attribute on the target methods.
                    bool isDoFlush = Util.StackTraceHelper.DoesStackTraceContainMethod("Flush");
                    bool isClose = Util.StackTraceHelper.DoesStackTraceContainMethod("Close") ||
                        Util.StackTraceHelper.DoesStackTraceContainMethod("Dispose");

                    if (isDoFlush && !isClose && Random().NextBoolean())
                    {
                        HitExc = true;
                        throw new IOException(Thread.CurrentThread.Name + ": now failing during flush");
                    }
                }
            }
        }

        // Make sure running BG merges still work fine even when
        // we are hitting exceptions during flushing.
        [Test]
        public virtual void TestFlushExceptions()
        {
            MockDirectoryWrapper directory = NewMockDirectory();
            FailOnlyOnFlush failure = new FailOnlyOnFlush(this);
            directory.FailOn(failure);

            IndexWriter writer = new IndexWriter(directory, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
            Document doc = new Document();
            Field idField = NewStringField("id", "", Field.Store.YES);
            doc.Add(idField);
            int extraCount = 0;

            for (int i = 0; i < 10; i++)
            {
                if (VERBOSE)
                {
                    Console.WriteLine("TEST: iter=" + i);
                }

                for (int j = 0; j < 20; j++)
                {
                    idField.SetStringValue(Convert.ToString(i * 20 + j));
                    writer.AddDocument(doc);
                }

                // must cycle here because sometimes the merge flushes
                // the doc we just added and so there's nothing to
                // flush, and we don't hit the exception
                while (true)
                {
                    writer.AddDocument(doc);
                    failure.SetDoFail();
                    try
                    {
                        writer.Flush(true, true);
                        if (failure.HitExc)
                        {
                            Assert.Fail("failed to hit IOException");
                        }
                        extraCount++;
                    }
                    catch (IOException ioe)
                    {
                        if (VERBOSE)
                        {
                            Console.WriteLine(ioe.StackTrace);
                        }
                        failure.ClearDoFail();
                        break;
                    }
                }
                Assert.AreEqual(20 * (i + 1) + extraCount, writer.NumDocs);
            }

            writer.Dispose();
            IndexReader reader = DirectoryReader.Open(directory);
            Assert.AreEqual(200 + extraCount, reader.NumDocs);
            reader.Dispose();
            directory.Dispose();
        }

        // Test that deletes committed after a merge started and
        // before it finishes, are correctly merged back:
        [Test]
        public virtual void TestDeleteMerging()
        {
            Directory directory = NewDirectory();

            LogDocMergePolicy mp = new LogDocMergePolicy();
            // Force degenerate merging so we can get a mix of
            // merging of segments with and without deletes at the
            // start:
            mp.MinMergeDocs = 1000;
            IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(mp));

            Document doc = new Document();
            Field idField = NewStringField("id", "", Field.Store.YES);
            doc.Add(idField);
            for (int i = 0; i < 10; i++)
            {
                if (VERBOSE)
                {
                    Console.WriteLine("\nTEST: cycle");
                }
                for (int j = 0; j < 100; j++)
                {
                    idField.SetStringValue(Convert.ToString(i * 100 + j));
                    writer.AddDocument(doc);
                }

                int delID = i;
                while (delID < 100 * (1 + i))
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("TEST: del " + delID);
                    }
                    writer.DeleteDocuments(new Term("id", "" + delID));
                    delID += 10;
                }

                writer.Commit();
            }

            writer.Dispose();
            IndexReader reader = DirectoryReader.Open(directory);
            // Verify that we did not lose any deletes...
            Assert.AreEqual(450, reader.NumDocs);
            reader.Dispose();
            directory.Dispose();
        }

        [Test]
        public virtual void TestNoExtraFiles()
        {
            Directory directory = NewDirectory();
            IndexWriter writer = new IndexWriter(directory, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));

            for (int iter = 0; iter < 7; iter++)
            {
                if (VERBOSE)
                {
                    Console.WriteLine("TEST: iter=" + iter);
                }

                for (int j = 0; j < 21; j++)
                {
                    Document doc = new Document();
                    doc.Add(NewTextField("content", "a b c", Field.Store.NO));
                    writer.AddDocument(doc);
                }

                writer.Dispose();
                TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles");

                // Reopen
                writer = new IndexWriter(directory, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND).SetMaxBufferedDocs(2));
            }

            writer.Dispose();

            directory.Dispose();
        }

        [Test]
        public virtual void TestNoWaitClose()
        {
            Directory directory = NewDirectory();
            Document doc = new Document();
            Field idField = NewStringField("id", "", Field.Store.YES);
            doc.Add(idField);

            IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(100)));

            for (int iter = 0; iter < 10; iter++)
            {
                for (int j = 0; j < 201; j++)
                {
                    idField.SetStringValue(Convert.ToString(iter * 201 + j));
                    writer.AddDocument(doc);
                }

                int delID = iter * 201;
                for (int j = 0; j < 20; j++)
                {
                    writer.DeleteDocuments(new Term("id", Convert.ToString(delID)));
                    delID += 5;
                }

                // Force a bunch of merge threads to kick off so we
                // stress out aborting them on close:
                ((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 3;
                writer.AddDocument(doc);
                writer.Commit();

                writer.Dispose(false);

                IndexReader reader = DirectoryReader.Open(directory);
                Assert.AreEqual((1 + iter) * 182, reader.NumDocs);
                reader.Dispose();

                // Reopen
                writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND).SetMergePolicy(NewLogMergePolicy(100)));
            }
            writer.Dispose();

            directory.Dispose();
        }

        // LUCENE-4544
        [Test]
        public virtual void TestMaxMergeCount()
        {
            Directory dir = NewDirectory();
            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));

            int maxMergeCount = TestUtil.NextInt(Random(), 1, 5);
            int maxMergeThreads = TestUtil.NextInt(Random(), 1, maxMergeCount);
            CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount);
            AtomicInt32 runningMergeCount = new AtomicInt32(0);
            AtomicBoolean failed = new AtomicBoolean();

            if (VERBOSE)
            {
                Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads);
            }

            ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed);
            cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
            iwc.SetMergeScheduler(cms);
            iwc.SetMaxBufferedDocs(2);

            TieredMergePolicy tmp = new TieredMergePolicy();
            iwc.SetMergePolicy(tmp);
            tmp.MaxMergeAtOnce = 2;
            tmp.SegmentsPerTier = 2;

            IndexWriter w = new IndexWriter(dir, iwc);
            Document doc = new Document();
            doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED));
            while (enoughMergesWaiting.CurrentCount != 0 && !failed.Get())
            {
                for (int i = 0; i < 10; i++)
                {
                    w.AddDocument(doc);
                }
            }
            w.Dispose(false);
            dir.Dispose();
        }

        private class ConcurrentMergeSchedulerAnonymousInnerClassHelper : ConcurrentMergeScheduler
        {
            private readonly TestConcurrentMergeScheduler OuterInstance;

            private int maxMergeCount;
            private CountdownEvent EnoughMergesWaiting;
            private AtomicInt32 RunningMergeCount;
            private AtomicBoolean Failed;

            public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInt32 runningMergeCount, AtomicBoolean failed)
            {
                this.OuterInstance = outerInstance;
                this.maxMergeCount = maxMergeCount;
                this.EnoughMergesWaiting = enoughMergesWaiting;
                this.RunningMergeCount = runningMergeCount;
                this.Failed = failed;
            }

            protected override void DoMerge(MergePolicy.OneMerge merge)
            {
                try
                {
                    // Stall all incoming merges until we see
                    // maxMergeCount:
                    int count = RunningMergeCount.IncrementAndGet();
                    try
                    {
                        Assert.IsTrue(count <= maxMergeCount, "count=" + count + " vs maxMergeCount=" + maxMergeCount);
                        EnoughMergesWaiting.Signal();

                        // Stall this merge until we see exactly
                        // maxMergeCount merges waiting
                        while (true)
                        {
                            // wait for 10 milliseconds
                            if (EnoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || Failed.Get())
                            {
                                break;
                            }
                        }
                        // Then sleep a bit to give a chance for the bug
                        // (too many pending merges) to appear:
                        Thread.Sleep(20);
                        base.DoMerge(merge);
                    }
                    finally
                    {
                        RunningMergeCount.DecrementAndGet();
                    }
                }
                catch (Exception /*t*/)
                {
                    Failed.Set(true);
                    m_writer.MergeFinish(merge);

                    // LUCENENET specific - throwing an exception on a background thread causes the test
                    // runner to crash on .NET Core 2.0.
                    //throw new Exception(t.ToString(), t);
                }
            }
        }

        private class TrackingCMS : ConcurrentMergeScheduler
        {
            internal long TotMergedBytes;

            public TrackingCMS()
            {
                SetMaxMergesAndThreads(5, 5);
            }

            protected override void DoMerge(MergePolicy.OneMerge merge)
            {
                TotMergedBytes += merge.TotalBytesSize;
                base.DoMerge(merge);
            }
        }

        [Test]
        public virtual void TestTotalBytesSize()
        {
            Directory d = NewDirectory();
            if (d is MockDirectoryWrapper)
            {
                ((MockDirectoryWrapper)d).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
            }
            IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
            iwc.SetMaxBufferedDocs(5);
            iwc.SetMergeScheduler(new TrackingCMS());
            if (TestUtil.GetPostingsFormat("id").Equals("SimpleText", StringComparison.Ordinal))
            {
                // no
                iwc.SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()));
            }
            RandomIndexWriter w = new RandomIndexWriter(Random(), d, iwc);
            for (int i = 0; i < 1000; i++)
            {
                Document doc = new Document();
                doc.Add(new StringField("id", "" + i, Field.Store.NO));
                w.AddDocument(doc);

                if (Random().NextBoolean())
                {
                    w.DeleteDocuments(new Term("id", "" + Random().Next(i + 1)));
                }
            }
            Assert.IsTrue(((TrackingCMS)w.w.Config.MergeScheduler).TotMergedBytes != 0);
            w.Dispose();
            d.Dispose();
        }


        // LUCENENET specific
        private class FailOnlyOnMerge : MockDirectoryWrapper.Failure
        {
            public override void Eval(MockDirectoryWrapper dir)
            {
                // LUCENENET specific: for these to work in release mode, we have added [MethodImpl(MethodImplOptions.NoInlining)]
                // to each possible target of the StackTraceHelper. If these change, so must the attribute on the target methods.
                if (StackTraceHelper.DoesStackTraceContainMethod("DoMerge"))
                {
                    throw new IOException("now failing during merge");
                }
            }
        }

        // LUCENENET-603
        [Test, LuceneNetSpecific]
        public void TestExceptionOnBackgroundThreadIsPropagatedToCallingThread()
        {
            using (MockDirectoryWrapper dir = NewMockDirectory())
            {
                dir.FailOn(new FailOnlyOnMerge());

                Document doc = new Document();
                Field idField = NewStringField("id", "", Field.Store.YES);
                doc.Add(idField);

                var mergeScheduler = new ConcurrentMergeScheduler();
                using (IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(mergeScheduler).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy())))
                {
                    LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy;
                    logMP.MergeFactor = 10;
                    for (int i = 0; i < 20; i++)
                    {
                        writer.AddDocument(doc);
                    }

                    bool exceptionHit = false;
                    try
                    {
                        mergeScheduler.Sync();
                    }
                    catch (MergePolicy.MergeException)
                    {
                        exceptionHit = true;
                    }

                    assertTrue(exceptionHit);
                }
            }
        }
    }
}
#endif