/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;

import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomLongBetween;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.AssertingDirectoryReader;
import org.apache.lucene.tests.index.DocHelper;
import org.apache.lucene.tests.store.MockDirectoryWrapper;
import org.apache.lucene.tests.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.Version;
import org.junit.Test;

@SuppressCodecs("SimpleText") // too slow here
public class TestIndexWriterReader extends LuceneTestCase {

  private final int numThreads = TEST_NIGHTLY ? 5 : 2;

  public static int count(Term t, IndexReader r) throws IOException {
    int count = 0;
    PostingsEnum td = TestUtil.docs(random(), r, t.field(), new BytesRef(t.text()), null, 0);

    if (td != null) {
      final Bits liveDocs = MultiBits.getLiveDocs(r);
      while (td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
        td.docID();
        if (liveDocs == null || liveDocs.get(td.docID())) {
          count++;
        }
      }
    }
    return count;
  }

  public void testAddCloseOpen() throws IOException {
    // Can't use assertNoDeletes: this test pulls a non-NRT
    // reader in the end:
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));

    IndexWriter writer = new IndexWriter(dir1, iwc);
    for (int i = 0; i < 97; i++) {
      DirectoryReader reader = DirectoryReader.open(writer);
      if (i == 0) {
        writer.addDocument(DocHelper.createDocument(i, "x", 1 + random().nextInt(5)));
      } else {
        int previous = random().nextInt(i);
        // a check if the reader is current here could fail since there might be
        // merges going on.
        switch (random().nextInt(5)) {
          case 0:
          case 1:
          case 2:
            writer.addDocument(DocHelper.createDocument(i, "x", 1 + random().nextInt(5)));
            break;
          case 3:
            writer.updateDocument(
                new Term("id", "" + previous),
                DocHelper.createDocument(previous, "x", 1 + random().nextInt(5)));
            break;
          case 4:
            writer.deleteDocuments(new Term("id", "" + previous));
        }
      }
      assertFalse(reader.isCurrent());
      reader.close();
    }
    writer.forceMerge(1); // make sure all merging is done etc.
    DirectoryReader reader = DirectoryReader.open(writer);
    writer.commit(); // no changes that are not visible to the reader

    // A commit is now seen as a change to an NRT reader:
    assertFalse(reader.isCurrent());
    reader.close();
    reader = DirectoryReader.open(writer);
    assertTrue(reader.isCurrent());
    writer.close();

    assertTrue(reader.isCurrent());
    iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    writer = new IndexWriter(dir1, iwc);
    assertTrue(reader.isCurrent());
    writer.addDocument(DocHelper.createDocument(1, "x", 1 + random().nextInt(5)));
    assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
    writer.close();
    assertFalse(reader.isCurrent()); // segments written
    reader.close();
    dir1.close();
  }

  public void testUpdateDocument() throws Exception {
    boolean doFullMerge = true;

    Directory dir1 = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    if (iwc.getMaxBufferedDocs() < 20) {
      iwc.setMaxBufferedDocs(20);
    }
    // no merging
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    if (VERBOSE) {
      System.out.println("TEST: make index");
    }
    IndexWriter writer = new IndexWriter(dir1, iwc);

    // create the index
    createIndexNoClose(!doFullMerge, "index1", writer);

    // writer.flush(false, true, true);

    // get a reader
    DirectoryReader r1 = DirectoryReader.open(writer);
    assertTrue(r1.isCurrent());

    String id10 = r1.storedFields().document(10).getField("id").stringValue();

    Document newDoc = r1.storedFields().document(10);
    newDoc.removeField("id");
    newDoc.add(new Field("id", Integer.toString(8000), DocHelper.STRING_TYPE_STORED_WITH_TVS));
    writer.updateDocument(new Term("id", id10), newDoc);
    assertFalse(r1.isCurrent());

    System.out.println("TEST: now get reader");
    DirectoryReader r2 = DirectoryReader.open(writer);
    assertTrue(r2.isCurrent());
    assertEquals(0, count(new Term("id", id10), r2));
    if (VERBOSE) {
      System.out.println("TEST: verify id");
    }
    assertEquals(1, count(new Term("id", Integer.toString(8000)), r2));

    r1.close();
    assertTrue(r2.isCurrent());
    writer.close();
    // writer.close wrote a new commit
    assertFalse(r2.isCurrent());

    DirectoryReader r3 = DirectoryReader.open(dir1);
    assertTrue(r3.isCurrent());
    assertFalse(r2.isCurrent());
    assertEquals(0, count(new Term("id", id10), r3));
    assertEquals(1, count(new Term("id", Integer.toString(8000)), r3));

    writer = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    doc.add(newTextField("field", "a b c", Field.Store.NO));
    writer.addDocument(doc);
    assertFalse(r2.isCurrent());
    assertTrue(r3.isCurrent());

    writer.close();

    assertFalse(r2.isCurrent());
    assertTrue(!r3.isCurrent());

    r2.close();
    r3.close();

    dir1.close();
  }

  public void testIsCurrent() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));

    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(newTextField("field", "a b c", Field.Store.NO));
    writer.addDocument(doc);
    writer.close();

    iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    writer = new IndexWriter(dir, iwc);
    doc = new Document();
    doc.add(newTextField("field", "a b c", Field.Store.NO));
    DirectoryReader nrtReader = DirectoryReader.open(writer);
    assertTrue(nrtReader.isCurrent());
    writer.addDocument(doc);
    assertFalse(nrtReader.isCurrent()); // should see the changes
    writer.forceMerge(1); // make sure we don't have a merge going on
    assertFalse(nrtReader.isCurrent());
    nrtReader.close();

    DirectoryReader dirReader = DirectoryReader.open(dir);
    nrtReader = DirectoryReader.open(writer);

    assertTrue(dirReader.isCurrent());
    assertTrue(nrtReader.isCurrent()); // nothing was committed yet so we are still current
    assertEquals(2, nrtReader.maxDoc()); // sees the actual document added
    assertEquals(1, dirReader.maxDoc());
    writer.close(); // close is actually a commit both should see the changes
    assertFalse(nrtReader.isCurrent());
    assertFalse(
        dirReader
            .isCurrent()); // this reader has been opened before the writer was closed / committed

    dirReader.close();
    nrtReader.close();
    dir.close();
  }

  /** Test using IW.addIndexes */
  public void testAddIndexes() throws Exception {
    boolean doFullMerge = false;

    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriterConfig iwc =
        newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0);
    if (iwc.getMaxBufferedDocs() < 20) {
      iwc.setMaxBufferedDocs(20);
    }
    // no merging
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    IndexWriter writer = new IndexWriter(dir1, iwc);

    // create the index
    createIndexNoClose(!doFullMerge, "index1", writer);
    writer.flush(false, true);

    // create a 2nd index
    Directory dir2 = newDirectory();
    IndexWriter writer2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random())));
    createIndexNoClose(!doFullMerge, "index2", writer2);
    writer2.close();

    DirectoryReader r0 = DirectoryReader.open(writer);
    assertTrue(r0.isCurrent());
    writer.addIndexes(dir2);
    assertFalse(r0.isCurrent());
    r0.close();

    DirectoryReader r1 = DirectoryReader.open(writer);
    assertTrue(r1.isCurrent());

    writer.commit();

    // A commit is seen as a change to NRT reader:
    assertFalse(r1.isCurrent());

    assertEquals(200, r1.maxDoc());

    int index2df = r1.docFreq(new Term("indexname", "index2"));

    assertEquals(100, index2df);

    // verify the docs are from different indexes
    Document doc5 = r1.storedFields().document(5);
    assertEquals("index1", doc5.get("indexname"));
    Document doc150 = r1.storedFields().document(150);
    assertEquals("index2", doc150.get("indexname"));
    r1.close();
    writer.close();
    dir1.close();
    dir2.close();
  }

  public void testAddIndexes2() throws Exception {
    boolean doFullMerge = false;

    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));

    // create a 2nd index
    Directory dir2 = newDirectory();
    IndexWriter writer2 =
        new IndexWriter(
            dir2,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));
    createIndexNoClose(!doFullMerge, "index2", writer2);
    writer2.close();

    writer.addIndexes(dir2);
    writer.addIndexes(dir2);
    writer.addIndexes(dir2);
    writer.addIndexes(dir2);
    writer.addIndexes(dir2);

    IndexReader r1 = DirectoryReader.open(writer);
    assertEquals(500, r1.maxDoc());

    r1.close();
    writer.close();
    dir1.close();
    dir2.close();
  }

  /** Deletes using IW.deleteDocuments */
  public void testDeleteFromIndexWriter() throws Exception {
    boolean doFullMerge = true;

    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));
    // create the index
    createIndexNoClose(!doFullMerge, "index1", writer);
    writer.flush(false, true);
    // get a reader
    IndexReader r1 = DirectoryReader.open(writer);

    String id10 = r1.storedFields().document(10).getField("id").stringValue();

    // deleted IW docs should not show up in the next getReader
    writer.deleteDocuments(new Term("id", id10));
    IndexReader r2 = DirectoryReader.open(writer);
    assertEquals(1, count(new Term("id", id10), r1));
    assertEquals(0, count(new Term("id", id10), r2));

    String id50 = r1.storedFields().document(50).getField("id").stringValue();
    assertEquals(1, count(new Term("id", id50), r1));

    writer.deleteDocuments(new Term("id", id50));

    IndexReader r3 = DirectoryReader.open(writer);
    assertEquals(0, count(new Term("id", id10), r3));
    assertEquals(0, count(new Term("id", id50), r3));

    String id75 = r1.storedFields().document(75).getField("id").stringValue();
    writer.deleteDocuments(new TermQuery(new Term("id", id75)));
    IndexReader r4 = DirectoryReader.open(writer);
    assertEquals(1, count(new Term("id", id75), r3));
    assertEquals(0, count(new Term("id", id75), r4));

    r1.close();
    r2.close();
    r3.close();
    r4.close();
    writer.close();

    // reopen the writer to verify the delete made it to the directory
    writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));
    IndexReader w2r1 = DirectoryReader.open(writer);
    assertEquals(0, count(new Term("id", id10), w2r1));
    w2r1.close();
    writer.close();
    dir1.close();
  }

  public void testAddIndexesAndDoDeletesThreads() throws Throwable {
    final int numIter = TEST_NIGHTLY ? 2 : 1;
    int numDirs = TEST_NIGHTLY ? 3 : 2;

    Directory mainDir = getAssertNoDeletesDirectory(newDirectory());

    IndexWriter mainWriter =
        new IndexWriter(
            mainDir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMergePolicy(newLogMergePolicy())
                .setMaxFullFlushMergeWaitMillis(0));
    TestUtil.reduceOpenFiles(mainWriter);

    AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(numIter, mainWriter);
    addDirThreads.launchThreads(numDirs);
    addDirThreads.joinThreads();

    // assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.numThreads
    //    * addDirThreads.NUM_INIT_DOCS, addDirThreads.mainwriter.getDocStats().numDocs);
    assertEquals(addDirThreads.count.intValue(), addDirThreads.mainWriter.getDocStats().numDocs);

    addDirThreads.close(true);

    assertTrue(addDirThreads.failures.isEmpty());

    TestUtil.checkIndex(mainDir);

    IndexReader reader = DirectoryReader.open(mainDir);
    assertEquals(addDirThreads.count.intValue(), reader.numDocs());
    // assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.numThreads
    //    * addDirThreads.NUM_INIT_DOCS, reader.numDocs());
    reader.close();

    addDirThreads.closeDir();
    mainDir.close();
  }

  private class AddDirectoriesThreads {
    Directory addDir;
    static final int NUM_INIT_DOCS = 100;
    int numDirs;
    final Thread[] threads = new Thread[numThreads];
    IndexWriter mainWriter;
    final List<Throwable> failures = new ArrayList<>();
    DirectoryReader[] readers;
    AtomicInteger count = new AtomicInteger(0);
    AtomicInteger numaddIndexes = new AtomicInteger(0);

    public AddDirectoriesThreads(int numDirs, IndexWriter mainWriter) throws Throwable {
      this.numDirs = numDirs;
      this.mainWriter = mainWriter;
      addDir = newDirectory();
      IndexWriter writer =
          new IndexWriter(
              addDir,
              newIndexWriterConfig(new MockAnalyzer(random()))
                  .setMaxFullFlushMergeWaitMillis(0)
                  .setMaxBufferedDocs(2));
      TestUtil.reduceOpenFiles(writer);
      for (int i = 0; i < NUM_INIT_DOCS; i++) {
        Document doc = DocHelper.createDocument(i, "addindex", 4);
        writer.addDocument(doc);
      }

      writer.close();

      readers = new DirectoryReader[numDirs];
      for (int i = 0; i < numDirs; i++) readers[i] = DirectoryReader.open(addDir);
    }

    void joinThreads() {
      for (int i = 0; i < numThreads; i++)
        try {
          threads[i].join();
        } catch (InterruptedException ie) {
          throw new ThreadInterruptedException(ie);
        }
    }

    void close(boolean doWait) throws Throwable {
      if (doWait) {
        mainWriter.close();
      } else {
        mainWriter.rollback();
      }
    }

    void closeDir() throws Throwable {
      for (int i = 0; i < numDirs; i++) {
        readers[i].close();
      }
      addDir.close();
    }

    void handle(Throwable t) {
      t.printStackTrace(System.out);
      synchronized (failures) {
        failures.add(t);
      }
    }

    void launchThreads(final int numIter) {
      for (int i = 0; i < numThreads; i++) {
        threads[i] =
            new Thread() {
              @Override
              public void run() {
                try {
                  final Directory[] dirs = new Directory[numDirs];
                  for (int k = 0; k < numDirs; k++)
                    dirs[k] = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(addDir));
                  // int j = 0;
                  // while (true) {
                  // System.out.println(Thread.currentThread().getName() + ": iter
                  // j=" + j);
                  for (int x = 0; x < numIter; x++) {
                    // only do addIndexes
                    doBody(x, dirs);
                  }
                  // if (numIter > 0 && j == numIter)
                  //  break;
                  // doBody(j++, dirs);
                  // doBody(5, dirs);
                  // }
                } catch (Throwable t) {
                  handle(t);
                }
              }
            };
      }
      for (int i = 0; i < numThreads; i++) threads[i].start();
    }

    void doBody(int j, Directory[] dirs) throws Throwable {
      switch (j % 4) {
        case 0:
          mainWriter.addIndexes(dirs);
          mainWriter.forceMerge(1);
          break;
        case 1:
          mainWriter.addIndexes(dirs);
          numaddIndexes.incrementAndGet();
          break;
        case 2:
          TestUtil.addIndexesSlowly(mainWriter, readers);
          break;
        case 3:
          mainWriter.commit();
      }
      count.addAndGet(dirs.length * NUM_INIT_DOCS);
    }
  }

  public void testIndexWriterReopenSegmentFullMerge() throws Exception {
    doTestIndexWriterReopenSegment(true);
  }

  public void testIndexWriterReopenSegment() throws Exception {
    doTestIndexWriterReopenSegment(false);
  }

  /** Tests creating a segment, then check to insure the segment can be seen via IW.getReader */
  public void doTestIndexWriterReopenSegment(boolean doFullMerge) throws Exception {
    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));
    IndexReader r1 = DirectoryReader.open(writer);
    assertEquals(0, r1.maxDoc());
    createIndexNoClose(false, "index1", writer);
    writer.flush(!doFullMerge, true);

    IndexReader iwr1 = DirectoryReader.open(writer);
    assertEquals(100, iwr1.maxDoc());

    IndexReader r2 = DirectoryReader.open(writer);
    assertEquals(r2.maxDoc(), 100);
    // add 100 documents
    for (int x = 10000; x < 10000 + 100; x++) {
      Document d = DocHelper.createDocument(x, "index1", 5);
      writer.addDocument(d);
    }
    writer.flush(false, true);
    // verify the reader was reopened internally
    IndexReader iwr2 = DirectoryReader.open(writer);
    assertTrue(iwr2 != r1);
    assertEquals(200, iwr2.maxDoc());
    // should have flushed out a segment
    IndexReader r3 = DirectoryReader.open(writer);
    assertTrue(r2 != r3);
    assertEquals(200, r3.maxDoc());

    // dec ref the readers rather than close them because
    // closing flushes changes to the writer
    r1.close();
    iwr1.close();
    r2.close();
    r3.close();
    iwr2.close();
    writer.close();

    // test whether the changes made it to the directory
    writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));
    IndexReader w2r1 = DirectoryReader.open(writer);
    // insure the deletes were actually flushed to the directory
    assertEquals(200, w2r1.maxDoc());
    w2r1.close();
    writer.close();

    dir1.close();
  }

  /*
   * Delete a document by term and return the doc id
   *
   * public static int deleteDocument(Term term, IndexWriter writer) throws
   * IOException { IndexReader reader = writer.getReader(); TermDocs td =
   * reader.termDocs(term); int doc = -1; //if (td.next()) { // doc = td.storedFields().document();
   * //} //writer.deleteDocuments(term); td.close(); return doc; }
   */

  public static void createIndex(
      Random random, Directory dir1, String indexName, boolean multiSegment) throws IOException {
    IndexWriter w =
        new IndexWriter(
            dir1,
            LuceneTestCase.newIndexWriterConfig(random, new MockAnalyzer(random))
                .setMergePolicy(new LogDocMergePolicy()));
    for (int i = 0; i < 100; i++) {
      w.addDocument(DocHelper.createDocument(i, indexName, 4));
    }
    if (!multiSegment) {
      w.forceMerge(1);
    }
    w.close();
  }

  public static void createIndexNoClose(boolean multiSegment, String indexName, IndexWriter w)
      throws IOException {
    for (int i = 0; i < 100; i++) {
      w.addDocument(DocHelper.createDocument(i, indexName, 4));
    }
    if (!multiSegment) {
      w.forceMerge(1);
    }
  }

  public void testMergeWarmer() throws Exception {
    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    // Enroll warmer
    AtomicInteger warmCount = new AtomicInteger();
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setMaxFullFlushMergeWaitMillis(0)
                .setMergedSegmentWarmer((leafReader) -> warmCount.incrementAndGet())
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMergePolicy(newLogMergePolicy()));

    // create the index
    createIndexNoClose(false, "test", writer);

    // get a reader to put writer into near real-time mode
    IndexReader r1 = DirectoryReader.open(writer);

    ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);

    int num = TEST_NIGHTLY ? atLeast(100) : atLeast(10);
    for (int i = 0; i < num; i++) {
      writer.addDocument(DocHelper.createDocument(i, "test", 4));
    }
    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();

    assertTrue(warmCount.get() > 0);
    final int count = warmCount.get();

    writer.addDocument(DocHelper.createDocument(17, "test", 4));
    writer.forceMerge(1);
    assertTrue(warmCount.get() > count);

    writer.close();
    r1.close();
    dir1.close();
  }

  public void testAfterCommit() throws Exception {
    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMaxFullFlushMergeWaitMillis(0));
    writer.commit();

    // create the index
    createIndexNoClose(false, "test", writer);

    // get a reader to put writer into near real-time mode
    DirectoryReader r1 = DirectoryReader.open(writer);
    TestUtil.checkIndex(dir1);
    writer.commit();
    TestUtil.checkIndex(dir1);
    assertEquals(100, r1.numDocs());

    for (int i = 0; i < 10; i++) {
      writer.addDocument(DocHelper.createDocument(i, "test", 4));
    }
    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();

    DirectoryReader r2 = DirectoryReader.openIfChanged(r1);
    if (r2 != null) {
      r1.close();
      r1 = r2;
    }
    assertEquals(110, r1.numDocs());
    writer.close();
    r1.close();
    dir1.close();
  }

  // Make sure reader remains usable even if IndexWriter closes
  public void testAfterClose() throws Exception {
    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));

    // create the index
    createIndexNoClose(false, "test", writer);

    DirectoryReader r = DirectoryReader.open(writer);
    writer.close();

    TestUtil.checkIndex(dir1);

    // reader should remain usable even after IndexWriter is closed:
    assertEquals(100, r.numDocs());
    Query q = new TermQuery(new Term("indexname", "test"));
    IndexSearcher searcher = newSearcher(r);
    assertEquals(100, searcher.count(q));

    expectThrows(AlreadyClosedException.class, () -> DirectoryReader.openIfChanged(r));

    r.close();
    dir1.close();
  }

  // Stress test reopen during addIndexes
  @Nightly
  public void testDuringAddIndexes() throws Exception {
    Directory dir1 = getAssertNoDeletesDirectory(newDirectory());
    final IndexWriter writer =
        new IndexWriter(
            dir1,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxFullFlushMergeWaitMillis(0)
                .setMergePolicy(newLogMergePolicy(2)));

    // create the index
    createIndexNoClose(false, "test", writer);
    writer.commit();

    final Directory[] dirs = new Directory[10];
    for (int i = 0; i < 10; i++) {
      dirs[i] = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(dir1));
    }

    DirectoryReader r = DirectoryReader.open(writer);

    final int numIterations = 10;
    final List<Throwable> excs = Collections.synchronizedList(new ArrayList<>());

    // Only one thread can addIndexes at a time, because
    // IndexWriter acquires a write lock in each directory:
    final Thread[] threads = new Thread[1];
    final AtomicBoolean threadDone = new AtomicBoolean(false);
    for (int i = 0; i < threads.length; i++) {
      threads[i] =
          new Thread() {
            @Override
            public void run() {
              int count = 0;
              do {
                count++;
                try {
                  writer.addIndexes(dirs);
                  writer.maybeMerge();
                } catch (Throwable t) {
                  excs.add(t);
                  throw new RuntimeException(t);
                }
              } while (count < numIterations);
              threadDone.set(true);
            }
          };
      threads[i].setDaemon(true);
      threads[i].start();
    }

    long lastCount = 0;
    while (threadDone.get() == false) {
      DirectoryReader r2 = DirectoryReader.openIfChanged(r);
      if (r2 != null) {
        r.close();
        r = r2;
        Query q = new TermQuery(new Term("indexname", "test"));
        IndexSearcher searcher = newSearcher(r);
        final long count = searcher.count(q);
        assertTrue(count >= lastCount);
        lastCount = count;
      }
    }

    for (Thread thread : threads) {
      thread.join();
    }
    // final check
    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    if (r2 != null) {
      r.close();
      r = r2;
    }
    Query q = new TermQuery(new Term("indexname", "test"));
    IndexSearcher searcher = newSearcher(r);
    final long count = searcher.count(q);
    assertTrue(count >= lastCount);

    assertEquals(0, excs.size());
    r.close();
    if (dir1 instanceof MockDirectoryWrapper) {
      final Collection<String> openDeletedFiles =
          ((MockDirectoryWrapper) dir1).getOpenDeletedFiles();
      assertEquals("openDeleted=" + openDeletedFiles, 0, openDeletedFiles.size());
    }

    writer.close();

    dir1.close();
  }

  private Directory getAssertNoDeletesDirectory(Directory directory) {
    if (directory instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) directory).setAssertNoDeleteOpenFile(true);
    }
    return directory;
  }

  // Stress test reopen during add/delete
  public void testDuringAddDelete() throws Exception {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc =
        newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(2));
    if (TEST_NIGHTLY) {
      // if we have a ton of iterations we need to make sure we don't do unnecessary
      // extra flushing otherwise we will time out on nightly
      iwc.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
      iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
    }
    final IndexWriter writer = new IndexWriter(dir1, iwc);

    // create the index
    createIndexNoClose(false, "test", writer);
    writer.commit();

    DirectoryReader r = DirectoryReader.open(writer);

    final int iters = TEST_NIGHTLY ? 1000 : 10;
    final List<Throwable> excs = Collections.synchronizedList(new ArrayList<>());

    final Thread[] threads = new Thread[numThreads];
    final AtomicInteger remainingThreads = new AtomicInteger(numThreads);
    for (int i = 0; i < numThreads; i++) {
      threads[i] =
          new Thread() {
            final Random r = new Random(random().nextLong());

            @Override
            public void run() {
              int count = 0;
              do {
                try {
                  for (int docUpto = 0; docUpto < 10; docUpto++) {
                    writer.addDocument(DocHelper.createDocument(10 * count + docUpto, "test", 4));
                  }
                  count++;
                  final int limit = count * 10;
                  for (int delUpto = 0; delUpto < 5; delUpto++) {
                    int x = r.nextInt(limit);
                    writer.deleteDocuments(new Term("field3", "b" + x));
                  }
                } catch (Throwable t) {
                  excs.add(t);
                  throw new RuntimeException(t);
                }
              } while (count < iters);
              remainingThreads.decrementAndGet();
            }
          };
      threads[i].setDaemon(true);
      threads[i].start();
    }

    int sum = 0;
    while (remainingThreads.get() > 0) {
      DirectoryReader r2 = DirectoryReader.openIfChanged(r);
      if (r2 != null) {
        r.close();
        r = r2;
        Query q = new TermQuery(new Term("indexname", "test"));
        IndexSearcher searcher = newSearcher(r);
        sum += searcher.count(q);
      }
    }

    for (int i = 0; i < numThreads; i++) {
      threads[i].join();
    }
    // at least search once
    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    if (r2 != null) {
      r.close();
      r = r2;
    }
    Query q = new TermQuery(new Term("indexname", "test"));
    IndexSearcher searcher = newSearcher(r);
    sum += searcher.count(q);
    assertTrue("no documents found at all", sum > 0);

    assertEquals(0, excs.size());
    writer.close();

    r.close();
    dir1.close();
  }

  public void testForceMergeDeletes() throws Throwable {
    Directory dir = newDirectory();
    final IndexWriter w =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
    Document doc = new Document();
    doc.add(newTextField("field", "a b c", Field.Store.NO));
    Field id = newStringField("id", "", Field.Store.NO);
    doc.add(id);
    id.setStringValue("0");
    w.addDocument(doc);
    id.setStringValue("1");
    w.addDocument(doc);
    w.deleteDocuments(new Term("id", "0"));

    IndexReader r = DirectoryReader.open(w);
    w.forceMergeDeletes();
    w.close();
    r.close();
    r = DirectoryReader.open(dir);
    assertEquals(1, r.numDocs());
    assertFalse(r.hasDeletions());
    r.close();
    dir.close();
  }

  public void testDeletesNumDocs() throws Throwable {
    Directory dir = newDirectory();
    final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    doc.add(newTextField("field", "a b c", Field.Store.NO));
    Field id = newStringField("id", "", Field.Store.NO);
    doc.add(id);
    id.setStringValue("0");
    w.addDocument(doc);
    id.setStringValue("1");
    w.addDocument(doc);
    IndexReader r = DirectoryReader.open(w);
    assertEquals(2, r.numDocs());
    r.close();

    w.deleteDocuments(new Term("id", "0"));
    r = DirectoryReader.open(w);
    assertEquals(1, r.numDocs());
    r.close();

    w.deleteDocuments(new Term("id", "1"));
    r = DirectoryReader.open(w);
    assertEquals(0, r.numDocs());
    r.close();

    w.close();
    dir.close();
  }

  public void testEmptyIndex() throws Exception {
    // Ensures that getReader works on an empty index, which hasn't been committed yet.
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    IndexReader r = DirectoryReader.open(w);
    assertEquals(0, r.numDocs());
    r.close();
    w.close();
    dir.close();
  }

  public void testSegmentWarmer() throws Exception {
    Directory dir = newDirectory();
    final AtomicBoolean didWarm = new AtomicBoolean();
    LogMergePolicy mp = newLogMergePolicy(10);
    mp.setTargetSearchConcurrency(1);
    IndexWriter w =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setReaderPooling(true)
                .setMergedSegmentWarmer(
                    (r) -> {
                      IndexSearcher s = newSearcher(r);
                      int count = s.count(new TermQuery(new Term("foo", "bar")));
                      assertEquals(20, count);
                      didWarm.set(true);
                    })
                .setMergePolicy(mp));

    Document doc = new Document();
    doc.add(newStringField("foo", "bar", Field.Store.NO));
    for (int i = 0; i < 20; i++) {
      w.addDocument(doc);
    }
    w.waitForMerges();
    w.close();
    dir.close();
    assertTrue(didWarm.get());
  }

  public void testSimpleMergedSegmentWarmer() throws Exception {
    Directory dir = newDirectory();
    final AtomicBoolean didWarm = new AtomicBoolean();
    InfoStream infoStream =
        new InfoStream() {
          @Override
          public void close() throws IOException {}

          @Override
          public void message(String component, String message) {
            if ("SMSW".equals(component)) {
              didWarm.set(true);
            }
          }

          @Override
          public boolean isEnabled(String component) {
            return true;
          }
        };
    LogMergePolicy mp = newLogMergePolicy(10);
    mp.setTargetSearchConcurrency(1);
    IndexWriter w =
        new IndexWriter(
            dir,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setReaderPooling(true)
                .setInfoStream(infoStream)
                .setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream))
                .setMergePolicy(mp));

    Document doc = new Document();
    doc.add(newStringField("foo", "bar", Field.Store.NO));
    for (int i = 0; i < 20; i++) {
      w.addDocument(doc);
    }
    w.waitForMerges();
    w.close();
    dir.close();
    assertTrue(didWarm.get());
  }

  public void testReopenAfterNoRealChange() throws Exception {
    Directory d = getAssertNoDeletesDirectory(newDirectory());
    IndexWriter w =
        new IndexWriter(
            d, newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0));

    DirectoryReader r = DirectoryReader.open(w); // start pooling readers

    DirectoryReader r2 = DirectoryReader.openIfChanged(r);
    assertNull(r2);

    w.addDocument(new Document());
    DirectoryReader r3 = DirectoryReader.openIfChanged(r);
    assertNotNull(r3);
    assertTrue(r3.getVersion() != r.getVersion());
    assertTrue(r3.isCurrent());

    // Deletes nothing in reality...:
    w.deleteDocuments(new Term("foo", "bar"));

    // ... but IW marks this as not current:
    assertFalse(r3.isCurrent());
    DirectoryReader r4 = DirectoryReader.openIfChanged(r3);
    assertNull(r4);

    // Deletes nothing in reality...:
    w.deleteDocuments(new Term("foo", "bar"));
    DirectoryReader r5 = DirectoryReader.openIfChanged(r3, w);
    assertNull(r5);

    r3.close();

    w.close();
    d.close();
  }

  @Test
  public void testNRTOpenExceptions() throws Exception {
    // LUCENE-5262: test that several failed attempts to obtain an NRT reader
    // don't leak file handles.
    MockDirectoryWrapper dir =
        (MockDirectoryWrapper) getAssertNoDeletesDirectory(newMockDirectory());
    final AtomicBoolean shouldFail = new AtomicBoolean();
    dir.failOn(
        new MockDirectoryWrapper.Failure() {
          @Override
          public void eval(MockDirectoryWrapper dir) throws IOException {
            if (shouldFail.get()) {
              if (callStackContainsAnyOf("getReadOnlyClone")) {
                if (VERBOSE) {
                  System.out.println("TEST: now fail; exc:");
                  new Throwable().printStackTrace(System.out);
                }
                shouldFail.set(false);
                throw new FakeIOException();
              }
            }
          }
        });

    IndexWriterConfig conf =
        newIndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0);
    conf.setMergePolicy(NoMergePolicy.INSTANCE); // prevent merges from getting in the way
    IndexWriter writer = new IndexWriter(dir, conf);

    // create a segment and open an NRT reader
    writer.addDocument(new Document());
    DirectoryReader.open(writer).close();

    // add a new document so a new NRT reader is required
    writer.addDocument(new Document());

    // try to obtain an NRT reader twice: first time it fails and closes all the
    // other NRT readers. second time it fails, but also fails to close the
    // other NRT reader, since it is already marked closed!
    for (int i = 0; i < 2; i++) {
      shouldFail.set(true);
      expectThrows(FakeIOException.class, () -> DirectoryReader.open(writer).close());
    }

    writer.close();
    dir.close();
  }

  /** Make sure if all we do is open NRT reader against writer, we don't see merge starvation. */
  public void testTooManySegments() throws Exception {
    Directory dir = getAssertNoDeletesDirectory(new ByteBuffersDirectory());
    // Don't use newIndexWriterConfig, because we need a
    // "sane" mergePolicy:
    IndexWriterConfig iwc =
        new IndexWriterConfig(new MockAnalyzer(random())).setMaxFullFlushMergeWaitMillis(0);
    IndexWriter w = new IndexWriter(dir, iwc);
    // Create 500 segments:
    for (int i = 0; i < 500; i++) {
      Document doc = new Document();
      doc.add(newStringField("id", "" + i, Field.Store.NO));
      w.addDocument(doc);
      IndexReader r = DirectoryReader.open(w);
      // Make sure segment count never exceeds 100:
      assertTrue(r.leaves().size() < 100);
      r.close();
    }
    w.close();
    dir.close();
  }

  // LUCENE-5912: make sure when you reopen an NRT reader using a commit point, the SegmentReaders
  // are in fact shared:
  public void testReopenNRTReaderOnCommit() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    w.addDocument(new Document());

    // Pull NRT reader; it has 1 segment:
    DirectoryReader r1 = DirectoryReader.open(w);
    assertEquals(1, r1.leaves().size());
    w.addDocument(new Document());
    w.commit();

    List<IndexCommit> commits = DirectoryReader.listCommits(dir);
    assertEquals(1, commits.size());
    DirectoryReader r2 = DirectoryReader.openIfChanged(r1, commits.get(0));
    assertEquals(2, r2.leaves().size());

    // Make sure we shared same instance of SegmentReader w/ first reader:
    assertTrue(r1.leaves().get(0).reader() == r2.leaves().get(0).reader());
    r1.close();
    r2.close();
    w.close();
    dir.close();
  }

  public void testIndexReaderWriterWithLeafSorter() throws IOException {
    final String FIELD_NAME = "field1";
    final boolean ASC_SORT = randomBoolean();
    final long MISSING_VALUE =
        ASC_SORT ? Long.MAX_VALUE : Long.MIN_VALUE; // missing values at the end

    // create a comparator that sort leaf readers according to
    // the min value (asc sort) or max value (desc sort) of its points
    Comparator<LeafReader> leafSorter =
        Comparator.comparingLong(
            r -> {
              try {
                PointValues points = r.getPointValues(FIELD_NAME);
                if (points != null) {
                  byte[] sortValue =
                      ASC_SORT ? points.getMinPackedValue() : points.getMaxPackedValue();
                  return LongPoint.decodeDimension(sortValue, 0);
                }
              } catch (
                  @SuppressWarnings("unused")
                  IOException e) {
              }
              return MISSING_VALUE;
            });
    if (ASC_SORT == false) {
      leafSorter = leafSorter.reversed();
    }

    final int NUM_DOCS = atLeast(30);
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig();
    iwc.setLeafSorter(leafSorter);
    IndexWriter writer = new IndexWriter(dir, iwc);
    for (int i = 0; i < NUM_DOCS; ++i) {
      final Document doc = new Document();
      doc.add(new LongPoint(FIELD_NAME, randomLongBetween(1, 99)));
      writer.addDocument(doc);
      if (i > 0 && i % 10 == 0) writer.flush();
    }

    // Test1: test that leafReaders are sorted according to leafSorter provided in IndexWriterConfig
    {
      try (DirectoryReader reader = DirectoryReader.open(writer)) {
        assertLeavesSorted(reader, leafSorter);

        // add more documents that should be sorted first
        final long FIRST_VALUE = ASC_SORT ? 0 : 100;
        for (int i = 0; i < 10; ++i) {
          final Document doc = new Document();
          doc.add(new LongPoint(FIELD_NAME, FIRST_VALUE));
          writer.addDocument(doc);
        }
        writer.commit();

        // and open again
        try (DirectoryReader reader2 = DirectoryReader.openIfChanged(reader)) {
          assertLeavesSorted(reader2, leafSorter);
        }
      }
    }

    // Test2: test that leafReaders are sorted according to the provided leafSorter when opened from
    // directory
    {
      try (DirectoryReader reader = DirectoryReader.open(dir, leafSorter)) {
        assertLeavesSorted(reader, leafSorter);

        // add more documents that should be sorted first
        final long FIRST_VALUE = ASC_SORT ? 0 : 100;
        for (int i = 0; i < 10; ++i) {
          final Document doc = new Document();
          doc.add(new LongPoint(FIELD_NAME, FIRST_VALUE));
          writer.addDocument(doc);
        }
        writer.commit();

        // and open again
        try (DirectoryReader reader2 = DirectoryReader.openIfChanged(reader)) {
          assertLeavesSorted(reader2, leafSorter);
        }
      }
    }

    // Test3: test that FilterDirectoryReader sorts leaves according
    // to leafSorter of its wrapped reader
    {
      try (DirectoryReader reader =
          new AssertingDirectoryReader(DirectoryReader.open(dir, leafSorter))) {
        assertLeavesSorted(reader, leafSorter);

        // add more documents that should be sorted first
        final long FIRST_VALUE = ASC_SORT ? 0 : 100;
        for (int i = 0; i < 10; ++i) {
          final Document doc = new Document();
          doc.add(new LongPoint(FIELD_NAME, FIRST_VALUE));
          writer.addDocument(doc);
        }
        writer.commit();

        // and open again
        try (DirectoryReader reader2 = DirectoryReader.openIfChanged(reader)) {
          assertLeavesSorted(reader2, leafSorter);
        }
      }
    }

    // Test4: test that leafReaders are sorted according to the provided leafSorter when opened from
    // commit
    {
      List<IndexCommit> commits = DirectoryReader.listCommits(dir);
      IndexCommit latestCommit = commits.get(commits.size() - 1);
      try (DirectoryReader reader =
          DirectoryReader.open(latestCommit, Version.MIN_SUPPORTED_MAJOR, leafSorter)) {
        assertLeavesSorted(reader, leafSorter);

        // add more documents that should be sorted first
        final long FIRST_VALUE = ASC_SORT ? 0 : 100;
        for (int i = 0; i < 10; ++i) {
          final Document doc = new Document();
          doc.add(new LongPoint(FIELD_NAME, FIRST_VALUE));
          writer.addDocument(doc);
        }
        writer.commit();

        // and open again
        try (DirectoryReader reader2 = DirectoryReader.openIfChanged(reader)) {
          assertLeavesSorted(reader2, leafSorter);
        }
      }
    }

    writer.close();
    dir.close();
  }

  // assert that the leaf readers of the provided directory reader are sorted according to the
  // provided leafSorter
  private static void assertLeavesSorted(
      DirectoryReader reader, Comparator<LeafReader> leafSorter) {
    List<LeafReader> lrs = reader.leaves().stream().map(LeafReaderContext::reader).toList();
    List<LeafReader> expectedSortedlrs =
        reader.leaves().stream().map(LeafReaderContext::reader).sorted(leafSorter).toList();
    assertEquals(expectedSortedlrs, lrs);
  }
}
