/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.facet.taxonomy;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.SlowDirectory;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.SuppressForbidden;
import org.junit.Test;

@SuppressCodecs("SimpleText")
public class TestTaxonomyCombined extends FacetTestCase {

  /**
   * The following categories will be added to the taxonomy by fillTaxonomy(), and tested by all
   * tests below:
   */
  private static final String[][] categories = {
    {"Author", "Tom Clancy"},
    {"Author", "Richard Dawkins"},
    {"Author", "Richard Adams"},
    {"Price", "10", "11"},
    {"Price", "10", "12"},
    {"Price", "20", "27"},
    {"Date", "2006", "05"},
    {"Date", "2005"},
    {"Date", "2006"},
    {"Subject", "Nonfiction", "Children", "Animals"},
    {"Author", "Stephen Jay Gould"},
    {"Author", "\u05e0\u05d3\u05d1\u3042\u0628"},
  };

  /**
   * When adding the above categories with TaxonomyWriter.addCategory(), the following paths are
   * expected to be returned: (note that currently the full path is not returned, and therefore not
   * tested - rather, just the last component, the ordinal, is returned and tested.
   */
  private static final int[][] expectedPaths = {
    {1, 2},
    {1, 3},
    {1, 4},
    {5, 6, 7},
    {5, 6, 8},
    {5, 9, 10},
    {11, 12, 13},
    {11, 14},
    {11, 12},
    {15, 16, 17, 18},
    {1, 19},
    {1, 20}
  };

  /**
   * The taxonomy index is expected to then contain the following generated categories, with
   * increasing ordinals (note how parent categories are be added automatically when subcategories
   * are added).
   */
  private static final String[][] expectedCategories = {
    {}, // the root category
    {"Author"},
    {"Author", "Tom Clancy"},
    {"Author", "Richard Dawkins"},
    {"Author", "Richard Adams"},
    {"Price"},
    {"Price", "10"},
    {"Price", "10", "11"},
    {"Price", "10", "12"},
    {"Price", "20"},
    {"Price", "20", "27"},
    {"Date"},
    {"Date", "2006"},
    {"Date", "2006", "05"},
    {"Date", "2005"},
    {"Subject"},
    {"Subject", "Nonfiction"},
    {"Subject", "Nonfiction", "Children"},
    {"Subject", "Nonfiction", "Children", "Animals"},
    {"Author", "Stephen Jay Gould"},
    {"Author", "\u05e0\u05d3\u05d1\u3042\u0628"},
  };

  /**
   * fillTaxonomy adds the categories in the categories[] array, and asserts that the additions
   * return exactly the ordinals (in the past - paths) specified in expectedPaths[]. Note that this
   * assumes that fillTaxonomy() is called on an empty taxonomy index. Calling it after something
   * else was already added to the taxonomy index will surely have this method fail.
   */
  public static void fillTaxonomy(TaxonomyWriter tw) throws IOException {
    for (int i = 0; i < categories.length; i++) {
      int ordinal = tw.addCategory(new FacetLabel(categories[i]));
      int expectedOrdinal = expectedPaths[i][expectedPaths[i].length - 1];
      if (ordinal != expectedOrdinal) {
        fail(
            "For category "
                + showcat(categories[i])
                + " expected ordinal "
                + expectedOrdinal
                + ", but got "
                + ordinal);
      }
    }
  }

  public static String showcat(String[] path) {
    if (path == null) {
      return "<null>";
    }
    if (path.length == 0) {
      return "<empty>";
    }
    if (path.length == 1 && path[0].length() == 0) {
      return "<\"\">";
    }
    StringBuilder sb = new StringBuilder(path[0]);
    for (int i = 1; i < path.length; i++) {
      sb.append('/');
      sb.append(path[i]);
    }
    return sb.toString();
  }

  private String showcat(FacetLabel path) {
    if (path == null) {
      return "<null>";
    }
    if (path.length == 0) {
      return "<empty>";
    }
    return "<" + path + ">";
  }

  /**
   * Basic tests for TaxonomyWriter. Basically, we test that IndexWriter.addCategory works, i.e.
   * returns the expected ordinals (this is tested by calling the fillTaxonomy() method above). We
   * do not test here that after writing the index can be read - this will be done in more tests
   * below.
   */
  @Test
  public void testWriter() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    // Also check TaxonomyWriter.getSize() - see that the taxonomy's size
    // is what we expect it to be.
    assertEquals(expectedCategories.length, tw.getSize());
    tw.close();
    indexDir.close();
  }

  /**
   * testWriterTwice is exactly like testWriter, except that after adding all the categories, we add
   * them again, and see that we get the same old ids again - not new categories.
   */
  @Test
  public void testWriterTwice() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    // run fillTaxonomy again - this will try to add the same categories
    // again, and check that we see the same ordinal paths again, not
    // different ones.
    fillTaxonomy(tw);
    // Let's check the number of categories again, to see that no
    // extraneous categories were created:
    assertEquals(expectedCategories.length, tw.getSize());
    tw.close();
    indexDir.close();
  }

  /**
   * testWriterTwice2 is similar to testWriterTwice, except that the index is closed and reopened
   * before attempting to write to it the same categories again. While testWriterTwice can get along
   * with writing and reading correctly just to the cache, testWriterTwice2 checks also the actual
   * disk read part of the writer:
   */
  @Test
  public void testWriterTwice2() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    tw = new DirectoryTaxonomyWriter(indexDir);
    // run fillTaxonomy again - this will try to add the same categories
    // again, and check that we see the same ordinals again, not different
    // ones, and that the number of categories hasn't grown by the new
    // additions
    fillTaxonomy(tw);
    assertEquals(expectedCategories.length, tw.getSize());
    tw.close();
    indexDir.close();
  }

  /**
   * testWriterTwice3 is yet another test which tests creating a taxonomy in two separate writing
   * sessions. This test used to fail because of a bug involving commit(), explained below, and now
   * should succeed.
   */
  @Test
  public void testWriterTwice3() throws Exception {
    Directory indexDir = newDirectory();
    // First, create and fill the taxonomy
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    // Now, open the same taxonomy and add the same categories again.
    // After a few categories, the LuceneTaxonomyWriter implementation
    // will stop looking for each category on disk, and rather read them
    // all into memory and close its reader. The bug was that it closed
    // the reader, but forgot that it did (because it didn't set the reader
    // reference to null).
    tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    // Add one new category, just to make commit() do something:
    tw.addCategory(new FacetLabel("hi"));
    // Do a commit(). Here was a bug - if tw had a reader open, it should
    // be reopened after the commit. However, in our case the reader should
    // not be open (as explained above) but because it was not set to null,
    // we forgot that, tried to reopen it, and got an AlreadyClosedException.
    tw.commit();
    assertEquals(expectedCategories.length + 1, tw.getSize());
    tw.close();
    indexDir.close();
  }

  /**
   * Another set of tests for the writer, which don't use an array and try to distill the different
   * cases, and therefore may be more helpful for debugging a problem than testWriter() which is
   * hard to know why or where it failed.
   */
  @Test
  public void testWriterSimpler() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    assertEquals(1, tw.getSize()); // the root only
    // Test that adding a new top-level category works
    assertEquals(1, tw.addCategory(new FacetLabel("a")));
    assertEquals(2, tw.getSize());
    // Test that adding the same category again is noticed, and the
    // same ordinal (and not a new one) is returned.
    assertEquals(1, tw.addCategory(new FacetLabel("a")));
    assertEquals(2, tw.getSize());
    // Test that adding another top-level category returns a new ordinal,
    // not the same one
    assertEquals(2, tw.addCategory(new FacetLabel("b")));
    assertEquals(3, tw.getSize());
    // Test that adding a category inside one of the above adds just one
    // new ordinal:
    assertEquals(3, tw.addCategory(new FacetLabel("a", "c")));
    assertEquals(4, tw.getSize());
    // Test that adding the same second-level category doesn't do anything:
    assertEquals(3, tw.addCategory(new FacetLabel("a", "c")));
    assertEquals(4, tw.getSize());
    // Test that adding a second-level category with two new components
    // indeed adds two categories
    assertEquals(5, tw.addCategory(new FacetLabel("d", "e")));
    assertEquals(6, tw.getSize());
    // Verify that the parents were added above in the order we expected
    assertEquals(4, tw.addCategory(new FacetLabel("d")));
    // Similar, but inside a category that already exists:
    assertEquals(7, tw.addCategory(new FacetLabel("b", "d", "e")));
    assertEquals(8, tw.getSize());
    // And now inside two levels of categories that already exist:
    assertEquals(8, tw.addCategory(new FacetLabel("b", "d", "f")));
    assertEquals(9, tw.getSize());

    tw.close();
    indexDir.close();
  }

  /**
   * Test writing an empty index, and seeing that a reader finds in it the root category, and only
   * it. We check all the methods on that root category return the expected results.
   */
  @Test
  public void testRootOnly() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    // right after opening the index, it should already contain the
    // root, so have size 1:
    assertEquals(1, tw.getSize());
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    assertEquals(1, tr.getSize());
    assertEquals(0, tr.getPath(0).length);
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParallelTaxonomyArrays().parents().get(0));
    assertEquals(0, tr.getOrdinal(new FacetLabel()));
    tr.close();
    indexDir.close();
  }

  /**
   * The following test is exactly the same as testRootOnly, except we do not close the writer
   * before opening the reader. We want to see that the root is visible to the reader not only after
   * the writer is closed, but immediately after it is created.
   */
  @Test
  public void testRootOnly2() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    assertEquals(1, tr.getSize());
    assertEquals(0, tr.getPath(0).length);
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getParallelTaxonomyArrays().parents().get(0));
    assertEquals(0, tr.getOrdinal(new FacetLabel()));
    tw.close();
    tr.close();
    indexDir.close();
  }

  /**
   * Basic tests for TaxonomyReader's category &lt;=&gt; ordinal transformations (getSize(),
   * getCategory() and getOrdinal()). We test that after writing the index, it can be read and all
   * the categories and ordinals are there just as we expected them to be.
   */
  @Test
  public void testReaderBasic() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    // test TaxonomyReader.getSize():
    assertEquals(expectedCategories.length, tr.getSize());

    // test round trips of ordinal => category => ordinal
    for (int i = 0; i < tr.getSize(); i++) {
      assertEquals(i, tr.getOrdinal(tr.getPath(i)));
    }

    // test TaxonomyReader.getCategory():
    for (int i = 1; i < tr.getSize(); i++) {
      FacetLabel expectedCategory = new FacetLabel(expectedCategories[i]);
      FacetLabel category = tr.getPath(i);
      if (!expectedCategory.equals(category)) {
        fail(
            "For ordinal "
                + i
                + " expected category "
                + showcat(expectedCategory)
                + ", but got "
                + showcat(category));
      }
    }
    //  (also test invalid ordinals:)
    expectThrows(IllegalArgumentException.class, () -> tr.getPath(-1));
    expectThrows(IllegalArgumentException.class, () -> tr.getPath(tr.getSize()));
    expectThrows(IllegalArgumentException.class, () -> tr.getPath(TaxonomyReader.INVALID_ORDINAL));

    // test TaxonomyReader.getOrdinal():
    for (int i = 1; i < expectedCategories.length; i++) {
      int expectedOrdinal = i;
      int ordinal = tr.getOrdinal(new FacetLabel(expectedCategories[i]));
      if (expectedOrdinal != ordinal) {
        fail(
            "For category "
                + showcat(expectedCategories[i])
                + " expected ordinal "
                + expectedOrdinal
                + ", but got "
                + ordinal);
      }
    }
    // (also test invalid categories:)
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(new FacetLabel("non-existant")));
    assertEquals(
        TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(new FacetLabel("Author", "Jules Verne")));

    tr.close();
    indexDir.close();
  }

  /**
   * Tests for TaxonomyReader's getParent() method. We check it by comparing its results to those we
   * could have gotten by looking at the category string paths (where the parentage is obvious).
   * Note that after testReaderBasic(), we already know we can trust the ordinal &lt;=&gt; category
   * conversions.
   *
   * <p>Note: At the moment, the parent methods in the reader are deprecated, but this does not mean
   * they should not be tested! Until they are removed (*if* they are removed), these tests should
   * remain to see that they still work correctly.
   */
  @Test
  public void testReaderParent() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    // check that the parent of the root ordinal is the invalid ordinal:
    ParallelTaxonomyArrays.IntArray parents = tr.getParallelTaxonomyArrays().parents();
    assertEquals(TaxonomyReader.INVALID_ORDINAL, parents.get(0));

    // check parent of non-root ordinals:
    for (int ordinal = 1; ordinal < tr.getSize(); ordinal++) {
      FacetLabel me = tr.getPath(ordinal);
      int parentOrdinal = parents.get(ordinal);
      FacetLabel parent = tr.getPath(parentOrdinal);
      if (parent == null) {
        fail(
            "Parent of "
                + ordinal
                + " is "
                + parentOrdinal
                + ", but this is not a valid category.");
      }
      // verify that the parent is indeed my parent, according to the strings
      if (!me.subpath(me.length - 1).equals(parent)) {
        fail(
            "Got parent "
                + parentOrdinal
                + " for ordinal "
                + ordinal
                + " but categories are "
                + showcat(parent)
                + " and "
                + showcat(me)
                + " respectively.");
      }
    }

    tr.close();
    indexDir.close();
  }

  /**
   * Tests for TaxonomyWriter's getParent() method. We check it by comparing its results to those we
   * could have gotten by looking at the category string paths using a TaxonomyReader (where the
   * parentage is obvious). Note that after testReaderBasic(), we already know we can trust the
   * ordinal &lt;=&gt; category conversions from TaxonomyReader.
   *
   * <p>The difference between testWriterParent1 and testWriterParent2 is that the former closes the
   * taxonomy writer before reopening it, while the latter does not.
   *
   * <p>This test code is virtually identical to that of testReaderParent().
   */
  @Test
  public void testWriterParent1() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    tw = new DirectoryTaxonomyWriter(indexDir);
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    checkWriterParent(tr, tw);

    tw.close();
    tr.close();
    indexDir.close();
  }

  @Test
  public void testWriterParent2() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    checkWriterParent(tr, tw);

    tw.close();
    tr.close();
    indexDir.close();
  }

  private void checkWriterParent(TaxonomyReader tr, TaxonomyWriter tw) throws Exception {
    // check that the parent of the root ordinal is the invalid ordinal:
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tw.getParent(0));

    // check parent of non-root ordinals:
    for (int ordinal = 1; ordinal < tr.getSize(); ordinal++) {
      FacetLabel me = tr.getPath(ordinal);
      int parentOrdinal = tw.getParent(ordinal);
      FacetLabel parent = tr.getPath(parentOrdinal);
      if (parent == null) {
        fail(
            "Parent of "
                + ordinal
                + " is "
                + parentOrdinal
                + ", but this is not a valid category.");
      }
      // verify that the parent is indeed my parent, according to the
      // strings
      if (!me.subpath(me.length - 1).equals(parent)) {
        fail(
            "Got parent "
                + parentOrdinal
                + " for ordinal "
                + ordinal
                + " but categories are "
                + showcat(parent)
                + " and "
                + showcat(me)
                + " respectively.");
      }
    }

    // check parent of of invalid ordinals:
    expectThrows(IndexOutOfBoundsException.class, () -> tw.getParent(-1));
    expectThrows(
        IndexOutOfBoundsException.class, () -> tw.getParent(TaxonomyReader.INVALID_ORDINAL));
    expectThrows(IndexOutOfBoundsException.class, () -> tw.getParent(tr.getSize()));
  }

  /**
   * Test TaxonomyReader's child browsing method, getChildrenArrays() This only tests for
   * correctness of the data on one example - we have below further tests on data refresh etc.
   */
  @Test
  public void testChildrenArrays() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    ParallelTaxonomyArrays ca = tr.getParallelTaxonomyArrays();
    ParallelTaxonomyArrays.IntArray youngestChildArray = ca.children();
    assertEquals(tr.getSize(), youngestChildArray.length());
    ParallelTaxonomyArrays.IntArray olderSiblingArray = ca.siblings();
    assertEquals(tr.getSize(), olderSiblingArray.length());
    for (int i = 0; i < expectedCategories.length; i++) {
      // find expected children by looking at all expectedCategories
      // for children
      ArrayList<Integer> expectedChildren = new ArrayList<>();
      for (int j = expectedCategories.length - 1; j >= 0; j--) {
        if (expectedCategories[j].length != expectedCategories[i].length + 1) {
          continue; // not longer by 1, so can't be a child
        }
        boolean ischild = true;
        for (int k = 0; k < expectedCategories[i].length; k++) {
          if (!expectedCategories[j][k].equals(expectedCategories[i][k])) {
            ischild = false;
            break;
          }
        }
        if (ischild) {
          expectedChildren.add(j);
        }
      }
      // check that children and expectedChildren are the same, with the
      // correct reverse (youngest to oldest) order:
      if (expectedChildren.size() == 0) {
        assertEquals(TaxonomyReader.INVALID_ORDINAL, youngestChildArray.get(i));
      } else {
        int child = youngestChildArray.get(i);
        assertEquals(expectedChildren.get(0).intValue(), child);
        for (int j = 1; j < expectedChildren.size(); j++) {
          child = olderSiblingArray.get(child);
          assertEquals(expectedChildren.get(j).intValue(), child);
          // if child is INVALID_ORDINAL we should stop, but
          // assertEquals would fail in this case anyway.
        }
        // When we're done comparing, olderSiblingArray should now point
        // to INVALID_ORDINAL, saying there are no more children. If it
        // doesn't, we found too many children...
        assertEquals(-1, olderSiblingArray.get(child));
      }
    }
    tr.close();
    indexDir.close();
  }

  /**
   * Similar to testChildrenArrays, except rather than look at expected results, we test for several
   * "invariants" that the results should uphold, e.g., that a child of a category indeed has this
   * category as its parent. This sort of test can more easily be extended to larger example
   * taxonomies, because we do not need to build the expected list of categories like we did in the
   * above test.
   */
  @Test
  public void testChildrenArraysInvariants() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    ParallelTaxonomyArrays ca = tr.getParallelTaxonomyArrays();
    ParallelTaxonomyArrays.IntArray children = ca.children();
    assertEquals(tr.getSize(), children.length());
    ParallelTaxonomyArrays.IntArray olderSiblingArray = ca.siblings();
    assertEquals(tr.getSize(), olderSiblingArray.length());

    // test that the "youngest child" of every category is indeed a child:
    ParallelTaxonomyArrays.IntArray parents = tr.getParallelTaxonomyArrays().parents();
    for (int i = 0; i < tr.getSize(); i++) {
      int youngestChild = children.get(i);
      if (youngestChild != TaxonomyReader.INVALID_ORDINAL) {
        assertEquals(i, parents.get(youngestChild));
      }
    }

    // test that the "older sibling" of every category is indeed older (lower)
    // (it can also be INVALID_ORDINAL, which is lower than any ordinal)
    for (int i = 0; i < tr.getSize(); i++) {
      assertTrue("olderSiblingArray[" + i + "] should be <" + i, olderSiblingArray.get(i) < i);
    }

    // test that the "older sibling" of every category is indeed a sibling
    // (they share the same parent)
    for (int i = 0; i < tr.getSize(); i++) {
      int sibling = olderSiblingArray.get(i);
      if (sibling == TaxonomyReader.INVALID_ORDINAL) {
        continue;
      }
      assertEquals(parents.get(i), parents.get(sibling));
    }

    // And now for slightly more complex (and less "invariant-like"...)
    // tests:

    // test that the "youngest child" is indeed the youngest (so we don't
    // miss the first children in the chain)
    for (int i = 0; i < tr.getSize(); i++) {
      // Find the really youngest child:
      int j;
      for (j = tr.getSize() - 1; j > i; j--) {
        if (parents.get(j) == i) {
          break; // found youngest child
        }
      }
      if (j == i) { // no child found
        j = TaxonomyReader.INVALID_ORDINAL;
      }
      assertEquals(j, children.get(i));
    }

    // test that the "older sibling" is indeed the least oldest one - and
    // not a too old one or -1 (so we didn't miss some children in the
    // middle or the end of the chain).
    for (int i = 0; i < tr.getSize(); i++) {
      // Find the youngest older sibling:
      int j;
      for (j = i - 1; j >= 0; j--) {
        if (parents.get(j) == parents.get(i)) {
          break; // found youngest older sibling
        }
      }
      if (j < 0) { // no sibling found
        j = TaxonomyReader.INVALID_ORDINAL;
      }
      assertEquals(j, olderSiblingArray.get(i));
    }

    tr.close();
    indexDir.close();
  }

  private static void assertArrayEquals(int[] expected, ParallelTaxonomyArrays.IntArray actual) {
    for (int i = 0; i < expected.length; i++) {
      assertEquals(expected[i], actual.get(i));
    }
  }

  /** Test how getChildrenArrays() deals with the taxonomy's growth: */
  @Test
  public void testChildrenArraysGrowth() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.addCategory(new FacetLabel("hi", "there"));
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    ParallelTaxonomyArrays ca = tr.getParallelTaxonomyArrays();
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.siblings().length());
    assertEquals(3, ca.children().length());
    assertArrayEquals(new int[] {1, 2, -1}, ca.children());
    assertArrayEquals(new int[] {-1, -1, -1}, ca.siblings());
    tw.addCategory(new FacetLabel("hi", "ho"));
    tw.addCategory(new FacetLabel("hello"));
    tw.commit();
    // Before refresh, nothing changed..
    ParallelTaxonomyArrays newca = tr.getParallelTaxonomyArrays();
    assertSame(newca, ca); // we got exactly the same object
    assertEquals(3, tr.getSize());
    assertEquals(3, ca.siblings().length());
    assertEquals(3, ca.children().length());
    // After the refresh, things change:
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newtr);
    tr.close();
    tr = newtr;
    ca = tr.getParallelTaxonomyArrays();
    assertEquals(5, tr.getSize());
    assertEquals(5, ca.siblings().length());
    assertEquals(5, ca.children().length());
    assertArrayEquals(new int[] {4, 3, -1, -1, -1}, ca.children());
    assertArrayEquals(new int[] {-1, -1, -1, 2, 1}, ca.siblings());
    tw.close();
    tr.close();
    indexDir.close();
  }

  // Test that getParentArrays is valid when retrieved during refresh
  @Test
  public void testTaxonomyReaderRefreshRaces() throws Exception {
    // compute base child arrays - after first chunk, and after the other
    Directory indexDirBase = newDirectory();
    TaxonomyWriter twBase = new DirectoryTaxonomyWriter(indexDirBase);
    twBase.addCategory(new FacetLabel("a", "0"));
    final FacetLabel abPath = new FacetLabel("a", "b");
    twBase.addCategory(abPath);
    twBase.commit();
    TaxonomyReader trBase = new DirectoryTaxonomyReader(indexDirBase);

    final ParallelTaxonomyArrays ca1 = trBase.getParallelTaxonomyArrays();

    final int abOrd = trBase.getOrdinal(abPath);
    final int abYoungChildBase1 = ca1.children().get(abOrd);

    final int numCategories = atLeast(200);
    for (int i = 0; i < numCategories; i++) {
      twBase.addCategory(new FacetLabel("a", "b", Integer.toString(i)));
    }
    twBase.close();

    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(trBase);
    assertNotNull(newTaxoReader);
    trBase.close();
    trBase = newTaxoReader;

    final ParallelTaxonomyArrays ca2 = trBase.getParallelTaxonomyArrays();
    final int abYoungChildBase2 = ca2.children().get(abOrd);

    int numRetries = atLeast(10);
    for (int retry = 0; retry < numRetries; retry++) {
      assertConsistentYoungestChild(
          abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, numCategories);
    }

    trBase.close();
    indexDirBase.close();
  }

  @SuppressForbidden(reason = "Thread sleep")
  private void assertConsistentYoungestChild(
      final FacetLabel abPath,
      final int abOrd,
      final int abYoungChildBase1,
      final int abYoungChildBase2,
      final int retry,
      int numCategories)
      throws Exception {
    SlowDirectory indexDir = new SlowDirectory(-1, null); // no slowness for initialization
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.addCategory(new FacetLabel("a", "0"));
    tw.addCategory(abPath);
    tw.commit();

    final DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    for (int i = 0; i < numCategories; i++) {
      final FacetLabel cp = new FacetLabel("a", "b", Integer.toString(i));
      tw.addCategory(cp);
      assertEquals(
          "Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed",
          TaxonomyReader.INVALID_ORDINAL,
          tr.getOrdinal(cp));
    }
    tw.close();

    final AtomicBoolean stop = new AtomicBoolean(false);
    final Throwable[] error = new Throwable[] {null};
    final int[] retrieval = {0};

    Thread thread =
        new Thread("Child Arrays Verifier") {
          @SuppressForbidden(reason = "Thread sleep")
          @Override
          public void run() {
            setPriority(1 + getPriority());
            try {
              while (!stop.get()) {
                int lastOrd = tr.getParallelTaxonomyArrays().parents().length() - 1;
                assertNotNull(
                    "path of last-ord " + lastOrd + " is not found!", tr.getPath(lastOrd));
                assertChildrenArrays(tr.getParallelTaxonomyArrays(), retry, retrieval[0]++);
                sleep(10); // don't starve refresh()'s CPU, which sleeps every 50 bytes for 1 ms
              }
            } catch (Throwable e) {
              error[0] = e;
              stop.set(true);
            }
          }

          private void assertChildrenArrays(ParallelTaxonomyArrays ca, int retry, int retrieval) {
            final int abYoungChild = ca.children().get(abOrd);
            assertTrue(
                "Retry "
                    + retry
                    + ": retrieval: "
                    + retrieval
                    + ": wrong youngest child for category "
                    + abPath
                    + " (ord="
                    + abOrd
                    + ") - must be either "
                    + abYoungChildBase1
                    + " or "
                    + abYoungChildBase2
                    + " but was: "
                    + abYoungChild,
                abYoungChildBase1 == abYoungChild || abYoungChildBase2 == ca.children().get(abOrd));
          }
        };
    thread.start();

    indexDir.setSleepMillis(1); // some delay for refresh
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
    if (newTaxoReader != null) {
      newTaxoReader.close();
    }

    stop.set(true);
    thread.join();
    assertNull(
        "Unexpcted exception at retry "
            + retry
            + " retrieval "
            + retrieval[0]
            + ": \n"
            + stackTraceStr(error[0]),
        error[0]);

    tr.close();
  }

  /**
   * Grab the stack trace into a string since the exception was thrown in a thread and we want the
   * assert outside the thread to show the stack trace in case of failure.
   */
  private String stackTraceStr(final Throwable error) {
    if (error == null) {
      return "";
    }
    StringWriter sw = new StringWriter();
    PrintWriter pw = new PrintWriter(sw);
    error.printStackTrace(pw);
    pw.close();
    return sw.toString();
  }

  /**
   * Test that if separate reader and writer objects are opened, new categories written into the
   * writer are available to a reader only after a commit(). Note that this test obviously doesn't
   * cover all the different concurrency scenarios, all different methods, and so on. We may want to
   * write more tests of this sort.
   *
   * <p>This test simulates what would happen when there are two separate processes, one doing
   * indexing, and the other searching, and each opens its own object (with obviously no connection
   * between the objects) using the same disk files. Note, though, that this test does not test what
   * happens when the two processes do their actual work at exactly the same time. It also doesn't
   * test multi-threading.
   */
  @Test
  public void testSeparateReaderAndWriter() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    assertEquals(1, tr.getSize()); // the empty taxonomy has size 1 (the root)
    tw.addCategory(new FacetLabel("Author"));
    assertEquals(1, tr.getSize()); // still root only...
    assertNull(
        TaxonomyReader.openIfChanged(
            tr)); // this is not enough, because tw.commit() hasn't been done yet
    assertEquals(1, tr.getSize()); // still root only...
    tw.commit();
    assertEquals(1, tr.getSize()); // still root only...
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newTaxoReader);
    tr.close();
    tr = newTaxoReader;

    int author = 1;
    try {
      assertEquals(
          TaxonomyReader.ROOT_ORDINAL, tr.getParallelTaxonomyArrays().parents().get(author));
      // ok
    } catch (
        @SuppressWarnings("unused")
        ArrayIndexOutOfBoundsException e) {
      fail(
          "After category addition, commit() and refresh(), getParent for "
              + author
              + " should NOT throw exception");
    }
    assertEquals(2, tr.getSize()); // finally, see there are two categories

    // now, add another category, and verify that after commit and refresh
    // the parent of this category is correct (this requires the reader
    // to correctly update its prefetched parent vector), and that the
    // old information also wasn't ruined:
    tw.addCategory(new FacetLabel("Author", "Richard Dawkins"));
    int dawkins = 2;
    tw.commit();
    newTaxoReader = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newTaxoReader);
    tr.close();
    tr = newTaxoReader;
    ParallelTaxonomyArrays.IntArray parents = tr.getParallelTaxonomyArrays().parents();
    assertEquals(author, parents.get(dawkins));
    assertEquals(TaxonomyReader.ROOT_ORDINAL, parents.get(author));
    assertEquals(TaxonomyReader.INVALID_ORDINAL, parents.get(TaxonomyReader.ROOT_ORDINAL));
    assertEquals(3, tr.getSize());
    tw.close();
    tr.close();
    indexDir.close();
  }

  @Test
  public void testSeparateReaderAndWriter2() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);

    // Test getOrdinal():
    FacetLabel author = new FacetLabel("Author");

    assertEquals(1, tr.getSize()); // the empty taxonomy has size 1 (the root)
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(author));
    tw.addCategory(author);
    // before commit and refresh, no change:
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(author));
    assertEquals(1, tr.getSize()); // still root only...
    assertNull(
        TaxonomyReader.openIfChanged(
            tr)); // this is not enough, because tw.commit() hasn't been done yet
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(author));
    assertEquals(1, tr.getSize()); // still root only...
    tw.commit();
    // still not enough before refresh:
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(author));
    assertEquals(1, tr.getSize()); // still root only...
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newTaxoReader);
    tr.close();
    tr = newTaxoReader;
    assertEquals(1, tr.getOrdinal(author));
    assertEquals(2, tr.getSize());
    tw.close();
    tr.close();
    indexDir.close();
  }

  /**
   * fillTaxonomyCheckPaths adds the categories in the categories[] array, and asserts that the
   * additions return exactly paths specified in expectedPaths[]. This is the same add
   * fillTaxonomy() but also checks the correctness of getParent(), not just addCategory(). Note
   * that this assumes that fillTaxonomyCheckPaths() is called on an empty taxonomy index. Calling
   * it after something else was already added to the taxonomy index will surely have this method
   * fail.
   */
  public static void fillTaxonomyCheckPaths(TaxonomyWriter tw) throws IOException {
    for (int i = 0; i < categories.length; i++) {
      int ordinal = tw.addCategory(new FacetLabel(categories[i]));
      int expectedOrdinal = expectedPaths[i][expectedPaths[i].length - 1];
      if (ordinal != expectedOrdinal) {
        fail(
            "For category "
                + showcat(categories[i])
                + " expected ordinal "
                + expectedOrdinal
                + ", but got "
                + ordinal);
      }
      for (int j = expectedPaths[i].length - 2; j >= 0; j--) {
        ordinal = tw.getParent(ordinal);
        expectedOrdinal = expectedPaths[i][j];
        if (ordinal != expectedOrdinal) {
          fail(
              "For category "
                  + showcat(categories[i])
                  + " expected ancestor level "
                  + (expectedPaths[i].length - 1 - j)
                  + " was "
                  + expectedOrdinal
                  + ", but got "
                  + ordinal);
        }
      }
    }
  }

  // After fillTaxonomy returned successfully, checkPaths() checks that
  // the getParent() calls return as expected, from the table
  public static void checkPaths(TaxonomyWriter tw) throws IOException {
    for (int i = 0; i < categories.length; i++) {
      int ordinal = expectedPaths[i][expectedPaths[i].length - 1];
      for (int j = expectedPaths[i].length - 2; j >= 0; j--) {
        ordinal = tw.getParent(ordinal);
        int expectedOrdinal = expectedPaths[i][j];
        if (ordinal != expectedOrdinal) {
          fail(
              "For category "
                  + showcat(categories[i])
                  + " expected ancestor level "
                  + (expectedPaths[i].length - 1 - j)
                  + " was "
                  + expectedOrdinal
                  + ", but got "
                  + ordinal);
        }
      }
      assertEquals(TaxonomyReader.ROOT_ORDINAL, tw.getParent(expectedPaths[i][0]));
    }
    assertEquals(TaxonomyReader.INVALID_ORDINAL, tw.getParent(TaxonomyReader.ROOT_ORDINAL));
  }

  /**
   * Basic test for TaxonomyWriter.getParent(). This is similar to testWriter above, except we also
   * check the parents of the added categories, not just the categories themselves.
   */
  @Test
  public void testWriterCheckPaths() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomyCheckPaths(tw);
    // Also check TaxonomyWriter.getSize() - see that the taxonomy's size
    // is what we expect it to be.
    assertEquals(expectedCategories.length, tw.getSize());
    tw.close();
    indexDir.close();
  }

  /**
   * testWriterCheckPaths2 is the path-checking variant of testWriterTwice and testWriterTwice2.
   * After adding all the categories, we add them again, and see that we get the same old ids and
   * paths. We repeat the path checking yet again after closing and opening the index for writing
   * again - to see that the reading of existing data from disk works as well.
   */
  @Test
  public void testWriterCheckPaths2() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    checkPaths(tw);
    fillTaxonomy(tw);
    checkPaths(tw);
    tw.close();

    tw = new DirectoryTaxonomyWriter(indexDir);
    checkPaths(tw);
    fillTaxonomy(tw);
    checkPaths(tw);
    tw.close();
    indexDir.close();
  }

  @Test
  public void testNRT() throws Exception {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    TaxonomyReader reader = new DirectoryTaxonomyReader(writer);

    FacetLabel cp = new FacetLabel("a");
    writer.addCategory(cp);
    TaxonomyReader newReader = TaxonomyReader.openIfChanged(reader);
    assertNotNull("expected a new instance", newReader);
    assertEquals(2, newReader.getSize());
    assertNotSame(TaxonomyReader.INVALID_ORDINAL, newReader.getOrdinal(cp));
    reader.close();
    reader = newReader;

    writer.close();
    reader.close();

    dir.close();
  }

  private static String[][] manyCategories(int count, int roundSize) {
    String[][] result = new String[count / roundSize + 1][];
    int k = 0;
    do {
      k += roundSize;
      List<String> round = new ArrayList<>();
      for (int i = k - roundSize + 1; i <= k && i < count; i++) {
        round.add(String.format(Locale.ROOT, "category %d of %d", i, k));
      }
      result[k / roundSize - 1] = round.toArray(String[]::new);
    } while (k <= count);
    return result;
  }

  public void testThousandsOfCategories() throws IOException {
    int roundSize = random().nextInt(2, 4);
    int size = random().nextInt(16384, 32768);
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    String[][] manyCategories = manyCategories(size, roundSize);
    for (String[] elem : manyCategories) {
      if (elem == null) {
        throw new IllegalStateException(
            "Got null array with size = " + size + " and roundSize = " + roundSize);
      } else if (elem.length > 0) {
        tw.addCategory(new FacetLabel(elem));
      }
    }
    tw.close();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    ParallelTaxonomyArrays ca = tr.getParallelTaxonomyArrays();
    ParallelTaxonomyArrays.IntArray parents = ca.parents();
    ParallelTaxonomyArrays.IntArray children = ca.children();
    assertEquals(size, parents.length());
    assertEquals(size, children.length());
    for (int j = 1; j < size - roundSize; j += roundSize) {
      // Top level categories all have root as their parent.
      assertEquals(0, parents.get(j));
      for (int i = j; i < j + roundSize - 1; i++) {
        // Children extend in a chain from the top level category.
        // The parent/child relationships are symmetric.
        assertEquals(i + 1, children.get(i));
        if (i > j) {
          assertEquals(i - 1, parents.get(i));
        }
      }
    }
    ParallelTaxonomyArrays.IntArray siblings = ca.siblings();
    assertEquals(size, siblings.length());
    for (int i = 1; i < size - roundSize; i += roundSize) {
      // Each top-level category (after the first) has the previous top-level category as their
      // older sibling.
      assertEquals(i, siblings.get(i + roundSize));
    }
    tr.close();
    indexDir.close();
  }

  //  TODO (Facet): test multiple readers, one writer. Have the multiple readers
  //  using the same object (simulating threads) or different objects
  //  (simulating processes).
}
