/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.tests.index;

import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.function.LongSupplier;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;

/**
 * Abstract class to do basic tests for a norms format. NOTE: This test focuses on the norms impl,
 * nothing else. The [stretch] goal is for this test to be so thorough in testing a new NormsFormat
 * that if this test passes, then all Lucene tests should also pass. Ie, if there is some bug in a
 * given NormsFormat that this test fails to catch then this test needs to be improved!
 */
public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCase {

  /** Whether the codec supports sparse values. */
  protected boolean codecSupportsSparsity() {
    return true;
  }

  public void testByteRange() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
            }
          });
    }
  }

  public void testSparseByteRange() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
            }
          });
    }
  }

  public void testShortRange() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Short.MIN_VALUE, Short.MAX_VALUE);
            }
          });
    }
  }

  public void testSparseShortRange() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Short.MIN_VALUE, Short.MAX_VALUE);
            }
          });
    }
  }

  public void testLongRange() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE);
            }
          });
    }
  }

  public void testSparseLongRange() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE);
            }
          });
    }
  }

  public void testFullLongRange() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              int thingToDo = r.nextInt(3);
              switch (thingToDo) {
                case 0:
                  return Long.MIN_VALUE;
                case 1:
                  return Long.MAX_VALUE;
                default:
                  return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE);
              }
            }
          });
    }
  }

  public void testSparseFullLongRange() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              int thingToDo = r.nextInt(3);
              switch (thingToDo) {
                case 0:
                  return Long.MIN_VALUE;
                case 1:
                  return Long.MAX_VALUE;
                default:
                  return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE);
              }
            }
          });
    }
  }

  public void testFewValues() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextBoolean() ? 20 : 3;
            }
          });
    }
  }

  public void testFewSparseValues() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextBoolean() ? 20 : 3;
            }
          });
    }
  }

  public void testFewLargeValues() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextBoolean() ? 1000000L : -5000;
            }
          });
    }
  }

  public void testFewSparseLargeValues() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextBoolean() ? 1000000L : -5000;
            }
          });
    }
  }

  public void testAllZeros() throws Exception {
    int iterations = atLeast(1);
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return 0;
            }
          });
    }
  }

  public void testSparseAllZeros() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return 0;
            }
          });
    }
  }

  public void testMostZeros() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : 0;
            }
          });
    }
  }

  public void testOutliers() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextInt(100) == 0
                  ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE)
                  : commonValue;
            }
          });
    }
  }

  public void testSparseOutliers() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextInt(100) == 0
                  ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE)
                  : commonValue;
            }
          });
    }
  }

  public void testOutliers2() throws Exception {
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      final long uncommonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      doTestNormsVersusDocValues(
          1,
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextInt(100) == 0 ? uncommonValue : commonValue;
            }
          });
    }
  }

  public void testSparseOutliers2() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; i++) {
      final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      final long uncommonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
      doTestNormsVersusDocValues(
          random().nextDouble(),
          new LongSupplier() {
            @Override
            public long getAsLong() {
              return r.nextInt(100) == 0 ? uncommonValue : commonValue;
            }
          });
    }
  }

  public void testNCommon() throws Exception {
    final Random r = random();
    final int N = TestUtil.nextInt(r, 2, 15);
    final long[] commonValues = new long[N];
    for (int j = 0; j < N; ++j) {
      commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
    }
    final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N);
    final long[] otherValues = new long[numOtherValues];
    for (int j = 0; j < numOtherValues; ++j) {
      otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
    }
    doTestNormsVersusDocValues(
        1,
        new LongSupplier() {
          @Override
          public long getAsLong() {
            return r.nextInt(100) == 0
                ? otherValues[r.nextInt(numOtherValues - 1)]
                : commonValues[r.nextInt(N - 1)];
          }
        });
  }

  public void testSparseNCommon() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    final Random r = random();
    final int N = TestUtil.nextInt(r, 2, 15);
    final long[] commonValues = new long[N];
    for (int j = 0; j < N; ++j) {
      commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
    }
    final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N);
    final long[] otherValues = new long[numOtherValues];
    for (int j = 0; j < numOtherValues; ++j) {
      otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
    }
    doTestNormsVersusDocValues(
        random().nextDouble(),
        new LongSupplier() {
          @Override
          public long getAsLong() {
            return r.nextInt(100) == 0
                ? otherValues[r.nextInt(numOtherValues - 1)]
                : commonValues[r.nextInt(N - 1)];
          }
        });
  }

  /** a more thorough n-common that tests all low bpv */
  @Nightly
  public void testNCommonBig() throws Exception {
    final int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; ++i) {
      // 16 is 4 bpv, the max before we jump to 8bpv
      for (int n = 2; n < 16; ++n) {
        final int N = n;
        final long[] commonValues = new long[N];
        for (int j = 0; j < N; ++j) {
          commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
        }
        final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N);
        final long[] otherValues = new long[numOtherValues];
        for (int j = 0; j < numOtherValues; ++j) {
          otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
        }
        doTestNormsVersusDocValues(
            1,
            new LongSupplier() {
              @Override
              public long getAsLong() {
                return r.nextInt(100) == 0
                    ? otherValues[r.nextInt(numOtherValues - 1)]
                    : commonValues[r.nextInt(N - 1)];
              }
            });
      }
    }
  }

  /** a more thorough n-common that tests all low bpv and sparse docs */
  @Nightly
  public void testSparseNCommonBig() throws Exception {
    assumeTrue("Requires sparse norms support", codecSupportsSparsity());
    final int iterations = atLeast(1);
    final Random r = random();
    for (int i = 0; i < iterations; ++i) {
      // 16 is 4 bpv, the max before we jump to 8bpv
      for (int n = 2; n < 16; ++n) {
        final int N = n;
        final long[] commonValues = new long[N];
        for (int j = 0; j < N; ++j) {
          commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
        }
        final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N);
        final long[] otherValues = new long[numOtherValues];
        for (int j = 0; j < numOtherValues; ++j) {
          otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
        }
        doTestNormsVersusDocValues(
            random().nextDouble(),
            new LongSupplier() {
              @Override
              public long getAsLong() {
                return r.nextInt(100) == 0
                    ? otherValues[r.nextInt(numOtherValues - 1)]
                    : commonValues[r.nextInt(N - 1)];
              }
            });
      }
    }
  }

  private void doTestNormsVersusDocValues(double density, LongSupplier longs) throws Exception {
    int numDocs = atLeast(500);
    final FixedBitSet docsWithField = new FixedBitSet(numDocs);
    final int numDocsWithField = Math.max(1, (int) (density * numDocs));
    if (numDocsWithField == numDocs) {
      docsWithField.set(0, numDocs);
    } else {
      int i = 0;
      while (i < numDocsWithField) {
        int doc = random().nextInt(numDocs);
        if (docsWithField.get(doc) == false) {
          docsWithField.set(doc);
          ++i;
        }
      }
    }
    long[] norms = new long[numDocsWithField];
    for (int i = 0; i < numDocsWithField; i++) {
      norms[i] = longs.getAsLong();
    }

    Directory dir = applyCreatedVersionMajor(newDirectory());
    Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
    IndexWriterConfig conf = newIndexWriterConfig(analyzer);
    CannedNormSimilarity sim = new CannedNormSimilarity(norms);
    conf.setSimilarity(sim);
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    Document doc = new Document();
    Field idField = new StringField("id", "", Field.Store.NO);
    Field indexedField = new TextField("indexed", "", Field.Store.NO);
    Field dvField = new NumericDocValuesField("dv", 0);
    doc.add(idField);
    doc.add(indexedField);
    doc.add(dvField);

    for (int i = 0, j = 0; i < numDocs; i++) {
      idField.setStringValue(Integer.toString(i));
      if (docsWithField.get(i) == false) {
        Document doc2 = new Document();
        doc2.add(idField);
        writer.addDocument(doc2);
      } else {
        long value = norms[j++];
        dvField.setLongValue(value);
        // only empty fields may have 0 as a norm
        indexedField.setStringValue(value == 0 ? "" : "a");
        writer.addDocument(doc);
      }
      if (random().nextInt(31) == 0) {
        writer.commit();
      }
    }

    // delete some docs
    int numDeletions = random().nextInt(numDocs / 20);
    for (int i = 0; i < numDeletions; i++) {
      int id = random().nextInt(numDocs);
      writer.deleteDocuments(new Term("id", Integer.toString(id)));
    }

    writer.commit();

    // compare
    DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    checkNormsVsDocValues(ir);
    ir.close();

    writer.forceMerge(1);

    // compare again
    ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    checkNormsVsDocValues(ir);

    writer.close();
    ir.close();
    dir.close();
  }

  private void checkNormsVsDocValues(IndexReader ir) throws IOException {
    for (LeafReaderContext context : ir.leaves()) {
      LeafReader r = context.reader();
      NumericDocValues expected = r.getNumericDocValues("dv");
      NumericDocValues actual = r.getNormValues("indexed");
      assertEquals(expected == null, actual == null);
      if (expected != null) {
        for (int d = expected.nextDoc();
            d != DocIdSetIterator.NO_MORE_DOCS;
            d = expected.nextDoc()) {
          assertEquals(d, actual.nextDoc());
          assertEquals("doc " + d, expected.longValue(), actual.longValue());
        }
        assertEquals(NO_MORE_DOCS, actual.nextDoc());

        // Now check bulk fetching
        expected = r.getNumericDocValues("dv");
        actual = r.getNormValues("indexed");

        int[] docs = new int[16];
        long[] expectedValues = new long[16];
        long[] actualValues = new long[16];
        for (int doc = -1; doc < r.maxDoc(); ) {
          int size = 0;
          for (int j = 0; j < docs.length; ++j) {
            doc += 1 + (j & 0x03);
            if (doc >= r.maxDoc()) {
              break;
            }
            docs[size++] = doc;
          }

          expected.longValues(size, docs, expectedValues, size);
          actual.longValues(size, docs, actualValues, size);
          assertArrayEquals(expectedValues, actualValues);
        }
      }
    }
  }

  static class CannedNormSimilarity extends Similarity {
    final long[] norms;
    int index = 0;

    CannedNormSimilarity(long[] norms) {
      this.norms = norms;
    }

    @Override
    public long computeNorm(FieldInvertState state) {
      assert state.getLength() > 0;
      while (true) {
        long norm = norms[index++];
        if (norm != 0) {
          return norm;
        }
      }
    }

    @Override
    public SimScorer scorer(
        float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
      throw new UnsupportedOperationException();
    }
  }

  @Override
  protected void addRandomFields(Document doc) {
    // TODO: improve
    doc.add(new TextField("foobar", TestUtil.randomSimpleString(random()), Field.Store.NO));
  }

  @Override
  public void testMergeStability() throws Exception {
    // TODO: can we improve this base test to just have subclasses declare the extensions to check,
    // rather than a blacklist to exclude? we need to index stuff to get norms, but we dont care
    // about testing
    // the PFs actually doing that...
    assumeTrue("The MockRandom PF randomizes content on the fly, so we can't check it", false);
  }

  // TODO: test thread safety (e.g. across different fields) explicitly here

  /*
   * LUCENE-6006: Tests undead norms.
   *                                 .....
   *                             C C  /
   *                            /<   /
   *             ___ __________/_#__=o
   *            /(- /(\_\________   \
   *            \ ) \ )_      \o     \
   *            /|\ /|\       |'     |
   *                          |     _|
   *                          /o   __\
   *                         / '     |
   *                        / /      |
   *                       /_/\______|
   *                      (   _(    <
   *                       \    \    \
   *                        \    \    |
   *                         \____\____\
   *                         ____\_\__\_\
   *                       /`   /`     o\
   *                       |___ |_______|
   *
   */
  public void testUndeadNorms() throws Exception {
    Directory dir = applyCreatedVersionMajor(newDirectory());
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    int numDocs = atLeast(500);
    List<Integer> toDelete = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(new StringField("id", "" + i, Field.Store.NO));
      if (random().nextInt(5) == 1) {
        toDelete.add(i);
        doc.add(new TextField("content", "some content", Field.Store.NO));
      }
      w.addDocument(doc);
    }
    for (Integer id : toDelete) {
      w.deleteDocuments(new Term("id", "" + id));
    }
    w.forceMerge(1);
    IndexReader r = maybeWrapWithMergingReader(w.getReader());
    assertFalse(r.hasDeletions());

    // Confusingly, norms should exist, and should all be 0, even though we deleted all docs that
    // had the field "content".  They should not
    // be undead:
    NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
    assertNotNull(norms);
    if (codecSupportsSparsity()) {
      assertEquals(DocIdSetIterator.NO_MORE_DOCS, norms.nextDoc());
    } else {
      for (int i = 0; i < r.maxDoc(); i++) {
        assertEquals(i, norms.nextDoc());
        assertEquals(0, norms.longValue());
      }
    }

    r.close();
    w.close();
    dir.close();
  }

  public void testThreads() throws Exception {
    float density =
        codecSupportsSparsity() == false || random().nextBoolean() ? 1f : random().nextFloat();
    int numDocs = atLeast(500);
    final FixedBitSet docsWithField = new FixedBitSet(numDocs);
    final int numDocsWithField = Math.max(1, (int) (density * numDocs));
    if (numDocsWithField == numDocs) {
      docsWithField.set(0, numDocs);
    } else {
      int i = 0;
      while (i < numDocsWithField) {
        int doc = random().nextInt(numDocs);
        if (docsWithField.get(doc) == false) {
          docsWithField.set(doc);
          ++i;
        }
      }
    }

    long[] norms = new long[numDocsWithField];
    for (int i = 0; i < numDocsWithField; i++) {
      norms[i] = random().nextLong();
    }

    Directory dir = applyCreatedVersionMajor(newDirectory());
    Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
    IndexWriterConfig conf = newIndexWriterConfig(analyzer);
    conf.setMergePolicy(NoMergePolicy.INSTANCE);
    conf.setSimilarity(new CannedNormSimilarity(norms));
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    Document doc = new Document();
    Field idField = new StringField("id", "", Field.Store.NO);
    Field indexedField = new TextField("indexed", "", Field.Store.NO);
    Field dvField = new NumericDocValuesField("dv", 0);
    doc.add(idField);
    doc.add(indexedField);
    doc.add(dvField);

    for (int i = 0, j = 0; i < numDocs; i++) {
      idField.setStringValue(Integer.toString(i));
      if (docsWithField.get(i) == false) {
        Document doc2 = new Document();
        doc2.add(idField);
        writer.addDocument(doc2);
      } else {
        long value = norms[j++];
        dvField.setLongValue(value);
        indexedField.setStringValue(value == 0 ? "" : "a");
        writer.addDocument(doc);
      }
      if (random().nextInt(31) == 0) {
        writer.commit();
      }
    }

    DirectoryReader reader = maybeWrapWithMergingReader(writer.getReader());
    writer.close();

    final int numThreads = TestUtil.nextInt(random(), 3, 30);
    Thread[] threads = new Thread[numThreads];
    final CountDownLatch latch = new CountDownLatch(1);
    for (int i = 0; i < numThreads; ++i) {
      threads[i] =
          new Thread(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    latch.await();
                    checkNormsVsDocValues(reader);
                    TestUtil.checkReader(reader);
                  } catch (Exception e) {
                    throw new RuntimeException(e);
                  }
                }
              });
    }

    for (Thread thread : threads) {
      thread.start();
    }
    latch.countDown();
    for (Thread thread : threads) {
      thread.join();
    }

    reader.close();
    dir.close();
  }

  public void testIndependantIterators() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
    CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
    conf.setSimilarity(sim);
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    Document doc = new Document();
    Field indexedField = new TextField("indexed", "a", Field.Store.NO);
    doc.add(indexedField);
    for (int i = 0; i < 3; ++i) {
      writer.addDocument(doc);
    }
    writer.forceMerge(1);
    LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
    NumericDocValues n1 = r.getNormValues("indexed");
    NumericDocValues n2 = r.getNormValues("indexed");
    assertEquals(0, n1.nextDoc());
    assertEquals(42, n1.longValue());
    assertEquals(1, n1.nextDoc());
    assertEquals(10, n1.longValue());
    assertEquals(0, n2.nextDoc());
    assertEquals(42, n2.longValue());
    assertEquals(1, n2.nextDoc());
    assertEquals(10, n2.longValue());
    assertEquals(2, n2.nextDoc());
    assertEquals(20, n2.longValue());
    assertEquals(2, n1.nextDoc());
    assertEquals(20, n1.longValue());
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
    IOUtils.close(r, writer, dir);
  }

  public void testIndependantSparseIterators() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
    CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
    conf.setSimilarity(sim);
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    Document doc = new Document();
    Field indexedField = new TextField("indexed", "a", Field.Store.NO);
    doc.add(indexedField);
    Document emptyDoc = new Document();
    for (int i = 0; i < 3; ++i) {
      writer.addDocument(doc);
      writer.addDocument(emptyDoc);
    }
    writer.forceMerge(1);
    LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
    NumericDocValues n1 = r.getNormValues("indexed");
    NumericDocValues n2 = r.getNormValues("indexed");
    assertEquals(0, n1.nextDoc());
    assertEquals(42, n1.longValue());
    assertEquals(2, n1.nextDoc());
    assertEquals(10, n1.longValue());
    assertEquals(0, n2.nextDoc());
    assertEquals(42, n2.longValue());
    assertEquals(2, n2.nextDoc());
    assertEquals(10, n2.longValue());
    assertEquals(4, n2.nextDoc());
    assertEquals(20, n2.longValue());
    assertEquals(4, n1.nextDoc());
    assertEquals(20, n1.longValue());
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
    assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
    IOUtils.close(r, writer, dir);
  }
}
