@Test(expected = ND4JIllegalStateException.class, timeout = 20000L)
    public void testParallelFlag2() throws Exception {
        val collection = new ArrayList<String>();
        collection.add("First string");
        collection.add("Second string");
        collection.add("Third string");
        collection.add("");
        collection.add("Fifth string");
//        collection.add("caboom");

        val vectorizer = new TfidfVectorizer.Builder()
                .allowParallelTokenization(false)
                .setIterator(new CollectionSentenceIterator(collection))
                .setTokenizerFactory(new ExplodingTokenizerFactory(8, -1))
                .build();

        vectorizer.buildVocab();


        log.info("Fitting vectorizer...");

        vectorizer.fit();
    }
 

@Test(expected = ND4JIllegalStateException.class, timeout = 20000L)
public void testParallelFlag3() throws Exception {
    val collection = new ArrayList<String>();
    collection.add("First string");
    collection.add("Second string");
    collection.add("Third string");
    collection.add("");
    collection.add("Fifth string");
    collection.add("Long long long string");
    collection.add("Sixth string");

    val vectorizer = new TfidfVectorizer.Builder()
            .allowParallelTokenization(false)
            .setIterator(new CollectionSentenceIterator(collection))
            .setTokenizerFactory(new ExplodingTokenizerFactory(-1, 4))
            .build();

    vectorizer.buildVocab();


    log.info("Fitting vectorizer...");

    vectorizer.fit();
}
 

public static void main(String[] args) throws IOException {
    List<String> sentences = Arrays.asList(
            "No ,  he says now .",
            "And what did he do ?",
            "The money 's there .",
            "That was less than a year ago .",
            "But he made only the first .",
            "There 's still time for them to do it .",
            "But he should nt have .",
            " They have to come down to the people .",
            "I do nt know where that is .",
            "No , I would nt .",
            "Who Will It Be ?",
            "And no , I was not the one ."
    );
    SentenceIterator iterator = new CollectionSentenceIterator(sentences);
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}
 

public static void main(String[] args) throws IOException {
    List<String> sentences = Arrays.asList(
            "No ,  he says now .",
            "And what did he do ?",
            "The money 's there .",
            "That was less than a year ago .",
            "But he made only the first .",
            "There 's still time for them to do it .",
            "But he should nt have .",
            " They have to come down to the people .",
            "I do nt know where that is .",
            "No , I would nt .",
            "Who Will It Be ?",
            "And no , I was not the one ."
    );
    SentenceIterator iterator = new CollectionSentenceIterator(sentences);
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}