001/*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.    
018 */
019package org.apache.wiki.search;
020
021import org.apache.commons.lang3.StringUtils;
022import org.apache.logging.log4j.LogManager;
023import org.apache.logging.log4j.Logger;
024import org.apache.lucene.analysis.Analyzer;
025import org.apache.lucene.analysis.TokenStream;
026import org.apache.lucene.analysis.classic.ClassicAnalyzer;
027import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
028import org.apache.lucene.document.Document;
029import org.apache.lucene.document.Field;
030import org.apache.lucene.document.StringField;
031import org.apache.lucene.document.TextField;
032import org.apache.lucene.index.DirectoryReader;
033import org.apache.lucene.index.IndexReader;
034import org.apache.lucene.index.IndexWriter;
035import org.apache.lucene.index.IndexWriterConfig;
036import org.apache.lucene.index.IndexWriterConfig.OpenMode;
037import org.apache.lucene.index.StoredFields;
038import org.apache.lucene.index.Term;
039import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper;
040import org.apache.lucene.queries.spans.SpanNearQuery;
041import org.apache.lucene.queries.spans.SpanQuery;
042import org.apache.lucene.queries.spans.SpanTermQuery;
043import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
044import org.apache.lucene.queryparser.classic.ParseException;
045import org.apache.lucene.queryparser.classic.QueryParser;
046import org.apache.lucene.search.BooleanClause;
047import org.apache.lucene.search.BooleanQuery;
048import org.apache.lucene.search.IndexSearcher;
049import org.apache.lucene.search.MultiTermQuery;
050import org.apache.lucene.search.PrefixQuery;
051import org.apache.lucene.search.Query;
052import org.apache.lucene.search.ScoreDoc;
053import org.apache.lucene.search.TermQuery;
054import org.apache.lucene.search.TopDocs;
055import org.apache.lucene.search.WildcardQuery;
056import org.apache.lucene.search.highlight.Highlighter;
057import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
058import org.apache.lucene.search.highlight.QueryScorer;
059import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
060import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
061import org.apache.lucene.store.Directory;
062import org.apache.lucene.store.NIOFSDirectory;
063import org.apache.wiki.InternalWikiException;
064import org.apache.wiki.WatchDog;
065import org.apache.wiki.WikiBackgroundThread;
066import org.apache.wiki.api.core.Attachment;
067import org.apache.wiki.api.core.Context;
068import org.apache.wiki.api.core.Engine;
069import org.apache.wiki.api.core.Page;
070import org.apache.wiki.api.exceptions.NoRequiredPropertyException;
071import org.apache.wiki.api.exceptions.ProviderException;
072import org.apache.wiki.api.providers.PageProvider;
073import org.apache.wiki.api.providers.WikiProvider;
074import org.apache.wiki.api.search.SearchResult;
075import org.apache.wiki.api.spi.Wiki;
076import org.apache.wiki.attachment.AttachmentManager;
077import org.apache.wiki.auth.AuthorizationManager;
078import org.apache.wiki.auth.permissions.PagePermission;
079import org.apache.wiki.pages.PageManager;
080import org.apache.wiki.util.ClassUtil;
081import org.apache.wiki.util.FileUtil;
082import org.apache.wiki.util.Synchronizer;
083import org.apache.wiki.util.TextUtil;
084
085import java.io.File;
086import java.io.IOException;
087import java.io.InputStream;
088import java.io.InputStreamReader;
089import java.io.StringReader;
090import java.io.StringWriter;
091import java.util.ArrayList;
092import java.util.Arrays;
093import java.util.Collection;
094import java.util.Collections;
095import java.util.Date;
096import java.util.List;
097import java.util.Properties;
098import java.util.concurrent.Executor;
099import java.util.concurrent.Executors;
100import java.util.concurrent.locks.ReentrantLock;
101import java.util.stream.Collectors;
102
103
104/**
105 * Interface for the search providers that handle searching the Wiki
106 *
107 * @since 2.2.21.
108 */
109public class LuceneSearchProvider implements SearchProvider {
110
111    protected static final Logger LOG = LogManager.getLogger( LuceneSearchProvider.class );
112
113    private Engine m_engine;
114    private Executor searchExecutor;
115
116    // Lucene properties.
117
118    /** Which analyzer to use.  Default is StandardAnalyzer. */
119    public static final String PROP_LUCENE_ANALYZER      = "jspwiki.lucene.analyzer";
120    private static final String PROP_LUCENE_INDEXDELAY   = "jspwiki.lucene.indexdelay";
121    private static final String PROP_LUCENE_INITIALDELAY = "jspwiki.lucene.initialdelay";
122
123    private String m_analyzerClass = ClassicAnalyzer.class.getName();
124
125    private static final String LUCENE_DIR = "lucene";
126
127    /** These attachment file suffixes will be indexed. */
128    public static final String[] SEARCHABLE_FILE_SUFFIXES = new String[] { ".txt", ".ini", ".xml", ".html", "htm", ".mm", ".htm",
129                                                                           ".xhtml", ".java", ".c", ".cpp", ".php", ".asm", ".sh",
130                                                                           ".properties", ".kml", ".gpx", ".loc", ".md", ".xml" };
131
132    protected static final String LUCENE_ID            = "id";
133    protected static final String LUCENE_PAGE_CONTENTS = "contents";
134    protected static final String LUCENE_AUTHOR        = "author";
135    protected static final String LUCENE_ATTACHMENTS   = "attachment";
136    protected static final String LUCENE_PAGE_NAME     = "name";
137    protected static final String LUCENE_PAGE_KEYWORDS = "keywords";
138
139    private String m_luceneDirectory;
140    protected final List< Object[] > m_updates = Collections.synchronizedList( new ArrayList<>() );
141
142    /** Maximum number of fragments from search matches. */
143    private static final int MAX_FRAGMENTS = 3;
144
145    /** The maximum number of hits to return from searches. */
146    public static final int MAX_SEARCH_HITS = 99_999;
147
148    private static final String PUNCTUATION_TO_SPACES = StringUtils.repeat( " ", TextUtil.PUNCTUATION_CHARS_ALLOWED.length() );
149
150    /**
151     * A lock used to ensure thread safety when accessing shared resources.
152     * This lock provides more flexibility and capabilities than the intrinsic locking mechanism,
153     * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread
154     * waiting to acquire a lock.
155     *
156     * @see java.util.concurrent.locks.ReentrantLock
157     */
158    private final ReentrantLock lock = new ReentrantLock();
159
160    /** {@inheritDoc} */
161    @Override
162    public void initialize( final Engine engine, final Properties props ) throws NoRequiredPropertyException, IOException {
163        m_engine = engine;
164        searchExecutor = Executors.newCachedThreadPool();
165
166        m_luceneDirectory = engine.getWorkDir() + File.separator + LUCENE_DIR;
167
168        final int initialDelay = TextUtil.getIntegerProperty( props, PROP_LUCENE_INITIALDELAY, LuceneUpdater.INITIAL_DELAY );
169        final int indexDelay   = TextUtil.getIntegerProperty( props, PROP_LUCENE_INDEXDELAY, LuceneUpdater.INDEX_DELAY );
170
171        m_analyzerClass = TextUtil.getStringProperty( props, PROP_LUCENE_ANALYZER, m_analyzerClass );
172        // FIXME: Just to be simple for now, we will do full reindex only if no files are in lucene directory.
173
174        final File dir = new File( m_luceneDirectory );
175        LOG.info( "Lucene enabled, cache will be in: {}", dir.getAbsolutePath() );
176        try {
177            if( !dir.exists() ) {
178                dir.mkdirs();
179            }
180
181            if( !dir.exists() || !dir.canWrite() || !dir.canRead() ) {
182                LOG.error( "Cannot write to Lucene directory, disabling Lucene: {}", dir.getAbsolutePath() );
183                throw new IOException( "Invalid Lucene directory." );
184            }
185
186            final String[] filelist = dir.list();
187            if( filelist == null ) {
188                throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath() );
189            }
190        } catch( final IOException e ) {
191            LOG.error( "Problem while creating Lucene index - not using Lucene.", e );
192        }
193
194        // Start the Lucene update thread, which waits first for a little while before starting to go through
195        // the Lucene "pages that need updating".
196        final LuceneUpdater updater = new LuceneUpdater( m_engine, this, initialDelay, indexDelay );
197        updater.start();
198    }
199
200    /**
201     * Returns the handling engine.
202     *
203     * @return Current Engine
204     */
205    protected Engine getEngine() {
206        return m_engine;
207    }
208
209    /**
210     * Performs a full Lucene reindex, if necessary.
211     *
212     * @throws IOException If there's a problem during indexing
213     */
214    protected void doFullLuceneReindex() throws IOException {
215        final File dir = new File( m_luceneDirectory );
216        final String[] filelist = dir.list();
217        if( filelist == null ) {
218            throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath() );
219        }
220
221        try {
222            if( filelist.length == 0 ) {
223                //
224                //  No files? Reindex!
225                //
226                final Date start = new Date();
227
228                LOG.info( "Starting Lucene reindexing, this can take a couple of minutes..." );
229
230                final Directory luceneDir = new NIOFSDirectory( dir.toPath() );
231                try( final IndexWriter writer = getIndexWriter( luceneDir ) ) {
232                    long pagesIndexed = 0L;
233                    final Collection< Page > allPages = m_engine.getManager( PageManager.class ).getAllPages();
234                    for( final Page page : allPages ) {
235                        try {
236                            final String text = m_engine.getManager( PageManager.class ).getPageText( page.getName(), WikiProvider.LATEST_VERSION );
237                            luceneIndexPage( page, text, writer );
238                            pagesIndexed++;
239                        } catch( final IOException e ) {
240                            LOG.warn( "Unable to index page {}, continuing to next ", page.getName(), e );
241                        }
242                    }
243                    LOG.info( "Indexed {} pages", pagesIndexed );
244
245                    long attachmentsIndexed = 0L;
246                    final Collection< Attachment > allAttachments = m_engine.getManager( AttachmentManager.class ).getAllAttachments();
247                    for( final Attachment att : allAttachments ) {
248                        try {
249                            final String text = getAttachmentContent( att.getName(), WikiProvider.LATEST_VERSION );
250                            luceneIndexPage( att, text, writer );
251                            attachmentsIndexed++;
252                        } catch( final IOException e ) {
253                            LOG.warn( "Unable to index attachment {}, continuing to next", att.getName(), e );
254                        }
255                    }
256                    LOG.info( "Indexed {} attachments", attachmentsIndexed );
257                }
258
259                final Date end = new Date();
260                LOG.info( "Full Lucene index finished in {} milliseconds.", end.getTime() - start.getTime() );
261            } else {
262                LOG.info( "Files found in Lucene directory, not reindexing." );
263            }
264        } catch( final IOException e ) {
265            LOG.error( "Problem while creating Lucene index - not using Lucene.", e );
266        } catch( final ProviderException e ) {
267            LOG.error( "Problem reading pages while creating Lucene index (JSPWiki won't start.)", e );
268            throw new IllegalArgumentException( "unable to create Lucene index" );
269        } catch( final Exception e ) {
270            LOG.error( "Unable to start lucene", e );
271        }
272
273    }
274
275    /**
276     * Fetches the attachment content from the repository.
277     * Content is flat text that can be used for indexing/searching or display
278     *
279     * @param attachmentName Name of the attachment.
280     * @param version        The version of the attachment.
281     * @return the content of the Attachment as a String.
282     */
283    protected String getAttachmentContent( final String attachmentName, final int version ) {
284        final AttachmentManager mgr = m_engine.getManager( AttachmentManager.class );
285        try {
286            final Attachment att = mgr.getAttachmentInfo( attachmentName, version );
287            //FIXME: Find out why sometimes att is null
288            if( att != null ) {
289                return getAttachmentContent( att );
290            }
291        } catch( final ProviderException e ) {
292            LOG.error( "Attachment cannot be loaded", e );
293        }
294        return null;
295    }
296
297    /**
298     * @param att Attachment to get content for. Filename extension is used to determine the type of the attachment.
299     * @return String representing the content of the file.
300     * FIXME This is a very simple implementation of some text-based attachment, mainly used for testing.
301     * This should be replaced /moved to Attachment search providers or some other 'pluggable' way to search attachments
302     */
303    protected String getAttachmentContent( final Attachment att ) {
304        final AttachmentManager mgr = m_engine.getManager( AttachmentManager.class );
305        //FIXME: Add attachment plugin structure
306
307        final String filename = att.getFileName();
308
309        boolean searchSuffix = Arrays.stream(SEARCHABLE_FILE_SUFFIXES).anyMatch(filename::endsWith);
310
311        String out = filename;
312        if( searchSuffix ) {
313            try( final InputStream attStream = mgr.getAttachmentStream( att ); final StringWriter sout = new StringWriter() ) {
314                FileUtil.copyContents( new InputStreamReader( attStream ), sout );
315                out = out + " " + sout;
316            } catch( final ProviderException | IOException e ) {
317                LOG.error( "Attachment cannot be loaded", e );
318            }
319        }
320
321        return out;
322    }
323
324    /**
325     * Updates the lucene index for a single page.
326     *
327     * @param page The WikiPage to check
328     * @param text The page text to index.
329     */
330    protected void updateLuceneIndex( final Page page, final String text ) {
331        Synchronizer.synchronize(lock, () -> {
332            LOG.debug( "Updating Lucene index for page '{}'...", page.getName() );
333            pageRemoved( page );
334
335            // Now add back the new version.
336            try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() );
337                 final IndexWriter writer = getIndexWriter( luceneDir ) ) {
338                luceneIndexPage( page, text, writer );
339            } catch( final IOException e ) {
340                LOG.error( "Unable to update page '{}' from Lucene index", page.getName(), e );
341                // reindexPage( page );
342            } catch( final Exception e ) {
343                LOG.error( "Unexpected Lucene exception - please check configuration!", e );
344                // reindexPage( page );
345            }
346
347            LOG.debug( "Done updating Lucene index for page '{}'.", page.getName() );
348        });
349    }
350
351    private Analyzer getLuceneAnalyzer() throws ProviderException {
352        try {
353            return ClassUtil.buildInstance( m_analyzerClass );
354        } catch( final Exception e ) {
355            final String msg = "Could not get LuceneAnalyzer class " + m_analyzerClass + ", reason: ";
356            LOG.error( msg, e );
357            throw new ProviderException( msg + e );
358        }
359    }
360
361    /**
362     * Indexes page using the given IndexWriter.
363     *
364     * @param page   WikiPage
365     * @param text   Page text to index
366     * @param writer The Lucene IndexWriter to use for indexing
367     * @return the created index Document
368     * @throws IOException If there's an indexing problem
369     */
370    protected Document luceneIndexPage( final Page page, final String text, final IndexWriter writer ) throws IOException {
371        LOG.debug( "Indexing {}...", page.getName() );
372
373        // make a new, empty document
374        final Document doc = new Document();
375        if( text == null ) {
376            return doc;
377        }
378
379        final String indexedText = text.replace( "__", " " ); // be nice to Language Analyzers - cfr. JSPWIKI-893
380
381        // Raw name is the keyword we'll use to refer to this document for updates.
382        Field field = new Field( LUCENE_ID, page.getName(), StringField.TYPE_STORED );
383        doc.add( field );
384
385        // Body text.  It is stored in the doc for search contexts.
386        field = new Field( LUCENE_PAGE_CONTENTS, indexedText, TextField.TYPE_STORED );
387        doc.add( field );
388
389        // Allow searching by page name. Both beautified and raw
390        final String unTokenizedTitle = StringUtils.replaceChars( page.getName(), TextUtil.PUNCTUATION_CHARS_ALLOWED, PUNCTUATION_TO_SPACES );
391        field = new Field( LUCENE_PAGE_NAME, TextUtil.beautifyString( page.getName() ) + " " + unTokenizedTitle, TextField.TYPE_STORED );
392        doc.add( field );
393
394        // Allow searching by authorname
395        if( page.getAuthor() != null ) {
396            field = new Field( LUCENE_AUTHOR, page.getAuthor(), TextField.TYPE_STORED );
397            doc.add( field );
398        }
399
400        // Now add the names of the attachments of this page
401        try {
402            final List< Attachment > attachments = m_engine.getManager( AttachmentManager.class ).listAttachments( page );
403            final String attachmentNames = attachments.stream().map(att -> att.getName() + ";").collect(Collectors.joining());
404
405            field = new Field( LUCENE_ATTACHMENTS, attachmentNames, TextField.TYPE_STORED );
406            doc.add( field );
407
408        } catch( final ProviderException e ) {
409            // Unable to read attachments
410            LOG.error( "Failed to get attachments for page", e );
411        }
412
413        // also index page keywords, if available
414        if( page.getAttribute( "keywords" ) != null ) {
415            field = new Field( LUCENE_PAGE_KEYWORDS, page.getAttribute( "keywords" ).toString(), TextField.TYPE_STORED );
416            doc.add( field );
417        }
418        writer.addDocument( doc );
419
420        return doc;
421    }
422
423    /**
424     * {@inheritDoc}
425     */
426    @Override
427    public void pageRemoved( final Page page ) {
428        try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() );
429             final IndexWriter writer = getIndexWriter( luceneDir ) ) {
430            final Query query = new TermQuery( new Term( LUCENE_ID, page.getName() ) );
431            writer.deleteDocuments( query );
432        } catch( final Exception e ) {
433            LOG.error( "Unable to remove page '{}' from Lucene index", page.getName(), e );
434        }
435    }
436
437    IndexWriter getIndexWriter( final Directory luceneDir ) throws IOException, ProviderException {
438        final IndexWriterConfig writerConfig = new IndexWriterConfig( getLuceneAnalyzer() );
439        writerConfig.setOpenMode( OpenMode.CREATE_OR_APPEND );
440        return new IndexWriter( luceneDir, writerConfig );
441    }
442
443    /**
444     * Adds a page-text pair to the lucene update queue.  Safe to call always
445     *
446     * @param page WikiPage to add to the update queue.
447     */
448    @Override
449    public void reindexPage( final Page page ) {
450        if( page != null ) {
451            final String text;
452
453            // TODO: Think if this was better done in the thread itself?
454            if( page instanceof Attachment ) {
455                text = getAttachmentContent( ( Attachment ) page );
456            } else {
457                text = m_engine.getManager( PageManager.class ).getPureText( page );
458            }
459
460            if( text != null ) {
461                // Add work item to m_updates queue.
462                final Object[] pair = new Object[ 2 ];
463                pair[ 0 ] = page;
464                pair[ 1 ] = text;
465                m_updates.add( pair );
466                LOG.debug( "Scheduling page {} for index update", page.getName() );
467            }
468        }
469    }
470
471    /** {@inheritDoc} */
472    @Override
473    public Collection< SearchResult > findPages( final String query, final Context wikiContext ) throws ProviderException {
474        return findPages( query, FLAG_CONTEXTS, wikiContext );
475    }
476
477    /** Create contexts also. Generating contexts can be expensive, so they're not on by default. */
478    public static final int FLAG_CONTEXTS = 0x01;
479
480    /**
481     * Searches pages using a particular combination of flags.
482     *
483     * @param query The query to perform in Lucene query language
484     * @param flags A set of flags
485     * @return A Collection of SearchResult instances
486     * @throws ProviderException if there is a problem with the backend
487     */
488    public Collection< SearchResult > findPages( final String query, final int flags, final Context wikiContext ) throws ProviderException {
489        ArrayList< SearchResult > list = null;
490        Highlighter highlighter = null;
491
492        try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() );
493             final IndexReader reader = DirectoryReader.open( luceneDir ) ) {
494            final String[] queryfields = { LUCENE_PAGE_CONTENTS, LUCENE_PAGE_NAME, LUCENE_AUTHOR, LUCENE_ATTACHMENTS, LUCENE_PAGE_KEYWORDS };
495            final QueryParser qp = new SeparatorAwareQueryParser( queryfields, getLuceneAnalyzer() );
496            final Query luceneQuery = qp.parse( query );
497            final IndexSearcher searcher = new IndexSearcher( reader, searchExecutor );
498
499            if( ( flags & FLAG_CONTEXTS ) != 0 ) {
500                highlighter = new Highlighter( new SimpleHTMLFormatter( "<span class=\"searchmatch\">", "</span>" ),
501                                               new SimpleHTMLEncoder(),
502                                               new QueryScorer( luceneQuery ) );
503            }
504
505            final AuthorizationManager mgr = m_engine.getManager( AuthorizationManager.class );
506            final TopDocs hits = searcher.search( luceneQuery, MAX_SEARCH_HITS );
507            final StoredFields storedFields = reader.storedFields();
508
509            list = new ArrayList<>( hits.scoreDocs.length );
510            for( final ScoreDoc hit : hits.scoreDocs ) {
511                final Document doc = storedFields.document( hit.doc );
512                final String pageName = doc.get( LUCENE_ID );
513                final Page page = m_engine.getManager( PageManager.class ).getPage( pageName, PageProvider.LATEST_VERSION );
514
515                if( page != null ) {
516                    final PagePermission pp = new PagePermission( page, PagePermission.VIEW_ACTION );
517                    if( mgr.checkPermission( wikiContext.getWikiSession(), pp ) ) {
518                        final int score = hits.scoreDocs[ 0 ].score > 0 ? Math.round( hit.score / hits.scoreDocs[ 0 ].score * 100 ) : 0;
519
520                        // Get highlighted search contexts
521                        final String text = doc.get( LUCENE_PAGE_CONTENTS );
522
523                        String[] fragments = new String[ 0 ];
524                        if( text != null && highlighter != null ) {
525                            final TokenStream tokenStream = getLuceneAnalyzer().tokenStream( LUCENE_PAGE_CONTENTS, new StringReader( text ) );
526                            fragments = highlighter.getBestFragments( tokenStream, text, MAX_FRAGMENTS );
527                        }
528
529                        final SearchResult result = new SearchResultImpl( page, score, fragments );
530                        list.add( result );
531                    }
532                } else {
533                    LOG.error( "Lucene found a result page '{}' that could not be loaded, removing from Lucene cache",  pageName );
534                    pageRemoved( Wiki.contents().page( m_engine, pageName ) );
535                }
536            }
537        } catch( final IOException e ) {
538            LOG.error( "Failed during lucene search", e );
539        } catch( final ParseException e ) {
540            LOG.error( "Broken query; cannot parse query: {}", query, e );
541            throw new ProviderException( "You have entered a query Lucene cannot process [" + query + "]: " + e.getMessage() );
542        } catch( final InvalidTokenOffsetsException e ) {
543            LOG.error( "Tokens are incompatible with provided text ", e );
544        }
545
546        return list;
547    }
548
549    /** {@inheritDoc} */
550    @Override
551    public String getProviderInfo() {
552        return "LuceneSearchProvider";
553    }
554
555    /**
556     * Updater thread that updates Lucene indexes.
557     */
558    private static final class LuceneUpdater extends WikiBackgroundThread {
559        static final int INDEX_DELAY    = 5;
560        static final int INITIAL_DELAY = 60;
561        private final LuceneSearchProvider m_provider;
562
563        private final int m_initialDelay;
564
565        private WatchDog m_watchdog;
566
567        /**
568         * A lock used to ensure thread safety when accessing shared resources.
569         * This lock provides more flexibility and capabilities than the intrinsic locking mechanism,
570         * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread
571         * waiting to acquire a lock.
572         *
573         * @see java.util.concurrent.locks.ReentrantLock
574         */
575        private static final ReentrantLock lock = new ReentrantLock();
576
577        private LuceneUpdater( final Engine engine, final LuceneSearchProvider provider, final int initialDelay, final int indexDelay ) {
578            super( engine, indexDelay );
579            m_provider = provider;
580            m_initialDelay = initialDelay;
581            setName( "JSPWiki Lucene Indexer" );
582        }
583
584        @Override
585        public void startupTask() throws Exception {
586            m_watchdog = WatchDog.getCurrentWatchDog( getEngine() );
587
588            // Sleep initially...
589            try {
590                Thread.sleep( m_initialDelay * 1000L );
591            } catch( final InterruptedException e ) {
592                throw new InternalWikiException( "Interrupted while waiting to start.", e );
593            }
594
595            m_watchdog.enterState( "Full reindex" );
596            // Reindex everything
597            m_provider.doFullLuceneReindex();
598            m_watchdog.exitState();
599        }
600
601        @Override
602        public void backgroundTask() {
603            m_watchdog.enterState( "Emptying index queue", 60 );
604            Synchronizer.synchronize(lock, () -> {
605                while( !m_provider.m_updates.isEmpty() ) {
606                    final Object[] pair = m_provider.m_updates.remove( 0 );
607                    final Page page = ( Page )pair[ 0 ];
608                    final String text = ( String )pair[ 1 ];
609                    m_provider.updateLuceneIndex( page, text );
610                }
611            });
612
613
614            m_watchdog.exitState();
615        }
616
617    }
618
619    /**
620     * Query parser that lets wildcard terms cross the separators the analyzer splits on. The index holds {@code snake} and {@code case}
621     * for {@code snake_case}, so a term like {@code snake_ca*} would never find it. Such a term is split on the same separators and
622     * matched as adjacent tokens instead, with the wildcard applied to the token it is in. A trailing separator, as in
623     * {@code snake_*}, only requires the tokens before it.
624     */
625    static final class SeparatorAwareQueryParser extends MultiFieldQueryParser {
626
627        private static final String NON_SEPARATORS = "[\\p{L}\\p{N}*?]*";
628        private static final String SEPARATORS = "[^\\p{L}\\p{N}*?]+";
629
630        SeparatorAwareQueryParser( final String[] fields, final Analyzer analyzer ) {
631            super( fields, analyzer );
632        }
633
634        @Override
635        protected Query getPrefixQuery( final String field, final String termStr ) throws ParseException {
636            final Query separated = field != null ? separated( field, termStr, true ) : null;
637            if( separated == null ) {
638                return super.getPrefixQuery( field, termStr );
639            }
640            try {
641                return either( super.getPrefixQuery( field, termStr ), separated );
642            } catch( final ParseException e ) {
643                return separated; // the default handling does not accept the term, the separated one does
644            }
645        }
646
647        /**
648         * A page name term with a {@code *} between other characters does not need a {@code *} at its end: {@code blog*26} is read as
649         * {@code blog*26*}, as a token of a name, such as {@code blogentry_021126_1}, goes on after the digits that are asked for.
650         */
651        @Override
652        protected Query getWildcardQuery( final String field, final String termStr ) throws ParseException {
653            final String term = LUCENE_PAGE_NAME.equals( field ) && termStr.matches( "[^*].*\\*.*[^*]" ) ? termStr + "*" : termStr;
654            final Query separated = field != null ? separated( field, term, false ) : null;
655            if( separated == null ) {
656                return super.getWildcardQuery( field, term );
657            }
658            try {
659                return either( super.getWildcardQuery( field, term ), separated );
660            } catch( final ParseException e ) {
661                return separated; // the default handling does not accept the term, the separated one does
662            }
663        }
664
665        /**
666         * Matches what either query matches. The default query is kept because the analyzer does not always split on a separator, so
667         * a token like {@code blogentry_021126_1} can hold it and only the default query finds it.
668         */
669        private static Query either( final Query defaultQuery, final Query separated ) {
670            return new BooleanQuery.Builder()
671                    .add( defaultQuery, BooleanClause.Occur.SHOULD )
672                    .add( separated, BooleanClause.Occur.SHOULD )
673                    .build();
674        }
675
676        /**
677         * Builds the query for a term that holds separators, or returns {@code null} if the default handling applies.
678         *
679         * @param prefix {@code true} if the term is a prefix whose trailing {@code *} has already been removed
680         */
681        private Query separated( final String field, final String termStr, final boolean prefix ) throws ParseException {
682            if( termStr.matches( NON_SEPARATORS ) ) {
683                return null;
684            }
685
686            final boolean endsWithSeparator = termStr.matches( ".*" + SEPARATORS );
687            final List< SpanQuery > clauses = new ArrayList<>();
688            final String[] segments = Arrays.stream( termStr.split( SEPARATORS ) ).filter( segment -> !segment.isEmpty() ).toArray( String[]::new );
689            for( int i = 0; i < segments.length; i++ ) {
690                final String segment = segments[ i ];
691                if( segment.indexOf( '*' ) != -1 || segment.indexOf( '?' ) != -1 ) {
692                    final Term term = new Term( field, getAnalyzer().normalize( field, segment ) );
693                    clauses.add( new SpanMultiTermQueryWrapper<>( new WildcardQuery( term ) ) );
694                } else if( prefix && i == segments.length - 1 && !endsWithSeparator ) {
695                    final Term term = new Term( field, getAnalyzer().normalize( field, segment ) );
696                    clauses.add( new SpanMultiTermQueryWrapper<>( new PrefixQuery( term ) ) );
697                } else {
698                    for( final String token : tokens( field, segment ) ) {
699                        clauses.add( new SpanTermQuery( new Term( field, token ) ) );
700                    }
701                }
702            }
703
704            if( clauses.isEmpty() ) {
705                return null;
706            }
707            return clauses.size() == 1 ? clauses.get( 0 ) : new SpanNearQuery( clauses.toArray( new SpanQuery[ 0 ] ), 0, true );
708        }
709
710        private List< String > tokens( final String field, final String text ) throws ParseException {
711            final List< String > tokens = new ArrayList<>();
712            try( final TokenStream stream = getAnalyzer().tokenStream( field, text ) ) {
713                final CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
714                stream.reset();
715                while( stream.incrementToken() ) {
716                    tokens.add( term.toString() );
717                }
718                stream.end();
719            } catch( final IOException e ) {
720                throw new ParseException( "Cannot analyze '" + text + "': " + e.getMessage() );
721            }
722            return tokens;
723        }
724
725    }
726
727    // FIXME: This class is dumb; needs to have a better implementation
728    private static class SearchResultImpl implements SearchResult {
729
730        private final Page m_page;
731        private final int m_score;
732        private final String[] m_contexts;
733
734        public SearchResultImpl( final Page page, final int score, final String[] contexts ) {
735            m_page = page;
736            m_score = score;
737            m_contexts = contexts != null ? contexts.clone() : null;
738        }
739
740        @Override
741        public Page getPage() {
742            return m_page;
743        }
744
745        /* (non-Javadoc)
746         * @see org.apache.wiki.SearchResult#getScore()
747         */
748        @Override
749        public int getScore() {
750            return m_score;
751        }
752
753
754        @Override
755        public String[] getContexts() {
756            return m_contexts;
757        }
758    }
759
760}