001/* 002 Licensed to the Apache Software Foundation (ASF) under one 003 or more contributor license agreements. See the NOTICE file 004 distributed with this work for additional information 005 regarding copyright ownership. The ASF licenses this file 006 to you under the Apache License, Version 2.0 (the 007 "License"); you may not use this file except in compliance 008 with the License. You may obtain a copy of the License at 009 010 http://www.apache.org/licenses/LICENSE-2.0 011 012 Unless required by applicable law or agreed to in writing, 013 software distributed under the License is distributed on an 014 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 KIND, either express or implied. See the License for the 016 specific language governing permissions and limitations 017 under the License. 018 */ 019package org.apache.wiki.search; 020 021import org.apache.commons.lang3.StringUtils; 022import org.apache.logging.log4j.LogManager; 023import org.apache.logging.log4j.Logger; 024import org.apache.lucene.analysis.Analyzer; 025import org.apache.lucene.analysis.TokenStream; 026import org.apache.lucene.analysis.classic.ClassicAnalyzer; 027import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; 028import org.apache.lucene.document.Document; 029import org.apache.lucene.document.Field; 030import org.apache.lucene.document.StringField; 031import org.apache.lucene.document.TextField; 032import org.apache.lucene.index.DirectoryReader; 033import org.apache.lucene.index.IndexReader; 034import org.apache.lucene.index.IndexWriter; 035import org.apache.lucene.index.IndexWriterConfig; 036import org.apache.lucene.index.IndexWriterConfig.OpenMode; 037import org.apache.lucene.index.StoredFields; 038import org.apache.lucene.index.Term; 039import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper; 040import org.apache.lucene.queries.spans.SpanNearQuery; 041import org.apache.lucene.queries.spans.SpanQuery; 042import org.apache.lucene.queries.spans.SpanTermQuery; 043import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; 044import org.apache.lucene.queryparser.classic.ParseException; 045import org.apache.lucene.queryparser.classic.QueryParser; 046import org.apache.lucene.search.BooleanClause; 047import org.apache.lucene.search.BooleanQuery; 048import org.apache.lucene.search.IndexSearcher; 049import org.apache.lucene.search.MultiTermQuery; 050import org.apache.lucene.search.PrefixQuery; 051import org.apache.lucene.search.Query; 052import org.apache.lucene.search.ScoreDoc; 053import org.apache.lucene.search.TermQuery; 054import org.apache.lucene.search.TopDocs; 055import org.apache.lucene.search.WildcardQuery; 056import org.apache.lucene.search.highlight.Highlighter; 057import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; 058import org.apache.lucene.search.highlight.QueryScorer; 059import org.apache.lucene.search.highlight.SimpleHTMLEncoder; 060import org.apache.lucene.search.highlight.SimpleHTMLFormatter; 061import org.apache.lucene.store.Directory; 062import org.apache.lucene.store.NIOFSDirectory; 063import org.apache.wiki.InternalWikiException; 064import org.apache.wiki.WatchDog; 065import org.apache.wiki.WikiBackgroundThread; 066import org.apache.wiki.api.core.Attachment; 067import org.apache.wiki.api.core.Context; 068import org.apache.wiki.api.core.Engine; 069import org.apache.wiki.api.core.Page; 070import org.apache.wiki.api.exceptions.NoRequiredPropertyException; 071import org.apache.wiki.api.exceptions.ProviderException; 072import org.apache.wiki.api.providers.PageProvider; 073import org.apache.wiki.api.providers.WikiProvider; 074import org.apache.wiki.api.search.SearchResult; 075import org.apache.wiki.api.spi.Wiki; 076import org.apache.wiki.attachment.AttachmentManager; 077import org.apache.wiki.auth.AuthorizationManager; 078import org.apache.wiki.auth.permissions.PagePermission; 079import org.apache.wiki.pages.PageManager; 080import org.apache.wiki.util.ClassUtil; 081import org.apache.wiki.util.FileUtil; 082import org.apache.wiki.util.Synchronizer; 083import org.apache.wiki.util.TextUtil; 084 085import java.io.File; 086import java.io.IOException; 087import java.io.InputStream; 088import java.io.InputStreamReader; 089import java.io.StringReader; 090import java.io.StringWriter; 091import java.util.ArrayList; 092import java.util.Arrays; 093import java.util.Collection; 094import java.util.Collections; 095import java.util.Date; 096import java.util.List; 097import java.util.Properties; 098import java.util.concurrent.Executor; 099import java.util.concurrent.Executors; 100import java.util.concurrent.locks.ReentrantLock; 101import java.util.stream.Collectors; 102 103 104/** 105 * Interface for the search providers that handle searching the Wiki 106 * 107 * @since 2.2.21. 108 */ 109public class LuceneSearchProvider implements SearchProvider { 110 111 protected static final Logger LOG = LogManager.getLogger( LuceneSearchProvider.class ); 112 113 private Engine m_engine; 114 private Executor searchExecutor; 115 116 // Lucene properties. 117 118 /** Which analyzer to use. Default is StandardAnalyzer. */ 119 public static final String PROP_LUCENE_ANALYZER = "jspwiki.lucene.analyzer"; 120 private static final String PROP_LUCENE_INDEXDELAY = "jspwiki.lucene.indexdelay"; 121 private static final String PROP_LUCENE_INITIALDELAY = "jspwiki.lucene.initialdelay"; 122 123 private String m_analyzerClass = ClassicAnalyzer.class.getName(); 124 125 private static final String LUCENE_DIR = "lucene"; 126 127 /** These attachment file suffixes will be indexed. */ 128 public static final String[] SEARCHABLE_FILE_SUFFIXES = new String[] { ".txt", ".ini", ".xml", ".html", "htm", ".mm", ".htm", 129 ".xhtml", ".java", ".c", ".cpp", ".php", ".asm", ".sh", 130 ".properties", ".kml", ".gpx", ".loc", ".md", ".xml" }; 131 132 protected static final String LUCENE_ID = "id"; 133 protected static final String LUCENE_PAGE_CONTENTS = "contents"; 134 protected static final String LUCENE_AUTHOR = "author"; 135 protected static final String LUCENE_ATTACHMENTS = "attachment"; 136 protected static final String LUCENE_PAGE_NAME = "name"; 137 protected static final String LUCENE_PAGE_KEYWORDS = "keywords"; 138 139 private String m_luceneDirectory; 140 protected final List< Object[] > m_updates = Collections.synchronizedList( new ArrayList<>() ); 141 142 /** Maximum number of fragments from search matches. */ 143 private static final int MAX_FRAGMENTS = 3; 144 145 /** The maximum number of hits to return from searches. */ 146 public static final int MAX_SEARCH_HITS = 99_999; 147 148 private static final String PUNCTUATION_TO_SPACES = StringUtils.repeat( " ", TextUtil.PUNCTUATION_CHARS_ALLOWED.length() ); 149 150 /** 151 * A lock used to ensure thread safety when accessing shared resources. 152 * This lock provides more flexibility and capabilities than the intrinsic locking mechanism, 153 * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread 154 * waiting to acquire a lock. 155 * 156 * @see java.util.concurrent.locks.ReentrantLock 157 */ 158 private final ReentrantLock lock = new ReentrantLock(); 159 160 /** {@inheritDoc} */ 161 @Override 162 public void initialize( final Engine engine, final Properties props ) throws NoRequiredPropertyException, IOException { 163 m_engine = engine; 164 searchExecutor = Executors.newCachedThreadPool(); 165 166 m_luceneDirectory = engine.getWorkDir() + File.separator + LUCENE_DIR; 167 168 final int initialDelay = TextUtil.getIntegerProperty( props, PROP_LUCENE_INITIALDELAY, LuceneUpdater.INITIAL_DELAY ); 169 final int indexDelay = TextUtil.getIntegerProperty( props, PROP_LUCENE_INDEXDELAY, LuceneUpdater.INDEX_DELAY ); 170 171 m_analyzerClass = TextUtil.getStringProperty( props, PROP_LUCENE_ANALYZER, m_analyzerClass ); 172 // FIXME: Just to be simple for now, we will do full reindex only if no files are in lucene directory. 173 174 final File dir = new File( m_luceneDirectory ); 175 LOG.info( "Lucene enabled, cache will be in: {}", dir.getAbsolutePath() ); 176 try { 177 if( !dir.exists() ) { 178 dir.mkdirs(); 179 } 180 181 if( !dir.exists() || !dir.canWrite() || !dir.canRead() ) { 182 LOG.error( "Cannot write to Lucene directory, disabling Lucene: {}", dir.getAbsolutePath() ); 183 throw new IOException( "Invalid Lucene directory." ); 184 } 185 186 final String[] filelist = dir.list(); 187 if( filelist == null ) { 188 throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath() ); 189 } 190 } catch( final IOException e ) { 191 LOG.error( "Problem while creating Lucene index - not using Lucene.", e ); 192 } 193 194 // Start the Lucene update thread, which waits first for a little while before starting to go through 195 // the Lucene "pages that need updating". 196 final LuceneUpdater updater = new LuceneUpdater( m_engine, this, initialDelay, indexDelay ); 197 updater.start(); 198 } 199 200 /** 201 * Returns the handling engine. 202 * 203 * @return Current Engine 204 */ 205 protected Engine getEngine() { 206 return m_engine; 207 } 208 209 /** 210 * Performs a full Lucene reindex, if necessary. 211 * 212 * @throws IOException If there's a problem during indexing 213 */ 214 protected void doFullLuceneReindex() throws IOException { 215 final File dir = new File( m_luceneDirectory ); 216 final String[] filelist = dir.list(); 217 if( filelist == null ) { 218 throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath() ); 219 } 220 221 try { 222 if( filelist.length == 0 ) { 223 // 224 // No files? Reindex! 225 // 226 final Date start = new Date(); 227 228 LOG.info( "Starting Lucene reindexing, this can take a couple of minutes..." ); 229 230 final Directory luceneDir = new NIOFSDirectory( dir.toPath() ); 231 try( final IndexWriter writer = getIndexWriter( luceneDir ) ) { 232 long pagesIndexed = 0L; 233 final Collection< Page > allPages = m_engine.getManager( PageManager.class ).getAllPages(); 234 for( final Page page : allPages ) { 235 try { 236 final String text = m_engine.getManager( PageManager.class ).getPageText( page.getName(), WikiProvider.LATEST_VERSION ); 237 luceneIndexPage( page, text, writer ); 238 pagesIndexed++; 239 } catch( final IOException e ) { 240 LOG.warn( "Unable to index page {}, continuing to next ", page.getName(), e ); 241 } 242 } 243 LOG.info( "Indexed {} pages", pagesIndexed ); 244 245 long attachmentsIndexed = 0L; 246 final Collection< Attachment > allAttachments = m_engine.getManager( AttachmentManager.class ).getAllAttachments(); 247 for( final Attachment att : allAttachments ) { 248 try { 249 final String text = getAttachmentContent( att.getName(), WikiProvider.LATEST_VERSION ); 250 luceneIndexPage( att, text, writer ); 251 attachmentsIndexed++; 252 } catch( final IOException e ) { 253 LOG.warn( "Unable to index attachment {}, continuing to next", att.getName(), e ); 254 } 255 } 256 LOG.info( "Indexed {} attachments", attachmentsIndexed ); 257 } 258 259 final Date end = new Date(); 260 LOG.info( "Full Lucene index finished in {} milliseconds.", end.getTime() - start.getTime() ); 261 } else { 262 LOG.info( "Files found in Lucene directory, not reindexing." ); 263 } 264 } catch( final IOException e ) { 265 LOG.error( "Problem while creating Lucene index - not using Lucene.", e ); 266 } catch( final ProviderException e ) { 267 LOG.error( "Problem reading pages while creating Lucene index (JSPWiki won't start.)", e ); 268 throw new IllegalArgumentException( "unable to create Lucene index" ); 269 } catch( final Exception e ) { 270 LOG.error( "Unable to start lucene", e ); 271 } 272 273 } 274 275 /** 276 * Fetches the attachment content from the repository. 277 * Content is flat text that can be used for indexing/searching or display 278 * 279 * @param attachmentName Name of the attachment. 280 * @param version The version of the attachment. 281 * @return the content of the Attachment as a String. 282 */ 283 protected String getAttachmentContent( final String attachmentName, final int version ) { 284 final AttachmentManager mgr = m_engine.getManager( AttachmentManager.class ); 285 try { 286 final Attachment att = mgr.getAttachmentInfo( attachmentName, version ); 287 //FIXME: Find out why sometimes att is null 288 if( att != null ) { 289 return getAttachmentContent( att ); 290 } 291 } catch( final ProviderException e ) { 292 LOG.error( "Attachment cannot be loaded", e ); 293 } 294 return null; 295 } 296 297 /** 298 * @param att Attachment to get content for. Filename extension is used to determine the type of the attachment. 299 * @return String representing the content of the file. 300 * FIXME This is a very simple implementation of some text-based attachment, mainly used for testing. 301 * This should be replaced /moved to Attachment search providers or some other 'pluggable' way to search attachments 302 */ 303 protected String getAttachmentContent( final Attachment att ) { 304 final AttachmentManager mgr = m_engine.getManager( AttachmentManager.class ); 305 //FIXME: Add attachment plugin structure 306 307 final String filename = att.getFileName(); 308 309 boolean searchSuffix = Arrays.stream(SEARCHABLE_FILE_SUFFIXES).anyMatch(filename::endsWith); 310 311 String out = filename; 312 if( searchSuffix ) { 313 try( final InputStream attStream = mgr.getAttachmentStream( att ); final StringWriter sout = new StringWriter() ) { 314 FileUtil.copyContents( new InputStreamReader( attStream ), sout ); 315 out = out + " " + sout; 316 } catch( final ProviderException | IOException e ) { 317 LOG.error( "Attachment cannot be loaded", e ); 318 } 319 } 320 321 return out; 322 } 323 324 /** 325 * Updates the lucene index for a single page. 326 * 327 * @param page The WikiPage to check 328 * @param text The page text to index. 329 */ 330 protected void updateLuceneIndex( final Page page, final String text ) { 331 Synchronizer.synchronize(lock, () -> { 332 LOG.debug( "Updating Lucene index for page '{}'...", page.getName() ); 333 pageRemoved( page ); 334 335 // Now add back the new version. 336 try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() ); 337 final IndexWriter writer = getIndexWriter( luceneDir ) ) { 338 luceneIndexPage( page, text, writer ); 339 } catch( final IOException e ) { 340 LOG.error( "Unable to update page '{}' from Lucene index", page.getName(), e ); 341 // reindexPage( page ); 342 } catch( final Exception e ) { 343 LOG.error( "Unexpected Lucene exception - please check configuration!", e ); 344 // reindexPage( page ); 345 } 346 347 LOG.debug( "Done updating Lucene index for page '{}'.", page.getName() ); 348 }); 349 } 350 351 private Analyzer getLuceneAnalyzer() throws ProviderException { 352 try { 353 return ClassUtil.buildInstance( m_analyzerClass ); 354 } catch( final Exception e ) { 355 final String msg = "Could not get LuceneAnalyzer class " + m_analyzerClass + ", reason: "; 356 LOG.error( msg, e ); 357 throw new ProviderException( msg + e ); 358 } 359 } 360 361 /** 362 * Indexes page using the given IndexWriter. 363 * 364 * @param page WikiPage 365 * @param text Page text to index 366 * @param writer The Lucene IndexWriter to use for indexing 367 * @return the created index Document 368 * @throws IOException If there's an indexing problem 369 */ 370 protected Document luceneIndexPage( final Page page, final String text, final IndexWriter writer ) throws IOException { 371 LOG.debug( "Indexing {}...", page.getName() ); 372 373 // make a new, empty document 374 final Document doc = new Document(); 375 if( text == null ) { 376 return doc; 377 } 378 379 final String indexedText = text.replace( "__", " " ); // be nice to Language Analyzers - cfr. JSPWIKI-893 380 381 // Raw name is the keyword we'll use to refer to this document for updates. 382 Field field = new Field( LUCENE_ID, page.getName(), StringField.TYPE_STORED ); 383 doc.add( field ); 384 385 // Body text. It is stored in the doc for search contexts. 386 field = new Field( LUCENE_PAGE_CONTENTS, indexedText, TextField.TYPE_STORED ); 387 doc.add( field ); 388 389 // Allow searching by page name. Both beautified and raw 390 final String unTokenizedTitle = StringUtils.replaceChars( page.getName(), TextUtil.PUNCTUATION_CHARS_ALLOWED, PUNCTUATION_TO_SPACES ); 391 field = new Field( LUCENE_PAGE_NAME, TextUtil.beautifyString( page.getName() ) + " " + unTokenizedTitle, TextField.TYPE_STORED ); 392 doc.add( field ); 393 394 // Allow searching by authorname 395 if( page.getAuthor() != null ) { 396 field = new Field( LUCENE_AUTHOR, page.getAuthor(), TextField.TYPE_STORED ); 397 doc.add( field ); 398 } 399 400 // Now add the names of the attachments of this page 401 try { 402 final List< Attachment > attachments = m_engine.getManager( AttachmentManager.class ).listAttachments( page ); 403 final String attachmentNames = attachments.stream().map(att -> att.getName() + ";").collect(Collectors.joining()); 404 405 field = new Field( LUCENE_ATTACHMENTS, attachmentNames, TextField.TYPE_STORED ); 406 doc.add( field ); 407 408 } catch( final ProviderException e ) { 409 // Unable to read attachments 410 LOG.error( "Failed to get attachments for page", e ); 411 } 412 413 // also index page keywords, if available 414 if( page.getAttribute( "keywords" ) != null ) { 415 field = new Field( LUCENE_PAGE_KEYWORDS, page.getAttribute( "keywords" ).toString(), TextField.TYPE_STORED ); 416 doc.add( field ); 417 } 418 writer.addDocument( doc ); 419 420 return doc; 421 } 422 423 /** 424 * {@inheritDoc} 425 */ 426 @Override 427 public void pageRemoved( final Page page ) { 428 try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() ); 429 final IndexWriter writer = getIndexWriter( luceneDir ) ) { 430 final Query query = new TermQuery( new Term( LUCENE_ID, page.getName() ) ); 431 writer.deleteDocuments( query ); 432 } catch( final Exception e ) { 433 LOG.error( "Unable to remove page '{}' from Lucene index", page.getName(), e ); 434 } 435 } 436 437 IndexWriter getIndexWriter( final Directory luceneDir ) throws IOException, ProviderException { 438 final IndexWriterConfig writerConfig = new IndexWriterConfig( getLuceneAnalyzer() ); 439 writerConfig.setOpenMode( OpenMode.CREATE_OR_APPEND ); 440 return new IndexWriter( luceneDir, writerConfig ); 441 } 442 443 /** 444 * Adds a page-text pair to the lucene update queue. Safe to call always 445 * 446 * @param page WikiPage to add to the update queue. 447 */ 448 @Override 449 public void reindexPage( final Page page ) { 450 if( page != null ) { 451 final String text; 452 453 // TODO: Think if this was better done in the thread itself? 454 if( page instanceof Attachment ) { 455 text = getAttachmentContent( ( Attachment ) page ); 456 } else { 457 text = m_engine.getManager( PageManager.class ).getPureText( page ); 458 } 459 460 if( text != null ) { 461 // Add work item to m_updates queue. 462 final Object[] pair = new Object[ 2 ]; 463 pair[ 0 ] = page; 464 pair[ 1 ] = text; 465 m_updates.add( pair ); 466 LOG.debug( "Scheduling page {} for index update", page.getName() ); 467 } 468 } 469 } 470 471 /** {@inheritDoc} */ 472 @Override 473 public Collection< SearchResult > findPages( final String query, final Context wikiContext ) throws ProviderException { 474 return findPages( query, FLAG_CONTEXTS, wikiContext ); 475 } 476 477 /** Create contexts also. Generating contexts can be expensive, so they're not on by default. */ 478 public static final int FLAG_CONTEXTS = 0x01; 479 480 /** 481 * Searches pages using a particular combination of flags. 482 * 483 * @param query The query to perform in Lucene query language 484 * @param flags A set of flags 485 * @return A Collection of SearchResult instances 486 * @throws ProviderException if there is a problem with the backend 487 */ 488 public Collection< SearchResult > findPages( final String query, final int flags, final Context wikiContext ) throws ProviderException { 489 ArrayList< SearchResult > list = null; 490 Highlighter highlighter = null; 491 492 try( final Directory luceneDir = new NIOFSDirectory( new File( m_luceneDirectory ).toPath() ); 493 final IndexReader reader = DirectoryReader.open( luceneDir ) ) { 494 final String[] queryfields = { LUCENE_PAGE_CONTENTS, LUCENE_PAGE_NAME, LUCENE_AUTHOR, LUCENE_ATTACHMENTS, LUCENE_PAGE_KEYWORDS }; 495 final QueryParser qp = new SeparatorAwareQueryParser( queryfields, getLuceneAnalyzer() ); 496 final Query luceneQuery = qp.parse( query ); 497 final IndexSearcher searcher = new IndexSearcher( reader, searchExecutor ); 498 499 if( ( flags & FLAG_CONTEXTS ) != 0 ) { 500 highlighter = new Highlighter( new SimpleHTMLFormatter( "<span class=\"searchmatch\">", "</span>" ), 501 new SimpleHTMLEncoder(), 502 new QueryScorer( luceneQuery ) ); 503 } 504 505 final AuthorizationManager mgr = m_engine.getManager( AuthorizationManager.class ); 506 final TopDocs hits = searcher.search( luceneQuery, MAX_SEARCH_HITS ); 507 final StoredFields storedFields = reader.storedFields(); 508 509 list = new ArrayList<>( hits.scoreDocs.length ); 510 for( final ScoreDoc hit : hits.scoreDocs ) { 511 final Document doc = storedFields.document( hit.doc ); 512 final String pageName = doc.get( LUCENE_ID ); 513 final Page page = m_engine.getManager( PageManager.class ).getPage( pageName, PageProvider.LATEST_VERSION ); 514 515 if( page != null ) { 516 final PagePermission pp = new PagePermission( page, PagePermission.VIEW_ACTION ); 517 if( mgr.checkPermission( wikiContext.getWikiSession(), pp ) ) { 518 final int score = hits.scoreDocs[ 0 ].score > 0 ? Math.round( hit.score / hits.scoreDocs[ 0 ].score * 100 ) : 0; 519 520 // Get highlighted search contexts 521 final String text = doc.get( LUCENE_PAGE_CONTENTS ); 522 523 String[] fragments = new String[ 0 ]; 524 if( text != null && highlighter != null ) { 525 final TokenStream tokenStream = getLuceneAnalyzer().tokenStream( LUCENE_PAGE_CONTENTS, new StringReader( text ) ); 526 fragments = highlighter.getBestFragments( tokenStream, text, MAX_FRAGMENTS ); 527 } 528 529 final SearchResult result = new SearchResultImpl( page, score, fragments ); 530 list.add( result ); 531 } 532 } else { 533 LOG.error( "Lucene found a result page '{}' that could not be loaded, removing from Lucene cache", pageName ); 534 pageRemoved( Wiki.contents().page( m_engine, pageName ) ); 535 } 536 } 537 } catch( final IOException e ) { 538 LOG.error( "Failed during lucene search", e ); 539 } catch( final ParseException e ) { 540 LOG.error( "Broken query; cannot parse query: {}", query, e ); 541 throw new ProviderException( "You have entered a query Lucene cannot process [" + query + "]: " + e.getMessage() ); 542 } catch( final InvalidTokenOffsetsException e ) { 543 LOG.error( "Tokens are incompatible with provided text ", e ); 544 } 545 546 return list; 547 } 548 549 /** {@inheritDoc} */ 550 @Override 551 public String getProviderInfo() { 552 return "LuceneSearchProvider"; 553 } 554 555 /** 556 * Updater thread that updates Lucene indexes. 557 */ 558 private static final class LuceneUpdater extends WikiBackgroundThread { 559 static final int INDEX_DELAY = 5; 560 static final int INITIAL_DELAY = 60; 561 private final LuceneSearchProvider m_provider; 562 563 private final int m_initialDelay; 564 565 private WatchDog m_watchdog; 566 567 /** 568 * A lock used to ensure thread safety when accessing shared resources. 569 * This lock provides more flexibility and capabilities than the intrinsic locking mechanism, 570 * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread 571 * waiting to acquire a lock. 572 * 573 * @see java.util.concurrent.locks.ReentrantLock 574 */ 575 private static final ReentrantLock lock = new ReentrantLock(); 576 577 private LuceneUpdater( final Engine engine, final LuceneSearchProvider provider, final int initialDelay, final int indexDelay ) { 578 super( engine, indexDelay ); 579 m_provider = provider; 580 m_initialDelay = initialDelay; 581 setName( "JSPWiki Lucene Indexer" ); 582 } 583 584 @Override 585 public void startupTask() throws Exception { 586 m_watchdog = WatchDog.getCurrentWatchDog( getEngine() ); 587 588 // Sleep initially... 589 try { 590 Thread.sleep( m_initialDelay * 1000L ); 591 } catch( final InterruptedException e ) { 592 throw new InternalWikiException( "Interrupted while waiting to start.", e ); 593 } 594 595 m_watchdog.enterState( "Full reindex" ); 596 // Reindex everything 597 m_provider.doFullLuceneReindex(); 598 m_watchdog.exitState(); 599 } 600 601 @Override 602 public void backgroundTask() { 603 m_watchdog.enterState( "Emptying index queue", 60 ); 604 Synchronizer.synchronize(lock, () -> { 605 while( !m_provider.m_updates.isEmpty() ) { 606 final Object[] pair = m_provider.m_updates.remove( 0 ); 607 final Page page = ( Page )pair[ 0 ]; 608 final String text = ( String )pair[ 1 ]; 609 m_provider.updateLuceneIndex( page, text ); 610 } 611 }); 612 613 614 m_watchdog.exitState(); 615 } 616 617 } 618 619 /** 620 * Query parser that lets wildcard terms cross the separators the analyzer splits on. The index holds {@code snake} and {@code case} 621 * for {@code snake_case}, so a term like {@code snake_ca*} would never find it. Such a term is split on the same separators and 622 * matched as adjacent tokens instead, with the wildcard applied to the token it is in. A trailing separator, as in 623 * {@code snake_*}, only requires the tokens before it. 624 */ 625 static final class SeparatorAwareQueryParser extends MultiFieldQueryParser { 626 627 private static final String NON_SEPARATORS = "[\\p{L}\\p{N}*?]*"; 628 private static final String SEPARATORS = "[^\\p{L}\\p{N}*?]+"; 629 630 SeparatorAwareQueryParser( final String[] fields, final Analyzer analyzer ) { 631 super( fields, analyzer ); 632 } 633 634 @Override 635 protected Query getPrefixQuery( final String field, final String termStr ) throws ParseException { 636 final Query separated = field != null ? separated( field, termStr, true ) : null; 637 if( separated == null ) { 638 return super.getPrefixQuery( field, termStr ); 639 } 640 try { 641 return either( super.getPrefixQuery( field, termStr ), separated ); 642 } catch( final ParseException e ) { 643 return separated; // the default handling does not accept the term, the separated one does 644 } 645 } 646 647 /** 648 * A page name term with a {@code *} between other characters does not need a {@code *} at its end: {@code blog*26} is read as 649 * {@code blog*26*}, as a token of a name, such as {@code blogentry_021126_1}, goes on after the digits that are asked for. 650 */ 651 @Override 652 protected Query getWildcardQuery( final String field, final String termStr ) throws ParseException { 653 final String term = LUCENE_PAGE_NAME.equals( field ) && termStr.matches( "[^*].*\\*.*[^*]" ) ? termStr + "*" : termStr; 654 final Query separated = field != null ? separated( field, term, false ) : null; 655 if( separated == null ) { 656 return super.getWildcardQuery( field, term ); 657 } 658 try { 659 return either( super.getWildcardQuery( field, term ), separated ); 660 } catch( final ParseException e ) { 661 return separated; // the default handling does not accept the term, the separated one does 662 } 663 } 664 665 /** 666 * Matches what either query matches. The default query is kept because the analyzer does not always split on a separator, so 667 * a token like {@code blogentry_021126_1} can hold it and only the default query finds it. 668 */ 669 private static Query either( final Query defaultQuery, final Query separated ) { 670 return new BooleanQuery.Builder() 671 .add( defaultQuery, BooleanClause.Occur.SHOULD ) 672 .add( separated, BooleanClause.Occur.SHOULD ) 673 .build(); 674 } 675 676 /** 677 * Builds the query for a term that holds separators, or returns {@code null} if the default handling applies. 678 * 679 * @param prefix {@code true} if the term is a prefix whose trailing {@code *} has already been removed 680 */ 681 private Query separated( final String field, final String termStr, final boolean prefix ) throws ParseException { 682 if( termStr.matches( NON_SEPARATORS ) ) { 683 return null; 684 } 685 686 final boolean endsWithSeparator = termStr.matches( ".*" + SEPARATORS ); 687 final List< SpanQuery > clauses = new ArrayList<>(); 688 final String[] segments = Arrays.stream( termStr.split( SEPARATORS ) ).filter( segment -> !segment.isEmpty() ).toArray( String[]::new ); 689 for( int i = 0; i < segments.length; i++ ) { 690 final String segment = segments[ i ]; 691 if( segment.indexOf( '*' ) != -1 || segment.indexOf( '?' ) != -1 ) { 692 final Term term = new Term( field, getAnalyzer().normalize( field, segment ) ); 693 clauses.add( new SpanMultiTermQueryWrapper<>( new WildcardQuery( term ) ) ); 694 } else if( prefix && i == segments.length - 1 && !endsWithSeparator ) { 695 final Term term = new Term( field, getAnalyzer().normalize( field, segment ) ); 696 clauses.add( new SpanMultiTermQueryWrapper<>( new PrefixQuery( term ) ) ); 697 } else { 698 for( final String token : tokens( field, segment ) ) { 699 clauses.add( new SpanTermQuery( new Term( field, token ) ) ); 700 } 701 } 702 } 703 704 if( clauses.isEmpty() ) { 705 return null; 706 } 707 return clauses.size() == 1 ? clauses.get( 0 ) : new SpanNearQuery( clauses.toArray( new SpanQuery[ 0 ] ), 0, true ); 708 } 709 710 private List< String > tokens( final String field, final String text ) throws ParseException { 711 final List< String > tokens = new ArrayList<>(); 712 try( final TokenStream stream = getAnalyzer().tokenStream( field, text ) ) { 713 final CharTermAttribute term = stream.addAttribute( CharTermAttribute.class ); 714 stream.reset(); 715 while( stream.incrementToken() ) { 716 tokens.add( term.toString() ); 717 } 718 stream.end(); 719 } catch( final IOException e ) { 720 throw new ParseException( "Cannot analyze '" + text + "': " + e.getMessage() ); 721 } 722 return tokens; 723 } 724 725 } 726 727 // FIXME: This class is dumb; needs to have a better implementation 728 private static class SearchResultImpl implements SearchResult { 729 730 private final Page m_page; 731 private final int m_score; 732 private final String[] m_contexts; 733 734 public SearchResultImpl( final Page page, final int score, final String[] contexts ) { 735 m_page = page; 736 m_score = score; 737 m_contexts = contexts != null ? contexts.clone() : null; 738 } 739 740 @Override 741 public Page getPage() { 742 return m_page; 743 } 744 745 /* (non-Javadoc) 746 * @see org.apache.wiki.SearchResult#getScore() 747 */ 748 @Override 749 public int getScore() { 750 return m_score; 751 } 752 753 754 @Override 755 public String[] getContexts() { 756 return m_contexts; 757 } 758 } 759 760}