001/* 002 Licensed to the Apache Software Foundation (ASF) under one 003 or more contributor license agreements. See the NOTICE file 004 distributed with this work for additional information 005 regarding copyright ownership. The ASF licenses this file 006 to you under the Apache License, Version 2.0 (the 007 "License"); you may not use this file except in compliance 008 with the License. You may obtain a copy of the License at 009 010 http://www.apache.org/licenses/LICENSE-2.0 011 012 Unless required by applicable law or agreed to in writing, 013 software distributed under the License is distributed on an 014 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 KIND, either express or implied. See the License for the 016 specific language governing permissions and limitations 017 under the License. 018 */ 019package org.apache.wiki.search.kendra; 020 021import com.amazonaws.services.kendra.AWSkendra; 022import com.amazonaws.services.kendra.AWSkendraClientBuilder; 023import com.amazonaws.services.kendra.model.*; 024import com.amazonaws.util.IOUtils; 025import com.google.gson.Gson; 026import com.google.gson.GsonBuilder; 027import com.google.gson.reflect.TypeToken; 028import org.apache.commons.io.FilenameUtils; 029import org.apache.commons.lang3.Strings; 030import org.apache.logging.log4j.LogManager; 031import org.apache.logging.log4j.Logger; 032import org.apache.wiki.InternalWikiException; 033import org.apache.wiki.WatchDog; 034import org.apache.wiki.WikiBackgroundThread; 035import org.apache.wiki.api.core.Attachment; 036import org.apache.wiki.api.core.Context; 037import org.apache.wiki.api.core.Engine; 038import org.apache.wiki.api.core.Page; 039import org.apache.wiki.api.exceptions.NoRequiredPropertyException; 040import org.apache.wiki.api.exceptions.ProviderException; 041import org.apache.wiki.api.providers.PageProvider; 042import org.apache.wiki.api.search.SearchResult; 043import org.apache.wiki.api.spi.Wiki; 044import org.apache.wiki.attachment.AttachmentManager; 045import org.apache.wiki.auth.AuthorizationManager; 046import org.apache.wiki.auth.permissions.PagePermission; 047import org.apache.wiki.pages.PageManager; 048import org.apache.wiki.search.SearchProvider; 049import org.apache.wiki.util.Synchronizer; 050import org.apache.wiki.util.TextUtil; 051 052import java.io.IOException; 053import java.io.InputStream; 054import java.io.InputStreamReader; 055import java.lang.reflect.Type; 056import java.nio.ByteBuffer; 057import java.nio.charset.StandardCharsets; 058import java.util.ArrayList; 059import java.util.Collection; 060import java.util.Collections; 061import java.util.HashMap; 062import java.util.List; 063import java.util.Map; 064import java.util.Properties; 065import java.util.concurrent.locks.ReentrantLock; 066 067import static java.lang.String.format; 068 069/** 070 * Search provider that implements {link SearchProvider} using AWS Kendra for 071 * indexing. Note that we are using a Custom DataSource which limits the 072 * attributes that can be uploaded / searched for each page (as per 073 * https://docs.aws.amazon.com/kendra/latest/dg/custom-attributes.html). This 074 * could be overcome by using an S3 bucket where any custom attributes can be 075 * added. 076 * 077 * @since 2.11.0 078 */ 079public class KendraSearchProvider implements SearchProvider { 080 081 private static final Logger LOG = LogManager.getLogger( KendraSearchProvider.class ); 082 private Engine engine; 083 private Properties properties; 084 private Map< String, Object > contentTypes; 085 private AWSkendra kendra; 086 private String indexName; 087 private String indexId; 088 private String dataSourceName; 089 private String dataSourceId; 090 091 private final List< Page > updates = Collections.synchronizedList( new ArrayList<>() ); 092 093 private static final String PROP_KENDRA_INDEX_NAME = "jspwiki.kendra.indexName"; 094 private static final String PROP_KENDRA_DATA_SOURCE_NAME = "jspwiki.kendra.dataSourceName"; 095 private static final String PROP_KENDRA_INDEXDELAY = "jspwiki.kendra.indexdelay"; 096 private static final String PROP_KENDRA_INITIALDELAY = "jspwiki.kendra.initialdelay"; 097 098 /** 099 * A lock used to ensure thread safety when accessing shared resources. 100 * This lock provides more flexibility and capabilities than the intrinsic locking mechanism, 101 * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread 102 * waiting to acquire a lock. 103 * 104 * @see java.util.concurrent.locks.ReentrantLock 105 */ 106 private final ReentrantLock lock = new ReentrantLock(); 107 108 /** 109 * {@inheritDoc} 110 */ 111 @Override 112 public void initialize( final Engine engine, final Properties properties ) throws NoRequiredPropertyException, IOException { 113 this.engine = engine; 114 this.properties = properties; 115 this.contentTypes = getContentTypes(); 116 117 setKendra( buildClient() ); 118 119 this.indexName = TextUtil.getRequiredProperty( this.properties, PROP_KENDRA_INDEX_NAME ); 120 this.dataSourceName = TextUtil.getRequiredProperty( this.properties, PROP_KENDRA_DATA_SOURCE_NAME ); 121 final int initialDelay = TextUtil.getIntegerProperty( this.properties, PROP_KENDRA_INITIALDELAY, KendraUpdater.INITIAL_DELAY ); 122 final int indexDelay = TextUtil.getIntegerProperty( this.properties, PROP_KENDRA_INDEXDELAY, KendraUpdater.INDEX_DELAY ); 123 124 // Start the Kendra update thread, which waits first for a little while 125 // before starting to go through the "pages that need updating". 126 if( initialDelay >= 0 ) { 127 final KendraUpdater updater = new KendraUpdater( engine, this, initialDelay, indexDelay ); 128 updater.start(); 129 } 130 } 131 132 private Map< String, Object > getContentTypes() { 133 final Gson gson = new GsonBuilder().create(); 134 try( final InputStream in = KendraSearchProvider.class.getResourceAsStream( "content_types.json" ) ) { 135 if ( in != null ) { 136 final Type collectionType = new TypeToken< HashMap< String, Object > >() { 137 }.getType(); 138 return gson.fromJson( new InputStreamReader( in ), collectionType ); 139 } 140 } catch ( final IOException e ) { 141 LOG.error( "Unable to load default propertyfile 'content_types.json': {}", e.getMessage(), e ); 142 } 143 return null; 144 } 145 146 /** 147 * {@inheritDoc} 148 */ 149 @Override 150 public String getProviderInfo() { 151 return "KendraSearchProvider"; 152 } 153 154 /** 155 * {@inheritDoc} 156 */ 157 @Override 158 public void pageRemoved( final Page page ) { 159 final String pageName = page.getName(); 160 final BatchDeleteDocumentRequest request = new BatchDeleteDocumentRequest().withIndexId( indexId ).withDocumentIdList( pageName ); 161 final BatchDeleteDocumentResult result = getKendra().batchDeleteDocument( request ); 162 if (result.getFailedDocuments().isEmpty()) { 163 LOG.debug( "Page '{}' was removed from index", pageName ); 164 } else { 165 LOG.error( "Failed to remove Page '{}' from index", pageName ); 166 } 167 } 168 169 /** 170 * {@inheritDoc} 171 */ 172 @Override 173 public void reindexPage( final Page page ) { 174 if( page != null ) { 175 updates.add( page ); 176 LOG.debug( "Scheduling page '{}' for indexing ...", page.getName() ); 177 } 178 } 179 180 /** 181 * {@inheritDoc} 182 */ 183 @Override 184 public Collection< SearchResult > findPages( final String query, final Context wikiContext ) throws ProviderException, IOException { 185 final QueryRequest request = new QueryRequest().withIndexId( indexId ).withQueryText( query ); 186 final List< QueryResultItem > items; 187 try { 188 items = getKendra().query( request ).getResultItems(); 189 } catch ( final ThrottlingException e ) { 190 LOG.error( "ThrottlingException. Skipping..." ); 191 return new ArrayList<>(); 192 } 193 final List< SearchResult > searchResults = new ArrayList<>( items.size() ); 194 final AuthorizationManager mgr = engine.getManager( AuthorizationManager.class ); 195 196 for ( final QueryResultItem item : items ) { 197 switch( QueryResultType.fromValue( item.getType() ) ) { 198 case DOCUMENT: 199 final String documentId = item.getDocumentId(); 200 final String documentExcerpt = item.getDocumentExcerpt().getText(); 201 final String scoreConfidence = item.getScoreAttributes().getScoreConfidence(); 202 final Page page = this.engine.getManager( PageManager.class ).getPage( documentId, PageProvider.LATEST_VERSION ); 203 if ( page != null ) { 204 final PagePermission pp = new PagePermission( page, PagePermission.VIEW_ACTION ); 205 if ( mgr.checkPermission( wikiContext.getWikiSession(), pp ) ) { 206 final SearchResult searchResult = new SearchResultImpl( page, confidence2score( scoreConfidence ), 207 new String[]{ documentExcerpt } ); 208 searchResults.add( searchResult ); 209 } else { 210 LOG.error( "Page '{}' is not accessible", documentId ); 211 } 212 } else { 213 LOG.error( "Kendra found a result page '{}' that could not be loaded, removing from index", documentId ); 214 pageRemoved( Wiki.contents().page( this.engine, documentId ) ); 215 } 216 break; 217 default: 218 LOG.error( "Unknown query result type: {}", item.getType() ); 219 } 220 } 221 return searchResults; 222 } 223 224 /** 225 * This method initialize the AWS Kendra Index and Datasources to be used. 226 */ 227 public void initializeIndexAndDataSource() { 228 this.indexId = getIndexId( indexName ); 229 if ( this.indexId == null ) { 230 final String message = format( "Index '%s' does not exist", indexName ); 231 LOG.error( message ); 232 throw new IllegalArgumentException( message ); 233 } 234 this.dataSourceId = getDatasourceId( this.indexId, dataSourceName ); 235 if ( this.dataSourceId == null ) { 236 final String message = format( "Datasource '%s' does not exist in index %s", dataSourceName, indexName ); 237 LOG.error( message ); 238 throw new IllegalArgumentException( message ); 239 } 240 } 241 242 /** 243 * Given an Kendra's Index name, returns the corresponding Index Id, or 244 * {@code null} if it does not exists 245 * 246 * @param indexName the name of the index to look up 247 * @return the index id or {@code null} 248 */ 249 private String getIndexId( final String indexName ) { 250 ListIndicesRequest request = new ListIndicesRequest(); 251 ListIndicesResult result = getKendra().listIndices( request ); 252 String nextToken = ""; 253 while ( nextToken != null ) { 254 final List< IndexConfigurationSummary > items = result.getIndexConfigurationSummaryItems(); 255 if ( items == null || items.isEmpty() ) { 256 return null; 257 } 258 for ( final IndexConfigurationSummary item : items ) { 259 if ( Strings.CS.equals( item.getName(), indexName ) ) { 260 return item.getId(); 261 } 262 } 263 nextToken = result.getNextToken(); 264 request = new ListIndicesRequest().withNextToken( result.getNextToken() ); 265 result = getKendra().listIndices( request ); 266 } 267 return null; 268 } 269 270 /** 271 * Given an Kendra's Datasource name, returns the corresponding Datasource Id, 272 * or {@code null} if it does not exists 273 * 274 * @param dataSourceName the name of the datasource to look up 275 * @return the datasource id or {@code null} 276 */ 277 private String getDatasourceId( final String indexId, final String dataSourceName ) { 278 ListDataSourcesRequest request = new ListDataSourcesRequest().withIndexId( indexId ); 279 ListDataSourcesResult result = getKendra().listDataSources( request ); 280 String nextToken = ""; 281 while ( nextToken != null ) { 282 final List< DataSourceSummary > items = result.getSummaryItems(); 283 if( items == null || items.isEmpty() ) { 284 return null; 285 } 286 287 for( final DataSourceSummary item : items ) { 288 if( Strings.CS.equals( item.getName(), dataSourceName ) ) { 289 return item.getId(); 290 } 291 } 292 nextToken = result.getNextToken(); 293 request = new ListDataSourcesRequest().withNextToken( result.getNextToken() ); 294 result = getKendra().listDataSources( request ); 295 } 296 return null; 297 } 298 299 /* 300 * Converts a SCORE Confidence from Kendra to an "equivalent" integer score 301 */ 302 private int confidence2score( final String scoreConfidence ) { 303 return switch( ScoreConfidence.fromValue( scoreConfidence ) ) { 304 case VERY_HIGH -> 100; 305 case HIGH -> 75; 306 case MEDIUM -> 50; 307 case LOW -> 25; 308 default -> 0; 309 }; 310 } 311 312 /** 313 * This method re-index all the pages found in the Wiki. It is mainly used at 314 * startup. 315 * 316 * @throws IOException in case some page can not be read 317 */ 318 private void doFullReindex() throws IOException { 319 try { 320 final Collection< Page > pages = engine.getManager( PageManager.class ).getAllPages(); 321 if ( pages.isEmpty() ) { 322 return; 323 } 324 LOG.debug( "Indexing all {} pages. Please wait ...", pages.size() ); 325 final String executionId = startExecution(); 326 for ( final Page page : pages ) { 327 // Since I do not want to handle the size limit 328 // (https://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/BatchPutDocument) 329 // uploading documents one at a time 330 indexOnePage( page, executionId ); 331 } 332 } catch ( final ProviderException e ) { 333 LOG.error( e.getMessage() ); 334 throw new IOException( e ); 335 } finally { 336 stopExecution(); 337 } 338 } 339 340 /** 341 * This method re-index all pages marked as updated. It is used to periodically 342 * index pages that have been modified 343 */ 344 private void doPartialReindex() { 345 if (updates.isEmpty()) { 346 return; 347 } 348 LOG.debug("Indexing updated pages. Please wait ..."); 349 final String executionId = startExecution(); 350 351 Synchronizer.synchronize(lock, () -> { 352 try { 353 while (!updates.isEmpty()) { 354 indexOnePage(updates.remove(0), executionId); 355 } 356 } finally { 357 stopExecution(); 358 } 359 }); 360 } 361 362 /** 363 * Returns an ExecutiuonId that is required to keep track of the modifed 364 * documents 365 * 366 * @return The execution id 367 */ 368 private String startExecution() { 369 final StartDataSourceSyncJobRequest request = new StartDataSourceSyncJobRequest().withIndexId( indexId ) 370 .withId( dataSourceId ); 371 final StartDataSourceSyncJobResult result = getKendra().startDataSourceSyncJob( request ); 372 return result.getExecutionId(); 373 } 374 375 /** 376 * Stop the execution for the given index Id and DataSource Id. 377 */ 378 private void stopExecution() { 379 final StopDataSourceSyncJobRequest request = new StopDataSourceSyncJobRequest().withIndexId( indexId ).withId( dataSourceId ); 380 getKendra().stopDataSourceSyncJob( request ); 381 } 382 383 /** 384 * Index on single {@link Page} into the Kendra Index 385 * 386 * @param page the {@link Page} to index 387 * @param executionId The Execution Id 388 */ 389 private void indexOnePage( final Page page, final String executionId ) { 390 final String pageName = page.getName(); 391 try { 392 final Document document = newDocument( page, executionId ); 393 final BatchPutDocumentRequest request = new BatchPutDocumentRequest().withIndexId( indexId ).withDocuments( document ); 394 final BatchPutDocumentResult result = getKendra().batchPutDocument( request ); 395 if ( result.getFailedDocuments().isEmpty() ) { 396 LOG.info( "Successfully indexed Page '{}' as {}", page.getName(), document.getContentType() ); 397 } else { 398 for ( final BatchPutDocumentResponseFailedDocument failedDocument : result.getFailedDocuments() ) { 399 LOG.error( "Failed to index Page '{}': {}", failedDocument.getId(), failedDocument.getErrorMessage() ); 400 } 401 } 402 } catch ( final IOException e ) { 403 LOG.error( "Failed to index Page '{}': {}", pageName, e.getMessage() ); 404 } 405 } 406 407 408 /** 409 * Given a {@link Page}, returns the corresponding Kendra {@link Document}. 410 * 411 * @param page the {@link Page} to be indexed 412 * @param executionId an execution id to identify when the {@link Page} was 413 * indexed for the last time. 414 * @return a {@link Document} containing the searchable attributes. 415 * @throws IOException if the {@link Page}'s {@link Attachment} can not be read. 416 */ 417 private Document newDocument( final Page page, final String executionId ) throws IOException { 418 final String pageName = page.getName(); 419 final List< DocumentAttribute > attrs = new ArrayList<>(); 420 // These 2 are required as per 421 // https://docs.aws.amazon.com/kendra/latest/dg/data-source-custom.html#custom-required-attributes 422 attrs.add( newAttribute( "_data_source_id", dataSourceId ) ); 423 attrs.add( newAttribute( "_data_source_sync_job_execution_id", executionId ) ); 424 425 final String title = TextUtil.beautifyString( pageName ); 426 ByteBuffer blob; 427 ContentType contentType = ContentType.PLAIN_TEXT; 428 if ( page instanceof final Attachment attachment ) { 429 InputStream is = null; 430 try { 431 final String filename = attachment.getFileName(); 432 contentType = getContentType( filename ); 433 is = engine.getManager( AttachmentManager.class ).getAttachmentStream( attachment ); 434 blob = ByteBuffer.wrap( IOUtils.toByteArray( is ) ); 435 } catch ( final ProviderException e ) { 436 throw new IOException( e ); 437 } finally { 438 IOUtils.closeQuietly( is, null ); 439 } 440 // contentType should be set to its real value 441 } else { 442 final String text = engine.getManager( PageManager.class ).getPureText( page ); 443 blob = ByteBuffer.wrap( text.getBytes( StandardCharsets.UTF_8 ) ); 444 } 445 return new Document().withId( pageName ).withTitle( title ).withAttributes( attrs ).withBlob( blob ) 446 .withContentType( contentType ); 447 } 448 449 private DocumentAttribute newAttribute( final String key, final String value ) { 450 return new DocumentAttribute().withKey( key ).withValue( new DocumentAttributeValue().withStringValue( value ) ); 451 } 452 453 @SuppressWarnings( "unchecked" ) 454 private ContentType getContentType( final String filename ) { 455 final String extention = FilenameUtils.getExtension( filename ); 456 final Map< String, String > ct = ( Map< String, String > ) this.contentTypes.get( "ContentTypes" ); 457 return ContentType.fromValue( ct.getOrDefault( extention, ContentType.PLAIN_TEXT.name() ) ); 458 } 459 460 /** 461 * Updater thread that updates Kendra indexes. 462 */ 463 private static final class KendraUpdater extends WikiBackgroundThread { 464 static final int INDEX_DELAY = 5; 465 static final int INITIAL_DELAY = 10; 466 private final KendraSearchProvider provider; 467 468 private final int initialDelay; 469 470 private WatchDog watchdog; 471 472 private KendraUpdater( final Engine engine, final KendraSearchProvider provider, final int initialDelay, final int indexDelay ) { 473 super( engine, indexDelay ); 474 this.provider = provider; 475 this.initialDelay = initialDelay; 476 setName( "JSPWiki Kendra Indexer" ); 477 } 478 479 @Override 480 public void startupTask() throws Exception { 481 watchdog = WatchDog.getCurrentWatchDog( getEngine() ); 482 try { 483 Thread.sleep( initialDelay * 1000L ); 484 } catch ( final InterruptedException e ) { 485 throw new InternalWikiException( "Interrupted while waiting to start.", e ); 486 } 487 watchdog.enterState( "Full reindex" ); 488 provider.initializeIndexAndDataSource(); 489 provider.doFullReindex(); 490 watchdog.exitState(); 491 } 492 493 @Override 494 public void backgroundTask() { 495 watchdog.enterState( "Reindexing ...", 60 ); 496 provider.doPartialReindex(); 497 watchdog.exitState(); 498 } 499 } 500 501 private static class SearchResultImpl implements SearchResult { 502 503 private final Page page; 504 private final int score; 505 private final String[] contexts; 506 507 public SearchResultImpl( final Page page, final int score, final String[] contexts ) { 508 this.page = page; 509 this.score = score; 510 this.contexts = contexts != null ? contexts.clone() : null; 511 } 512 513 @Override 514 public Page getPage() { 515 return this.page; 516 } 517 518 @Override 519 public int getScore() { 520 return this.score; 521 } 522 523 @Override 524 public String[] getContexts() { 525 return this.contexts; 526 } 527 } 528 529 public AWSkendra getKendra() { 530 return kendra; 531 } 532 533 public void setKendra( final AWSkendra kendra ) { 534 this.kendra = kendra; 535 } 536 537 protected AWSkendra buildClient() { 538 return AWSkendraClientBuilder.defaultClient(); 539 } 540 541 public String getIndexName() { 542 return indexName; 543 } 544 545 public String getDataSourceName() { 546 return dataSourceName; 547 } 548 549}