001/*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.    
018 */
019package org.apache.wiki.search.kendra;
020
021import com.amazonaws.services.kendra.AWSkendra;
022import com.amazonaws.services.kendra.AWSkendraClientBuilder;
023import com.amazonaws.services.kendra.model.*;
024import com.amazonaws.util.IOUtils;
025import com.google.gson.Gson;
026import com.google.gson.GsonBuilder;
027import com.google.gson.reflect.TypeToken;
028import org.apache.commons.io.FilenameUtils;
029import org.apache.commons.lang3.Strings;
030import org.apache.logging.log4j.LogManager;
031import org.apache.logging.log4j.Logger;
032import org.apache.wiki.InternalWikiException;
033import org.apache.wiki.WatchDog;
034import org.apache.wiki.WikiBackgroundThread;
035import org.apache.wiki.api.core.Attachment;
036import org.apache.wiki.api.core.Context;
037import org.apache.wiki.api.core.Engine;
038import org.apache.wiki.api.core.Page;
039import org.apache.wiki.api.exceptions.NoRequiredPropertyException;
040import org.apache.wiki.api.exceptions.ProviderException;
041import org.apache.wiki.api.providers.PageProvider;
042import org.apache.wiki.api.search.SearchResult;
043import org.apache.wiki.api.spi.Wiki;
044import org.apache.wiki.attachment.AttachmentManager;
045import org.apache.wiki.auth.AuthorizationManager;
046import org.apache.wiki.auth.permissions.PagePermission;
047import org.apache.wiki.pages.PageManager;
048import org.apache.wiki.search.SearchProvider;
049import org.apache.wiki.util.Synchronizer;
050import org.apache.wiki.util.TextUtil;
051
052import java.io.IOException;
053import java.io.InputStream;
054import java.io.InputStreamReader;
055import java.lang.reflect.Type;
056import java.nio.ByteBuffer;
057import java.nio.charset.StandardCharsets;
058import java.util.ArrayList;
059import java.util.Collection;
060import java.util.Collections;
061import java.util.HashMap;
062import java.util.List;
063import java.util.Map;
064import java.util.Properties;
065import java.util.concurrent.locks.ReentrantLock;
066
067import static java.lang.String.format;
068
069/**
070 * Search provider that implements {link SearchProvider} using AWS Kendra for
071 * indexing. Note that we are using a Custom DataSource which limits the
072 * attributes that can be uploaded / searched for each page (as per
073 * https://docs.aws.amazon.com/kendra/latest/dg/custom-attributes.html). This
074 * could be overcome by using an S3 bucket where any custom attributes can be
075 * added.
076 *
077 * @since 2.11.0
078 */
079public class KendraSearchProvider implements SearchProvider {
080
081    private static final Logger LOG = LogManager.getLogger( KendraSearchProvider.class );
082    private Engine engine;
083    private Properties properties;
084    private Map< String, Object > contentTypes;
085    private AWSkendra kendra;
086    private String indexName;
087    private String indexId;
088    private String dataSourceName;
089    private String dataSourceId;
090
091    private final List< Page > updates = Collections.synchronizedList( new ArrayList<>() );
092
093    private static final String PROP_KENDRA_INDEX_NAME = "jspwiki.kendra.indexName";
094    private static final String PROP_KENDRA_DATA_SOURCE_NAME = "jspwiki.kendra.dataSourceName";
095    private static final String PROP_KENDRA_INDEXDELAY = "jspwiki.kendra.indexdelay";
096    private static final String PROP_KENDRA_INITIALDELAY = "jspwiki.kendra.initialdelay";
097
098    /**
099     * A lock used to ensure thread safety when accessing shared resources.
100     * This lock provides more flexibility and capabilities than the intrinsic locking mechanism,
101     * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread
102     * waiting to acquire a lock.
103     *
104     * @see java.util.concurrent.locks.ReentrantLock
105     */
106    private final ReentrantLock lock = new ReentrantLock();
107
108    /**
109     * {@inheritDoc}
110     */
111    @Override
112    public void initialize( final Engine engine, final Properties properties ) throws NoRequiredPropertyException, IOException {
113        this.engine = engine;
114        this.properties = properties;
115        this.contentTypes = getContentTypes();
116
117        setKendra( buildClient() );
118
119        this.indexName = TextUtil.getRequiredProperty( this.properties, PROP_KENDRA_INDEX_NAME );
120        this.dataSourceName = TextUtil.getRequiredProperty( this.properties, PROP_KENDRA_DATA_SOURCE_NAME );
121        final int initialDelay = TextUtil.getIntegerProperty( this.properties, PROP_KENDRA_INITIALDELAY, KendraUpdater.INITIAL_DELAY );
122        final int indexDelay = TextUtil.getIntegerProperty( this.properties, PROP_KENDRA_INDEXDELAY, KendraUpdater.INDEX_DELAY );
123
124        // Start the Kendra update thread, which waits first for a little while
125        // before starting to go through the "pages that need updating".
126        if( initialDelay >= 0 ) {
127            final KendraUpdater updater = new KendraUpdater( engine, this, initialDelay, indexDelay );
128            updater.start();
129        }
130    }
131
132    private Map< String, Object > getContentTypes() {
133        final Gson gson = new GsonBuilder().create();
134        try( final InputStream in = KendraSearchProvider.class.getResourceAsStream( "content_types.json" ) ) {
135            if ( in != null ) {
136                final Type collectionType = new TypeToken< HashMap< String, Object > >() {
137                }.getType();
138                return gson.fromJson( new InputStreamReader( in ), collectionType );
139            }
140        } catch ( final IOException e ) {
141            LOG.error( "Unable to load default propertyfile 'content_types.json': {}", e.getMessage(), e );
142        }
143        return null;
144    }
145
146    /**
147     * {@inheritDoc}
148     */
149    @Override
150    public String getProviderInfo() {
151        return "KendraSearchProvider";
152    }
153
154    /**
155     * {@inheritDoc}
156     */
157    @Override
158    public void pageRemoved( final Page page ) {
159        final String pageName = page.getName();
160        final BatchDeleteDocumentRequest request = new BatchDeleteDocumentRequest().withIndexId( indexId ).withDocumentIdList( pageName );
161        final BatchDeleteDocumentResult result = getKendra().batchDeleteDocument( request );
162        if (result.getFailedDocuments().isEmpty()) {
163            LOG.debug( "Page '{}' was removed from index", pageName );
164        } else {
165            LOG.error( "Failed to remove Page '{}' from index", pageName );
166        }
167    }
168
169    /**
170     * {@inheritDoc}
171     */
172    @Override
173    public void reindexPage( final Page page ) {
174        if( page != null ) {
175            updates.add( page );
176            LOG.debug( "Scheduling page '{}' for indexing ...", page.getName() );
177        }
178    }
179
180    /**
181     * {@inheritDoc}
182     */
183    @Override
184    public Collection< SearchResult > findPages( final String query, final Context wikiContext ) throws ProviderException, IOException {
185        final QueryRequest request = new QueryRequest().withIndexId( indexId ).withQueryText( query );
186        final List< QueryResultItem > items;
187        try {
188            items = getKendra().query( request ).getResultItems();
189        } catch ( final ThrottlingException e ) {
190            LOG.error( "ThrottlingException. Skipping..." );
191            return new ArrayList<>();
192        }
193        final List< SearchResult > searchResults = new ArrayList<>( items.size() );
194        final AuthorizationManager mgr = engine.getManager( AuthorizationManager.class );
195
196        for ( final QueryResultItem item : items ) {
197            switch( QueryResultType.fromValue( item.getType() ) ) {
198                case DOCUMENT:
199                    final String documentId = item.getDocumentId();
200                    final String documentExcerpt = item.getDocumentExcerpt().getText();
201                    final String scoreConfidence = item.getScoreAttributes().getScoreConfidence();
202                    final Page page = this.engine.getManager( PageManager.class ).getPage( documentId, PageProvider.LATEST_VERSION );
203                    if ( page != null ) {
204                        final PagePermission pp = new PagePermission( page, PagePermission.VIEW_ACTION );
205                        if ( mgr.checkPermission( wikiContext.getWikiSession(), pp ) ) {
206                            final SearchResult searchResult = new SearchResultImpl( page, confidence2score( scoreConfidence ),
207                                    new String[]{ documentExcerpt } );
208                            searchResults.add( searchResult );
209                        } else {
210                            LOG.error( "Page '{}' is not accessible", documentId );
211                        }
212                    } else {
213                        LOG.error( "Kendra found a result page '{}' that could not be loaded, removing from index", documentId );
214                        pageRemoved( Wiki.contents().page( this.engine, documentId ) );
215                    }
216                    break;
217                default:
218                    LOG.error( "Unknown query result type: {}", item.getType() );
219            }
220        }
221        return searchResults;
222    }
223
224    /**
225     * This method initialize the AWS Kendra Index and Datasources to be used.
226     */
227    public void initializeIndexAndDataSource() {
228        this.indexId = getIndexId( indexName );
229        if ( this.indexId == null ) {
230            final String message = format( "Index '%s' does not exist", indexName );
231            LOG.error( message );
232            throw new IllegalArgumentException( message );
233        }
234        this.dataSourceId = getDatasourceId( this.indexId, dataSourceName );
235        if ( this.dataSourceId == null ) {
236            final String message = format( "Datasource '%s' does not exist in index %s", dataSourceName, indexName );
237            LOG.error( message );
238            throw new IllegalArgumentException( message );
239        }
240    }
241
242    /**
243     * Given an Kendra's Index name, returns the corresponding Index Id, or
244     * {@code null} if it does not exists
245     *
246     * @param indexName the name of the index to look up
247     * @return the index id or {@code null}
248     */
249    private String getIndexId( final String indexName ) {
250        ListIndicesRequest request = new ListIndicesRequest();
251        ListIndicesResult result = getKendra().listIndices( request );
252        String nextToken = "";
253        while ( nextToken != null ) {
254            final List< IndexConfigurationSummary > items = result.getIndexConfigurationSummaryItems();
255            if ( items == null || items.isEmpty() ) {
256                return null;
257            }
258            for ( final IndexConfigurationSummary item : items ) {
259                if ( Strings.CS.equals( item.getName(), indexName ) ) {
260                    return item.getId();
261                }
262            }
263            nextToken = result.getNextToken();
264            request = new ListIndicesRequest().withNextToken( result.getNextToken() );
265            result = getKendra().listIndices( request );
266        }
267        return null;
268    }
269
270    /**
271     * Given an Kendra's Datasource name, returns the corresponding Datasource Id,
272     * or {@code null} if it does not exists
273     *
274     * @param dataSourceName the name of the datasource to look up
275     * @return the datasource id or {@code null}
276     */
277    private String getDatasourceId( final String indexId, final String dataSourceName ) {
278        ListDataSourcesRequest request = new ListDataSourcesRequest().withIndexId( indexId );
279        ListDataSourcesResult result = getKendra().listDataSources( request );
280        String nextToken = "";
281        while ( nextToken != null ) {
282            final List< DataSourceSummary > items = result.getSummaryItems();
283            if( items == null || items.isEmpty() ) {
284                return null;
285            }
286
287            for( final DataSourceSummary item : items ) {
288                if( Strings.CS.equals( item.getName(), dataSourceName ) ) {
289                    return item.getId();
290                }
291            }
292            nextToken = result.getNextToken();
293            request = new ListDataSourcesRequest().withNextToken( result.getNextToken() );
294            result = getKendra().listDataSources( request );
295        }
296        return null;
297    }
298
299    /*
300     * Converts a SCORE Confidence from Kendra to an "equivalent" integer score
301     */
302    private int confidence2score( final String scoreConfidence ) {
303        return switch( ScoreConfidence.fromValue( scoreConfidence ) ) {
304            case VERY_HIGH -> 100;
305            case HIGH -> 75;
306            case MEDIUM -> 50;
307            case LOW -> 25;
308            default -> 0;
309        };
310    }
311
312    /**
313     * This method re-index all the pages found in the Wiki. It is mainly used at
314     * startup.
315     *
316     * @throws IOException in case some page can not be read
317     */
318    private void doFullReindex() throws IOException {
319        try {
320            final Collection< Page > pages = engine.getManager( PageManager.class ).getAllPages();
321            if ( pages.isEmpty() ) {
322                return;
323            }
324            LOG.debug( "Indexing all {} pages. Please wait ...", pages.size() );
325            final String executionId = startExecution();
326            for ( final Page page : pages ) {
327                // Since I do not want to handle the size limit
328                // (https://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/BatchPutDocument)
329                // uploading documents one at a time
330                indexOnePage( page, executionId );
331            }
332        } catch ( final ProviderException e ) {
333            LOG.error( e.getMessage() );
334            throw new IOException( e );
335        } finally {
336            stopExecution();
337        }
338    }
339
340    /**
341     * This method re-index all pages marked as updated. It is used to periodically
342     * index pages that have been modified
343     */
344    private void doPartialReindex() {
345        if (updates.isEmpty()) {
346            return;
347        }
348        LOG.debug("Indexing updated pages. Please wait ...");
349        final String executionId = startExecution();
350
351        Synchronizer.synchronize(lock, () -> {
352            try {
353                while (!updates.isEmpty()) {
354                    indexOnePage(updates.remove(0), executionId);
355                }
356            } finally {
357                stopExecution();
358            }
359        });
360    }
361
362    /**
363     * Returns an ExecutiuonId that is required to keep track of the modifed
364     * documents
365     *
366     * @return The execution id
367     */
368    private String startExecution() {
369        final StartDataSourceSyncJobRequest request = new StartDataSourceSyncJobRequest().withIndexId( indexId )
370                .withId( dataSourceId );
371        final StartDataSourceSyncJobResult result = getKendra().startDataSourceSyncJob( request );
372        return result.getExecutionId();
373    }
374
375    /**
376     * Stop the execution for the given index Id and DataSource Id.
377     */
378    private void stopExecution() {
379        final StopDataSourceSyncJobRequest request = new StopDataSourceSyncJobRequest().withIndexId( indexId ).withId( dataSourceId );
380        getKendra().stopDataSourceSyncJob( request );
381    }
382
383    /**
384     * Index on single {@link Page} into the Kendra Index
385     *
386     * @param page        the {@link Page} to index
387     * @param executionId The Execution Id
388     */
389    private void indexOnePage( final Page page, final String executionId ) {
390        final String pageName = page.getName();
391        try {
392            final Document document = newDocument( page, executionId );
393            final BatchPutDocumentRequest request = new BatchPutDocumentRequest().withIndexId( indexId ).withDocuments( document );
394            final BatchPutDocumentResult result = getKendra().batchPutDocument( request );
395            if ( result.getFailedDocuments().isEmpty() ) {
396                LOG.info( "Successfully indexed Page '{}' as {}", page.getName(), document.getContentType() );
397            } else {
398                for ( final BatchPutDocumentResponseFailedDocument failedDocument : result.getFailedDocuments() ) {
399                    LOG.error( "Failed to index Page '{}': {}", failedDocument.getId(), failedDocument.getErrorMessage() );
400                }
401            }
402        } catch ( final IOException e ) {
403            LOG.error( "Failed to index Page '{}': {}", pageName, e.getMessage() );
404        }
405    }
406
407
408    /**
409     * Given a {@link Page}, returns the corresponding Kendra {@link Document}.
410     *
411     * @param page        the {@link Page} to be indexed
412     * @param executionId an execution id to identify when the {@link Page} was
413     *                    indexed for the last time.
414     * @return a {@link Document} containing the searchable attributes.
415     * @throws IOException if the {@link Page}'s {@link Attachment} can not be read.
416     */
417    private Document newDocument( final Page page, final String executionId ) throws IOException {
418        final String pageName = page.getName();
419        final List< DocumentAttribute > attrs = new ArrayList<>();
420        // These 2 are required as per
421        // https://docs.aws.amazon.com/kendra/latest/dg/data-source-custom.html#custom-required-attributes
422        attrs.add( newAttribute( "_data_source_id", dataSourceId ) );
423        attrs.add( newAttribute( "_data_source_sync_job_execution_id", executionId ) );
424
425        final String title = TextUtil.beautifyString( pageName );
426        ByteBuffer blob;
427        ContentType contentType = ContentType.PLAIN_TEXT;
428        if ( page instanceof final Attachment attachment ) {
429            InputStream is = null;
430            try {
431                final String filename = attachment.getFileName();
432                contentType = getContentType( filename );
433                is = engine.getManager( AttachmentManager.class ).getAttachmentStream( attachment );
434                blob = ByteBuffer.wrap( IOUtils.toByteArray( is ) );
435            } catch ( final ProviderException e ) {
436                throw new IOException( e );
437            } finally {
438                IOUtils.closeQuietly( is, null );
439            }
440            // contentType should be set to its real value
441        } else {
442            final String text = engine.getManager( PageManager.class ).getPureText( page );
443            blob = ByteBuffer.wrap( text.getBytes( StandardCharsets.UTF_8 ) );
444        }
445        return new Document().withId( pageName ).withTitle( title ).withAttributes( attrs ).withBlob( blob )
446                .withContentType( contentType );
447    }
448
449    private DocumentAttribute newAttribute( final String key, final String value ) {
450        return new DocumentAttribute().withKey( key ).withValue( new DocumentAttributeValue().withStringValue( value ) );
451    }
452
453    @SuppressWarnings( "unchecked" )
454    private ContentType getContentType( final String filename ) {
455        final String extention = FilenameUtils.getExtension( filename );
456        final Map< String, String > ct = ( Map< String, String > ) this.contentTypes.get( "ContentTypes" );
457        return ContentType.fromValue( ct.getOrDefault( extention, ContentType.PLAIN_TEXT.name() ) );
458    }
459
460    /**
461     * Updater thread that updates Kendra indexes.
462     */
463    private static final class KendraUpdater extends WikiBackgroundThread {
464        static final int INDEX_DELAY = 5;
465        static final int INITIAL_DELAY = 10;
466        private final KendraSearchProvider provider;
467
468        private final int initialDelay;
469
470        private WatchDog watchdog;
471
472        private KendraUpdater( final Engine engine, final KendraSearchProvider provider, final int initialDelay, final int indexDelay ) {
473            super( engine, indexDelay );
474            this.provider = provider;
475            this.initialDelay = initialDelay;
476            setName( "JSPWiki Kendra Indexer" );
477        }
478
479        @Override
480        public void startupTask() throws Exception {
481            watchdog = WatchDog.getCurrentWatchDog( getEngine() );
482            try {
483                Thread.sleep( initialDelay * 1000L );
484            } catch ( final InterruptedException e ) {
485                throw new InternalWikiException( "Interrupted while waiting to start.", e );
486            }
487            watchdog.enterState( "Full reindex" );
488            provider.initializeIndexAndDataSource();
489            provider.doFullReindex();
490            watchdog.exitState();
491        }
492
493        @Override
494        public void backgroundTask() {
495            watchdog.enterState( "Reindexing ...", 60 );
496            provider.doPartialReindex();
497            watchdog.exitState();
498        }
499    }
500
501    private static class SearchResultImpl implements SearchResult {
502
503        private final Page page;
504        private final int score;
505        private final String[] contexts;
506
507        public SearchResultImpl( final Page page, final int score, final String[] contexts ) {
508            this.page = page;
509            this.score = score;
510            this.contexts = contexts != null ? contexts.clone() : null;
511        }
512
513        @Override
514        public Page getPage() {
515            return this.page;
516        }
517
518        @Override
519        public int getScore() {
520            return this.score;
521        }
522
523        @Override
524        public String[] getContexts() {
525            return this.contexts;
526        }
527    }
528
529    public AWSkendra getKendra() {
530        return kendra;
531    }
532
533    public void setKendra( final AWSkendra kendra ) {
534        this.kendra = kendra;
535    }
536
537    protected AWSkendra buildClient() {
538        return AWSkendraClientBuilder.defaultClient();
539    }
540
541    public String getIndexName() {
542        return indexName;
543    }
544
545    public String getDataSourceName() {
546        return dataSourceName;
547    }
548
549}