001/*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.    
018 */
019package org.apache.wiki.search.tika;
020
021import org.apache.logging.log4j.LogManager;
022import org.apache.logging.log4j.Logger;
023import org.apache.tika.exception.TikaException;
024import org.apache.tika.exception.WriteLimitReachedException;
025import org.apache.tika.metadata.ClimateForcast;
026import org.apache.tika.metadata.CreativeCommons;
027import org.apache.tika.metadata.Database;
028import org.apache.tika.metadata.HttpHeaders;
029import org.apache.tika.metadata.IPTC;
030import org.apache.tika.metadata.Metadata;
031import org.apache.tika.metadata.Office;
032import org.apache.tika.metadata.OfficeOpenXMLCore;
033import org.apache.tika.metadata.PDF;
034import org.apache.tika.metadata.TikaCoreProperties;
035import org.apache.tika.parser.AutoDetectParser;
036import org.apache.tika.sax.BodyContentHandler;
037import org.apache.wiki.api.core.Attachment;
038import org.apache.wiki.api.exceptions.ProviderException;
039import org.apache.wiki.attachment.AttachmentManager;
040import org.apache.wiki.search.LuceneSearchProvider;
041import org.xml.sax.ContentHandler;
042import org.xml.sax.SAXException;
043
044import java.io.IOException;
045import java.io.InputStream;
046import java.util.HashSet;
047import java.util.Set;
048
049/**
050 * Search provider that extends {link LuceneSearchProvider} using Apache Tika for indexing attachment content.
051 *
052 * @since 2.11.0
053 * @see <a href="https://issues.apache.org/jira/browse/JSPWIKI-469">JSPWIKI-469</a>
054 */
055public class TikaSearchProvider extends LuceneSearchProvider {
056
057    private static final Logger LOG = LogManager.getLogger( TikaSearchProvider.class );
058    final AutoDetectParser parser;
059    final Set< String > textualMetadataFields;
060
061    /**
062     * Maximum number of characters extracted from a single attachment for indexing. Bounds the memory used at index
063     * time regardless of attachment content: a small compressed attachment (e.g. a zip bomb, which expands to
064     * gigabytes of text) must not be able to grow into an unbounded in-memory string and exhaust the heap.
065     */
066    static final int MAX_EXTRACTED_CHARS = 1_000_000;
067
068    public TikaSearchProvider() {
069        parser = new AutoDetectParser();
070
071        // metadata fields that also are indexed
072        textualMetadataFields = new HashSet<>();
073        textualMetadataFields.add( TikaCoreProperties.TITLE.getName() );
074        textualMetadataFields.add( TikaCoreProperties.COMMENTS.getName() );
075        textualMetadataFields.add( TikaCoreProperties.SUBJECT.getName() );
076        textualMetadataFields.add( TikaCoreProperties.DESCRIPTION.getName() );
077        textualMetadataFields.add( TikaCoreProperties.TYPE.getName() );
078        textualMetadataFields.add( TikaCoreProperties.RESOURCE_NAME_KEY );
079        textualMetadataFields.add( PDF.DOC_INFO_TITLE.getName() );
080        textualMetadataFields.add( PDF.DOC_INFO_KEY_WORDS.getName() );
081        textualMetadataFields.add( PDF.DOC_INFO_SUBJECT.getName() );
082        textualMetadataFields.add( OfficeOpenXMLCore.SUBJECT.getName() );
083        textualMetadataFields.add( Office.KEYWORDS.getName() );
084        textualMetadataFields.add( TikaCoreProperties.TYPE.getName() );
085        textualMetadataFields.add( HttpHeaders.CONTENT_TYPE );
086        textualMetadataFields.add( IPTC.HEADLINE.getName() );
087        textualMetadataFields.add( Database.COLUMN_NAME.getName() );
088        textualMetadataFields.add( Database.TABLE_NAME.getName() );
089        textualMetadataFields.add( CreativeCommons.WORK_TYPE );
090        textualMetadataFields.add( ClimateForcast.COMMENT );
091        textualMetadataFields.add( ClimateForcast.HISTORY );
092        textualMetadataFields.add( ClimateForcast.INSTITUTION );
093    }
094
095    /**
096     * {@inheritDoc}
097     *
098     * @param att Attachment to get content for. Filename extension is used to determine the type of the attachment.
099     * @return String representing the content of the file.
100     */
101    @Override
102    protected String getAttachmentContent( final Attachment att ) {
103        final AttachmentManager mgr = getEngine().getManager( AttachmentManager.class );
104        final StringBuilder out = new StringBuilder();
105
106        try( final InputStream attStream = mgr.getAttachmentStream( att ) ) {
107            final Metadata metadata = new Metadata();
108            metadata.set( TikaCoreProperties.RESOURCE_NAME_KEY, att.getFileName() );
109
110            final ContentHandler handler = new BodyContentHandler( MAX_EXTRACTED_CHARS );
111            try {
112                parser.parse( attStream, handler, metadata );
113            } catch( final SAXException e ) {
114                if( !WriteLimitReachedException.isWriteLimitReached( e ) ) {
115                    throw e;
116                }
117                // keep what was extracted so far, so oversized attachments still get (truncated) indexing
118                LOG.warn( "Attachment {} reached the maximum of {} extracted characters, indexing truncated content",
119                          att.getFileName(), MAX_EXTRACTED_CHARS );
120            }
121            out.append( handler );
122
123            final String[] names = metadata.names();
124            for( final String name : names ) {
125                if( textualMetadataFields.contains( name ) ) {
126                    out.append( " " ).append( metadata.get( name ) );
127                }
128            }
129        } catch( final TikaException | SAXException e ) {
130            LOG.error( "Attachment cannot be parsed", e );
131        } catch( final ProviderException | IOException e ) {
132            LOG.error( "Attachment cannot be loaded", e );
133        }
134
135        return out.toString();
136    }
137
138}