001/* 002 Licensed to the Apache Software Foundation (ASF) under one 003 or more contributor license agreements. See the NOTICE file 004 distributed with this work for additional information 005 regarding copyright ownership. The ASF licenses this file 006 to you under the Apache License, Version 2.0 (the 007 "License"); you may not use this file except in compliance 008 with the License. You may obtain a copy of the License at 009 010 http://www.apache.org/licenses/LICENSE-2.0 011 012 Unless required by applicable law or agreed to in writing, 013 software distributed under the License is distributed on an 014 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 KIND, either express or implied. See the License for the 016 specific language governing permissions and limitations 017 under the License. 018 */ 019package org.apache.wiki.search.tika; 020 021import org.apache.logging.log4j.LogManager; 022import org.apache.logging.log4j.Logger; 023import org.apache.tika.exception.TikaException; 024import org.apache.tika.exception.WriteLimitReachedException; 025import org.apache.tika.metadata.ClimateForcast; 026import org.apache.tika.metadata.CreativeCommons; 027import org.apache.tika.metadata.Database; 028import org.apache.tika.metadata.HttpHeaders; 029import org.apache.tika.metadata.IPTC; 030import org.apache.tika.metadata.Metadata; 031import org.apache.tika.metadata.Office; 032import org.apache.tika.metadata.OfficeOpenXMLCore; 033import org.apache.tika.metadata.PDF; 034import org.apache.tika.metadata.TikaCoreProperties; 035import org.apache.tika.parser.AutoDetectParser; 036import org.apache.tika.sax.BodyContentHandler; 037import org.apache.wiki.api.core.Attachment; 038import org.apache.wiki.api.exceptions.ProviderException; 039import org.apache.wiki.attachment.AttachmentManager; 040import org.apache.wiki.search.LuceneSearchProvider; 041import org.xml.sax.ContentHandler; 042import org.xml.sax.SAXException; 043 044import java.io.IOException; 045import java.io.InputStream; 046import java.util.HashSet; 047import java.util.Set; 048 049/** 050 * Search provider that extends {link LuceneSearchProvider} using Apache Tika for indexing attachment content. 051 * 052 * @since 2.11.0 053 * @see <a href="https://issues.apache.org/jira/browse/JSPWIKI-469">JSPWIKI-469</a> 054 */ 055public class TikaSearchProvider extends LuceneSearchProvider { 056 057 private static final Logger LOG = LogManager.getLogger( TikaSearchProvider.class ); 058 final AutoDetectParser parser; 059 final Set< String > textualMetadataFields; 060 061 /** 062 * Maximum number of characters extracted from a single attachment for indexing. Bounds the memory used at index 063 * time regardless of attachment content: a small compressed attachment (e.g. a zip bomb, which expands to 064 * gigabytes of text) must not be able to grow into an unbounded in-memory string and exhaust the heap. 065 */ 066 static final int MAX_EXTRACTED_CHARS = 1_000_000; 067 068 public TikaSearchProvider() { 069 parser = new AutoDetectParser(); 070 071 // metadata fields that also are indexed 072 textualMetadataFields = new HashSet<>(); 073 textualMetadataFields.add( TikaCoreProperties.TITLE.getName() ); 074 textualMetadataFields.add( TikaCoreProperties.COMMENTS.getName() ); 075 textualMetadataFields.add( TikaCoreProperties.SUBJECT.getName() ); 076 textualMetadataFields.add( TikaCoreProperties.DESCRIPTION.getName() ); 077 textualMetadataFields.add( TikaCoreProperties.TYPE.getName() ); 078 textualMetadataFields.add( TikaCoreProperties.RESOURCE_NAME_KEY ); 079 textualMetadataFields.add( PDF.DOC_INFO_TITLE.getName() ); 080 textualMetadataFields.add( PDF.DOC_INFO_KEY_WORDS.getName() ); 081 textualMetadataFields.add( PDF.DOC_INFO_SUBJECT.getName() ); 082 textualMetadataFields.add( OfficeOpenXMLCore.SUBJECT.getName() ); 083 textualMetadataFields.add( Office.KEYWORDS.getName() ); 084 textualMetadataFields.add( TikaCoreProperties.TYPE.getName() ); 085 textualMetadataFields.add( HttpHeaders.CONTENT_TYPE ); 086 textualMetadataFields.add( IPTC.HEADLINE.getName() ); 087 textualMetadataFields.add( Database.COLUMN_NAME.getName() ); 088 textualMetadataFields.add( Database.TABLE_NAME.getName() ); 089 textualMetadataFields.add( CreativeCommons.WORK_TYPE ); 090 textualMetadataFields.add( ClimateForcast.COMMENT ); 091 textualMetadataFields.add( ClimateForcast.HISTORY ); 092 textualMetadataFields.add( ClimateForcast.INSTITUTION ); 093 } 094 095 /** 096 * {@inheritDoc} 097 * 098 * @param att Attachment to get content for. Filename extension is used to determine the type of the attachment. 099 * @return String representing the content of the file. 100 */ 101 @Override 102 protected String getAttachmentContent( final Attachment att ) { 103 final AttachmentManager mgr = getEngine().getManager( AttachmentManager.class ); 104 final StringBuilder out = new StringBuilder(); 105 106 try( final InputStream attStream = mgr.getAttachmentStream( att ) ) { 107 final Metadata metadata = new Metadata(); 108 metadata.set( TikaCoreProperties.RESOURCE_NAME_KEY, att.getFileName() ); 109 110 final ContentHandler handler = new BodyContentHandler( MAX_EXTRACTED_CHARS ); 111 try { 112 parser.parse( attStream, handler, metadata ); 113 } catch( final SAXException e ) { 114 if( !WriteLimitReachedException.isWriteLimitReached( e ) ) { 115 throw e; 116 } 117 // keep what was extracted so far, so oversized attachments still get (truncated) indexing 118 LOG.warn( "Attachment {} reached the maximum of {} extracted characters, indexing truncated content", 119 att.getFileName(), MAX_EXTRACTED_CHARS ); 120 } 121 out.append( handler ); 122 123 final String[] names = metadata.names(); 124 for( final String name : names ) { 125 if( textualMetadataFields.contains( name ) ) { 126 out.append( " " ).append( metadata.get( name ) ); 127 } 128 } 129 } catch( final TikaException | SAXException e ) { 130 LOG.error( "Attachment cannot be parsed", e ); 131 } catch( final ProviderException | IOException e ) { 132 LOG.error( "Attachment cannot be loaded", e ); 133 } 134 135 return out.toString(); 136 } 137 138}