001/*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.  
018 */
019package org.apache.wiki.filters;
020
021import com.github.difflib.DiffUtils;
022import com.github.difflib.patch.AbstractDelta;
023import com.github.difflib.patch.Patch;
024import org.apache.commons.lang3.StringUtils;
025import org.apache.commons.lang3.time.StopWatch;
026import org.apache.logging.log4j.LogManager;
027import org.apache.logging.log4j.Logger;
028import org.apache.oro.text.regex.MalformedPatternException;
029import org.apache.oro.text.regex.MatchResult;
030import org.apache.oro.text.regex.Pattern;
031import org.apache.oro.text.regex.PatternCompiler;
032import org.apache.oro.text.regex.PatternMatcher;
033import org.apache.oro.text.regex.Perl5Compiler;
034import org.apache.oro.text.regex.Perl5Matcher;
035import org.apache.wiki.InternalWikiException;
036import org.apache.wiki.api.core.Attachment;
037import org.apache.wiki.api.core.Context;
038import org.apache.wiki.api.core.ContextEnum;
039import org.apache.wiki.api.core.Engine;
040import org.apache.wiki.api.core.Page;
041import org.apache.wiki.api.exceptions.ProviderException;
042import org.apache.wiki.api.exceptions.RedirectException;
043import org.apache.wiki.api.filters.BasePageFilter;
044import org.apache.wiki.api.providers.WikiProvider;
045import org.apache.wiki.attachment.AttachmentManager;
046import org.apache.wiki.auth.user.UserProfile;
047import org.apache.wiki.pages.PageManager;
048import org.apache.wiki.ui.EditorManager;
049import org.apache.wiki.util.FileUtil;
050import org.apache.wiki.util.HttpUtil;
051import org.apache.wiki.util.Synchronizer;
052import org.apache.wiki.util.TextUtil;
053
054import jakarta.servlet.http.HttpServletRequest;
055import jakarta.servlet.http.HttpServletResponse;
056import jakarta.servlet.jsp.PageContext;
057import java.io.BufferedReader;
058import java.io.IOException;
059import java.io.InputStream;
060import java.io.InputStreamReader;
061import java.io.StringReader;
062import java.io.StringWriter;
063import java.nio.charset.StandardCharsets;
064import java.util.ArrayList;
065import java.util.Arrays;
066import java.util.Collection;
067import java.util.Date;
068import java.util.Iterator;
069import java.util.List;
070import java.util.Properties;
071import java.util.Random;
072import java.util.StringTokenizer;
073import java.util.concurrent.CopyOnWriteArrayList;
074import java.util.concurrent.ThreadLocalRandom;
075import net.thauvin.erik.akismet.Akismet;
076import net.thauvin.erik.akismet.AkismetComment;
077import java.util.concurrent.locks.ReentrantLock;
078
079
080/**
081 *  This is Herb, the JSPWiki spamfilter that can also do choke modifications.
082 *
083 *  Parameters:
084 *  <ul>
085 *    <li>wordlist - Page name where the spamword regexps are found.  Use [{SET spamwords='regexp list separated with spaces'}] on
086 *     that page.  Default is "SpamFilterWordList".
087 *    <li>IPlist - Page name where the IP regexps are found.  Use [{SET ips='regexp list separated with spaces'}] on
088 *     that page.  Default is "SpamFilterIPList".
089 *    <li>maxpagenamelength - Maximum page name length. Default is 100.
090 *    <li>blacklist - The name of an attachment containing the list of spam patterns, one per line. Default is
091 *        "SpamFilterWordList/blacklist.txt"</li>
092 *    <li>errorpage - The page to which the user is redirected.  Has a special variable $msg which states the reason. Default is "RejectedMessage".
093 *    <li>pagechangesinminute - How many page changes are allowed/minute.  Default is 5.</li>
094 *    <li>similarchanges - How many similar page changes are allowed before the host is banned.  Default is 2.  (since 2.4.72)</li>
095 *    <li>bantime - How long an IP address stays on the temporary ban list (default is 60 for 60 minutes).</li>
096 *    <li>maxurls - How many URLs can be added to the page before it is considered spam (default is 5)</li>
097 *    <li>akismet-apikey - The Akismet API key (see akismet.org)</li>
098 *    <li>ignoreauthenticated - If set to "true", all authenticated users are ignored and never caught in SpamFilter</li>
099 *    <li>captcha - Sets the captcha technology to use.  Current allowed values are "none". "asirra" was previously supported however that service has been discontinued.</li>
100 *    <li>strategy - Sets the filtering strategy to use.  If set to "eager", will stop at the first probable
101 *        match, and won't consider any other tests.  This is the default, as it's considerably lighter. If set to "score", will go through all of the tests
102 *        and calculates a score for the spam, which is then compared to a filter level value.
103 *  </ul>
104 *
105 *  <p>Please see the default editors/plain.jsp for examples on how the SpamFilter integrates
106 *  with the editor system.</p>
107 *  
108 *  <p>Changes by admin users are ignored in any case.</p>
109 *
110 *  @since 2.1.112
111 */
112public class SpamFilter extends BasePageFilter {
113    
114    private static final String ATTR_SPAMFILTER_SCORE = "spamfilter.score";
115    private static final String REASON_REGEXP = "Regexp";
116    private static final String REASON_IP_BANNED_TEMPORARILY = "IPBannedTemporarily";
117    private static final String REASON_IP_BANNED_PERMANENTLY = "IPBannedPermanently";
118    private static final String REASON_BOT_TRAP = "BotTrap";
119    private static final String REASON_AKISMET = "Akismet";
120    private static final String REASON_TOO_MANY_URLS = "TooManyUrls";
121    private static final String REASON_SIMILAR_MODIFICATIONS = "SimilarModifications";
122    private static final String REASON_TOO_MANY_MODIFICATIONS = "TooManyModifications";
123    private static final String REASON_PAGENAME_TOO_LONG = "PageNameTooLong";
124    private static final String REASON_UTF8_TRAP = "UTF8Trap";
125
126    private static final String LISTVAR = "spamwords";
127    private static final String LISTIPVAR = "ips";
128
129    private static final Random RANDOM = ThreadLocalRandom.current();
130
131    /** The filter property name for specifying the page which contains the list of spamwords. Value is <tt>{@value}</tt>. */
132    public static final String  PROP_WORDLIST              = "wordlist";
133
134    /** The filter property name for specifying the page which contains the list of IPs to ban. Value is <tt>{@value}</tt>. */
135    public static final String  PROP_IPLIST                = "IPlist";
136
137    /** The filter property name for specifying the maximum page name length.  Value is <tt>{@value}</tt>. */
138    public static final String  PROP_MAX_PAGENAME_LENGTH   = "maxpagenamelength";
139
140    /** The filter property name for the page to which you are directed if Herb rejects your edit.  Value is <tt>{@value}</tt>. */
141    public static final String  PROP_ERRORPAGE             = "errorpage";
142    
143    /** The filter property name for specifying how many changes is any given IP address
144     *  allowed to do per minute.  Value is <tt>{@value}</tt>.
145     */
146    public static final String  PROP_PAGECHANGES           = "pagechangesinminute";
147    
148    /** The filter property name for specifying how many similar changes are allowed before a host is banned.  Value is <tt>{@value}</tt>. */
149    public static final String  PROP_SIMILARCHANGES        = "similarchanges";
150    
151    /** The filter property name for specifying how long a host is banned.  Value is <tt>{@value}</tt>.*/
152    public static final String  PROP_BANTIME               = "bantime";
153    
154    /** The filter property name for the attachment containing the blacklist.  Value is <tt>{@value}</tt>.*/
155    public static final String  PROP_BLACKLIST             = "blacklist";
156    
157    /** The filter property name for specifying how many URLs can any given edit contain. Value is <tt>{@value}</tt> */
158    public static final String  PROP_MAXURLS               = "maxurls";
159    
160    /** The filter property name for specifying the Akismet API-key.  Value is <tt>{@value}</tt>. */
161    public static final String  PROP_AKISMET_API_KEY       = "akismet-apikey";
162    
163    /** The filter property name for specifying whether authenticated users should be ignored. Value is <tt>{@value}</tt>. */
164    public static final String  PROP_IGNORE_AUTHENTICATED  = "ignoreauthenticated";
165
166    /** The filter property name for specifying groups allowed to bypass the spam filter. Value is <tt>{@value}</tt>. */
167    public static final String PROP_ALLOWED_GROUPS = "jspwiki.filters.spamfilter.allowedgroups";
168    
169    /** The filter property name for specifying which captcha technology should be used. Value is <tt>{@value}</tt>. */
170    public static final String  PROP_CAPTCHA               = "captcha";
171    
172    /** The filter property name for specifying which filter strategy should be used.  Value is <tt>{@value}</tt>. */
173    public static final String  PROP_FILTERSTRATEGY        = "strategy";
174
175    /** The string specifying the "eager" strategy. Value is <tt>{@value}</tt>. */
176    public static final String  STRATEGY_EAGER             = "eager";
177    
178    /** The string specifying the "score" strategy. Value is <tt>{@value}</tt>. */
179    public static final String  STRATEGY_SCORE             = "score";
180
181    private static final String URL_REGEXP = "(http://|https://|mailto:)([A-Za-z0-9_/\\.\\+\\?\\#\\-\\@=&;]+)";
182
183    private String          m_forbiddenWordsPage = "SpamFilterWordList";
184    private String          m_forbiddenIPsPage   = "SpamFilterIPList";
185    private String          m_pageNameMaxLength  = "100";
186    private String          m_errorPage          = "RejectedMessage";
187    private String          m_blacklist          = "SpamFilterWordList/blacklist.txt";
188
189    private final PatternMatcher  m_matcher = new Perl5Matcher();
190    private final PatternCompiler m_compiler = new Perl5Compiler();
191
192    private Collection<Pattern> m_spamPatterns;
193    private Collection<Pattern> m_IPPatterns;
194
195    private Date m_lastRebuild = new Date( 0L );
196
197    private static final Logger C_SPAMLOG = LogManager.getLogger( "SpamLog" );
198    private static final Logger LOG = LogManager.getLogger( SpamFilter.class );
199
200    private final List< Host > m_temporaryBanList = new CopyOnWriteArrayList<>();
201
202    private int             m_banTime = 60; // minutes
203
204    /** Guarded by this. */
205    private final List< Host > m_lastModifications = new ArrayList<>();
206
207    /** How many times a single IP address can change a page per minute? */
208    private int             m_limitSinglePageChanges = 5;
209
210    /** How many times can you add the exact same string to a page? */
211    private int             m_limitSimilarChanges = 2;
212
213    /** How many URLs can be added at maximum. */
214    private int             m_maxUrls = 10;
215
216    private Pattern         m_urlPattern;
217    private Akismet         m_akismet;
218
219    private String          m_akismetAPIKey;
220
221    /** The limit at which we consider something to be spam. */
222    private final int             m_scoreLimit = 1;
223
224    /** If set to true, will ignore anyone who is in Authenticated role. */
225    private boolean         m_ignoreAuthenticated;
226
227    /** Groups allowed to bypass the filter */
228    private String[]         m_allowedGroups;
229
230    private boolean         m_stopAtFirstMatch = true;
231
232    private static String   c_hashName;
233    private static long     c_lastUpdate;
234
235    /** The HASH_DELAY value is a maximum amount of time that a user can keep
236     *  a session open, because after the value has expired, we will invent a new
237     *  hash field name.  By default, this is {@value} hours, which should be ample
238     *  time for someone.
239     */
240    private static final long HASH_DELAY = 24;
241
242    /**
243     * A lock used to ensure thread safety when accessing shared resources.
244     * This lock provides more flexibility and capabilities than the intrinsic locking mechanism,
245     * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread
246     * waiting to acquire a lock.
247     *
248     * @see java.util.concurrent.locks.ReentrantLock
249     */
250    private final ReentrantLock lock = new ReentrantLock();
251
252    /**
253     *  {@inheritDoc}
254     */
255    @Override
256    public void initialize( final Engine engine, final Properties properties ) {
257        m_forbiddenWordsPage = properties.getProperty( PROP_WORDLIST, m_forbiddenWordsPage );
258        m_forbiddenIPsPage = properties.getProperty( PROP_IPLIST, m_forbiddenIPsPage);
259        m_pageNameMaxLength = properties.getProperty( PROP_MAX_PAGENAME_LENGTH, m_pageNameMaxLength);
260        m_errorPage = properties.getProperty( PROP_ERRORPAGE, m_errorPage );
261        m_limitSinglePageChanges = TextUtil.getIntegerProperty( properties, PROP_PAGECHANGES, m_limitSinglePageChanges );
262        
263        m_limitSimilarChanges = TextUtil.getIntegerProperty( properties, PROP_SIMILARCHANGES, m_limitSimilarChanges );
264
265        m_maxUrls = TextUtil.getIntegerProperty( properties, PROP_MAXURLS, m_maxUrls );
266        m_banTime = TextUtil.getIntegerProperty( properties, PROP_BANTIME, m_banTime );
267        m_blacklist = properties.getProperty( PROP_BLACKLIST, m_blacklist );
268
269        m_ignoreAuthenticated = TextUtil.getBooleanProperty( properties, PROP_IGNORE_AUTHENTICATED, m_ignoreAuthenticated );
270        m_allowedGroups = StringUtils.split( StringUtils.defaultString( properties.getProperty( PROP_ALLOWED_GROUPS, m_blacklist ) ), ',' );
271
272        try {
273            m_urlPattern = m_compiler.compile( URL_REGEXP );
274        } catch( final MalformedPatternException e ) {
275            LOG.fatal( "Internal error: Someone put in a faulty pattern.", e );
276            throw new InternalWikiException( "Faulty pattern." , e);
277        }
278
279        m_akismetAPIKey = TextUtil.getStringProperty( properties, PROP_AKISMET_API_KEY, m_akismetAPIKey );
280        m_stopAtFirstMatch = TextUtil.getStringProperty( properties, PROP_FILTERSTRATEGY, STRATEGY_EAGER ).equals( STRATEGY_EAGER );
281
282        LOG.info( "# Spam filter initialized.  Temporary ban time " + m_banTime +
283                  " mins, max page changes/minute: " + m_limitSinglePageChanges );
284    }
285
286    private static final int REJECT = 0;
287    private static final int ACCEPT = 1;
288    private static final int NOTE   = 2;
289
290    private static String log( final Context ctx, final int type, final String source, String message ) {
291        message = TextUtil.replaceString( message, "\r\n", "\\r\\n" );
292        message = TextUtil.replaceString( message, "\"", "\\\"" );
293
294        final String uid = getUniqueID();
295        final String page   = ctx.getPage().getName();
296        final String addr   = ctx.getHttpRequest() != null ? HttpUtil.getRemoteAddress( ctx.getHttpRequest() ) : "-";
297        final String reason;
298        switch( type ) {
299            case REJECT: reason = "REJECTED";
300                break;
301            case ACCEPT: reason = "ACCEPTED";
302                break;
303            case NOTE: reason = "NOTE";
304                break;
305            default: throw new InternalWikiException( "Illegal type " + type );
306        }
307        C_SPAMLOG.info( reason + " " + source + " " + uid + " " + addr + " \"" + page + "\" " + message );
308
309        return uid;
310    }
311
312    /** {@inheritDoc} */
313    @Override
314    public String preSave( final Context context, final String content ) throws RedirectException {
315        cleanBanList();
316        refreshBlacklists( context );
317        final Change change = getChange( context, content );
318
319        if( !ignoreThisUser( context ) ) {
320            checkBanList( context, change );
321            checkSinglePageChange( context, change );
322            checkIPList( context );
323            checkPatternList( context, change );
324            checkPageName( context);
325        }
326
327        if( !m_stopAtFirstMatch ) {
328            final Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE );
329
330            if( score != null && score >= m_scoreLimit ) {
331                throw new RedirectException( "Herb says you got too many points", getRedirectPage( context ) );
332            }
333        }
334
335        log( context, ACCEPT, "-", change.toString() );
336        return content;
337    }
338
339    private void checkPageName(final Context context ) throws RedirectException {
340        final Page page = context.getPage();
341        final String pageName = page.getName();
342        final int maxlength = Integer.parseInt(m_pageNameMaxLength);
343        if ( pageName.length() > maxlength) {
344            //
345            //  Spam filter has a match.
346            //
347
348            final String uid = log( context, REJECT, REASON_PAGENAME_TOO_LONG + "(" + m_pageNameMaxLength + ")" , pageName);
349
350            LOG.info("SPAM:PageNameTooLong (" + uid + "). The length of the page name is too large (" + pageName.length() + " , limit is " + m_pageNameMaxLength + ")");
351            checkStrategy( context, "Herb says '" + pageName + "' is a bad pageName and I trust Herb! (Incident code " + uid + ")" );
352
353        }
354    }
355
356    private void checkStrategy(final Context context, final String message ) throws RedirectException {
357        if( m_stopAtFirstMatch ) {
358            throw new RedirectException( message, getRedirectPage( context ) );
359        }
360
361        Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE );
362        if( score != null ) {
363            score = score + 1;
364        } else {
365            score = 1;
366        }
367
368        context.setVariable( ATTR_SPAMFILTER_SCORE, score );
369    }
370    
371    /**
372     *  Parses a list of patterns and returns a Collection of compiled Pattern objects.
373     *
374     * @param source page containing the list of patterns.
375     * @param list list of patterns.
376     * @return A Collection of the Patterns that were found from the lists.
377     */
378    private Collection< Pattern > parseWordList( final Page source, final String list ) {
379        final ArrayList< Pattern > compiledpatterns = new ArrayList<>();
380
381        if( list != null ) {
382            final StringTokenizer tok = new StringTokenizer( list, " \t\n" );
383
384            while( tok.hasMoreTokens() ) {
385                final String pattern = tok.nextToken();
386
387                try {
388                    compiledpatterns.add( m_compiler.compile( pattern ) );
389                } catch( final MalformedPatternException e ) {
390                    LOG.debug( "Malformed spam filter pattern " + pattern );
391                    source.setAttribute("error", "Malformed spam filter pattern " + pattern);
392                }
393            }
394        }
395
396        return compiledpatterns;
397    }
398
399    /**
400     *  Takes a MT-Blacklist -formatted blacklist and returns a list of compiled Pattern objects.
401     *
402     *  @param list list of patterns.
403     *  @return The parsed blacklist patterns.
404     */
405    private Collection< Pattern > parseBlacklist( final String list ) {
406        final ArrayList< Pattern > compiledpatterns = new ArrayList<>();
407
408        if( list != null ) {
409            try {
410                final BufferedReader in = new BufferedReader( new StringReader(list) );
411                String line;
412                while( (line = in.readLine() ) != null ) {
413                    line = line.trim();
414                    if( line.isEmpty() ) continue; // Empty line
415                    if( line.startsWith("#") ) continue; // It's a comment
416
417                    int ws = line.indexOf( ' ' );
418                    if( ws == -1 ) ws = line.indexOf( '\t' );
419                    if( ws != -1 ) line = line.substring( 0, ws );
420
421                    try {
422                        compiledpatterns.add( m_compiler.compile( line ) );
423                    } catch( final MalformedPatternException e ) {
424                        LOG.debug( "Malformed spam filter pattern " + line );
425                    }
426                }
427            } catch( final IOException e ) {
428                LOG.info( "Could not read patterns; returning what I got" , e );
429            }
430        }
431
432        return compiledpatterns;
433    }
434
435    /**
436     * Takes a single page change and performs a load of tests on the content change. An admin can modify anything.
437     *
438     * @param context page Context
439     * @param change page change
440     * @throws RedirectException spam filter rejects the page change.
441     */
442    private void checkSinglePageChange(final Context context, final Change change )
443            throws RedirectException {
444        Synchronizer.synchronize(lock, () -> {
445            final HttpServletRequest req = context.getHttpRequest();
446
447            if( req != null ) {
448                final String addr = HttpUtil.getRemoteAddress( req );
449                int hostCounter = 0;
450                int changeCounter = 0;
451
452                LOG.debug( "Change is " + change.m_change );
453
454                final long time = System.currentTimeMillis() - 60*1000L; // 1 minute
455
456                for( final Iterator< Host > i = m_lastModifications.iterator(); i.hasNext(); ) {
457                    final Host host = i.next();
458
459                    //  Check if this item is invalid
460                    if( host.getAddedTime() < time ) {
461                        LOG.debug( "Removed host " + host.getAddress() + " from modification queue (expired)" );
462                        i.remove();
463                        continue;
464                    }
465
466                    // Check if this IP address has been seen before
467                    if( host.getAddress().equals( addr ) ) {
468                        hostCounter++;
469                    }
470
471                    //  Check, if this change has been seen before
472                    if( host.getChange() != null && host.getChange().equals( change ) ) {
473                        changeCounter++;
474                    }
475                }
476
477                //  Now, let's check against the limits.
478                if( hostCounter >= m_limitSinglePageChanges ) {
479                    final Host host = new Host( addr, null );
480                    m_temporaryBanList.add( host );
481
482                    final String uid = log( context, REJECT, REASON_TOO_MANY_MODIFICATIONS, change.m_change );
483                    LOG.info( "SPAM:TooManyModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many modifications/minute" );
484                    checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" );
485                }
486
487                if( changeCounter >= m_limitSimilarChanges ) {
488                    final Host host = new Host( addr, null );
489                    m_temporaryBanList.add( host );
490
491                    final String uid = log( context, REJECT, REASON_SIMILAR_MODIFICATIONS, change.m_change );
492                    LOG.info( "SPAM:SimilarModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many similar modifications" );
493                    checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")");
494                }
495
496                //  Calculate the number of links in the addition.
497                String tstChange  = change.toString();
498                int urlCounter = 0;
499                while( m_matcher.contains( tstChange,m_urlPattern ) ) {
500                    final MatchResult m = m_matcher.getMatch();
501                    tstChange = tstChange.substring( m.endOffset(0) );
502                    urlCounter++;
503                }
504
505                if( urlCounter > m_maxUrls ) {
506                    final Host host = new Host( addr, null );
507                    m_temporaryBanList.add( host );
508
509                    final String uid = log( context, REJECT, REASON_TOO_MANY_URLS, change.toString() );
510                    LOG.info( "SPAM:TooManyUrls (" + uid + "). Added host " + addr + " to temporary ban list for adding too many URLs" );
511                    checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" );
512                }
513
514                //  Check bot trap
515                checkBotTrap( context, change );
516
517                //  Check UTF-8 mangling
518                checkUTF8( context, change );
519
520                //  Do Akismet check.  This is good to be the last, because this is the most expensive operation.
521                checkAkismet( context, change );
522
523                m_lastModifications.add( new Host( addr, change ) );
524            }
525        });
526    }
527
528
529    /**
530     *  Checks against the akismet system.
531     *
532     * @param context page Context
533     * @throws RedirectException spam filter rejects the page change.
534     */
535    private void checkAkismet( final Context context, final Change change ) throws RedirectException {
536        if( m_akismetAPIKey != null ) {
537            if( m_akismet == null ) {
538                LOG.info( "Initializing Akismet spam protection." );
539                String fullPageUrl = context.getHttpRequest().getRequestURL().toString();
540                String fragment = context.getEngine().getBaseURL();
541                fullPageUrl = fullPageUrl.substring(0, fullPageUrl.indexOf(fragment) + fragment.length());
542                m_akismet = new Akismet( m_akismetAPIKey, fullPageUrl );
543
544                if( !m_akismet.verifyKey() ) {
545                    LOG.error( "Akismet API key cannot be verified.  Please check your config." );
546                    m_akismetAPIKey = null;
547                    m_akismet = null;
548                }
549            }
550
551            final HttpServletRequest req = context.getHttpRequest();
552
553            //  Akismet will mark all empty statements as spam, so we'll just ignore them.
554            if( change.m_adds == 0 && change.m_removals > 0 ) {
555                return;
556            }
557            
558            if( req != null && m_akismet != null ) {
559                LOG.debug( "Calling Akismet to check for spam..." );
560
561                final StopWatch sw = new StopWatch();
562                sw.start();
563
564                final String ipAddress     = HttpUtil.getRemoteAddress( req );
565                final String userAgent     = req.getHeader( "User-Agent" );
566                final String referrer      = req.getHeader( "Referer");
567                final String permalink     = context.getViewURL( context.getPage().getName() );
568                final String commentType   = context.getRequestContext().equals( ContextEnum.PAGE_COMMENT.getRequestContext() ) ? "comment" : "edit";
569                final String commentAuthor = context.getCurrentUser().getName();
570                final String commentAuthorEmail = null;
571                final String commentAuthorURL   = null;
572                AkismetComment comment = new AkismetComment(ipAddress, userAgent);
573                comment.setAuthor(commentAuthor);
574                comment.setAuthorEmail(commentAuthorEmail);
575                comment.setAuthorUrl(commentAuthorURL);
576                comment.setContent(change.toString());
577                comment.setPermalink(permalink);
578                comment.setReferrer(referrer);
579                comment.setType(commentType);
580                
581                final boolean isSpam = m_akismet.checkComment(comment);
582
583                sw.stop();
584                LOG.debug( "Akismet request done in: " + sw );
585
586                if( isSpam ) {
587                    // Host host = new Host( ipAddress, null );
588                    // m_temporaryBanList.add( host );
589
590                    final String uid = log( context, REJECT, REASON_AKISMET, change.toString() );
591                    LOG.info( "SPAM:Akismet (" + uid + "). Akismet thinks this change is spam; added host to temporary ban list." );
592                    checkStrategy( context, "Akismet tells Herb you're a spammer, Herb trusts Akismet, and I trust Herb! (Incident code " + uid + ")" );
593                }
594            }
595        }
596    }
597
598    /**
599     * Returns a static string which can be used to detect spambots which just wildly fill in all the fields.
600     *
601     * @return A string
602     */
603    public static String getBotFieldName() {
604        return "submit_auth";
605    }
606
607    /**
608     * This checks whether an invisible field is available in the request, and whether it's contents are suspected spam.
609     *
610     * @param context page Context
611     * @param change page change
612     * @throws RedirectException spam filter rejects the page change.
613     */
614    private void checkBotTrap( final Context context, final Change change ) throws RedirectException {
615        final HttpServletRequest request = context.getHttpRequest();
616        if( request != null ) {
617            final String unspam = request.getParameter( getBotFieldName() );
618            if( unspam != null && !unspam.isEmpty() ) {
619                final String uid = log( context, REJECT, REASON_BOT_TRAP, change.toString() );
620
621                LOG.info( "SPAM:BotTrap (" + uid + ").  Wildly behaving bot detected." );
622                checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" );
623            }
624        }
625    }
626
627    private void checkUTF8( final Context context, final Change change ) throws RedirectException {
628        final HttpServletRequest request = context.getHttpRequest();
629        if( request != null ) {
630            final String utf8field = request.getParameter( "encodingcheck" );
631            if( utf8field != null && !utf8field.equals( "\u3041" ) ) {
632                final String uid = log( context, REJECT, REASON_UTF8_TRAP, change.toString() );
633
634                LOG.info( "SPAM:UTF8Trap (" + uid + ").  Wildly posting dumb bot detected." );
635                checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" );
636            }
637        }
638    }
639
640    /** Goes through the ban list and cleans away any host which has expired from it. */
641    private void cleanBanList() {
642        Synchronizer.synchronize( lock, () -> {
643            final long now = System.currentTimeMillis();
644            m_temporaryBanList.removeIf( host -> {
645                final boolean expired = host.getReleaseTime() < now;
646                if( expired ) {
647                    LOG.debug( "Removed host " + host.getAddress() + " from temporary ban list (expired)" );
648                }
649                return expired;
650            } );
651        } );
652    }
653
654    /**
655     *  Checks the ban list if the IP address of the changer is already on it.
656     *
657     *  @param context page context
658     *  @throws RedirectException spam filter rejects the page change.
659     */
660    private void checkBanList( final Context context, final Change change ) throws RedirectException {
661        final HttpServletRequest req = context.getHttpRequest();
662
663        if( req != null ) {
664            final String remote = HttpUtil.getRemoteAddress(req);
665            final long now = System.currentTimeMillis();
666
667            for( final Host host : m_temporaryBanList ) {
668                if( host.getAddress().equals( remote ) ) {
669                    final long timeleft = ( host.getReleaseTime() - now ) / 1000L;
670
671                    log( context, REJECT, REASON_IP_BANNED_TEMPORARILY, change.m_change );
672                    checkStrategy( context,
673                            "You have been temporarily banned from modifying this wiki. (" + timeleft + " seconds of ban left)" );
674                }
675            }
676        }
677    }
678
679    /**
680     *  If the spam filter notices changes in the black list page, it will refresh them automatically.
681     *
682     *  @param context associated WikiContext
683     */
684    private void refreshBlacklists( final Context context ) {
685        try {
686            boolean rebuild = false;
687
688            //  Rebuild, if the spam words page, the attachment or the IP ban page has changed since.
689            final Page sourceSpam = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenWordsPage );
690            if( sourceSpam != null ) {
691                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || sourceSpam.getLastModified().after( m_lastRebuild ) ) {
692                    rebuild = true;
693                }
694            }
695
696            final Attachment att = context.getEngine().getManager( AttachmentManager.class ).getAttachmentInfo( context, m_blacklist );
697            if( att != null ) {
698                if( m_spamPatterns == null || m_spamPatterns.isEmpty() || att.getLastModified().after( m_lastRebuild ) ) {
699                    rebuild = true;
700                }
701            }
702
703            final Page sourceIPs = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenIPsPage );
704            if( sourceIPs != null ) {
705                if( m_IPPatterns == null || m_IPPatterns.isEmpty() || sourceIPs.getLastModified().after( m_lastRebuild ) ) {
706                    rebuild = true;
707                }
708            }
709
710            //  Do the actual rebuilding.  For simplicity's sake, we always rebuild the complete filter list regardless of what changed.
711            if( rebuild ) {
712                m_lastRebuild = new Date();
713                m_spamPatterns = parseWordList( sourceSpam, ( sourceSpam != null ) ? sourceSpam.getAttribute( LISTVAR ) : null );
714
715                LOG.info( "Spam filter reloaded - recognizing " + m_spamPatterns.size() + " patterns from page " + m_forbiddenWordsPage );
716
717                m_IPPatterns = parseWordList( sourceIPs,  ( sourceIPs != null ) ? sourceIPs.getAttribute( LISTIPVAR ) : null );
718                LOG.info( "IP filter reloaded - recognizing " + m_IPPatterns.size() + " patterns from page " + m_forbiddenIPsPage );
719
720                if( att != null ) {
721                    final InputStream in = context.getEngine().getManager( AttachmentManager.class ).getAttachmentStream(att);
722                    final StringWriter out = new StringWriter();
723                    FileUtil.copyContents( new InputStreamReader( in, StandardCharsets.UTF_8 ), out );
724                    final Collection< Pattern > blackList = parseBlacklist( out.toString() );
725                    LOG.info( "...recognizing additional " + blackList.size() + " patterns from blacklist " + m_blacklist );
726                    m_spamPatterns.addAll( blackList );
727                }
728            }
729        } catch( final IOException ex ) {
730            LOG.info( "Unable to read attachment data, continuing...", ex );
731        } catch( final ProviderException ex ) {
732            LOG.info( "Failed to read spam filter attachment, continuing...", ex );
733        }
734    }
735
736    /**
737     * Does a check against a known pattern list.
738     *
739     * @param context page Context
740     * @param change page change
741     * @throws RedirectException spam filter rejects the page change.
742     */
743    private void checkPatternList( final Context context, final Change change ) throws RedirectException {
744        // If we have no spam patterns defined, or we're trying to save the page containing the patterns, just return.
745        if( m_spamPatterns == null || context.getPage().getName().equals( m_forbiddenWordsPage ) ) {
746            return;
747        }
748
749        String ch = change.toString();
750        if( context.getHttpRequest() != null ) {
751            ch += HttpUtil.getRemoteAddress( context.getHttpRequest() );
752        }
753
754        for( final Pattern p : m_spamPatterns ) {
755            // LOG.debug("Attempting to match page contents with "+p.getPattern());
756
757            if( m_matcher.contains( ch, p ) ) {
758                //  Spam filter has a match.
759                final String uid = log( context, REJECT, REASON_REGEXP + "(" + p.getPattern() + ")", ch );
760
761                LOG.info( "SPAM:Regexp (" + uid + "). Content matches the spam filter '" + p.getPattern() + "'" );
762                checkStrategy( context, "Herb says '" + p.getPattern() + "' is a bad spam word and I trust Herb! (Incident code " + uid + ")" );
763            }
764        }
765    }
766
767
768    /**
769     *  Does a check against a pattern list of IPs.
770     *
771     *  @param context page context
772     *  @throws RedirectException spam filter rejects the page change.
773     */
774    private void checkIPList( final Context context ) throws RedirectException {
775        //  If we have no IP patterns defined, or we're trying to save the page containing the IP patterns, just return.
776        if( m_IPPatterns == null || context.getPage().getName().equals( m_forbiddenIPsPage ) ) {
777            return;
778        }
779
780        final String remoteIP = HttpUtil.getRemoteAddress( context.getHttpRequest() );
781        LOG.info("Attempting to match remoteIP " + remoteIP + " against " + m_IPPatterns.size() + " patterns");
782
783        for( final Pattern p : m_IPPatterns ) {
784             LOG.debug("Attempting to match remoteIP with " + p.getPattern());
785
786            if( m_matcher.contains( remoteIP, p ) ) {
787
788                //  IP filter has a match.
789                //
790                final String uid = log( context, REJECT, REASON_IP_BANNED_PERMANENTLY + "(" + p.getPattern() + ")", remoteIP );
791
792                LOG.info( "SPAM:IPBanList (" + uid + "). remoteIP matches the IP filter '" + p.getPattern() + "'" );
793                checkStrategy( context, "Herb says '" + p.getPattern() + "' is a banned IP and I trust Herb! (Incident code " + uid + ")" );
794            }
795        }
796    }
797
798    private void checkPatternList( final Context context, final String change ) throws RedirectException {
799        final Change c = new Change();
800        c.m_change = change;
801        checkPatternList( context, c );
802    }
803 
804    /**
805     *  Creates a simple text string describing the added content.
806     *
807     *  @param context page context
808     *  @param newText added content
809     *  @return Empty string, if there is no change.
810     */
811    private static Change getChange( final Context context, final String newText ) {
812        final Page page = context.getPage();
813        final StringBuffer change = new StringBuffer();
814        final Engine engine = context.getEngine();
815        // Get current page version
816
817        final Change ch = new Change();
818        
819        try {
820            final String oldText = engine.getManager( PageManager.class ).getPureText( page.getName(), WikiProvider.LATEST_VERSION );
821            Patch<String> patch = DiffUtils.diffInline(oldText, newText);
822            
823        
824            if( patch == null ) {
825                return ch;
826            }
827            int lineNumber = 1;
828            int currentOriginalLine = 0;
829            int currentModifiedLine = 0;
830
831            for (AbstractDelta<String> delta : patch.getDeltas()) {
832                int originalPosition = delta.getSource().getPosition();
833                int modifiedPosition = delta.getTarget().getPosition();
834
835                // Output unchanged lines before the delta
836                while (currentOriginalLine < originalPosition && currentModifiedLine < modifiedPosition) {
837                    lineNumber++;
838                    currentOriginalLine++;
839                    currentModifiedLine++;
840                }
841
842                List<String> originalLines = delta.getSource().getLines();
843                List<String> revisedLines = delta.getTarget().getLines();
844
845                for (String line : originalLines) {
846                    change.append("- " + lineNumber + ": " + line + "\r\n");
847                    ch.m_removals++;
848                    lineNumber++;
849                    currentOriginalLine++;
850                }
851
852                for (String line : revisedLines) {
853                    change.append("+ " + lineNumber + ": " + line + "\r\n");
854                    lineNumber++;
855                    currentModifiedLine++;
856                    ch.m_adds++;
857
858                }
859            }
860
861        } catch (final Exception e) {
862            LOG.error("Diff failed", e);
863        }
864
865        //  Don't forget to include the change note, too
866        final String changeNote = page.getAttribute( Page.CHANGENOTE );
867        if( changeNote != null ) {
868            change.append( "\r\n" );
869            change.append( changeNote );
870        }
871
872        //  And author as well
873        if( page.getAuthor() != null ) {
874            change.append( "\r\n" ).append( page.getAuthor() );
875        }
876
877        ch.m_change = change.toString();
878        return ch;
879    }
880
881    /**
882     * Returns true, if this user should be ignored.  For example, admin users.
883     *
884     * @param context page context
885     * @return True, if this user should be ignored.
886     */
887    private boolean ignoreThisUser( final Context context ) {
888        if( context.hasAdminPermissions() ) {
889            return true;
890        }
891
892        final List< String > groups = Arrays.asList( m_allowedGroups );
893        if( Arrays.stream( context.getWikiSession().getRoles() ).anyMatch( role -> groups.contains( role.getName() ) ) ) {
894            return true;
895        }
896
897        if( m_ignoreAuthenticated && context.getWikiSession().isAuthenticated() ) {
898            return true;
899        }
900
901        return context.getVariable("captcha") != null;
902    }
903
904    /**
905     *  Returns a random string of six uppercase characters.
906     *
907     *  @return A random string
908     */
909    private static String getUniqueID() {
910        final StringBuilder sb = new StringBuilder();
911        for( int i = 0; i < 6; i++ ) {
912            final char x = ( char )( 'A' + RANDOM.nextInt( 26 ) );
913            sb.append( x );
914        }
915
916        return sb.toString();
917    }
918
919    /**
920     *  Returns a page to which we shall redirect, based on the current value of the "captcha" parameter.
921     *
922     *  @param ctx WikiContext
923     *  @return An URL to redirect to
924     */
925    private String getRedirectPage( final Context ctx ) {
926
927        return ctx.getURL( ContextEnum.PAGE_VIEW.getRequestContext(), m_errorPage );
928    }
929
930    /**
931     *  Checks whether the UserProfile matches certain checks.
932     *
933     *  @param profile The profile to check
934     *  @param context The WikiContext
935     *  @return False, if this userprofile is suspect and should not be allowed to be added.
936     *  @since 2.6.1
937     */
938    public boolean isValidUserProfile( final Context context, final UserProfile profile ) {
939        try {
940            checkPatternList( context, profile.getEmail() );
941            checkPatternList( context, profile.getFullname() );
942            checkPatternList( context, profile.getLoginName() );
943        } catch( final RedirectException e ) {
944            LOG.info("Detected attempt to create a spammer user account (see above for rejection reason)");
945            return false;
946        }
947
948        return true;
949    }
950
951    /**
952     *  This method is used to calculate an unique code when submitting the page to detect edit conflicts.  
953     *  It currently incorporates the last-modified date of the page, and the IP address of the submitter.
954     *
955     *  @param page The WikiPage under edit
956     *  @param request The HTTP Request
957     *  @since 2.6
958     *  @return A hash value for this page and session
959     */
960    public static String getSpamHash( final Page page, final HttpServletRequest request ) {
961        long lastModified = 0;
962
963        if( page.getLastModified() != null ) {
964            lastModified = page.getLastModified().getTime();
965        }
966        final long remote = HttpUtil.getRemoteAddress( request ).hashCode();
967
968        return Long.toString( lastModified ^ remote );
969    }
970
971    /**
972     *  Returns the name of the hash field to be used in this request. The value is unique per session, and once 
973     *  the session has expired, you cannot edit anymore.
974     *
975     *  @param request The page request
976     *  @return The name to be used in the hash field
977     *  @since  2.6
978     */
979    public static String getHashFieldName( final HttpServletRequest request ) {
980        String hash = null;
981
982        if( request.getSession() != null ) {
983            hash = ( String )request.getSession().getAttribute( "_hash" );
984
985            if( hash == null ) {
986                hash = c_hashName;
987                request.getSession().setAttribute( "_hash", hash );
988            }
989        }
990
991        if( c_hashName == null || c_lastUpdate < ( System.currentTimeMillis() - HASH_DELAY * 60 * 60 * 1000 ) ) {
992            c_hashName = getUniqueID().toLowerCase();
993            c_lastUpdate = System.currentTimeMillis();
994        }
995
996        return hash != null ? hash : c_hashName;
997    }
998
999
1000    /**
1001     *  This method checks if the hash value is still valid, i.e. if it exists at all. This can occur in two cases: 
1002     *  either this is a spam bot which is not adaptive, or it is someone who has been editing one page for too long, 
1003     *  and their session has expired.
1004     *  <p>
1005     *  This method puts a redirect to the http response field to page "SessionExpired" and logs the incident in 
1006     *  the spam log (it may or may not be spam, but it's rather likely that it is).
1007     *
1008     *  @param context The WikiContext
1009     *  @param pageContext The JSP PageContext.
1010     *  @return True, if hash is okay.  False, if hash is not okay, and you need to redirect.
1011     *  @throws IOException If redirection fails
1012     *  @since 2.6
1013     */
1014    public static boolean checkHash( final Context context, final PageContext pageContext ) throws IOException {
1015        final String hashName = getHashFieldName( (HttpServletRequest)pageContext.getRequest() );
1016        if( pageContext.getRequest().getParameter(hashName) == null ) {
1017            if( pageContext.getAttribute( hashName ) == null ) {
1018                final Change change = getChange( context, EditorManager.getEditedText( pageContext ) );
1019                log( context, REJECT, "MissingHash", change.m_change );
1020
1021                final String redirect = context.getURL( ContextEnum.PAGE_VIEW.getRequestContext(),"SessionExpired" );
1022                ( ( HttpServletResponse )pageContext.getResponse() ).sendRedirect( redirect );
1023                return false;
1024            }
1025        }
1026
1027        return true;
1028    }
1029
1030    /**
1031     * This helper method adds all the input fields to your editor that the SpamFilter requires
1032     * to check for spam.  This <i>must</i> be in your editor form if you intend to use the SpamFilter.
1033     *  
1034     * @param pageContext The PageContext
1035     * @return A HTML string which contains input fields for the SpamFilter.
1036     */
1037    public static String insertInputFields( final PageContext pageContext ) {
1038        final Context ctx = Context.findContext( pageContext );
1039        final Engine engine = ctx.getEngine();
1040        final StringBuilder sb = new StringBuilder();
1041        if( engine.getContentEncoding().equals( StandardCharsets.UTF_8 ) ) {
1042            sb.append( "<input name='encodingcheck' type='hidden' value='\u3041' />\n" );
1043        }
1044
1045        return sb.toString();
1046    }
1047    
1048    /**
1049     *  A local class for storing host information.
1050     */
1051    private class Host {
1052
1053        private final long m_addedTime = System.currentTimeMillis();
1054        private final long m_releaseTime;
1055        private final String m_address;
1056        private final Change m_change;
1057
1058        public String getAddress() {
1059            return m_address;
1060        }
1061
1062        public long getReleaseTime() {
1063            return m_releaseTime;
1064        }
1065
1066        public long getAddedTime() {
1067            return m_addedTime;
1068        }
1069
1070        public Change getChange() {
1071            return m_change;
1072        }
1073
1074        public Host( final String ipaddress, final Change change ) {
1075            m_address = ipaddress;
1076            m_change = change;
1077            m_releaseTime = System.currentTimeMillis() + m_banTime * 60 * 1000L;
1078        }
1079        
1080    }
1081    
1082    private static class Change {
1083        
1084        public String m_change;
1085        public int    m_adds;
1086        public int    m_removals;
1087
1088        @Override
1089        public String toString() {
1090            return m_change;
1091        }
1092
1093        @Override
1094        public boolean equals( final Object o ) {
1095            if( o instanceof Change ) {
1096                return m_change.equals( ( ( Change )o ).m_change );
1097            }
1098            return false;
1099        }
1100
1101        @Override
1102        public int hashCode() {
1103            return m_change.hashCode() + 17;
1104        }
1105        
1106    }
1107
1108}