001/* 002 Licensed to the Apache Software Foundation (ASF) under one 003 or more contributor license agreements. See the NOTICE file 004 distributed with this work for additional information 005 regarding copyright ownership. The ASF licenses this file 006 to you under the Apache License, Version 2.0 (the 007 "License"); you may not use this file except in compliance 008 with the License. You may obtain a copy of the License at 009 010 http://www.apache.org/licenses/LICENSE-2.0 011 012 Unless required by applicable law or agreed to in writing, 013 software distributed under the License is distributed on an 014 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 KIND, either express or implied. See the License for the 016 specific language governing permissions and limitations 017 under the License. 018 */ 019package org.apache.wiki.filters; 020 021import com.github.difflib.DiffUtils; 022import com.github.difflib.patch.AbstractDelta; 023import com.github.difflib.patch.Patch; 024import org.apache.commons.lang3.StringUtils; 025import org.apache.commons.lang3.time.StopWatch; 026import org.apache.logging.log4j.LogManager; 027import org.apache.logging.log4j.Logger; 028import org.apache.oro.text.regex.MalformedPatternException; 029import org.apache.oro.text.regex.MatchResult; 030import org.apache.oro.text.regex.Pattern; 031import org.apache.oro.text.regex.PatternCompiler; 032import org.apache.oro.text.regex.PatternMatcher; 033import org.apache.oro.text.regex.Perl5Compiler; 034import org.apache.oro.text.regex.Perl5Matcher; 035import org.apache.wiki.InternalWikiException; 036import org.apache.wiki.api.core.Attachment; 037import org.apache.wiki.api.core.Context; 038import org.apache.wiki.api.core.ContextEnum; 039import org.apache.wiki.api.core.Engine; 040import org.apache.wiki.api.core.Page; 041import org.apache.wiki.api.exceptions.ProviderException; 042import org.apache.wiki.api.exceptions.RedirectException; 043import org.apache.wiki.api.filters.BasePageFilter; 044import org.apache.wiki.api.providers.WikiProvider; 045import org.apache.wiki.attachment.AttachmentManager; 046import org.apache.wiki.auth.user.UserProfile; 047import org.apache.wiki.pages.PageManager; 048import org.apache.wiki.ui.EditorManager; 049import org.apache.wiki.util.FileUtil; 050import org.apache.wiki.util.HttpUtil; 051import org.apache.wiki.util.Synchronizer; 052import org.apache.wiki.util.TextUtil; 053 054import jakarta.servlet.http.HttpServletRequest; 055import jakarta.servlet.http.HttpServletResponse; 056import jakarta.servlet.jsp.PageContext; 057import java.io.BufferedReader; 058import java.io.IOException; 059import java.io.InputStream; 060import java.io.InputStreamReader; 061import java.io.StringReader; 062import java.io.StringWriter; 063import java.nio.charset.StandardCharsets; 064import java.util.ArrayList; 065import java.util.Arrays; 066import java.util.Collection; 067import java.util.Date; 068import java.util.Iterator; 069import java.util.List; 070import java.util.Properties; 071import java.util.Random; 072import java.util.StringTokenizer; 073import java.util.concurrent.CopyOnWriteArrayList; 074import java.util.concurrent.ThreadLocalRandom; 075import net.thauvin.erik.akismet.Akismet; 076import net.thauvin.erik.akismet.AkismetComment; 077import java.util.concurrent.locks.ReentrantLock; 078 079 080/** 081 * This is Herb, the JSPWiki spamfilter that can also do choke modifications. 082 * 083 * Parameters: 084 * <ul> 085 * <li>wordlist - Page name where the spamword regexps are found. Use [{SET spamwords='regexp list separated with spaces'}] on 086 * that page. Default is "SpamFilterWordList". 087 * <li>IPlist - Page name where the IP regexps are found. Use [{SET ips='regexp list separated with spaces'}] on 088 * that page. Default is "SpamFilterIPList". 089 * <li>maxpagenamelength - Maximum page name length. Default is 100. 090 * <li>blacklist - The name of an attachment containing the list of spam patterns, one per line. Default is 091 * "SpamFilterWordList/blacklist.txt"</li> 092 * <li>errorpage - The page to which the user is redirected. Has a special variable $msg which states the reason. Default is "RejectedMessage". 093 * <li>pagechangesinminute - How many page changes are allowed/minute. Default is 5.</li> 094 * <li>similarchanges - How many similar page changes are allowed before the host is banned. Default is 2. (since 2.4.72)</li> 095 * <li>bantime - How long an IP address stays on the temporary ban list (default is 60 for 60 minutes).</li> 096 * <li>maxurls - How many URLs can be added to the page before it is considered spam (default is 5)</li> 097 * <li>akismet-apikey - The Akismet API key (see akismet.org)</li> 098 * <li>ignoreauthenticated - If set to "true", all authenticated users are ignored and never caught in SpamFilter</li> 099 * <li>captcha - Sets the captcha technology to use. Current allowed values are "none". "asirra" was previously supported however that service has been discontinued.</li> 100 * <li>strategy - Sets the filtering strategy to use. If set to "eager", will stop at the first probable 101 * match, and won't consider any other tests. This is the default, as it's considerably lighter. If set to "score", will go through all of the tests 102 * and calculates a score for the spam, which is then compared to a filter level value. 103 * </ul> 104 * 105 * <p>Please see the default editors/plain.jsp for examples on how the SpamFilter integrates 106 * with the editor system.</p> 107 * 108 * <p>Changes by admin users are ignored in any case.</p> 109 * 110 * @since 2.1.112 111 */ 112public class SpamFilter extends BasePageFilter { 113 114 private static final String ATTR_SPAMFILTER_SCORE = "spamfilter.score"; 115 private static final String REASON_REGEXP = "Regexp"; 116 private static final String REASON_IP_BANNED_TEMPORARILY = "IPBannedTemporarily"; 117 private static final String REASON_IP_BANNED_PERMANENTLY = "IPBannedPermanently"; 118 private static final String REASON_BOT_TRAP = "BotTrap"; 119 private static final String REASON_AKISMET = "Akismet"; 120 private static final String REASON_TOO_MANY_URLS = "TooManyUrls"; 121 private static final String REASON_SIMILAR_MODIFICATIONS = "SimilarModifications"; 122 private static final String REASON_TOO_MANY_MODIFICATIONS = "TooManyModifications"; 123 private static final String REASON_PAGENAME_TOO_LONG = "PageNameTooLong"; 124 private static final String REASON_UTF8_TRAP = "UTF8Trap"; 125 126 private static final String LISTVAR = "spamwords"; 127 private static final String LISTIPVAR = "ips"; 128 129 private static final Random RANDOM = ThreadLocalRandom.current(); 130 131 /** The filter property name for specifying the page which contains the list of spamwords. Value is <tt>{@value}</tt>. */ 132 public static final String PROP_WORDLIST = "wordlist"; 133 134 /** The filter property name for specifying the page which contains the list of IPs to ban. Value is <tt>{@value}</tt>. */ 135 public static final String PROP_IPLIST = "IPlist"; 136 137 /** The filter property name for specifying the maximum page name length. Value is <tt>{@value}</tt>. */ 138 public static final String PROP_MAX_PAGENAME_LENGTH = "maxpagenamelength"; 139 140 /** The filter property name for the page to which you are directed if Herb rejects your edit. Value is <tt>{@value}</tt>. */ 141 public static final String PROP_ERRORPAGE = "errorpage"; 142 143 /** The filter property name for specifying how many changes is any given IP address 144 * allowed to do per minute. Value is <tt>{@value}</tt>. 145 */ 146 public static final String PROP_PAGECHANGES = "pagechangesinminute"; 147 148 /** The filter property name for specifying how many similar changes are allowed before a host is banned. Value is <tt>{@value}</tt>. */ 149 public static final String PROP_SIMILARCHANGES = "similarchanges"; 150 151 /** The filter property name for specifying how long a host is banned. Value is <tt>{@value}</tt>.*/ 152 public static final String PROP_BANTIME = "bantime"; 153 154 /** The filter property name for the attachment containing the blacklist. Value is <tt>{@value}</tt>.*/ 155 public static final String PROP_BLACKLIST = "blacklist"; 156 157 /** The filter property name for specifying how many URLs can any given edit contain. Value is <tt>{@value}</tt> */ 158 public static final String PROP_MAXURLS = "maxurls"; 159 160 /** The filter property name for specifying the Akismet API-key. Value is <tt>{@value}</tt>. */ 161 public static final String PROP_AKISMET_API_KEY = "akismet-apikey"; 162 163 /** The filter property name for specifying whether authenticated users should be ignored. Value is <tt>{@value}</tt>. */ 164 public static final String PROP_IGNORE_AUTHENTICATED = "ignoreauthenticated"; 165 166 /** The filter property name for specifying groups allowed to bypass the spam filter. Value is <tt>{@value}</tt>. */ 167 public static final String PROP_ALLOWED_GROUPS = "jspwiki.filters.spamfilter.allowedgroups"; 168 169 /** The filter property name for specifying which captcha technology should be used. Value is <tt>{@value}</tt>. */ 170 public static final String PROP_CAPTCHA = "captcha"; 171 172 /** The filter property name for specifying which filter strategy should be used. Value is <tt>{@value}</tt>. */ 173 public static final String PROP_FILTERSTRATEGY = "strategy"; 174 175 /** The string specifying the "eager" strategy. Value is <tt>{@value}</tt>. */ 176 public static final String STRATEGY_EAGER = "eager"; 177 178 /** The string specifying the "score" strategy. Value is <tt>{@value}</tt>. */ 179 public static final String STRATEGY_SCORE = "score"; 180 181 private static final String URL_REGEXP = "(http://|https://|mailto:)([A-Za-z0-9_/\\.\\+\\?\\#\\-\\@=&;]+)"; 182 183 private String m_forbiddenWordsPage = "SpamFilterWordList"; 184 private String m_forbiddenIPsPage = "SpamFilterIPList"; 185 private String m_pageNameMaxLength = "100"; 186 private String m_errorPage = "RejectedMessage"; 187 private String m_blacklist = "SpamFilterWordList/blacklist.txt"; 188 189 private final PatternMatcher m_matcher = new Perl5Matcher(); 190 private final PatternCompiler m_compiler = new Perl5Compiler(); 191 192 private Collection<Pattern> m_spamPatterns; 193 private Collection<Pattern> m_IPPatterns; 194 195 private Date m_lastRebuild = new Date( 0L ); 196 197 private static final Logger C_SPAMLOG = LogManager.getLogger( "SpamLog" ); 198 private static final Logger LOG = LogManager.getLogger( SpamFilter.class ); 199 200 private final List< Host > m_temporaryBanList = new CopyOnWriteArrayList<>(); 201 202 private int m_banTime = 60; // minutes 203 204 /** Guarded by this. */ 205 private final List< Host > m_lastModifications = new ArrayList<>(); 206 207 /** How many times a single IP address can change a page per minute? */ 208 private int m_limitSinglePageChanges = 5; 209 210 /** How many times can you add the exact same string to a page? */ 211 private int m_limitSimilarChanges = 2; 212 213 /** How many URLs can be added at maximum. */ 214 private int m_maxUrls = 10; 215 216 private Pattern m_urlPattern; 217 private Akismet m_akismet; 218 219 private String m_akismetAPIKey; 220 221 /** The limit at which we consider something to be spam. */ 222 private final int m_scoreLimit = 1; 223 224 /** If set to true, will ignore anyone who is in Authenticated role. */ 225 private boolean m_ignoreAuthenticated; 226 227 /** Groups allowed to bypass the filter */ 228 private String[] m_allowedGroups; 229 230 private boolean m_stopAtFirstMatch = true; 231 232 private static String c_hashName; 233 private static long c_lastUpdate; 234 235 /** The HASH_DELAY value is a maximum amount of time that a user can keep 236 * a session open, because after the value has expired, we will invent a new 237 * hash field name. By default, this is {@value} hours, which should be ample 238 * time for someone. 239 */ 240 private static final long HASH_DELAY = 24; 241 242 /** 243 * A lock used to ensure thread safety when accessing shared resources. 244 * This lock provides more flexibility and capabilities than the intrinsic locking mechanism, 245 * such as the ability to attempt to acquire a lock with a timeout, or to interrupt a thread 246 * waiting to acquire a lock. 247 * 248 * @see java.util.concurrent.locks.ReentrantLock 249 */ 250 private final ReentrantLock lock = new ReentrantLock(); 251 252 /** 253 * {@inheritDoc} 254 */ 255 @Override 256 public void initialize( final Engine engine, final Properties properties ) { 257 m_forbiddenWordsPage = properties.getProperty( PROP_WORDLIST, m_forbiddenWordsPage ); 258 m_forbiddenIPsPage = properties.getProperty( PROP_IPLIST, m_forbiddenIPsPage); 259 m_pageNameMaxLength = properties.getProperty( PROP_MAX_PAGENAME_LENGTH, m_pageNameMaxLength); 260 m_errorPage = properties.getProperty( PROP_ERRORPAGE, m_errorPage ); 261 m_limitSinglePageChanges = TextUtil.getIntegerProperty( properties, PROP_PAGECHANGES, m_limitSinglePageChanges ); 262 263 m_limitSimilarChanges = TextUtil.getIntegerProperty( properties, PROP_SIMILARCHANGES, m_limitSimilarChanges ); 264 265 m_maxUrls = TextUtil.getIntegerProperty( properties, PROP_MAXURLS, m_maxUrls ); 266 m_banTime = TextUtil.getIntegerProperty( properties, PROP_BANTIME, m_banTime ); 267 m_blacklist = properties.getProperty( PROP_BLACKLIST, m_blacklist ); 268 269 m_ignoreAuthenticated = TextUtil.getBooleanProperty( properties, PROP_IGNORE_AUTHENTICATED, m_ignoreAuthenticated ); 270 m_allowedGroups = StringUtils.split( StringUtils.defaultString( properties.getProperty( PROP_ALLOWED_GROUPS, m_blacklist ) ), ',' ); 271 272 try { 273 m_urlPattern = m_compiler.compile( URL_REGEXP ); 274 } catch( final MalformedPatternException e ) { 275 LOG.fatal( "Internal error: Someone put in a faulty pattern.", e ); 276 throw new InternalWikiException( "Faulty pattern." , e); 277 } 278 279 m_akismetAPIKey = TextUtil.getStringProperty( properties, PROP_AKISMET_API_KEY, m_akismetAPIKey ); 280 m_stopAtFirstMatch = TextUtil.getStringProperty( properties, PROP_FILTERSTRATEGY, STRATEGY_EAGER ).equals( STRATEGY_EAGER ); 281 282 LOG.info( "# Spam filter initialized. Temporary ban time " + m_banTime + 283 " mins, max page changes/minute: " + m_limitSinglePageChanges ); 284 } 285 286 private static final int REJECT = 0; 287 private static final int ACCEPT = 1; 288 private static final int NOTE = 2; 289 290 private static String log( final Context ctx, final int type, final String source, String message ) { 291 message = TextUtil.replaceString( message, "\r\n", "\\r\\n" ); 292 message = TextUtil.replaceString( message, "\"", "\\\"" ); 293 294 final String uid = getUniqueID(); 295 final String page = ctx.getPage().getName(); 296 final String addr = ctx.getHttpRequest() != null ? HttpUtil.getRemoteAddress( ctx.getHttpRequest() ) : "-"; 297 final String reason; 298 switch( type ) { 299 case REJECT: reason = "REJECTED"; 300 break; 301 case ACCEPT: reason = "ACCEPTED"; 302 break; 303 case NOTE: reason = "NOTE"; 304 break; 305 default: throw new InternalWikiException( "Illegal type " + type ); 306 } 307 C_SPAMLOG.info( reason + " " + source + " " + uid + " " + addr + " \"" + page + "\" " + message ); 308 309 return uid; 310 } 311 312 /** {@inheritDoc} */ 313 @Override 314 public String preSave( final Context context, final String content ) throws RedirectException { 315 cleanBanList(); 316 refreshBlacklists( context ); 317 final Change change = getChange( context, content ); 318 319 if( !ignoreThisUser( context ) ) { 320 checkBanList( context, change ); 321 checkSinglePageChange( context, change ); 322 checkIPList( context ); 323 checkPatternList( context, change ); 324 checkPageName( context); 325 } 326 327 if( !m_stopAtFirstMatch ) { 328 final Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE ); 329 330 if( score != null && score >= m_scoreLimit ) { 331 throw new RedirectException( "Herb says you got too many points", getRedirectPage( context ) ); 332 } 333 } 334 335 log( context, ACCEPT, "-", change.toString() ); 336 return content; 337 } 338 339 private void checkPageName(final Context context ) throws RedirectException { 340 final Page page = context.getPage(); 341 final String pageName = page.getName(); 342 final int maxlength = Integer.parseInt(m_pageNameMaxLength); 343 if ( pageName.length() > maxlength) { 344 // 345 // Spam filter has a match. 346 // 347 348 final String uid = log( context, REJECT, REASON_PAGENAME_TOO_LONG + "(" + m_pageNameMaxLength + ")" , pageName); 349 350 LOG.info("SPAM:PageNameTooLong (" + uid + "). The length of the page name is too large (" + pageName.length() + " , limit is " + m_pageNameMaxLength + ")"); 351 checkStrategy( context, "Herb says '" + pageName + "' is a bad pageName and I trust Herb! (Incident code " + uid + ")" ); 352 353 } 354 } 355 356 private void checkStrategy(final Context context, final String message ) throws RedirectException { 357 if( m_stopAtFirstMatch ) { 358 throw new RedirectException( message, getRedirectPage( context ) ); 359 } 360 361 Integer score = context.getVariable( ATTR_SPAMFILTER_SCORE ); 362 if( score != null ) { 363 score = score + 1; 364 } else { 365 score = 1; 366 } 367 368 context.setVariable( ATTR_SPAMFILTER_SCORE, score ); 369 } 370 371 /** 372 * Parses a list of patterns and returns a Collection of compiled Pattern objects. 373 * 374 * @param source page containing the list of patterns. 375 * @param list list of patterns. 376 * @return A Collection of the Patterns that were found from the lists. 377 */ 378 private Collection< Pattern > parseWordList( final Page source, final String list ) { 379 final ArrayList< Pattern > compiledpatterns = new ArrayList<>(); 380 381 if( list != null ) { 382 final StringTokenizer tok = new StringTokenizer( list, " \t\n" ); 383 384 while( tok.hasMoreTokens() ) { 385 final String pattern = tok.nextToken(); 386 387 try { 388 compiledpatterns.add( m_compiler.compile( pattern ) ); 389 } catch( final MalformedPatternException e ) { 390 LOG.debug( "Malformed spam filter pattern " + pattern ); 391 source.setAttribute("error", "Malformed spam filter pattern " + pattern); 392 } 393 } 394 } 395 396 return compiledpatterns; 397 } 398 399 /** 400 * Takes a MT-Blacklist -formatted blacklist and returns a list of compiled Pattern objects. 401 * 402 * @param list list of patterns. 403 * @return The parsed blacklist patterns. 404 */ 405 private Collection< Pattern > parseBlacklist( final String list ) { 406 final ArrayList< Pattern > compiledpatterns = new ArrayList<>(); 407 408 if( list != null ) { 409 try { 410 final BufferedReader in = new BufferedReader( new StringReader(list) ); 411 String line; 412 while( (line = in.readLine() ) != null ) { 413 line = line.trim(); 414 if( line.isEmpty() ) continue; // Empty line 415 if( line.startsWith("#") ) continue; // It's a comment 416 417 int ws = line.indexOf( ' ' ); 418 if( ws == -1 ) ws = line.indexOf( '\t' ); 419 if( ws != -1 ) line = line.substring( 0, ws ); 420 421 try { 422 compiledpatterns.add( m_compiler.compile( line ) ); 423 } catch( final MalformedPatternException e ) { 424 LOG.debug( "Malformed spam filter pattern " + line ); 425 } 426 } 427 } catch( final IOException e ) { 428 LOG.info( "Could not read patterns; returning what I got" , e ); 429 } 430 } 431 432 return compiledpatterns; 433 } 434 435 /** 436 * Takes a single page change and performs a load of tests on the content change. An admin can modify anything. 437 * 438 * @param context page Context 439 * @param change page change 440 * @throws RedirectException spam filter rejects the page change. 441 */ 442 private void checkSinglePageChange(final Context context, final Change change ) 443 throws RedirectException { 444 Synchronizer.synchronize(lock, () -> { 445 final HttpServletRequest req = context.getHttpRequest(); 446 447 if( req != null ) { 448 final String addr = HttpUtil.getRemoteAddress( req ); 449 int hostCounter = 0; 450 int changeCounter = 0; 451 452 LOG.debug( "Change is " + change.m_change ); 453 454 final long time = System.currentTimeMillis() - 60*1000L; // 1 minute 455 456 for( final Iterator< Host > i = m_lastModifications.iterator(); i.hasNext(); ) { 457 final Host host = i.next(); 458 459 // Check if this item is invalid 460 if( host.getAddedTime() < time ) { 461 LOG.debug( "Removed host " + host.getAddress() + " from modification queue (expired)" ); 462 i.remove(); 463 continue; 464 } 465 466 // Check if this IP address has been seen before 467 if( host.getAddress().equals( addr ) ) { 468 hostCounter++; 469 } 470 471 // Check, if this change has been seen before 472 if( host.getChange() != null && host.getChange().equals( change ) ) { 473 changeCounter++; 474 } 475 } 476 477 // Now, let's check against the limits. 478 if( hostCounter >= m_limitSinglePageChanges ) { 479 final Host host = new Host( addr, null ); 480 m_temporaryBanList.add( host ); 481 482 final String uid = log( context, REJECT, REASON_TOO_MANY_MODIFICATIONS, change.m_change ); 483 LOG.info( "SPAM:TooManyModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many modifications/minute" ); 484 checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" ); 485 } 486 487 if( changeCounter >= m_limitSimilarChanges ) { 488 final Host host = new Host( addr, null ); 489 m_temporaryBanList.add( host ); 490 491 final String uid = log( context, REJECT, REASON_SIMILAR_MODIFICATIONS, change.m_change ); 492 LOG.info( "SPAM:SimilarModifications (" + uid + "). Added host " + addr + " to temporary ban list for doing too many similar modifications" ); 493 checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code "+uid+")"); 494 } 495 496 // Calculate the number of links in the addition. 497 String tstChange = change.toString(); 498 int urlCounter = 0; 499 while( m_matcher.contains( tstChange,m_urlPattern ) ) { 500 final MatchResult m = m_matcher.getMatch(); 501 tstChange = tstChange.substring( m.endOffset(0) ); 502 urlCounter++; 503 } 504 505 if( urlCounter > m_maxUrls ) { 506 final Host host = new Host( addr, null ); 507 m_temporaryBanList.add( host ); 508 509 final String uid = log( context, REJECT, REASON_TOO_MANY_URLS, change.toString() ); 510 LOG.info( "SPAM:TooManyUrls (" + uid + "). Added host " + addr + " to temporary ban list for adding too many URLs" ); 511 checkStrategy( context, "Herb says you look like a spammer, and I trust Herb! (Incident code " + uid + ")" ); 512 } 513 514 // Check bot trap 515 checkBotTrap( context, change ); 516 517 // Check UTF-8 mangling 518 checkUTF8( context, change ); 519 520 // Do Akismet check. This is good to be the last, because this is the most expensive operation. 521 checkAkismet( context, change ); 522 523 m_lastModifications.add( new Host( addr, change ) ); 524 } 525 }); 526 } 527 528 529 /** 530 * Checks against the akismet system. 531 * 532 * @param context page Context 533 * @throws RedirectException spam filter rejects the page change. 534 */ 535 private void checkAkismet( final Context context, final Change change ) throws RedirectException { 536 if( m_akismetAPIKey != null ) { 537 if( m_akismet == null ) { 538 LOG.info( "Initializing Akismet spam protection." ); 539 String fullPageUrl = context.getHttpRequest().getRequestURL().toString(); 540 String fragment = context.getEngine().getBaseURL(); 541 fullPageUrl = fullPageUrl.substring(0, fullPageUrl.indexOf(fragment) + fragment.length()); 542 m_akismet = new Akismet( m_akismetAPIKey, fullPageUrl ); 543 544 if( !m_akismet.verifyKey() ) { 545 LOG.error( "Akismet API key cannot be verified. Please check your config." ); 546 m_akismetAPIKey = null; 547 m_akismet = null; 548 } 549 } 550 551 final HttpServletRequest req = context.getHttpRequest(); 552 553 // Akismet will mark all empty statements as spam, so we'll just ignore them. 554 if( change.m_adds == 0 && change.m_removals > 0 ) { 555 return; 556 } 557 558 if( req != null && m_akismet != null ) { 559 LOG.debug( "Calling Akismet to check for spam..." ); 560 561 final StopWatch sw = new StopWatch(); 562 sw.start(); 563 564 final String ipAddress = HttpUtil.getRemoteAddress( req ); 565 final String userAgent = req.getHeader( "User-Agent" ); 566 final String referrer = req.getHeader( "Referer"); 567 final String permalink = context.getViewURL( context.getPage().getName() ); 568 final String commentType = context.getRequestContext().equals( ContextEnum.PAGE_COMMENT.getRequestContext() ) ? "comment" : "edit"; 569 final String commentAuthor = context.getCurrentUser().getName(); 570 final String commentAuthorEmail = null; 571 final String commentAuthorURL = null; 572 AkismetComment comment = new AkismetComment(ipAddress, userAgent); 573 comment.setAuthor(commentAuthor); 574 comment.setAuthorEmail(commentAuthorEmail); 575 comment.setAuthorUrl(commentAuthorURL); 576 comment.setContent(change.toString()); 577 comment.setPermalink(permalink); 578 comment.setReferrer(referrer); 579 comment.setType(commentType); 580 581 final boolean isSpam = m_akismet.checkComment(comment); 582 583 sw.stop(); 584 LOG.debug( "Akismet request done in: " + sw ); 585 586 if( isSpam ) { 587 // Host host = new Host( ipAddress, null ); 588 // m_temporaryBanList.add( host ); 589 590 final String uid = log( context, REJECT, REASON_AKISMET, change.toString() ); 591 LOG.info( "SPAM:Akismet (" + uid + "). Akismet thinks this change is spam; added host to temporary ban list." ); 592 checkStrategy( context, "Akismet tells Herb you're a spammer, Herb trusts Akismet, and I trust Herb! (Incident code " + uid + ")" ); 593 } 594 } 595 } 596 } 597 598 /** 599 * Returns a static string which can be used to detect spambots which just wildly fill in all the fields. 600 * 601 * @return A string 602 */ 603 public static String getBotFieldName() { 604 return "submit_auth"; 605 } 606 607 /** 608 * This checks whether an invisible field is available in the request, and whether it's contents are suspected spam. 609 * 610 * @param context page Context 611 * @param change page change 612 * @throws RedirectException spam filter rejects the page change. 613 */ 614 private void checkBotTrap( final Context context, final Change change ) throws RedirectException { 615 final HttpServletRequest request = context.getHttpRequest(); 616 if( request != null ) { 617 final String unspam = request.getParameter( getBotFieldName() ); 618 if( unspam != null && !unspam.isEmpty() ) { 619 final String uid = log( context, REJECT, REASON_BOT_TRAP, change.toString() ); 620 621 LOG.info( "SPAM:BotTrap (" + uid + "). Wildly behaving bot detected." ); 622 checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" ); 623 } 624 } 625 } 626 627 private void checkUTF8( final Context context, final Change change ) throws RedirectException { 628 final HttpServletRequest request = context.getHttpRequest(); 629 if( request != null ) { 630 final String utf8field = request.getParameter( "encodingcheck" ); 631 if( utf8field != null && !utf8field.equals( "\u3041" ) ) { 632 final String uid = log( context, REJECT, REASON_UTF8_TRAP, change.toString() ); 633 634 LOG.info( "SPAM:UTF8Trap (" + uid + "). Wildly posting dumb bot detected." ); 635 checkStrategy( context, "Spamming attempt detected. (Incident code " + uid + ")" ); 636 } 637 } 638 } 639 640 /** Goes through the ban list and cleans away any host which has expired from it. */ 641 private void cleanBanList() { 642 Synchronizer.synchronize( lock, () -> { 643 final long now = System.currentTimeMillis(); 644 m_temporaryBanList.removeIf( host -> { 645 final boolean expired = host.getReleaseTime() < now; 646 if( expired ) { 647 LOG.debug( "Removed host " + host.getAddress() + " from temporary ban list (expired)" ); 648 } 649 return expired; 650 } ); 651 } ); 652 } 653 654 /** 655 * Checks the ban list if the IP address of the changer is already on it. 656 * 657 * @param context page context 658 * @throws RedirectException spam filter rejects the page change. 659 */ 660 private void checkBanList( final Context context, final Change change ) throws RedirectException { 661 final HttpServletRequest req = context.getHttpRequest(); 662 663 if( req != null ) { 664 final String remote = HttpUtil.getRemoteAddress(req); 665 final long now = System.currentTimeMillis(); 666 667 for( final Host host : m_temporaryBanList ) { 668 if( host.getAddress().equals( remote ) ) { 669 final long timeleft = ( host.getReleaseTime() - now ) / 1000L; 670 671 log( context, REJECT, REASON_IP_BANNED_TEMPORARILY, change.m_change ); 672 checkStrategy( context, 673 "You have been temporarily banned from modifying this wiki. (" + timeleft + " seconds of ban left)" ); 674 } 675 } 676 } 677 } 678 679 /** 680 * If the spam filter notices changes in the black list page, it will refresh them automatically. 681 * 682 * @param context associated WikiContext 683 */ 684 private void refreshBlacklists( final Context context ) { 685 try { 686 boolean rebuild = false; 687 688 // Rebuild, if the spam words page, the attachment or the IP ban page has changed since. 689 final Page sourceSpam = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenWordsPage ); 690 if( sourceSpam != null ) { 691 if( m_spamPatterns == null || m_spamPatterns.isEmpty() || sourceSpam.getLastModified().after( m_lastRebuild ) ) { 692 rebuild = true; 693 } 694 } 695 696 final Attachment att = context.getEngine().getManager( AttachmentManager.class ).getAttachmentInfo( context, m_blacklist ); 697 if( att != null ) { 698 if( m_spamPatterns == null || m_spamPatterns.isEmpty() || att.getLastModified().after( m_lastRebuild ) ) { 699 rebuild = true; 700 } 701 } 702 703 final Page sourceIPs = context.getEngine().getManager( PageManager.class ).getPage( m_forbiddenIPsPage ); 704 if( sourceIPs != null ) { 705 if( m_IPPatterns == null || m_IPPatterns.isEmpty() || sourceIPs.getLastModified().after( m_lastRebuild ) ) { 706 rebuild = true; 707 } 708 } 709 710 // Do the actual rebuilding. For simplicity's sake, we always rebuild the complete filter list regardless of what changed. 711 if( rebuild ) { 712 m_lastRebuild = new Date(); 713 m_spamPatterns = parseWordList( sourceSpam, ( sourceSpam != null ) ? sourceSpam.getAttribute( LISTVAR ) : null ); 714 715 LOG.info( "Spam filter reloaded - recognizing " + m_spamPatterns.size() + " patterns from page " + m_forbiddenWordsPage ); 716 717 m_IPPatterns = parseWordList( sourceIPs, ( sourceIPs != null ) ? sourceIPs.getAttribute( LISTIPVAR ) : null ); 718 LOG.info( "IP filter reloaded - recognizing " + m_IPPatterns.size() + " patterns from page " + m_forbiddenIPsPage ); 719 720 if( att != null ) { 721 final InputStream in = context.getEngine().getManager( AttachmentManager.class ).getAttachmentStream(att); 722 final StringWriter out = new StringWriter(); 723 FileUtil.copyContents( new InputStreamReader( in, StandardCharsets.UTF_8 ), out ); 724 final Collection< Pattern > blackList = parseBlacklist( out.toString() ); 725 LOG.info( "...recognizing additional " + blackList.size() + " patterns from blacklist " + m_blacklist ); 726 m_spamPatterns.addAll( blackList ); 727 } 728 } 729 } catch( final IOException ex ) { 730 LOG.info( "Unable to read attachment data, continuing...", ex ); 731 } catch( final ProviderException ex ) { 732 LOG.info( "Failed to read spam filter attachment, continuing...", ex ); 733 } 734 } 735 736 /** 737 * Does a check against a known pattern list. 738 * 739 * @param context page Context 740 * @param change page change 741 * @throws RedirectException spam filter rejects the page change. 742 */ 743 private void checkPatternList( final Context context, final Change change ) throws RedirectException { 744 // If we have no spam patterns defined, or we're trying to save the page containing the patterns, just return. 745 if( m_spamPatterns == null || context.getPage().getName().equals( m_forbiddenWordsPage ) ) { 746 return; 747 } 748 749 String ch = change.toString(); 750 if( context.getHttpRequest() != null ) { 751 ch += HttpUtil.getRemoteAddress( context.getHttpRequest() ); 752 } 753 754 for( final Pattern p : m_spamPatterns ) { 755 // LOG.debug("Attempting to match page contents with "+p.getPattern()); 756 757 if( m_matcher.contains( ch, p ) ) { 758 // Spam filter has a match. 759 final String uid = log( context, REJECT, REASON_REGEXP + "(" + p.getPattern() + ")", ch ); 760 761 LOG.info( "SPAM:Regexp (" + uid + "). Content matches the spam filter '" + p.getPattern() + "'" ); 762 checkStrategy( context, "Herb says '" + p.getPattern() + "' is a bad spam word and I trust Herb! (Incident code " + uid + ")" ); 763 } 764 } 765 } 766 767 768 /** 769 * Does a check against a pattern list of IPs. 770 * 771 * @param context page context 772 * @throws RedirectException spam filter rejects the page change. 773 */ 774 private void checkIPList( final Context context ) throws RedirectException { 775 // If we have no IP patterns defined, or we're trying to save the page containing the IP patterns, just return. 776 if( m_IPPatterns == null || context.getPage().getName().equals( m_forbiddenIPsPage ) ) { 777 return; 778 } 779 780 final String remoteIP = HttpUtil.getRemoteAddress( context.getHttpRequest() ); 781 LOG.info("Attempting to match remoteIP " + remoteIP + " against " + m_IPPatterns.size() + " patterns"); 782 783 for( final Pattern p : m_IPPatterns ) { 784 LOG.debug("Attempting to match remoteIP with " + p.getPattern()); 785 786 if( m_matcher.contains( remoteIP, p ) ) { 787 788 // IP filter has a match. 789 // 790 final String uid = log( context, REJECT, REASON_IP_BANNED_PERMANENTLY + "(" + p.getPattern() + ")", remoteIP ); 791 792 LOG.info( "SPAM:IPBanList (" + uid + "). remoteIP matches the IP filter '" + p.getPattern() + "'" ); 793 checkStrategy( context, "Herb says '" + p.getPattern() + "' is a banned IP and I trust Herb! (Incident code " + uid + ")" ); 794 } 795 } 796 } 797 798 private void checkPatternList( final Context context, final String change ) throws RedirectException { 799 final Change c = new Change(); 800 c.m_change = change; 801 checkPatternList( context, c ); 802 } 803 804 /** 805 * Creates a simple text string describing the added content. 806 * 807 * @param context page context 808 * @param newText added content 809 * @return Empty string, if there is no change. 810 */ 811 private static Change getChange( final Context context, final String newText ) { 812 final Page page = context.getPage(); 813 final StringBuffer change = new StringBuffer(); 814 final Engine engine = context.getEngine(); 815 // Get current page version 816 817 final Change ch = new Change(); 818 819 try { 820 final String oldText = engine.getManager( PageManager.class ).getPureText( page.getName(), WikiProvider.LATEST_VERSION ); 821 Patch<String> patch = DiffUtils.diffInline(oldText, newText); 822 823 824 if( patch == null ) { 825 return ch; 826 } 827 int lineNumber = 1; 828 int currentOriginalLine = 0; 829 int currentModifiedLine = 0; 830 831 for (AbstractDelta<String> delta : patch.getDeltas()) { 832 int originalPosition = delta.getSource().getPosition(); 833 int modifiedPosition = delta.getTarget().getPosition(); 834 835 // Output unchanged lines before the delta 836 while (currentOriginalLine < originalPosition && currentModifiedLine < modifiedPosition) { 837 lineNumber++; 838 currentOriginalLine++; 839 currentModifiedLine++; 840 } 841 842 List<String> originalLines = delta.getSource().getLines(); 843 List<String> revisedLines = delta.getTarget().getLines(); 844 845 for (String line : originalLines) { 846 change.append("- " + lineNumber + ": " + line + "\r\n"); 847 ch.m_removals++; 848 lineNumber++; 849 currentOriginalLine++; 850 } 851 852 for (String line : revisedLines) { 853 change.append("+ " + lineNumber + ": " + line + "\r\n"); 854 lineNumber++; 855 currentModifiedLine++; 856 ch.m_adds++; 857 858 } 859 } 860 861 } catch (final Exception e) { 862 LOG.error("Diff failed", e); 863 } 864 865 // Don't forget to include the change note, too 866 final String changeNote = page.getAttribute( Page.CHANGENOTE ); 867 if( changeNote != null ) { 868 change.append( "\r\n" ); 869 change.append( changeNote ); 870 } 871 872 // And author as well 873 if( page.getAuthor() != null ) { 874 change.append( "\r\n" ).append( page.getAuthor() ); 875 } 876 877 ch.m_change = change.toString(); 878 return ch; 879 } 880 881 /** 882 * Returns true, if this user should be ignored. For example, admin users. 883 * 884 * @param context page context 885 * @return True, if this user should be ignored. 886 */ 887 private boolean ignoreThisUser( final Context context ) { 888 if( context.hasAdminPermissions() ) { 889 return true; 890 } 891 892 final List< String > groups = Arrays.asList( m_allowedGroups ); 893 if( Arrays.stream( context.getWikiSession().getRoles() ).anyMatch( role -> groups.contains( role.getName() ) ) ) { 894 return true; 895 } 896 897 if( m_ignoreAuthenticated && context.getWikiSession().isAuthenticated() ) { 898 return true; 899 } 900 901 return context.getVariable("captcha") != null; 902 } 903 904 /** 905 * Returns a random string of six uppercase characters. 906 * 907 * @return A random string 908 */ 909 private static String getUniqueID() { 910 final StringBuilder sb = new StringBuilder(); 911 for( int i = 0; i < 6; i++ ) { 912 final char x = ( char )( 'A' + RANDOM.nextInt( 26 ) ); 913 sb.append( x ); 914 } 915 916 return sb.toString(); 917 } 918 919 /** 920 * Returns a page to which we shall redirect, based on the current value of the "captcha" parameter. 921 * 922 * @param ctx WikiContext 923 * @return An URL to redirect to 924 */ 925 private String getRedirectPage( final Context ctx ) { 926 927 return ctx.getURL( ContextEnum.PAGE_VIEW.getRequestContext(), m_errorPage ); 928 } 929 930 /** 931 * Checks whether the UserProfile matches certain checks. 932 * 933 * @param profile The profile to check 934 * @param context The WikiContext 935 * @return False, if this userprofile is suspect and should not be allowed to be added. 936 * @since 2.6.1 937 */ 938 public boolean isValidUserProfile( final Context context, final UserProfile profile ) { 939 try { 940 checkPatternList( context, profile.getEmail() ); 941 checkPatternList( context, profile.getFullname() ); 942 checkPatternList( context, profile.getLoginName() ); 943 } catch( final RedirectException e ) { 944 LOG.info("Detected attempt to create a spammer user account (see above for rejection reason)"); 945 return false; 946 } 947 948 return true; 949 } 950 951 /** 952 * This method is used to calculate an unique code when submitting the page to detect edit conflicts. 953 * It currently incorporates the last-modified date of the page, and the IP address of the submitter. 954 * 955 * @param page The WikiPage under edit 956 * @param request The HTTP Request 957 * @since 2.6 958 * @return A hash value for this page and session 959 */ 960 public static String getSpamHash( final Page page, final HttpServletRequest request ) { 961 long lastModified = 0; 962 963 if( page.getLastModified() != null ) { 964 lastModified = page.getLastModified().getTime(); 965 } 966 final long remote = HttpUtil.getRemoteAddress( request ).hashCode(); 967 968 return Long.toString( lastModified ^ remote ); 969 } 970 971 /** 972 * Returns the name of the hash field to be used in this request. The value is unique per session, and once 973 * the session has expired, you cannot edit anymore. 974 * 975 * @param request The page request 976 * @return The name to be used in the hash field 977 * @since 2.6 978 */ 979 public static String getHashFieldName( final HttpServletRequest request ) { 980 String hash = null; 981 982 if( request.getSession() != null ) { 983 hash = ( String )request.getSession().getAttribute( "_hash" ); 984 985 if( hash == null ) { 986 hash = c_hashName; 987 request.getSession().setAttribute( "_hash", hash ); 988 } 989 } 990 991 if( c_hashName == null || c_lastUpdate < ( System.currentTimeMillis() - HASH_DELAY * 60 * 60 * 1000 ) ) { 992 c_hashName = getUniqueID().toLowerCase(); 993 c_lastUpdate = System.currentTimeMillis(); 994 } 995 996 return hash != null ? hash : c_hashName; 997 } 998 999 1000 /** 1001 * This method checks if the hash value is still valid, i.e. if it exists at all. This can occur in two cases: 1002 * either this is a spam bot which is not adaptive, or it is someone who has been editing one page for too long, 1003 * and their session has expired. 1004 * <p> 1005 * This method puts a redirect to the http response field to page "SessionExpired" and logs the incident in 1006 * the spam log (it may or may not be spam, but it's rather likely that it is). 1007 * 1008 * @param context The WikiContext 1009 * @param pageContext The JSP PageContext. 1010 * @return True, if hash is okay. False, if hash is not okay, and you need to redirect. 1011 * @throws IOException If redirection fails 1012 * @since 2.6 1013 */ 1014 public static boolean checkHash( final Context context, final PageContext pageContext ) throws IOException { 1015 final String hashName = getHashFieldName( (HttpServletRequest)pageContext.getRequest() ); 1016 if( pageContext.getRequest().getParameter(hashName) == null ) { 1017 if( pageContext.getAttribute( hashName ) == null ) { 1018 final Change change = getChange( context, EditorManager.getEditedText( pageContext ) ); 1019 log( context, REJECT, "MissingHash", change.m_change ); 1020 1021 final String redirect = context.getURL( ContextEnum.PAGE_VIEW.getRequestContext(),"SessionExpired" ); 1022 ( ( HttpServletResponse )pageContext.getResponse() ).sendRedirect( redirect ); 1023 return false; 1024 } 1025 } 1026 1027 return true; 1028 } 1029 1030 /** 1031 * This helper method adds all the input fields to your editor that the SpamFilter requires 1032 * to check for spam. This <i>must</i> be in your editor form if you intend to use the SpamFilter. 1033 * 1034 * @param pageContext The PageContext 1035 * @return A HTML string which contains input fields for the SpamFilter. 1036 */ 1037 public static String insertInputFields( final PageContext pageContext ) { 1038 final Context ctx = Context.findContext( pageContext ); 1039 final Engine engine = ctx.getEngine(); 1040 final StringBuilder sb = new StringBuilder(); 1041 if( engine.getContentEncoding().equals( StandardCharsets.UTF_8 ) ) { 1042 sb.append( "<input name='encodingcheck' type='hidden' value='\u3041' />\n" ); 1043 } 1044 1045 return sb.toString(); 1046 } 1047 1048 /** 1049 * A local class for storing host information. 1050 */ 1051 private class Host { 1052 1053 private final long m_addedTime = System.currentTimeMillis(); 1054 private final long m_releaseTime; 1055 private final String m_address; 1056 private final Change m_change; 1057 1058 public String getAddress() { 1059 return m_address; 1060 } 1061 1062 public long getReleaseTime() { 1063 return m_releaseTime; 1064 } 1065 1066 public long getAddedTime() { 1067 return m_addedTime; 1068 } 1069 1070 public Change getChange() { 1071 return m_change; 1072 } 1073 1074 public Host( final String ipaddress, final Change change ) { 1075 m_address = ipaddress; 1076 m_change = change; 1077 m_releaseTime = System.currentTimeMillis() + m_banTime * 60 * 1000L; 1078 } 1079 1080 } 1081 1082 private static class Change { 1083 1084 public String m_change; 1085 public int m_adds; 1086 public int m_removals; 1087 1088 @Override 1089 public String toString() { 1090 return m_change; 1091 } 1092 1093 @Override 1094 public boolean equals( final Object o ) { 1095 if( o instanceof Change ) { 1096 return m_change.equals( ( ( Change )o ).m_change ); 1097 } 1098 return false; 1099 } 1100 1101 @Override 1102 public int hashCode() { 1103 return m_change.hashCode() + 17; 1104 } 1105 1106 } 1107 1108}