001    /*
002    Licensed to the Apache Software Foundation (ASF) under one
003    or more contributor license agreements.  See the NOTICE file
004    distributed with this work for additional information
005    regarding copyright ownership.  The ASF licenses this file
006    to you under the Apache License, Version 2.0 (the
007    "License"); you may not use this file except in compliance
008    with the License.  You may obtain a copy of the License at
009
010       http://www.apache.org/licenses/LICENSE-2.0
011
012    Unless required by applicable law or agreed to in writing,
013    software distributed under the License is distributed on an
014    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015    KIND, either express or implied.  See the License for the
016    specific language governing permissions and limitations
017    under the License.
018 */
019package org.apache.wiki.parser;
020
021    import org.apache.commons.lang3.Strings;
022    import org.apache.commons.text.StringEscapeUtils;
023    import org.apache.logging.log4j.LogManager;
024    import org.apache.logging.log4j.Logger;
025    import org.apache.oro.text.regex.MalformedPatternException;
026    import org.apache.oro.text.regex.MatchResult;
027    import org.apache.oro.text.regex.Pattern;
028    import org.apache.oro.text.regex.PatternCompiler;
029    import org.apache.oro.text.regex.PatternMatcher;
030    import org.apache.oro.text.regex.Perl5Compiler;
031    import org.apache.oro.text.regex.Perl5Matcher;
032    import org.apache.wiki.InternalWikiException;
033    import org.apache.wiki.StringTransmutator;
034    import org.apache.wiki.api.core.Acl;
035    import org.apache.wiki.api.core.Context;
036    import org.apache.wiki.api.core.ContextEnum;
037    import org.apache.wiki.api.core.Page;
038    import org.apache.wiki.api.exceptions.PluginException;
039    import org.apache.wiki.api.plugin.Plugin;
040    import org.apache.wiki.api.spi.Wiki;
041    import org.apache.wiki.attachment.AttachmentManager;
042    import org.apache.wiki.auth.AuthorizationManager;
043    import org.apache.wiki.auth.UserManager;
044    import org.apache.wiki.auth.WikiSecurityException;
045    import org.apache.wiki.auth.acl.AclManager;
046    import org.apache.wiki.i18n.InternationalizationManager;
047    import org.apache.wiki.preferences.Preferences;
048    import org.apache.wiki.util.TextUtil;
049    import org.apache.wiki.util.XmlUtil;
050    import org.apache.wiki.variables.VariableManager;
051    import org.jdom2.Attribute;
052    import org.jdom2.Content;
053    import org.jdom2.Element;
054    import org.jdom2.IllegalDataException;
055    import org.jdom2.ProcessingInstruction;
056    import org.jdom2.Verifier;
057
058    import javax.xml.transform.Result;
059    import java.io.IOException;
060    import java.io.Reader;
061    import java.io.StringReader;
062    import java.text.MessageFormat;
063    import java.util.ArrayList;
064    import java.util.Arrays;
065    import java.util.Collection;
066    import java.util.EmptyStackException;
067    import java.util.HashMap;
068    import java.util.Iterator;
069    import java.util.List;
070    import java.util.Map;
071    import java.util.Properties;
072    import java.util.ResourceBundle;
073    import java.util.Stack;
074
075/**
076 * Parses JSPWiki-style markup into a WikiDocument DOM tree.  This class is the heart and soul of JSPWiki : make
077 * sure you test properly anything that is added, or else it breaks down horribly.
078 *
079 *  @since  2.4
080 */
081public class JSPWikiMarkupParser extends MarkupParser {
082
083    protected static final int              READ          = 0;
084    protected static final int              EDIT          = 1;
085    protected static final int              EMPTY         = 2;  // Empty message
086    protected static final int              LOCAL         = 3;
087    protected static final int              LOCALREF      = 4;
088    protected static final int              IMAGE         = 5;
089    protected static final int              EXTERNAL      = 6;
090    protected static final int              INTERWIKI     = 7;
091    protected static final int              IMAGELINK     = 8;
092    protected static final int              IMAGEWIKILINK = 9;
093    protected static final int              ATTACHMENT    = 10;
094
095    private static final Logger LOG = LogManager.getLogger( JSPWikiMarkupParser.class );
096
097    private boolean        m_isbold;
098    private boolean        m_isitalic;
099    private boolean        m_istable;
100    private boolean        m_isPre;
101    private boolean        m_isEscaping;
102    private boolean        m_isdefinition;
103    private boolean        m_isPreBlock;
104
105    /** Contains style information, in multiple forms. */
106    private final Stack< Boolean > m_styleStack = new Stack<>();
107
108     // general list handling
109    private int m_genlistlevel;
110    private final StringBuilder m_genlistBulletBuffer = new StringBuilder( 10 );  // stores the # and * pattern
111    private final boolean m_allowPHPWikiStyleLists = true;
112
113    private boolean m_isOpenParagraph;
114
115    /** Parser for extended link functionality. */
116    private final LinkParser m_linkParser = new LinkParser();
117
118    /** Keeps track of any plain text that gets put in the Text nodes */
119    private StringBuilder m_plainTextBuf = new StringBuilder( 20 );
120
121    private Element m_currentElement;
122
123    /** Keep track of duplicate header names.  */
124    private final Map< String, Integer > m_titleSectionCounter = new HashMap<>();
125
126    /** If true, then considers CamelCase links as well. */
127    private boolean m_camelCaseLinks;
128
129    /** If true, then generate special output for wysiwyg editing in certain cases */
130    private boolean m_wysiwygEditorMode;
131
132    /** If true, consider URIs that have no brackets as well. */
133    // FIXME: Currently reserved, but not used.
134    private boolean m_plainUris;
135
136    /** If true, all outward links use a small link image. */
137    private boolean m_useOutlinkImage = true;
138
139    private boolean m_useAttachmentImage = true;
140
141    /** If true, allows raw HTML. */
142    private boolean m_allowHTML;
143
144    private boolean m_useRelNofollow;
145
146    private final PatternCompiler m_compiler = new Perl5Compiler();
147
148    static final String WIKIWORD_REGEX = "(^|[[:^alnum:]]+)([[:upper:]]+[[:lower:]]+[[:upper:]]+[[:alnum:]]*|(http://|https://|mailto:)([A-Za-z0-9_/\\.\\+\\?\\#\\-\\@=&;~%]+))";
149
150    private final PatternMatcher m_camelCaseMatcher = new Perl5Matcher();
151    private Pattern m_camelCasePattern;
152
153    private int m_rowNum = 1;
154
155    private Heading m_lastHeading;
156
157    private static final String CAMELCASE_PATTERN = "JSPWikiMarkupParser.camelCasePattern";
158
159    /**
160     *  Creates a markup parser.
161     *
162     *  @param context The WikiContext which controls the parsing
163     *  @param in Where the data is read from.
164     */
165    public JSPWikiMarkupParser( final Context context, final Reader in ) {
166        super( context, in );
167        initialize();
168    }
169
170    // FIXME: parsers should be pooled for better performance.
171    private void initialize() {
172        initInlineImagePatterns();
173
174        m_camelCasePattern = m_engine.getAttribute( CAMELCASE_PATTERN );
175        if( m_camelCasePattern == null ) {
176            try {
177                m_camelCasePattern = m_compiler.compile( WIKIWORD_REGEX,Perl5Compiler.DEFAULT_MASK|Perl5Compiler.READ_ONLY_MASK );
178            } catch( final MalformedPatternException e ) {
179                LOG.fatal("Internal error: Someone put in a faulty pattern.",e);
180                throw new InternalWikiException("Faulty camelcasepattern in TranslatorReader", e);
181            }
182            m_engine.setAttribute( CAMELCASE_PATTERN, m_camelCasePattern );
183        }
184
185        //  Set the properties.
186        final Properties props = m_engine.getWikiProperties();
187        final String cclinks = m_context.getPage().getAttribute( PROP_CAMELCASELINKS );
188
189        if( cclinks != null ) {
190            m_camelCaseLinks = TextUtil.isPositive( cclinks );
191        } else {
192            m_camelCaseLinks  = TextUtil.getBooleanProperty( props, PROP_CAMELCASELINKS, m_camelCaseLinks );
193        }
194
195        final Boolean wysiwygVariable = m_context.getVariable( Context.VAR_WYSIWYG_EDITOR_MODE );
196        if( wysiwygVariable != null ) {
197            m_wysiwygEditorMode = wysiwygVariable;
198        }
199
200        m_plainUris          = m_context.getBooleanWikiProperty( PROP_PLAINURIS, m_plainUris );
201        m_useOutlinkImage    = m_context.getBooleanWikiProperty( PROP_USEOUTLINKIMAGE, m_useOutlinkImage );
202        m_useAttachmentImage = m_context.getBooleanWikiProperty( PROP_USEATTACHMENTIMAGE, m_useAttachmentImage );
203        m_allowHTML          = m_context.getBooleanWikiProperty( PROP_ALLOWHTML, m_allowHTML );
204        m_useRelNofollow     = m_context.getBooleanWikiProperty( PROP_USERELNOFOLLOW, m_useRelNofollow );
205
206        if( m_engine.getManager( UserManager.class ).getUserDatabase() == null || m_engine.getManager( AuthorizationManager.class ) == null ) {
207            disableAccessRules();
208        }
209
210        m_context.getPage().setHasMetadata();
211    }
212
213    /**
214     *  Calls a transmutator chain.
215     *
216     *  @param list Chain to call
217     *  @param text Text that should be passed to the mutate() method of each of the mutators in the chain.
218     *  @return The result of the mutation.
219     */
220    protected String callMutatorChain( final Collection< StringTransmutator > list, String text ) {
221        if( list == null || list.isEmpty()) {
222            return text;
223        }
224
225        for( final StringTransmutator m : list ) {
226            text = m.mutate( m_context, text );
227        }
228
229        return text;
230    }
231
232    /**
233     * Calls the heading listeners.
234     *
235     * @param param A Heading object.
236     */
237    private void callHeadingListenerChain( final Heading param ) {
238        for( final HeadingListener h : m_headingListenerChain ) {
239            h.headingAdded( m_context, param );
240        }
241    }
242
243    /**
244     *  Creates a JDOM anchor element.  Can be overridden to change the URL creation, if you really know what you are doing.
245     *
246     *  @param type One of the types above
247     *  @param link URL to which to link to
248     *  @param text Link text
249     *  @param section If a particular section identifier is required.
250     *  @return An 'A' element.
251     *  @since 2.4.78
252     */
253    private Element createAnchor( final int type, final String link, String text, String section ) {
254        text = TextUtil.escapeHTMLEntities( text );
255        section = TextUtil.escapeHTMLEntities( section );
256        final Element el = new Element( "a" );
257        el.setAttribute( "class", CLASS_TYPES[ type ] );
258        el.setAttribute( "href", link + section );
259        el.addContent( text );
260        return el;
261    }
262
263    private Element makeLink( int type, final String link, String text, String section, final Iterator< Attribute > attributes ) {
264        Element el = null;
265        if( text == null ) {
266            text = link;
267        }
268        text = callMutatorChain( m_linkMutators, text );
269        section = (section != null) ? ("#"+section) : "";
270
271        // Make sure we make a link name that can be accepted  as a valid URL.
272        if( link.isEmpty() ) {
273            type = EMPTY;
274        }
275        final ResourceBundle rb = Preferences.getBundle( m_context, InternationalizationManager.CORE_BUNDLE );
276
277        switch( type ) {
278            case READ:
279                el = createAnchor( READ, m_context.getURL( ContextEnum.PAGE_VIEW.getRequestContext(), link), text, section );
280                break;
281
282            case EDIT:
283                el = createAnchor( EDIT, m_context.getURL( ContextEnum.PAGE_EDIT.getRequestContext(),link), text, "" );
284                el.setAttribute("title", MessageFormat.format( rb.getString( "markupparser.link.create" ), link ) );
285                break;
286
287            case EMPTY:
288                el = new Element("u").addContent(text);
289                break;
290
291            // These two are for local references - footnotes and references to footnotes.
292            // We embed the page name (or whatever WikiContext gives us) to make sure the links are unique across Wiki.
293            case LOCALREF:
294                el = createAnchor( LOCALREF, "#ref-"+m_context.getName()+"-"+link, "["+text+"]", "" );
295                break;
296
297            case LOCAL:
298                el = new Element( "a" ).setAttribute( "class", CLASS_FOOTNOTE );
299                el.setAttribute( "name", "ref-" + m_context.getName() + "-" + link.substring( 1 ) );
300                if( !m_allowHTML ) {
301                    el.addContent( "[" + TextUtil.escapeHTMLEntities( text ) + "]" );
302                } else {
303                    el.addContent( "[" + text + "]" );
304                }
305                break;
306
307                //  With the image, external and interwiki types we need to make sure nobody can put in Javascript or
308                //  something else annoying into the links themselves.  We do this by preventing a haxor from stopping
309                //  the link name short with quotes in fillBuffer().
310            case IMAGE:
311                el = new Element( "img" ).setAttribute( "class", "inline" );
312                el.setAttribute( "src", link );
313                el.setAttribute( "alt", text );
314                break;
315
316            case IMAGELINK:
317                el = new Element( "img" ).setAttribute( "class", "inline" );
318                el.setAttribute( "src", link );
319                el.setAttribute( "alt", text );
320                el = createAnchor( IMAGELINK, text, "", "" ).addContent( el );
321                break;
322
323            case IMAGEWIKILINK:
324                final String pagelink = m_context.getURL( ContextEnum.PAGE_VIEW.getRequestContext(), text );
325                el = new Element( "img" ).setAttribute( "class", "inline" );
326                el.setAttribute( "src", link );
327                el.setAttribute( "alt", text );
328                el = createAnchor( IMAGEWIKILINK, pagelink, "", "" ).addContent( el );
329                break;
330
331            case EXTERNAL:
332                el = createAnchor( EXTERNAL, link, text, section );
333                if( m_useRelNofollow ) {
334                    el.setAttribute( "rel", "nofollow" );
335                }
336                break;
337
338            case INTERWIKI:
339                el = createAnchor( INTERWIKI, link, text, section );
340                break;
341
342            case ATTACHMENT:
343                final String attlink = m_context.getURL( ContextEnum.PAGE_ATTACH.getRequestContext(), link );
344                final String infolink = m_context.getURL( ContextEnum.PAGE_INFO.getRequestContext(), link );
345                final String imglink = m_context.getURL( ContextEnum.PAGE_NONE.getRequestContext(), "images/attachment_small.png" );
346                el = createAnchor( ATTACHMENT, attlink, text, "" );
347                if(  m_engine.getManager( AttachmentManager.class ).forceDownload( attlink ) ) {
348                    el.setAttribute("download", "");
349                }
350
351                pushElement( el );
352                popElement( el.getName() );
353
354                if( m_useAttachmentImage ) {
355                    el = new Element( "img" ).setAttribute( "src", imglink );
356                    el.setAttribute( "border", "0" );
357                    el.setAttribute( "alt", "(info)" );
358
359                    el = new Element( "a" ).setAttribute( "href", infolink ).addContent( el );
360                    el.setAttribute( "class", "infolink" );
361                } else {
362                    el = null;
363                }
364                break;
365
366            default:
367                break;
368        }
369
370        if( el != null && attributes != null ) {
371            while( attributes.hasNext() ) {
372                final Attribute attr = attributes.next();
373                if( attr != null ) {
374                    el.setAttribute( attr );
375                }
376            }
377        }
378
379        if( el != null ) {
380            flushPlainText();
381            m_currentElement.addContent( el );
382        }
383        return el;
384    }
385
386    /**
387     *  These are all the HTML 4.01 block-level elements.
388     */
389    private static final String[] BLOCK_ELEMENTS = {
390        "address", "blockquote", "div", "dl", "fieldset", "form",
391        "h1", "h2", "h3", "h4", "h5", "h6",
392        "hr", "noscript", "ol", "p", "pre", "table", "ul"
393    };
394
395    private static boolean isBlockLevel( final String name ) {
396        return Arrays.binarySearch( BLOCK_ELEMENTS, name ) >= 0;
397    }
398
399    /**
400     *  This method peeks ahead in the stream until EOL and returns the result. It will keep the buffers untouched. Of a line longer than
401     *  the pushback buffer only the first {@link #PUSHBACK_BUFFER_SIZE} characters are looked at, as more could not be given back.
402     *
403     *  @return The string from the current position to the end of line, or its first {@link #PUSHBACK_BUFFER_SIZE} characters.
404     */
405    private String peekAheadLine() throws IOException {
406        final StringBuilder buf = new StringBuilder( 256 );
407        int ch;
408        while( buf.length() < PUSHBACK_BUFFER_SIZE && ( ch = nextToken() ) != -1 ) {
409            buf.append( ( char ) ch );
410            if( ch == '\n' ) {
411                break;
412            }
413        }
414        final String s = buf.toString();
415        pushBack( s );
416        return s;
417    }
418
419    private int flushPlainText() {
420        final int numChars = m_plainTextBuf.length();
421        if( numChars > 0 ) {
422            String buf;
423
424            if( !m_allowHTML ) {
425                buf = TextUtil.escapeHTMLEntities( m_plainTextBuf.toString() );
426            } else {
427                buf = m_plainTextBuf.toString();
428            }
429            //  We must first empty the buffer because the side effect of calling makeCamelCaseLink() is to call this routine.
430            m_plainTextBuf = new StringBuilder(20);
431            try {
432                // This is the heaviest part of parsing, and therefore we can do some optimization here.
433                // 1) Only when the length of the buffer is big enough, we try to do the match
434                if( m_camelCaseLinks && !m_isEscaping && buf.length() > 3 ) {
435                    while( m_camelCaseMatcher.contains( buf, m_camelCasePattern ) ) {
436                        final MatchResult result = m_camelCaseMatcher.getMatch();
437                        final String firstPart = buf.substring( 0, result.beginOffset( 0 ) );
438                        String prefix = result.group( 1 );
439                        if( prefix == null ) {
440                            prefix = "";
441                        }
442
443                        final String camelCase = result.group(2);
444                        final String protocol  = result.group(3);
445                        String uri       = protocol+result.group(4);
446                        buf              = buf.substring(result.endOffset(0));
447
448                        m_currentElement.addContent( firstPart );
449                        //  Check if the user does not wish to do URL or WikiWord expansion
450                        if( prefix.endsWith( "~" ) || prefix.indexOf( '[' ) != -1 ) {
451                            if( prefix.endsWith( "~" ) ) {
452                                if( m_wysiwygEditorMode ) {
453                                    m_currentElement.addContent( "~" );
454                                }
455                                prefix = prefix.substring( 0, prefix.length() - 1 );
456                            }
457                            if( camelCase != null ) {
458                                m_currentElement.addContent( prefix + camelCase );
459                            } else if( protocol != null ) {
460                                m_currentElement.addContent( prefix + uri );
461                            }
462                            continue;
463                        }
464
465                        // Fine, then let's check what kind of link this was and emit the proper elements
466                        if( protocol != null ) {
467                            final char c = uri.charAt( uri.length() - 1 );
468                            if( c == '.' || c == ',' ) {
469                                uri = uri.substring( 0, uri.length() - 1 );
470                                buf = c + buf;
471                            }
472                            // System.out.println("URI match "+uri);
473                            m_currentElement.addContent( prefix );
474                            makeDirectURILink( uri );
475                        } else {
476                            // System.out.println("Matched: '"+camelCase+"'");
477                            // System.out.println("Split to '"+firstPart+"', and '"+buf+"'");
478                            // System.out.println("prefix="+prefix);
479                            m_currentElement.addContent( prefix );
480                            makeCamelCaseLink( camelCase );
481                        }
482                    }
483                    m_currentElement.addContent( buf );
484                } else {
485                    //  No camelcase asked for, just add the elements
486                    m_currentElement.addContent( buf );
487                }
488            } catch( final IllegalDataException e ) {
489                // Sometimes it's possible that illegal XML chars is added to the data. Here we make sure it does not stop parsing.
490                m_currentElement.addContent( makeError(cleanupSuspectData( e.getMessage() )) );
491            }
492        }
493
494        return numChars;
495    }
496
497    private Element pushElement( final Element e ) {
498        flushPlainText();
499        m_currentElement.addContent( e );
500        m_currentElement = e;
501
502        return e;
503    }
504
505    private Element addElement( final Content e ) {
506        if( e != null ) {
507            flushPlainText();
508            m_currentElement.addContent( e );
509        }
510        return m_currentElement;
511    }
512
513    /**
514     *  All elements that can be empty by the HTML DTD.
515     */
516    //  Keep sorted.
517    private static final String[] EMPTY_ELEMENTS = {
518        "area", "base", "br", "col", "hr", "img", "input", "link", "meta", "p", "param"
519    };
520
521    /**
522     *  Goes through the current element stack and pops all elements until this
523     *  element is found - this essentially "closes" and element.
524     *
525     *  @param s element to be found.
526     *  @return The new current element, or null, if there was no such element in the entire stack.
527     */
528    private Element popElement( final String s ) {
529        final int flushedBytes = flushPlainText();
530        Element currEl = m_currentElement;
531        while( currEl.getParentElement() != null ) {
532            if( currEl.getName().equals( s ) && !currEl.isRootElement() ) {
533                m_currentElement = currEl.getParentElement();
534
535                //  Check if it's okay for this element to be empty.  Then we will
536                //  trick the JDOM generator into not generating an empty element,
537                //  by putting an empty string between the tags.  Yes, it's a kludge
538                //  but what'cha gonna do about it. :-)
539                if( flushedBytes == 0 && Arrays.binarySearch( EMPTY_ELEMENTS, s ) < 0 ) {
540                    currEl.addContent( "" );
541                }
542                return m_currentElement;
543            }
544            currEl = currEl.getParentElement();
545        }
546        return null;
547    }
548
549
550    /**
551     * Reads the stream until it meets one of the specified ending characters, or stream end. The ending
552     * character will be left in the stream.
553     */
554    private String readUntil( final String endChars ) throws IOException {
555        final StringBuilder sb = new StringBuilder( 80 );
556        int ch = nextToken();
557        while( ch != -1 ) {
558            if( ch == '\\' ) {
559                ch = nextToken();
560                if( ch == -1 ) {
561                    break;
562                }
563            } else {
564                if( endChars.indexOf( ( char )ch ) != -1 ) {
565                    pushBack( ch );
566                    break;
567                }
568            }
569            sb.append( ( char )ch );
570            ch = nextToken();
571        }
572
573        return sb.toString();
574    }
575
576    /**
577     *  Reads the stream while the characters that have been specified are
578     *  in the stream, returning then the result as a String.
579     */
580    private String readWhile( final String endChars ) throws IOException {
581        final StringBuilder sb = new StringBuilder( 80 );
582        int ch = nextToken();
583        while( ch != -1 ) {
584            if( endChars.indexOf( ( char ) ch ) == -1 ) {
585                pushBack( ch );
586                break;
587            }
588            sb.append( ( char ) ch );
589            ch = nextToken();
590        }
591
592        return sb.toString();
593    }
594
595    private JSPWikiMarkupParser m_cleanTranslator;
596
597    /**
598     *  Does a lazy init.  Otherwise, we would get into a situation where HTMLRenderer would try and boot a TranslatorReader before
599     *  the TranslatorReader it is contained by is up.
600     */
601    private JSPWikiMarkupParser getCleanTranslator() {
602        if( m_cleanTranslator == null ) {
603            final Context dummyContext = Wiki.context().create( m_engine, m_context.getHttpRequest(), m_context.getPage() );
604            m_cleanTranslator = new JSPWikiMarkupParser( dummyContext, null );
605            m_cleanTranslator.m_allowHTML = true;
606        }
607
608        return m_cleanTranslator;
609    }
610
611    /**
612     *  Modifies the "hd" parameter to contain proper values.  Because
613     *  an "id" tag may only contain [a-zA-Z0-9:_-], we'll replace the
614     *  % after url encoding with '_'.
615     *  <p>
616     *  Counts also duplicate headings (= headings with similar name), and
617     *  attaches a counter.
618     */
619    protected String makeHeadingAnchor( final String baseName, String title, final Heading hd ) {
620        hd.m_titleText = title;
621        title = MarkupParser.wikifyLink( title );
622        hd.m_titleSection = m_engine.encodeName(title);
623        if( m_titleSectionCounter.containsKey( hd.m_titleSection ) ) {
624            final Integer count = m_titleSectionCounter.get( hd.m_titleSection ) + 1;
625            m_titleSectionCounter.put( hd.m_titleSection, count );
626            hd.m_titleSection += "-" + count;
627        } else {
628            m_titleSectionCounter.put( hd.m_titleSection, 1 );
629        }
630
631        hd.m_titleAnchor = "section-" + m_engine.encodeName( baseName ) + "-" + hd.m_titleSection;
632        hd.m_titleAnchor = hd.m_titleAnchor.replace( '%', '_' );
633        hd.m_titleAnchor = hd.m_titleAnchor.replace( '/', '_' );
634
635        return hd.m_titleAnchor;
636    }
637
638    private String makeSectionTitle( String title ) {
639        title = title.trim();
640        try {
641            final JSPWikiMarkupParser dtr = getCleanTranslator();
642            dtr.setInputReader( new StringReader( title ) );
643            final WikiDocument doc = dtr.parse();
644            doc.setContext( m_context );
645
646            return XmlUtil.extractTextFromDocument( doc );
647        } catch( final IOException e ) {
648            LOG.fatal("Title parsing not working", e );
649            throw new InternalWikiException( "Xml text extraction not working as expected when cleaning title" + e.getMessage() , e );
650        }
651    }
652
653    /**
654     *  Returns XHTML for the heading.
655     *
656     *  @param level The level of the heading.  @see Heading
657     *  @param title the title for the heading
658     *  @param hd a List to which heading should be added
659     *  @return An Element containing the heading
660     */
661    public Element makeHeading( final int level, final String title, final Heading hd ) {
662        final Element el;
663        final String pageName = m_context.getPage().getName();
664        final String outTitle = makeSectionTitle( title );
665        hd.m_level = level;
666
667        switch( level ) {
668          case Heading.HEADING_SMALL:
669            el = new Element( "h4" ).setAttribute("id",makeHeadingAnchor( pageName, outTitle, hd ) );
670            break;
671
672          case Heading.HEADING_MEDIUM:
673            el = new Element( "h3" ).setAttribute("id",makeHeadingAnchor( pageName, outTitle, hd ) );
674            break;
675
676          case Heading.HEADING_LARGE:
677            el = new Element( "h2" ).setAttribute("id",makeHeadingAnchor( pageName, outTitle, hd ) );
678            break;
679
680          default:
681            throw new InternalWikiException( "Illegal heading type " + level );
682        }
683
684        return el;
685    }
686
687    /**
688     *  When given a link to a WikiName, we just return a proper HTML link for it.  The local link mutator
689     *  chain is also called.
690     */
691    private Element makeCamelCaseLink( final String wikiname ) {
692        final String matchedLink = m_linkParsingOperations.linkIfExists( wikiname );
693        callMutatorChain( m_localLinkMutatorChain, wikiname );
694        if( matchedLink != null ) {
695            makeLink( READ, matchedLink, wikiname, null, null );
696        } else {
697            makeLink( EDIT, wikiname, wikiname, null, null );
698        }
699
700        return m_currentElement;
701    }
702
703    /** Holds the image URL for the duration of this parser */
704    private String m_outlinkImageURL;
705
706    /**
707     * Returns an element for the external link image (out.png).  However, this method caches the URL for the lifetime
708     * of this MarkupParser, because it's commonly used, and we'll end up with possibly hundreds our thousands of
709     * references to it...  It's a lot faster, too.
710     *
711     * @return  An element containing the HTML for the outlink image.
712     */
713    private Element outlinkImage() {
714        Element el = null;
715        if( m_useOutlinkImage ) {
716            if( m_outlinkImageURL == null ) {
717                m_outlinkImageURL = m_context.getURL( ContextEnum.PAGE_NONE.getRequestContext(), OUTLINK_IMAGE );
718            }
719
720            el = new Element( "img" ).setAttribute( "class", OUTLINK );
721            el.setAttribute( "src", m_outlinkImageURL );
722            el.setAttribute( "alt","" );
723        }
724
725        return el;
726    }
727
728    /**
729     *  Takes a URL and turns it into a regular wiki link. Unfortunately, because of the way that flushPlainText()
730     *  works, it already encodes all the XML entities. But so does WikiContext.getURL(), so we
731     *  have to do a reverse-replace here, so that it can again be replaced in makeLink.
732     *  <p>
733     *  What a crappy problem.
734     *
735     * @param url provided url.
736     * @return An anchor Element containing the link.
737     */
738    private Element makeDirectURILink( String url ) {
739        final Element result;
740        String last = null;
741
742        if( url.endsWith( "," ) || url.endsWith( "." ) ) {
743            last = url.substring( url.length() - 1 );
744            url = url.substring( 0, url.length() - 1 );
745        }
746
747        callMutatorChain( m_externalLinkMutatorChain, url );
748
749        if( m_linkParsingOperations.isImageLink( url, isImageInlining(), getInlineImagePatterns() ) ) {
750            result = handleImageLink( Strings.CS.replace( url, "&amp;", "&" ), url, false );
751        } else {
752            result = makeLink( EXTERNAL, Strings.CS.replace( url, "&amp;", "&" ), url, null, null );
753            addElement( outlinkImage() );
754        }
755
756        if( last != null ) {
757            m_plainTextBuf.append( last );
758        }
759
760        return result;
761    }
762
763    /**
764     *  Image links are handled differently:
765     *  1. If the text is a WikiName of an existing page, it gets linked.
766     *  2. If the text is an external link, then it is inlined.
767     *  3. Otherwise, it becomes an ALT text.
768     *
769     *  @param reallink The link to the image.
770     *  @param link     Link text portion, may be a link to somewhere else.
771     *  @param hasLinkText If true, then the defined link had a link text available.
772     *                  This means that the link text may be a link to a wiki page,
773     *                  or an external resource.
774     */
775    private Element handleImageLink( final String reallink, final String link, final boolean hasLinkText ) {
776        final String possiblePage = MarkupParser.cleanLink( link );
777        if( m_linkParsingOperations.isExternalLink( link ) && hasLinkText ) {
778            return makeLink( IMAGELINK, reallink, link, null, null );
779        } else if( m_linkParsingOperations.linkExists( possiblePage ) && hasLinkText ) {
780            callMutatorChain( m_localLinkMutatorChain, possiblePage );
781            return makeLink( IMAGEWIKILINK, reallink, link, null, null );
782        } else {
783            // A data URL makes no sense as alternative text
784            return makeLink( IMAGE, reallink, LinkParsingOperations.isDataImage( link ) ? "" : link, null, null );
785        }
786    }
787
788    private Element handleAccessRule( String ruleLine ) {
789        if( m_wysiwygEditorMode ) {
790            m_currentElement.addContent( "[" + TextUtil.escapeHTMLEntities( ruleLine ) + "]" );
791        }
792        if( !m_parseAccessRules ) {
793            return m_currentElement;
794        }
795        final Page page = m_context.getRealPage();
796        // UserDatabase db = m_context.getEngine().getUserDatabase();
797
798        if( ruleLine.startsWith( "{" ) ) {
799            ruleLine = ruleLine.substring( 1 );
800        }
801
802        if( ruleLine.endsWith( "}" ) ) {
803            ruleLine = ruleLine.substring( 0, ruleLine.length() - 1 );
804        }
805
806        LOG.debug("page={}, ACL = {}", page.getName(), ruleLine);
807
808        try {
809            final Acl acl = m_engine.getManager( AclManager.class ).parseAcl( page, ruleLine );
810            page.setAcl( acl );
811            LOG.debug( acl.toString() );
812        } catch( final WikiSecurityException wse ) {
813            return makeError( wse.getMessage() );
814        }
815
816        return m_currentElement;
817    }
818
819    /**
820     *  Handles metadata setting [{SET foo=bar}]
821     */
822    private Element handleMetadata( final String link ) {
823        if( m_wysiwygEditorMode ) {
824            m_currentElement.addContent( "[" + TextUtil.escapeHTMLEntities( link ) + "]" );
825        }
826
827        try {
828            final String args = link.substring( link.indexOf(' '), link.length()-1 );
829            final String name = args.substring( 0, args.indexOf('=') ).trim();
830            String val  = args.substring( args.indexOf('=')+1 ).trim();
831
832            if( val.startsWith("'") ) {
833                val = val.substring( 1 );
834            }
835            if( val.endsWith("'") ) {
836                val = val.substring( 0, val.length()-1 );
837            }
838
839            // LOG.debug("SET name='"+name+"', value='"+val+"'.");
840
841            if( !name.isEmpty() && !val.isEmpty() ) {
842                val = m_engine.getManager( VariableManager.class ).expandVariables( m_context, val );
843                m_context.getPage().setAttribute( name, val );
844            }
845        } catch( final Exception e ) {
846            final ResourceBundle rb = Preferences.getBundle( m_context, InternationalizationManager.CORE_BUNDLE );
847            return makeError( MessageFormat.format( rb.getString( "markupparser.error.invalidset" ), link ) );
848        }
849
850        return m_currentElement;
851    }
852
853    /**
854     *  Emits a processing instruction that will disable markup escaping. This is
855     *  very useful if you want to emit HTML directly into the stream.
856     */
857    private void disableOutputEscaping() {
858        addElement( new ProcessingInstruction( Result.PI_DISABLE_OUTPUT_ESCAPING, "" ) );
859    }
860
861    /**
862     *  Gobbles up all hyperlinks that are encased in square brackets.
863     */
864    private Element handleHyperlinks( String linktext, final int pos ) {
865        final ResourceBundle rb = Preferences.getBundle( m_context, InternationalizationManager.CORE_BUNDLE );
866        final StringBuilder sb = new StringBuilder( linktext.length() + 80 );
867
868        if( m_linkParsingOperations.isAccessRule( linktext ) ) {
869            return handleAccessRule( linktext );
870        }
871
872        if( m_linkParsingOperations.isMetadata( linktext ) ) {
873            return handleMetadata( linktext );
874        }
875
876        if( m_linkParsingOperations.isPluginLink( linktext ) ) {
877            try {
878                final PluginContent pluginContent = PluginContent.parsePluginLine( m_context, linktext, pos );
879
880                // This might sometimes fail, especially if there is something which looks like a plugin invocation but is really not.
881                if( pluginContent != null ) {
882                    addElement( pluginContent );
883                    pluginContent.executeParse( m_context );
884                }
885            } catch( final PluginException e ) {
886                LOG.info( m_context.getRealPage().getWiki() + " : " + m_context.getRealPage().getName() + " - Failed to insert plugin: " + e.getMessage() );
887                //LOG.info( "Root cause:",e.getRootThrowable() );
888                if( !m_wysiwygEditorMode ) {
889                    final ResourceBundle rbPlugin = Preferences.getBundle( m_context, Plugin.CORE_PLUGINS_RESOURCEBUNDLE );
890                    return addElement( makeError( MessageFormat.format( rbPlugin.getString( "plugin.error.insertionfailed" ),
891                                                                        m_context.getRealPage().getWiki(),
892                                                                        m_context.getRealPage().getName(),
893                                                                        e.getMessage() ) ) );
894                }
895            }
896            return m_currentElement;
897        }
898
899        try {
900            final LinkParser.Link link = m_linkParser.parse( linktext );
901            linktext = link.getText();
902            String linkref = link.getReference();
903            //  Yes, we now have the components separated.
904            //  linktext = the text the link should have
905            //  linkref  = the url or page name.
906            //  In many cases these are the same.  [linktext|linkref].
907            if( m_linkParsingOperations.isVariableLink( linktext ) ) {
908                final Content el = new VariableContent( linktext );
909                addElement( el );
910            } else if( isImageInlining() && LinkParsingOperations.isDataImage( linkref ) ) {
911                // The image is in the link itself, so there is nothing to track as an external link
912                handleImageLink( linkref, linktext, link.hasReference() );
913            } else if( m_linkParsingOperations.isExternalLink( linkref ) ) {
914                // It's an external link, out of this Wiki
915                callMutatorChain( m_externalLinkMutatorChain, linkref );
916                if( m_linkParsingOperations.isImageLink( linkref, isImageInlining(), getInlineImagePatterns() ) ) {
917                    handleImageLink( linkref, linktext, link.hasReference() );
918                } else {
919                    makeLink( EXTERNAL, linkref, linktext, null, link.getAttributes() );
920                    addElement( outlinkImage() );
921                }
922            } else if( link.isInterwikiLink() ) {
923                // It's an interwiki link; InterWiki links also get added to external link chain after the links have been resolved.
924
925                // FIXME: There is an interesting issue here:  We probably should
926                //        URLEncode the wikiPage, but we can't since some of the
927                //        Wikis use slashes (/), which won't survive URLEncoding.
928                //        Besides, we don't know which character set the other Wiki
929                //        is using, so you'll have to write the entire name as it appears
930                //        in the URL.  Bugger.
931
932                final String extWiki = link.getExternalWiki();
933                final String wikiPage = link.getExternalWikiPage();
934                if( m_wysiwygEditorMode ) {
935                    makeLink( INTERWIKI, extWiki + ":" + wikiPage, linktext, null, link.getAttributes() );
936                } else {
937                    String urlReference = m_engine.getInterWikiURL( extWiki );
938                    if( urlReference != null ) {
939                        urlReference = TextUtil.replaceString( urlReference, "%s", wikiPage );
940                        urlReference = callMutatorChain( m_externalLinkMutatorChain, urlReference );
941
942                        if( m_linkParsingOperations.isImageLink( urlReference, isImageInlining(), getInlineImagePatterns() ) ) {
943                            handleImageLink( urlReference, linktext, link.hasReference() );
944                        } else {
945                            makeLink( INTERWIKI, urlReference, linktext, null, link.getAttributes() );
946                        }
947                        if( m_linkParsingOperations.isExternalLink( urlReference ) ) {
948                            addElement( outlinkImage() );
949                        }
950                    } else {
951                        final Object[] args = { TextUtil.escapeHTMLEntities( extWiki ) };
952                        addElement( makeError( MessageFormat.format( rb.getString( "markupparser.error.nointerwikiref" ), args ) ) );
953                    }
954                }
955            } else if( linkref.startsWith( "#" ) ) {
956                // It defines a local footnote
957                makeLink( LOCAL, linkref, linktext, null, link.getAttributes() );
958            } else if( TextUtil.isNumber( linkref ) ) {
959                // It defines a reference to a local footnote
960                makeLink( LOCALREF, linkref, linktext, null, link.getAttributes() );
961            } else {
962                final int hashMark;
963
964                // Internal wiki link, but is it an attachment link?
965                String attachment = m_engine.getManager( AttachmentManager.class ).getAttachmentInfoName( m_context, linkref );
966                if( attachment != null ) {
967                    callMutatorChain( m_attachmentLinkMutatorChain, attachment );
968                    if( m_linkParsingOperations.isImageLink( linkref, isImageInlining(), getInlineImagePatterns() ) ) {
969                        attachment = m_context.getURL( ContextEnum.PAGE_ATTACH.getRequestContext(), attachment );
970                        sb.append( handleImageLink( attachment, linktext, link.hasReference() ) );
971                    } else {
972                        makeLink( ATTACHMENT, attachment, linktext, null, link.getAttributes() );
973                    }
974                } else if( ( hashMark = linkref.indexOf( '#' ) ) != -1 ) {
975                    // It's an internal Wiki link, but to a named section
976                    final String namedSection = linkref.substring( hashMark + 1 );
977                    linkref = linkref.substring( 0, hashMark );
978                    linkref = MarkupParser.cleanLink( linkref );
979                    callMutatorChain( m_localLinkMutatorChain, linkref );
980                    final String matchedLink = m_linkParsingOperations.linkIfExists( linkref );
981                    if( matchedLink != null ) {
982                        String sectref = "section-" + m_engine.encodeName( matchedLink + "-" + wikifyLink( namedSection ) );
983                        sectref = sectref.replace( '%', '_' );
984                        makeLink( READ, matchedLink, linktext, sectref, link.getAttributes() );
985                    } else {
986                        makeLink( EDIT, linkref, linktext, null, link.getAttributes() );
987                    }
988                } else {
989                    // It's an internal Wiki link
990                    linkref = MarkupParser.cleanLink( linkref );
991                    callMutatorChain( m_localLinkMutatorChain, linkref );
992                    final String matchedLink = m_linkParsingOperations.linkIfExists( linkref );
993                    if( matchedLink != null ) {
994                        makeLink( READ, matchedLink, linktext, null, link.getAttributes() );
995                    } else {
996                        makeLink( EDIT, linkref, linktext, null, link.getAttributes() );
997                    }
998                }
999            }
1000
1001        } catch( final ParseException e ) {
1002            LOG.info( "Parser failure: ", e );
1003            final Object[] args = { e.getMessage() };
1004            addElement( makeError( MessageFormat.format( rb.getString( "markupparser.error.parserfailure" ), args ) ) );
1005        }
1006        return m_currentElement;
1007    }
1008
1009    /**
1010     *  Pushes back any string that has been read.  It will obviously be pushed back in a reverse order.
1011     *
1012     *  @since 2.1.77
1013     */
1014    private void pushBack( final String s ) throws IOException {
1015        for( int i = s.length()-1; i >= 0; i-- ) {
1016            pushBack( s.charAt(i) );
1017        }
1018    }
1019
1020    private Element handleBackslash() throws IOException {
1021        final int ch = nextToken();
1022        if( ch == '\\' ) {
1023            final int ch2 = nextToken();
1024            if( ch2 == '\\' ) {
1025                pushElement( new Element( "br" ).setAttribute( "clear", "all" ) );
1026                return popElement( "br" );
1027            }
1028            pushBack( ch2 );
1029            pushElement( new Element( "br" ) );
1030            return popElement( "br" );
1031        }
1032        pushBack( ch );
1033        return null;
1034    }
1035
1036    private Element handleUnderscore() throws IOException {
1037        final int ch = nextToken();
1038        Element el = null;
1039        if( ch == '_' ) {
1040            if( m_isbold ) {
1041                el = popElement( "b" );
1042            } else {
1043                el = pushElement( new Element( "b" ) );
1044            }
1045            m_isbold = !m_isbold;
1046        } else {
1047            pushBack( ch );
1048        }
1049
1050        return el;
1051    }
1052
1053
1054    /**
1055     *  For example: italics.
1056     */
1057    private Element handleApostrophe() throws IOException {
1058        final int ch = nextToken();
1059        Element el = null;
1060
1061        if( ch == '\'' ) {
1062            if( m_isitalic ) {
1063                el = popElement( "i" );
1064            } else {
1065                el = pushElement( new Element( "i" ) );
1066            }
1067            m_isitalic = !m_isitalic;
1068        } else {
1069            pushBack( ch );
1070        }
1071
1072        return el;
1073    }
1074
1075    private Element handleOpenbrace( final boolean isBlock ) throws IOException {
1076        final int ch = nextToken();
1077        if( ch == '{' ) {
1078            final int ch2 = nextToken();
1079            if( ch2 == '{' ) {
1080                m_isPre = true;
1081                m_isEscaping = true;
1082                m_isPreBlock = isBlock;
1083                if( isBlock ) {
1084                    startBlockLevel();
1085                    return pushElement( new Element( "pre" ) );
1086                }
1087
1088                return pushElement( new Element( "span" ).setAttribute( "class", "inline-code" ) );
1089            }
1090            pushBack( ch2 );
1091            return pushElement( new Element( "tt" ) );
1092        }
1093        pushBack( ch );
1094        return null;
1095    }
1096
1097    /**
1098     *  Handles both }} and }}}
1099     */
1100    private Element handleClosebrace() throws IOException {
1101        final int ch2 = nextToken();
1102        if( ch2 == '}' ) {
1103            final int ch3 = nextToken();
1104            if( ch3 == '}' ) {
1105                if( m_isPre ) {
1106                    if( m_isPreBlock ) {
1107                        popElement( "pre" );
1108                    } else {
1109                        popElement( "span" );
1110                    }
1111                    m_isPre = false;
1112                    m_isEscaping = false;
1113                    return m_currentElement;
1114                }
1115                m_plainTextBuf.append( "}}}" );
1116                return m_currentElement;
1117            }
1118            pushBack( ch3 );
1119            if( !m_isEscaping ) {
1120                return popElement( "tt" );
1121            }
1122        }
1123        pushBack( ch2 );
1124        return null;
1125    }
1126
1127    private Element handleDash() throws IOException {
1128        int ch = nextToken();
1129        if( ch == '-' ) {
1130            final int ch2 = nextToken();
1131            if( ch2 == '-' ) {
1132                final int ch3 = nextToken();
1133                if( ch3 == '-' ) {
1134                    // Empty away all the rest of the dashes.
1135                    // Do not forget to return the first non-match back.
1136                    do {
1137                        ch = nextToken();
1138                    } while ( ch == '-' );
1139
1140                    pushBack( ch );
1141                    startBlockLevel();
1142                    pushElement( new Element( "hr" ) );
1143                    return popElement( "hr" );
1144                }
1145                pushBack( ch3 );
1146            }
1147            pushBack( ch2 );
1148        }
1149        pushBack( ch );
1150        return null;
1151    }
1152
1153    private Element handleHeading() throws IOException {
1154        final Element el;
1155        final int ch  = nextToken();
1156        final Heading hd = new Heading();
1157        if( ch == '!' ) {
1158            final int ch2 = nextToken();
1159            if( ch2 == '!' ) {
1160                final String title = peekAheadLine();
1161                el = makeHeading( Heading.HEADING_LARGE, title, hd );
1162            } else {
1163                pushBack( ch2 );
1164                final String title = peekAheadLine();
1165                el = makeHeading( Heading.HEADING_MEDIUM, title, hd );
1166            }
1167        } else {
1168            pushBack( ch );
1169            final String title = peekAheadLine();
1170            el = makeHeading( Heading.HEADING_SMALL, title, hd );
1171        }
1172
1173        callHeadingListenerChain( hd );
1174        m_lastHeading = hd;
1175        if( el != null ) {
1176            pushElement( el );
1177        }
1178        return el;
1179    }
1180
1181    /** Controls whether italic is restarted after a paragraph shift */
1182
1183    private boolean m_restartitalic;
1184    private boolean m_restartbold;
1185
1186    private boolean m_newLine;
1187
1188    /**
1189     * Starts a block level element, therefore closing a potential open paragraph tag.
1190     */
1191    private void startBlockLevel() {
1192        // These may not continue over block level limits in XHTML
1193        popElement( "i" );
1194        popElement( "b" );
1195        popElement( "tt" );
1196        if( m_isOpenParagraph ) {
1197            m_isOpenParagraph = false;
1198            popElement( "p" );
1199            m_plainTextBuf.append( "\n" ); // Just small beautification
1200        }
1201        m_restartitalic = m_isitalic;
1202        m_restartbold   = m_isbold;
1203        m_isitalic = false;
1204        m_isbold   = false;
1205    }
1206
1207    private static String getListType( final char c ) {
1208        if( c == '*' ) {
1209            return "ul";
1210        } else if( c == '#' ) {
1211            return "ol";
1212        }
1213        throw new InternalWikiException( "Parser got faulty list type: " + c );
1214    }
1215    /**
1216     * Like original handleOrderedList() and handleUnorderedList(),
1217     * however handles both ordered ('#') and unordered ('*') mixed together.
1218     */
1219    // FIXME: Refactor this; it's a bit messy.
1220    private Element handleGeneralList() throws IOException {
1221         startBlockLevel();
1222         String strBullets = readWhile( "*#" );
1223         // String strBulletsRaw = strBullets;      // to know what was original before phpwiki style substitution
1224         final int numBullets = strBullets.length();
1225
1226         // override the beginning portion of bullet pattern to be like the previous to simulate PHPWiki style lists
1227
1228        if( m_allowPHPWikiStyleLists ) {
1229            // only substitute if different
1230            if( !( strBullets.substring( 0, Math.min( numBullets, m_genlistlevel ) ).equals( m_genlistBulletBuffer.substring( 0, Math.min( numBullets, m_genlistlevel ) ) ) ) ) {
1231                if( numBullets <= m_genlistlevel ) {
1232                    // Substitute all but the last character (keep the expressed bullet preference)
1233                    strBullets = ( numBullets > 1 ? m_genlistBulletBuffer.substring( 0, numBullets - 1 ) : "" ) +
1234                                 strBullets.charAt( numBullets - 1 );
1235                } else {
1236                    strBullets = m_genlistBulletBuffer + strBullets.substring( m_genlistlevel, numBullets );
1237                }
1238            }
1239        }
1240
1241         //  Check if this is still of the same type
1242        if( strBullets.substring( 0, Math.min( numBullets, m_genlistlevel ) ).equals( m_genlistBulletBuffer.substring( 0, Math.min( numBullets, m_genlistlevel ) ) ) ) {
1243            if( numBullets > m_genlistlevel ) {
1244                pushElement( new Element( getListType( strBullets.charAt( m_genlistlevel++ ) ) ) );
1245                for( ; m_genlistlevel < numBullets; m_genlistlevel++ ) {
1246                    // bullets are growing, get from new bullet list
1247                    pushElement( new Element( "li" ) );
1248                    pushElement( new Element( getListType( strBullets.charAt( m_genlistlevel ) ) ) );
1249                }
1250            } else if( numBullets < m_genlistlevel ) {
1251                //  Close the previous list item.
1252                popElement( "li" );
1253                for( ; m_genlistlevel > numBullets; m_genlistlevel-- ) {
1254                    // bullets are shrinking, get from old bullet list
1255                    popElement( getListType( m_genlistBulletBuffer.charAt( m_genlistlevel - 1 ) ) );
1256                    if( m_genlistlevel > 0 ) {
1257                        popElement( "li" );
1258                    }
1259                }
1260            } else {
1261                if( m_genlistlevel > 0 ) {
1262                    popElement( "li" );
1263                }
1264            }
1265        } else {
1266            //  The pattern has changed, unwind and restart
1267            int numEqualBullets;
1268            final int numCheckBullets;
1269
1270            // find out how much is the same
1271            numEqualBullets = 0;
1272            numCheckBullets = Math.min( numBullets, m_genlistlevel );
1273
1274            while( numEqualBullets < numCheckBullets ) {
1275                // if the bullets are equal so far, keep going
1276                if( strBullets.charAt( numEqualBullets ) == m_genlistBulletBuffer.charAt( numEqualBullets ) )
1277                    numEqualBullets++;
1278                    // otherwise giveup, we have found how many are equal
1279                else
1280                    break;
1281            }
1282
1283            //unwind
1284            for( ; m_genlistlevel > numEqualBullets; m_genlistlevel-- ) {
1285                popElement( getListType( m_genlistBulletBuffer.charAt( m_genlistlevel - 1 ) ) );
1286                if( m_genlistlevel > numBullets ) {
1287                    popElement( "li" );
1288                }
1289            }
1290
1291            //rewind
1292            pushElement( new Element( getListType( strBullets.charAt( numEqualBullets++ ) ) ) );
1293            for( int i = numEqualBullets; i < numBullets; i++ ) {
1294                pushElement( new Element( "li" ) );
1295                pushElement( new Element( getListType( strBullets.charAt( i ) ) ) );
1296            }
1297            m_genlistlevel = numBullets;
1298        }
1299
1300         // Push a new list item, and eat away any extra whitespace
1301        pushElement( new Element( "li" ) );
1302        readWhile( " " );
1303
1304        // work done, remember the new bullet list (in place of old one)
1305        m_genlistBulletBuffer.setLength( 0 );
1306        m_genlistBulletBuffer.append( strBullets );
1307        return m_currentElement;
1308    }
1309
1310    private Element unwindGeneralList() {
1311        // unwind
1312        for( ; m_genlistlevel > 0; m_genlistlevel-- ) {
1313            popElement( "li" );
1314            popElement( getListType( m_genlistBulletBuffer.charAt( m_genlistlevel - 1 ) ) );
1315        }
1316        m_genlistBulletBuffer.setLength( 0 );
1317        return null;
1318    }
1319
1320
1321    private Element handleDefinitionList() {
1322        if( !m_isdefinition ) {
1323            m_isdefinition = true;
1324            startBlockLevel();
1325            pushElement( new Element( "dl" ) );
1326            return pushElement( new Element( "dt" ) );
1327        }
1328        return null;
1329    }
1330
1331    private Element handleOpenbracket() throws IOException {
1332        final StringBuilder sb = new StringBuilder( 40 );
1333        final int pos = getPosition();
1334        int ch = nextToken();
1335        boolean isPlugin = false;
1336        if( ch == '[' ) {
1337            if( m_wysiwygEditorMode ) {
1338                sb.append( '[' );
1339            }
1340            sb.append( ( char )ch );
1341            while( ( ch = nextToken() ) == '[' ) {
1342                sb.append( ( char )ch );
1343            }
1344        }
1345
1346        if( ch == '{' ) {
1347            isPlugin = true;
1348        }
1349
1350        pushBack( ch );
1351
1352        if( sb.length() > 0 ) {
1353            m_plainTextBuf.append( sb );
1354            return m_currentElement;
1355        }
1356
1357        //  Find end of hyperlink
1358        ch = nextToken();
1359        int nesting = 1; // Check for nested plugins
1360        while( ch != -1 ) {
1361            final int ch2 = nextToken();
1362            pushBack( ch2 );
1363            if( isPlugin ) {
1364                if( ch == '[' && ch2 == '{' ) {
1365                    nesting++;
1366                } else if( nesting == 0 && ch == ']' && sb.charAt(sb.length()-1) == '}' ) {
1367                    break;
1368                } else if( ch == '}' && ch2 == ']' ) {
1369                    // NB: This will be decremented once at the end
1370                    nesting--;
1371                }
1372            } else {
1373                if( ch == ']' ) {
1374                    break;
1375                }
1376            }
1377
1378            sb.append( (char) ch );
1379
1380            ch = nextToken();
1381        }
1382
1383        //  If the link is never finished, do some tricks to display the rest of the line unchanged.
1384        if( ch == -1 ) {
1385            LOG.debug( "Warning: unterminated link detected!" );
1386            m_isEscaping = true;
1387            m_plainTextBuf.append( sb );
1388            flushPlainText();
1389            m_isEscaping = false;
1390            return m_currentElement;
1391        }
1392
1393        return handleHyperlinks( sb.toString(), pos );
1394    }
1395
1396    /**
1397     *  Reads the stream until the current brace is closed or stream end.
1398     */
1399    private String readBraceContent( final char opening, final char closing ) throws IOException {
1400        final StringBuilder sb = new StringBuilder( 40 );
1401        int braceLevel = 1;
1402        int ch;
1403        while( ( ch = nextToken() ) != -1 ) {
1404            if( ch == '\\' ) {
1405                continue;
1406            } else if( ch == opening ) {
1407                braceLevel++;
1408            } else if( ch == closing ) {
1409                braceLevel--;
1410                if( braceLevel == 0 ) {
1411                    break;
1412                }
1413            }
1414            sb.append( ( char ) ch );
1415        }
1416        return sb.toString();
1417    }
1418
1419
1420    /**
1421     * Handles constructs of type %%(style) and %%class
1422     * @param closing true when called for the closing marker /%, which never opens a style, whatever follows it
1423     * @return An Element containing the div or span, depending on the situation.
1424     * @throws IOException
1425     */
1426    private Element handleDiv( final boolean closing ) throws IOException {
1427        int ch = nextToken();
1428        Element el = null;
1429
1430        if( ch == '%' ) {
1431            String style = null;
1432            String clazz = null;
1433
1434            ch = nextToken();
1435
1436            //  Style or class?
1437            if( !closing && ch == '(' ) {
1438                style = readBraceContent('(',')');
1439            } else if( !closing && Character.isLetter( (char) ch ) ) {
1440                pushBack( ch );
1441                clazz = readUntil( "( \t\n\r" );
1442                //Note: ref.https://www.w3.org/TR/CSS21/syndata.html#characters
1443                //CSS Classnames can contain only the characters [a-zA-Z0-9] and
1444                //ISO 10646 characters U+00A0 and higher, plus the "-" and the "_".
1445                //They cannot start with a digit, two hyphens, or a hyphen followed by a digit.
1446
1447                //(1) replace '.' by spaces, allowing multiple classnames on a div or span
1448                //(2) remove any invalid character
1449                if( clazz != null ) {
1450                    clazz = clazz.replace( '.', ' ' )
1451                                 .replaceAll( "[^\\s-_\\w\\x200-\\x377]+", "" );
1452                }
1453                ch = nextToken();
1454
1455                // check for %%class1.class2( style information )
1456                if( ch == '(' ) {
1457                    style = readBraceContent( '(', ')' );
1458                //  Pop out only spaces, so that the upcoming EOL check does not check the next line.
1459                } else if( ch == '\n' || ch == '\r' ) {
1460                    pushBack( ch );
1461                }
1462            } else {
1463                // Anything else stops.
1464                pushBack( ch );
1465                try {
1466                    final Boolean isSpan = m_styleStack.pop();
1467                    if( isSpan == null ) {
1468                        // Fail quietly
1469                    } else if( isSpan ) {
1470                        el = popElement( "span" );
1471                    } else {
1472                        el = popElement( "div" );
1473                    }
1474                } catch( final EmptyStackException e ) {
1475                    LOG.debug( "Page '" + m_context.getName() + "' closes a %%-block that has not been opened." );
1476                    return m_currentElement;
1477                }
1478                return el;
1479            }
1480
1481            //  Check if there is an attempt to do something nasty
1482            try {
1483                style = StringEscapeUtils.unescapeHtml4(style);
1484                if( style != null && style.contains( "javascript:" ) ) {
1485                    LOG.debug( "Attempt to output javascript within CSS: {}", style );
1486                    final ResourceBundle rb = Preferences.getBundle( m_context, InternationalizationManager.CORE_BUNDLE );
1487                    return addElement( makeError( rb.getString( "markupparser.error.javascriptattempt" ) ) );
1488                }
1489            } catch( final NumberFormatException e ) {
1490                //  If there are unknown entities, we don't want the parser to stop.
1491                final ResourceBundle rb = Preferences.getBundle( m_context, InternationalizationManager.CORE_BUNDLE );
1492                final String msg = MessageFormat.format( rb.getString( "markupparser.error.parserfailure"), e.getMessage() );
1493                return addElement( makeError( msg ) );
1494            }
1495
1496            //  Decide if we should open a div or a span?
1497            final String eol = peekAheadLine();
1498
1499            if( !eol.trim().isEmpty() ) {
1500                // There is stuff after the class
1501                el = new Element("span");
1502                m_styleStack.push( Boolean.TRUE );
1503            } else {
1504                startBlockLevel();
1505                el = new Element("div");
1506                m_styleStack.push( Boolean.FALSE );
1507            }
1508
1509            if( style != null ) el.setAttribute("style", style);
1510            if( clazz != null ) el.setAttribute("class", clazz);
1511            return pushElement( el );
1512        }
1513        pushBack( ch );
1514        return el;
1515    }
1516
1517    private Element handleSlash( ) throws IOException {
1518        final int ch = nextToken();
1519        pushBack( ch );
1520        if( ch == '%' && !m_styleStack.isEmpty() ) {
1521            return handleDiv( true );
1522        }
1523
1524        return null;
1525    }
1526
1527    private Element handleBar( final boolean newLine ) throws IOException {
1528        Element el;
1529        if( !m_istable && !newLine ) {
1530            return null;
1531        }
1532
1533        //  If the bar is in the first column, we will either start a new table or continue the old one.
1534        if( newLine ) {
1535            if( !m_istable ) {
1536                startBlockLevel();
1537                el = pushElement( new Element("table").setAttribute("class","wikitable").setAttribute("border","1") );
1538                m_istable = true;
1539                m_rowNum = 0;
1540            }
1541
1542            m_rowNum++;
1543            final Element tr = ( m_rowNum % 2 != 0 )
1544                       ? new Element("tr").setAttribute("class", "odd")
1545                       : new Element("tr");
1546            el = pushElement( tr );
1547        }
1548
1549        //  Check out which table cell element to start; a header element (th) or a regular element (td).
1550        final int ch = nextToken();
1551        if( ch == '|' ) {
1552            if( !newLine ) {
1553                el = popElement("th");
1554                if( el == null ) popElement("td");
1555            }
1556            el = pushElement( new Element("th") );
1557        } else {
1558            if( !newLine ) {
1559                el = popElement( "td" );
1560                if( el == null ) popElement( "th" );
1561            }
1562            el = pushElement( new Element("td") );
1563            pushBack( ch );
1564        }
1565        return el;
1566    }
1567
1568    /**
1569     *  Generic escape of next character or entity.
1570     */
1571    private Element handleTilde() throws IOException {
1572        final int ch = nextToken();
1573
1574        if( ch == ' ' ) {
1575            if( m_wysiwygEditorMode ) {
1576                m_plainTextBuf.append( "~ " );
1577            }
1578            return m_currentElement;
1579        }
1580
1581        if( ch == '|' || ch == '~' || ch == '\\' || ch == '*' || ch == '#' ||
1582            ch == '-' || ch == '!' || ch == '\'' || ch == '_' || ch == '[' ||
1583            ch == '{' || ch == ']' || ch == '}' || ch == '%' ) {
1584            if( m_wysiwygEditorMode ) {
1585                m_plainTextBuf.append( '~' );
1586            }
1587            m_plainTextBuf.append( ( char ) ch );
1588            m_plainTextBuf.append( readWhile( "" + ( char ) ch ) );
1589            return m_currentElement;
1590        }
1591        // No escape.
1592        pushBack( ch );
1593        return null;
1594    }
1595
1596    private void fillBuffer( final Element startElement ) throws IOException {
1597        m_currentElement = startElement;
1598        m_newLine = true;
1599        boolean quitReading = false;
1600        disableOutputEscaping();
1601        while( !quitReading ) {
1602            final int ch = nextToken();
1603            if( ch == -1 ) {
1604                break;
1605            }
1606
1607            //  Check if we're actually ending the preformatted mode. We still must do an entity transformation here.
1608            if( m_isEscaping ) {
1609                if( ch == '}' ) {
1610                    if( handleClosebrace() == null ) m_plainTextBuf.append( (char) ch );
1611                } else if( ch == -1 ) {
1612                    quitReading = true;
1613                } else if( ch == '\r' ) {
1614                    // DOS line feeds we ignore.
1615                } else if( ch == '<' ) {
1616                    m_plainTextBuf.append( "&lt;" );
1617                } else if( ch == '>' ) {
1618                    m_plainTextBuf.append( "&gt;" );
1619                } else if( ch == '&' ) {
1620                    m_plainTextBuf.append( "&amp;" );
1621                } else if( ch == '~' ) {
1622                    String braces = readWhile( "}" );
1623                    if( braces.length() >= 3 ) {
1624                        m_plainTextBuf.append( "}}}" );
1625                        braces = braces.substring(3);
1626                    } else {
1627                        m_plainTextBuf.append( (char) ch );
1628                    }
1629
1630                    for( int i = braces.length()-1; i >= 0; i-- ) {
1631                        pushBack( braces.charAt( i ) );
1632                    }
1633                } else {
1634                    m_plainTextBuf.append( (char) ch );
1635                }
1636
1637                continue;
1638            }
1639
1640            //  An empty line stops a list
1641            if( m_newLine && ch != '*' && ch != '#' && ch != ' ' && m_genlistlevel > 0 ) {
1642                m_plainTextBuf.append(unwindGeneralList());
1643            }
1644
1645            if( m_newLine && ch != '|' && m_istable ) {
1646                popElement( "table" );
1647                m_istable = false;
1648            }
1649
1650            int skip = IGNORE;
1651            //  Do the actual parsing and catch any errors.
1652            try {
1653                skip = parseToken( ch );
1654            } catch( final IllegalDataException e ) {
1655                LOG.info( "Page {} contains data which cannot be added to DOM tree: {}", m_context.getPage().getName(), e.getMessage() );
1656                makeError( "Error: " + cleanupSuspectData( e.getMessage() ) );
1657            }
1658
1659            // The idea is as follows:  If the handler method returns an element (el != null), it is assumed that it
1660            // has been added in the stack.  Otherwise, the character is added as is to the plaintext buffer.
1661            //
1662            // For the transition phase, if s != null, it also gets added in the plaintext buffer.
1663            switch( skip ) {
1664                case ELEMENT:
1665                    m_newLine = false;
1666                    break;
1667
1668                case CHARACTER:
1669                    m_plainTextBuf.append( (char) ch );
1670                    m_newLine = false;
1671                    break;
1672
1673                case IGNORE:
1674                default:
1675                    break;
1676            }
1677        }
1678
1679        closeHeadings();
1680        popElement( "domroot" );
1681    }
1682
1683    private String cleanupSuspectData( final String s ) {
1684        final StringBuilder sb = new StringBuilder( s.length() );
1685        for( int i = 0; i < s.length(); i++ ) {
1686            final char c = s.charAt(i);
1687            if( Verifier.isXMLCharacter( c ) ) sb.append( c );
1688            else sb.append( "0x" ).append( Integer.toString( c, 16 ).toUpperCase() );
1689        }
1690
1691        return sb.toString();
1692    }
1693
1694    /** The token is a plain character. */
1695    protected static final int CHARACTER = 0;
1696
1697    /** The token is a wikimarkup element. */
1698    protected static final int ELEMENT   = 1;
1699
1700    /** The token is to be ignored. */
1701    protected static final int IGNORE    = 2;
1702
1703    /**
1704     *  Return CHARACTER, if you think this was a plain character; ELEMENT, if
1705     *  you think this was a wiki markup element, and IGNORE, if you think
1706     *  we should ignore this altogether.
1707     *  <p>
1708     *  To add your own MarkupParser, you can override this method, but it
1709     *  is recommended that you call super.parseToken() as well to gain advantage
1710     *  of JSPWiki's own markup.  You can call it at the start of your own
1711     *  parseToken() or end - it does not matter.
1712     *
1713     * @param ch The character under investigation
1714     * @return {@link #ELEMENT}, {@link #CHARACTER} or {@link #IGNORE}.
1715     * @throws IOException If parsing fails.
1716     */
1717    protected int parseToken( final int ch ) throws IOException {
1718        Element el = null;
1719        //  Now, check the incoming token.
1720        switch( ch ) {
1721          case '\r':
1722            // DOS linefeeds we forget
1723            return IGNORE;
1724
1725          case '\n':
1726            //  Close things like headings, etc.
1727            // FIXME: This is not really very fast
1728            closeHeadings();
1729
1730            popElement( "dl" ); // Close definition lists.
1731            if( m_istable ) {
1732                popElement("tr");
1733            }
1734            m_isdefinition = false;
1735            if( m_newLine ) {
1736                // Paragraph change.
1737                startBlockLevel();
1738                //  Figure out which elements cannot be enclosed inside a <p></p> pair according to XHTML rules.
1739                final String nextLine = peekAheadLine();
1740                if( nextLine.isEmpty() ||
1741                     ( !nextLine.isEmpty() &&
1742                       !nextLine.startsWith( "{{{" ) &&
1743                       !nextLine.startsWith( "----" ) &&
1744                       !nextLine.startsWith( "%%" ) &&
1745                       "*#!;".indexOf( nextLine.charAt( 0 ) ) == -1 ) ) {
1746                    pushElement( new Element( "p" ) );
1747                    m_isOpenParagraph = true;
1748
1749                    if( m_restartitalic ) {
1750                        pushElement( new Element( "i" ) );
1751                        m_isitalic = true;
1752                        m_restartitalic = false;
1753                    }
1754                    if( m_restartbold ) {
1755                        pushElement( new Element( "b" ) );
1756                        m_isbold = true;
1757                        m_restartbold = false;
1758                    }
1759                }
1760            } else {
1761                m_plainTextBuf.append("\n");
1762                m_newLine = true;
1763            }
1764            return IGNORE;
1765
1766          case '\\':
1767            el = handleBackslash();
1768            break;
1769
1770          case '_':
1771            el = handleUnderscore();
1772            break;
1773
1774          case '\'':
1775            el = handleApostrophe();
1776            break;
1777
1778          case '{':
1779            el = handleOpenbrace( m_newLine );
1780            break;
1781
1782          case '}':
1783            el = handleClosebrace();
1784            break;
1785
1786          case '-':
1787            if( m_newLine ) {
1788                el = handleDash();
1789            }
1790            break;
1791
1792          case '!':
1793            if( m_newLine ) {
1794                el = handleHeading();
1795            }
1796            break;
1797
1798          case ';':
1799            if( m_newLine ) {
1800                el = handleDefinitionList();
1801            }
1802            break;
1803
1804          case ':':
1805            if( m_isdefinition ) {
1806                popElement( "dt" );
1807                el = pushElement( new Element( "dd" ) );
1808                m_isdefinition = false;
1809            }
1810            break;
1811
1812          case '[':
1813            el = handleOpenbracket();
1814            break;
1815
1816          case '*':
1817            if( m_newLine ) {
1818                pushBack( '*' );
1819                el = handleGeneralList();
1820            }
1821            break;
1822
1823          case '#':
1824            if( m_newLine ) {
1825                pushBack( '#' );
1826                el = handleGeneralList();
1827            }
1828            break;
1829
1830          case '|':
1831            el = handleBar( m_newLine );
1832            break;
1833
1834          case '~':
1835            el = handleTilde();
1836            break;
1837
1838          case '%':
1839            el = handleDiv( false );
1840            break;
1841
1842          case '/':
1843            el = handleSlash();
1844            break;
1845
1846          default:
1847            break;
1848        }
1849
1850        return el != null ? ELEMENT : CHARACTER;
1851    }
1852
1853    private void closeHeadings() {
1854        if( m_lastHeading != null && !m_wysiwygEditorMode ) {
1855            // Add the hash anchor element at the end of the heading
1856            addElement( new Element("a").setAttribute( "class",HASHLINK )
1857                                              .setAttribute( "href","#" + m_lastHeading.m_titleAnchor )
1858                                              .setText( "#" ) );
1859            m_lastHeading = null;
1860        }
1861        popElement( "h2" );
1862        popElement( "h3" );
1863        popElement( "h4" );
1864    }
1865
1866    /**
1867     *  Parses the entire document from the Reader given in the constructor or set by {@link #setInputReader(Reader)}.
1868     *
1869     *  @return A WikiDocument, ready to be passed to the renderer.
1870     *  @throws IOException If parsing cannot be accomplished.
1871     */
1872    @Override
1873    public WikiDocument parse() throws IOException {
1874        final WikiDocument d = new WikiDocument( m_context.getPage() );
1875        d.setContext( m_context );
1876        final Element rootElement = new Element( "domroot" );
1877        d.setRootElement( rootElement );
1878        fillBuffer( rootElement );
1879        paragraphify( rootElement );
1880
1881        return d;
1882    }
1883
1884    /**
1885     *  Checks out that the first paragraph is correctly installed.
1886     *
1887     *  @param rootElement element to be checked.
1888     */
1889    private void paragraphify( final Element rootElement) {
1890        //  Add the paragraph tag to the first paragraph
1891        final List< Content > kids = rootElement.getContent();
1892        if( rootElement.getChild( "p" ) != null ) {
1893            final ArrayList<Content> ls = new ArrayList<>();
1894            int idxOfFirstContent = 0;
1895            int count = 0;
1896
1897            for( final Iterator< Content > i = kids.iterator(); i.hasNext(); count++ ) {
1898                final Content c = i.next();
1899                if( c instanceof Element ) {
1900                    final String name = ( ( Element )c ).getName();
1901                    if( isBlockLevel( name ) ) {
1902                        break;
1903                    }
1904                }
1905
1906                if( !( c instanceof ProcessingInstruction ) ) {
1907                    ls.add( c );
1908                    if( idxOfFirstContent == 0 ) {
1909                        idxOfFirstContent = count;
1910                    }
1911                }
1912            }
1913
1914            //  If there were any elements, then add a new <p> (unless it would be an empty one)
1915            if(!ls.isEmpty()) {
1916                final Element newel = new Element("p");
1917                for( final Content c : ls ) {
1918                    c.detach();
1919                    newel.addContent( c );
1920                }
1921
1922                // Make sure there are no empty <p/> tags added.
1923                if( !newel.getTextTrim().isEmpty() || !newel.getChildren().isEmpty() ) {
1924                    rootElement.addContent( idxOfFirstContent, newel );
1925                }
1926            }
1927        }
1928    }
1929
1930}