001// Copyright (c) 2001 Hursh Jain (http://www.mollypages.org) 
002// The Molly framework is freely distributable under the terms of an
003// MIT-style license. For details, see the molly pages web site at:
004// http://www.mollypages.org/. Use, modify, have fun !
005
006package fc.web.page;
007
008import java.io.*;
009import java.util.*;
010import java.util.regex.*;
011import fc.io.*;
012import fc.util.*;
013
014/*
015NOTES
016
017Code blocks of the form 
018 [...] 
019cause problems with java arrays
020
021String[] or foo[4] etc.., craps out. So we need to use 
022 [[...]] 
023for the molly code blocks
024
0251. If you are hacking this file, start with parseText()
026
0272. Turn the dbg flag to true to see how the parser works
028
0293. Keep in mind that the order of switch'es in a case statement in various
030methods is not always arbitrary (the order matters in this sort
031of recursive descent parsing)
032
0334. Read www.mollypages.org/page/grammar/index.mp for a intro
034to parsing
035
0365. This parser as shipped has a set of regression tests in the
037fc/web/page/test directory. These consist of a bunch of *.mp
038files and corresponding *.java files, each of which is known 
039to be generated properly. If you change stuff around, run these
040regression tests again by invoking "java fc.web.page.PageParserTest"
041Note, if you change things such that the .java output of the parser
042is different, then the tests will fail (since the new .java files
043of your parser will be different to the test ones shipped in
044fc/web/page/test. In this case, once you know that your parser works
045as you like it, then you should create a new baseline for your parser
046by invoking "java fc.web.page.PageParserTest -generateExpected" and
047then you can use *that* as the new baseline for further changes in
048your parser (you may have to modify the *.mp files in /fc/web/page/test
049to use your new page syntax).
050
0516. 
052When including files, previous versions of the parser constructed a new IncludeFile element
053which would be invoked when the page was written out. When invoked (via render), that element
054would creaet a new PageParser and set includeMode = true on that new parser. This new parser 
055would then parse/write out the subtree of the included file in-line.
056
057This inline processing had issues since the included file could not contain import statements,
058declarations, etc (since those had already been written out by the parent/top level parser).
059Another hack was to pass the child parser the parent/top level object and give access to the 
060top level parse root to the child parser (the child parser would have to be invoked immediaately
061anyway). Also, since inner classes for parse elements are non-static, the separate parser would
062create a parse tree, and although it would add those classes to the top most parse tree, the 
063classes themselves (when trying to write) would refer to the separate output stream of the child
064class (the output stream would also have to be set to the parent class). It was doaable but gets
065un-neccessarily complex.
066
067The only benefit to a seperate parser being able to print start/end sections:
068>> start include file
069 [..invoke child parser]
070>> end include file
071
072In the current/cleaner approach, I simply insert the included file into the character stream.
073But there isn't any easy way to track when that stream finishes and the original content starts 
074again. So we get:
075
076>> start include file
077[..include into original stream and continue parsing]
078-- no end include file line --
079
080*/
081
082/**
083Parses a page and writes out the corresponding java file to the specified
084output. The parser and scanner is combined into one class here for
085simplicity (a seperate scanner is overkill for a simple LL(1) grammar
086such as molly pages).
087
088@author hursh jain
089*/
090public final class PageParser
091{
092private static final boolean dbg    = false;
093private static final int     EOF    = -1;
094private              int     dbgtab = 0;
095
096String          classname;
097String          packagename = Page.PACKAGE_NAME;
098PageReader      in;
099PrintWriter     out;
100Log             log;
101File            inputFile;
102File            outputFile;
103File            contextRoot;
104boolean         includeMode = false;
105String          src_encoding;
106
107//Read data
108//we use these since stringbuffer/builders do not have a clear/reset function
109CharArrayWriter buf = new CharArrayWriter(4096);
110CharArrayWriter wsbuf = new CharArrayWriter(32);  // ^(whitespace)* 
111int c = EOF;
112
113//PageData
114List  decl       = new ArrayList();     //declarations
115List  inc_decl       = new ArrayList();     //external included declarations
116List  imps           = new ArrayList();     //imports
117List  tree           = new ArrayList();     //code, exp, text etc.
118Map   directives     = new HashMap();       //page options
119Set   circularityTrack = new HashSet();   //track pages already included to stop circular refs
120
121/** 
122The  name ("mimetype") of the[@ mimetype=....] directive. The value of
123<tt>none</tt> or an empty string will turn off writing any mimetype
124entirely (the user can then write a mimetype via the {@link
125javax.servlet.ServletResponse.setContentType} method manually).
126<p>
127Note, from {@link
128javax.servlet.ServletResponse.setContentType ServletResponse}
129<pre>
130Note that the character encoding cannot be communicated via HTTP headers
131if the servlet does not specify a content type; however, it is still used
132to encode text written via the servlet response's writer.
133</pre>
134*/
135public static String d_mimetype = "mimetype";
136
137/*
138this value (or an empty string) for mimetype means no mimetype
139will be specified (not even the default mimetype)
140*/
141public static String mimetype_none = "none";
142
143/** 
144The name ("encoding") of the [page encoding=....] directive. 
145*/
146public static String d_encoding = "encoding";
147
148/** 
149The name ("src-encoding") of the [page src-encoding=....] directive. 
150*/
151public static String d_src_encoding = "src-encoding";
152
153/** The name ("buffersize") of the [page buffersize=....] directive */
154public static String d_buffersize = "buffersize";
155
156/** The name ("out") of the [page out=....] directive */
157public static String d_out = "out";
158/** A value ("outputstream") of the [page out=outputstream] directive */
159public static String d_out_stream1 = "outputstream";
160/** A value ("outputstream") of the [page out=stream] directive */
161public static String d_out_stream2 = "stream";
162/** A value ("writer") of the [page out=writer] directive */
163public static String d_out_writer = "writer";
164/** The name of the  ("remove-initial-whitespace") directive */
165public static String d_remove_initial_emptylines = "remove-initial-emptylines";
166/** The name of the  ("remove-all-emptylines") directive */
167public static String d_remove_all_emptylines = "remove-all-emptylines";
168
169/* 
170This constructor for internal use.
171
172The parser can be invoked recursively to parse included files as
173well..that's what the includeMode() does (and this construtor is invoked
174when including). When including, we already have a output writer
175created, we use that writer (instead of creating a new one based on
176src_encoding as we do for in normal page parsing mode).
177*/
178private PageParser(
179 File contextRoot, File input, PrintWriter outputWriter, String classname, Log log) 
180throws IOException
181  {
182  this.contextRoot = contextRoot;
183  this.inputFile = input; 
184  this.in  = new PageReader(input);
185  this.out = outputWriter;
186  this.classname = classname;
187  this.log = log;
188
189  circularityTrack.add(input.getAbsolutePath());
190  }
191
192/**
193Creates a new page parser that will use the default log obtained by
194{@link Log#getDefault}
195
196@param  contextRoot absolute path to the webapp context root directory
197@param  input   absolute path to the input page file
198@param  input   absolute path to the output file (to be written to).
199@param  classname classname to give to the generated java class.
200*/
201public PageParser(File contextRoot, File input, File output, String classname) 
202throws IOException
203  {
204  this(contextRoot, input, output, classname, Log.getDefault());
205  }
206
207/**
208Creates a new page parser.
209
210@param  contextRoot absolute path to the webapp context root directory
211@param  input   absolute path to the input page file
212@param  output    absolute path to the output file (to be written to).
213@param  classname classname to give to the generated java class.
214@log  log     destination for internal logging output.
215*/
216public PageParser(
217  File contextRoot, File input, File output, String classname, Log log) 
218throws IOException
219  {
220  this.contextRoot = contextRoot;
221  this.inputFile = input; 
222  this.in  = new PageReader(input);
223  this.outputFile = output;
224  this.classname = classname;
225  this.log = log;
226
227  circularityTrack.add(input.getAbsolutePath());
228  }
229
230void append(final int c)
231  {
232  Argcheck.istrue(c >= 0, "Internal error: recieved c=" + c);
233  buf.append((char)c);
234  }
235
236void append(final char c)
237  {
238  buf.append(c);
239  }
240
241void append(final String str)
242  {
243  buf.append(str);
244  }
245
246/* not used anymore */
247PageParser includeMode()
248  {
249  includeMode = true;
250  return this;
251  }
252
253/**
254Parses the page. If the parse is successful, the java source will be
255generated.
256
257@throws IOException   a parse failure occurred. The java source file
258            may or may not be properly generated or written
259            in this case.
260*/
261public void parse() throws IOException
262  {
263  parseText();  
264  if (! includeMode)  {
265    writePage();
266    out.close();
267    }
268  else{
269    out.flush();
270    }
271  in.close();
272  }
273
274//util method for use in the case '[' branch of parseText below.
275private Text newTextNode()
276  {
277  Text text = new Text(buf);
278  tree.add(text);
279  buf.reset();
280  return text;
281  }
282  
283void parseText() throws IOException
284  {
285  if (dbg) dbgenter(); 
286      
287  while (true)
288    { 
289    c = in.read();
290    
291    if (c == EOF) {
292      tree.add(new Text(buf));
293      buf.reset();
294      break;
295      }
296      
297    switch (c)
298      { 
299      //Escape start tags
300      case '\\':
301        /*  we don't need to do this: previously, expressions
302        were [...] but now they are [=...], previously we needed
303        to escape \[[ entirely (since if we escaped \[ the second
304        [ would start an expression
305        */
306        /*        
307        if (in.match("[["))  
308          append("[[");
309        */
310        //escape only \[... otherwise leave \ alone
311        if (in.match("["))
312          append("[");
313        else
314          append(c);
315        break;
316
317      case '[':
318        /* suppose we have
319        \[[
320        escape handling above will capture \[
321        then the second '[' drops down here. Good so far.
322        But we must not create a new text object here by
323        default...only if we see another [[ or [= or [include or
324        whatever. 
325        */
326        /*
327        But creating a text object at the top is easier
328        then repeating this code at every if..else branch below
329        but this creates superfluous line breaks.
330        
331        hello[haha]world
332        -->prints as-->
333        hello  (text node 1)
334        [haha] (text node 2)
335        world  (text node 3)
336        --> we want
337        hello[haha]world (text node 1)
338        */
339          
340        if (in.match('[')) { 
341          newTextNode();
342          parseCode(); 
343          }
344        else if (in.match('=')) {
345          Text text = newTextNode();
346          parseExpression(text);
347          }
348        else if (in.match('!')) {
349          newTextNode();
350          parseDeclaration();
351          }
352        else if (in.match("/*")) {
353          newTextNode();
354          parseComment(); 
355          }
356        else if (in.matchIgnoreCase("page")) {
357          newTextNode();
358          parseDirective();
359          }
360        //longest match: "include-file" etc., last: "include"
361        else if (in.matchIgnoreCase("include-file")) {
362          newTextNode();
363          parseIncludeFile();
364          }
365        else if (in.matchIgnoreCase("include-decl")) {
366          newTextNode();
367          parseIncludeDecl();
368          }
369        else if (in.matchIgnoreCase("include")) {
370          newTextNode();
371          parseInclude();
372          }
373        else if (in.matchIgnoreCase("forward")) {
374          newTextNode();
375          parseForward();
376          }
377        else if (in.matchIgnoreCase("import")) {
378          newTextNode();
379          parseImport();
380          }
381        else  {
382          //System.out.println("c1=" + (char)c);
383          append(c);
384          }
385        break;  
386  
387      default:
388        //System.out.println("c2=" + (char)c);
389        append(c);
390        
391      } //switch    
392    } //while
393    
394  if (dbg) dbgexit(); 
395  }
396  
397void parseCode() throws IOException
398  {
399  if (dbg) dbgenter(); 
400
401  int startline = in.getLine();
402  int startcol = in.getCol();
403  
404  while (true)
405    {
406    c = in.read();  
407  
408    switch (c) /* the order of case tags is important. */
409      {
410      case EOF:
411        unclosed("code", startline, startcol);
412        if (dbg) dbgexit(); 
413        return;
414
415      case '/':   //Top level:  // and /* comments
416        append(c);
417        c = in.read();
418        append(c);
419        if (c == '/') 
420          appendCodeSlashComment();
421        else if (c == '*') 
422          appendCodeStarComment();
423          break;        
424    
425      case '"':     //strings outside of any comment
426        append(c);
427        appendCodeString();  
428        break;
429        
430      case '\'':
431        append(c);
432        appendCodeCharLiteral();
433        break;
434        
435      case ']':
436        if (in.match(']')) {
437          tree.add(new Code(buf));
438          buf.reset();
439          if (dbg) dbgexit(); 
440          return;
441          }
442        else {
443          append(c);
444          }
445        break;
446      
447      /* 
448      a hash by itself on a line starts a hash section.
449      whitespace before the # on that line is used as an
450      printing 'out' statements for that hash.
451      
452      for (int n = 0; n < ; n++) {
453      ....# foo #
454      | }
455      |=> 4 spaces 
456      so nice if generated code looked like:
457      
458      for (int n = 0; n < ; n++) {
459          out.print(" foo ");
460          }
461      */
462      case '\n':
463      case '\r':
464        append(c);       //the \n or \r just read
465        readToFirstNonWS();  //won't read past more newlines 
466        //is '#' is first non-ws on this line ?
467        c = in.read();
468        if (c == '#') {           
469          tree.add(new Code(buf));
470          buf.reset();
471          //whitespace provides indentation offset
472          parseHash(wsbuf.toString()); 
473          }
474        else{
475          append(wsbuf.toString());  //wsbuf contains codetext
476          //let other cases also handle first non-ws or EOF
477          in.unread();    
478          }
479        break;
480      
481      /* in this case, hash does not start on a new line, like:
482         for (...) { #
483      */
484      case '#':
485        tree.add(new Code(buf));
486        buf.reset();
487        parseHash(null);
488        break;  
489      
490      default:
491        append(c);
492      } //switch    
493    } //while
494  }
495
496void parseHash(String offset) throws IOException
497  {
498  if (dbg) dbgenter(); 
499
500  int startline = in.getLine();
501  int startcol = in.getCol();
502
503  while (true)
504    {
505    c = in.read();  
506  
507    switch (c)
508      {
509      case EOF: 
510        unclosed("hash", startline, startcol);
511        if (dbg) dbgexit(); 
512        return;
513
514      //special case: very common and would be a drag to escape
515      //this every time:
516      //  # <table bgcolor="#ffffff">....   #
517      //Now, all of:
518      //  bgcolor="#xxx"  
519      //  bgcolor='#xxx'
520      //  bgcolor="\#xxx" 
521      //will work the same and give: bgcolor="#xxx"
522      //1)
523      //However to get a:
524      //  bgcolor=#xxx    (no quoted around #xxx)
525      //we still have to say:
526      //  bgcolor=\#xxx   
527      //2)
528      //Of course, since we special case this, then:
529      //  #"bar"#
530      // that ending # is lost and we end up with
531      //  #"bar"  with no closing hash
532      //So we need to make sure that we write:
533      //  #"bar" #
534      // instead
535
536      case '\'':
537      case '"':
538        append(c);
539        if (in.match('#')) 
540          append('#');
541        break;
542        
543      case '\\':
544        if (in.match('[')) 
545          append('[');      
546        else if (in.match('#'))
547          append('#');
548        else
549          append(c);
550        break;
551        
552      case '[':
553        if (in.match('=')) {
554          Hash hash = new Hash(offset, buf);
555          tree.add(hash);
556          buf.reset();
557          parseExpression(hash);
558          }
559        else{
560          append(c);
561          }
562        break;
563
564      /*
565      this case is not needed but is a bit of a optimization
566      for (int n = 0; n < 1; n++) {
567        #
568        foo
569      ....#...NL
570        }
571      avoids printing the dots (spaces) and NL in this case
572      (the newline after foo is still printed)
573      */
574      case '\n':
575      case '\r':
576        append(c);
577        readToFirstNonWS(); 
578        c = in.read();
579        //'#' is first non-ws on the line
580        if (c == '#') {
581          tree.add(new Hash(offset, buf));
582          buf.reset();
583          //skipIfWhitespaceToEnd();
584          if (dbg) dbgexit(); 
585          return;
586          }
587        else {
588          append(wsbuf.toString());
589          in.unread(); //let other cases also handle first non-ws   
590          }
591        break;
592
593      case '#':
594        tree.add(new Hash(offset, buf));  
595        //skipIfWhitespaceToEnd();
596        buf.reset();
597        if (dbg) dbgexit(); 
598        return;
599        
600      default:
601        append(c);
602      }  //switch 
603    } //while
604  }
605
606/**
607[page <<<FOO]
608...as-is..no parse, no interpolation..
609FOO
610*/
611void parseHeredoc(StringBuilder directives_buf) throws IOException
612  {
613  if (dbg) dbgenter(); 
614
615  int startline = in.getLine();
616  int startcol = in.getCol();
617      
618  int i = directives_buf.indexOf("<<<"); /* "<<<".length = 3 */
619  CharSequence subseq = directives_buf.substring(
620            i+3, 
621            /*directives_buf does not have a ending ']' */
622            directives_buf.length() 
623            );
624    
625  final String      heredoc     = subseq.toString().trim();
626  final int         heredoc_len = heredoc.length();
627  final CharArrayWriter heredoc_buf = new CharArrayWriter(2048);
628
629  /* 
630  the ending heredoc after newline speeds things up a bit
631  which is why is traditionally used i guess, otherwise
632  we have to try a full match every first match. this 
633  implementation doesn't care where the ending heredoc
634  appears (can be anywhere)...simplifies the implementation.
635  */
636  
637  while (true)
638    { 
639    c = in.read();
640    
641    if (c == EOF) {
642      unclosed("heredoc: <<<"+heredoc, startline, startcol);
643      break;
644      }
645      
646    if (c == heredoc.charAt(0))
647      {
648      boolean matched = true;
649      if (heredoc_len > 1) {
650        matched = in.match(heredoc.substring(1));
651        }
652      if (matched) {  
653        tree.add(new Heredoc(heredoc_buf));
654        break;
655        }
656      }
657    
658    //default action
659    heredoc_buf.append((char)c);  
660    } //while
661    
662  if (dbg) dbgexit(); 
663  }
664
665/*
666Text is the parent node for the expression. A new expression is parsed,
667created and added to the text object by this method
668*/
669void parseExpression(Element parent) throws IOException
670  {
671  if (dbg) dbgenter(); 
672
673  int startline = in.getLine();
674  int startcol = in.getCol();
675
676  while (true)
677    {
678    c = in.read();      
679  
680    switch (c)
681      {
682      case EOF:
683        unclosed("expression", startline, startcol);
684        if (dbg) dbgexit(); 
685        return;
686
687      case '\\':
688        if (in.match(']')) 
689          append(']');    
690        else
691          append(c);
692        break;
693
694      case ']':
695        if (buf.toString().trim().length() == 0)
696          error("Empty expression not allowed", startline, startcol);
697        parent.addExp(new Exp(buf));
698        buf.reset();  
699        if (dbg) dbgexit(); 
700        return;
701        
702      default:
703        append(c);
704      }
705    }
706  }
707
708void parseComment() throws IOException
709  {
710  if (dbg) dbgenter(); 
711
712  int startline = in.getLine();
713  int startcol = in.getCol();
714
715  while (true)
716    {
717    c = in.read();      
718  
719    switch (c)
720      {
721      case EOF:
722        unclosed("comment", startline, startcol);
723        if (dbg) dbgexit(); 
724        return;
725        
726      case '*':
727        if (in.match("/]"))
728          {
729          tree.add(new Comment(buf));
730          buf.reset();  
731          if (dbg) dbgexit(); 
732          return;
733          }
734        else
735          append(c);  
736        break;
737      
738      default:
739        append(c);
740      }
741    }
742  }
743
744void parseDeclaration() throws IOException
745  {
746  if (dbg) dbgenter(); 
747  int startline = in.getLine();
748  int startcol = in.getCol();
749
750  while (true)
751    {
752    c = in.read();      
753  
754    switch (c)
755      {
756      case EOF:
757        unclosed("declaration", startline, startcol);
758        if (dbg) dbgexit(); 
759        return;
760      
761      case '!':
762        if (in.match(']')) {
763          decl.add(new Decl(buf));
764          buf.reset();  
765          if (dbg) dbgexit(); 
766          return;
767          }
768        else{
769          append(c);
770          }
771        break;
772
773      //top level // and /* comments, ']' (close decl tag)
774      //is ignored within them
775      case '/':   
776        append(c);
777        c = in.read();
778        append(c);
779        if (c == '/') 
780          appendCodeSlashComment();
781        else if (c == '*') 
782          appendCodeStarComment();
783          break;        
784    
785      //close tags are ignored within them
786      case '"':     //strings outside of any comment
787        append(c);
788        appendCodeString();  
789        break;
790        
791      case '\'':
792        append(c);
793        appendCodeCharLiteral();
794        break;
795            
796      default:
797        append(c);
798      }
799    }
800
801  }
802
803void parseDirective() throws IOException
804  {
805  if (dbg) dbgenter(); 
806
807  int startline = in.getLine();
808  int startcol = in.getCol();
809
810  StringBuilder directives_buf = new StringBuilder(1024);
811
812  while (true)
813    {
814    c = in.read();      
815  
816    switch (c)
817      {
818      case EOF:
819        unclosed("directive", startline, startcol);
820        if (dbg) dbgexit(); 
821        return;
822        
823      case ']':
824        if (directives_buf.indexOf("<<<") >= 0)  {
825          parseHeredoc(directives_buf); 
826          }
827        else{/* other directives used at page-generation time */
828          addDirectives(directives_buf);
829          }
830          
831        if (dbg) dbgexit(); 
832        return;
833      
834      default:
835        directives_buf.append((char)c);
836      }
837    }
838
839  }
840
841//[a-zA-Z_\-0-9] == ( \w | - )
842static final Pattern directive_pat = Pattern.compile(
843  //foo = "bar baz" (embd. spaces)
844  "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*\"((?:.|\r|\n)+?)\""  
845  + "|"
846  //foo = "bar$@#$" (no spaces) OR foo = bar (quotes optional)
847  + "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*(\\S+)" 
848  );
849  
850    
851void addDirectives(StringBuilder directives_buf) throws ParseException
852  {
853  if (dbg) {
854    dbgenter(); 
855    System.out.println("-------directives section--------");
856    System.out.println(directives_buf.toString());
857    System.out.println("-------end directives-------");
858    }
859  
860  String name, value;
861  try {
862    Matcher m = directive_pat.matcher(directives_buf);
863    while (m.find()) 
864      {
865      if (dbg) System.out.println(">>>>[0]->" + m.group() 
866        + "; [1]->" + m.group(1)  
867        + " [2]->" + m.group(2)  
868        + " [3]->" + m.group(3)  
869        + " [4]->" + m.group(4));
870        
871      name = m.group(1) != null ? m.group(1).toLowerCase() :
872                    m.group(3).toLowerCase();
873      value = m.group(2) != null ? m.group(2).toLowerCase() :
874                     m.group(4).toLowerCase();
875
876      if (name.equals(d_buffersize)) 
877        {
878        //can throw parse exception
879        directives.put(name, 
880          IOUtil.stringToFileSize(value.replace("\"|'",""))); 
881        }
882      else if (name.equals(d_encoding)) {
883        directives.put(name, value.replace("\"|'",""));       
884        }
885      else if (name.equals(d_src_encoding)) {
886        directives.put(name, value.replace("\"|'",""));       
887        } 
888      else if (name.equals(d_mimetype)) {
889        directives.put(name, value.replace("\"|'",""));       
890        }
891      else if (name.equals(d_out)) {
892        directives.put(name, value.replace("\"|'",""));       
893        } 
894      else if (name.equals(d_remove_initial_emptylines)) {
895        directives.put(name, value.replace("\"|'",""));       
896        } 
897      else if (name.equals(d_remove_all_emptylines)) {
898        directives.put(name, value.replace("\"|'",""));       
899        } 
900      //else if .... other directives here as needed....
901      else 
902        throw new Exception("Do not understand directive: " + m.group());
903      }
904    if (dbg) System.out.println("Added directives: " + directives);
905    }
906  catch (Exception e) {
907    throw new ParseException("File: " + inputFile.getAbsolutePath() 
908                  + ";\n" + e.toString());
909    }
910
911  if (dbg) dbgexit(); 
912  }
913
914void parseIncludeFile() throws IOException
915  {
916  if (dbg) dbgenter(); 
917
918  int startline = in.getLine();
919  int startcol = in.getCol();
920  String option = null;
921  
922  while (true)
923    {
924    c = in.read();      
925  
926    switch (c)
927      {
928      case EOF:
929        unclosed("include-file", startline, startcol);
930        if (dbg) dbgexit(); 
931        return;
932        
933      case '[':
934        if (in.match('=')) {
935  //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\"
936  //in [include-file... section starting at:", startline, startcol);
937  //instead of warn, we will error out. failing early is better.
938  //this does preclude having '[=' in the file name, but it's a good
939  //tradeoff
940          error("Expressions cannot exist in file includes. The offending static-include section starts at:", startline, startcol);
941          }
942        append(c);
943        break;
944      
945      case ']':
946        includeFile(buf, option); /* not added in the tree, just included in the stream */
947        buf.reset();  
948        if (dbg) dbgexit(); 
949        return;
950      
951      case 'o':
952        if (! in.match("ption"))
953          append(c);
954        else{
955          skipWS();
956          if (! in.match("=")) {
957            error("bad option parameter in file include: ", startline, startcol);
958            }
959          skipWS();
960          
961          int c2;
962          StringBuilder optionbuf = new StringBuilder();
963          while (true) {
964            c2 = in.read();
965            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
966              in.unread();
967              break;
968              }
969            optionbuf.append((char)c2);
970            }
971          
972          option = optionbuf.toString();
973          //System.out.println(option);
974          } //else
975        break;
976  
977      default:
978        append(c);
979      }
980    }
981  }
982
983void parseIncludeDecl() throws IOException
984  {
985  if (dbg) dbgenter(); 
986
987  int startline = in.getLine();
988  int startcol = in.getCol();
989  String option = null;
990  
991  while (true)
992    {
993    c = in.read();      
994  
995    switch (c)
996      {
997      case EOF:
998        unclosed("include-decl", startline, startcol);
999        if (dbg) dbgexit(); 
1000        return;
1001        
1002      case '[':
1003        if (in.match('=')) {
1004    //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\" in [include-static... section starting at:", startline, startcol);
1005    //we will throw an exception. failing early is better. this
1006    //does preclude having '[=' in the file name, but it's a good tradeoff
1007          error("Expressions cannot exist in include-decl. The offending static-include section starts at:", startline, startcol);
1008          }
1009        append(c);
1010        break;
1011      
1012      case ']':
1013        IncludeDecl i = new IncludeDecl(buf);
1014        if (option != null)
1015          i.setOption(option);
1016        inc_decl.add(i);
1017        buf.reset();  
1018        if (dbg) dbgexit(); 
1019        return;
1020      
1021      case 'o':
1022        if (! in.match("ption"))
1023          append(c);
1024        else{
1025          skipWS();
1026          if (! in.match("=")) {
1027            error("bad option parameter in include-code: ", startline, startcol);
1028            }
1029          skipWS();
1030          
1031          int c2;
1032          StringBuilder optionbuf = new StringBuilder();
1033          while (true) {
1034            c2 = in.read();
1035            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
1036              in.unread();
1037              break;
1038              }
1039            optionbuf.append((char)c2);
1040            }
1041          
1042          option = optionbuf.toString();
1043          //System.out.println(option);
1044          } //else
1045        break;
1046  
1047      default:
1048        append(c);
1049      }
1050    }
1051  }
1052
1053//the filename/url can be optionally double quoted. leading/trailing
1054//double quotes (if any) are ignored when an include is rendered...
1055//this way there isn't any additional parsing needed here...I could
1056//ignore the optional quote here (and that's the formal proper way) 
1057//and then not move the ignore quote logic into the render() method but
1058//this way is good too...and simpler..
1059//same goes for the other parseIncludeXX/ForwardXX functions.
1060void parseInclude() throws IOException
1061  {
1062  if (dbg) dbgenter(); 
1063
1064  int startline = in.getLine();
1065  int startcol = in.getCol();
1066  Include include = new Include();
1067  while (true)
1068    {
1069    c = in.read();      
1070  
1071    switch (c)
1072      {
1073      case EOF:
1074        unclosed("include", startline, startcol);
1075        if (dbg) dbgexit(); 
1076        return;
1077        
1078      case '[':
1079        if (in.match('=')) {
1080          include.add(buf);
1081          buf.reset();
1082          parseExpression(include);
1083          }
1084        else{
1085          append(c);
1086          }
1087        break;
1088      
1089      case ']':
1090        include.add(buf);
1091        tree.add(include);
1092        buf.reset();  
1093        if (dbg) dbgexit(); 
1094        return;
1095      
1096      default:
1097        append(c);
1098      }
1099    }
1100  }
1101
1102void parseForward() throws IOException
1103  {
1104  if (dbg) dbgenter(); 
1105
1106  int startline = in.getLine();
1107  int startcol = in.getCol();
1108
1109  Forward forward = new Forward();
1110  while (true)
1111    {
1112    c = in.read();      
1113  
1114    switch (c)
1115      {
1116      case EOF:
1117        unclosed("forward", startline, startcol);
1118        if (dbg) dbgexit(); 
1119        return;
1120        
1121      case '[':
1122        if (in.match('=')) {
1123          forward.add(buf);
1124          buf.reset();
1125          parseExpression(forward);
1126          }
1127        else{
1128          append(c);
1129          }
1130        break;
1131      
1132      case ']':
1133        forward.add(buf);
1134        tree.add(forward);
1135        buf.reset();  
1136        if (dbg) dbgexit(); 
1137        return;
1138      
1139      default:
1140        append(c);
1141      }
1142    }
1143  }
1144
1145//we need to parse imports seperately because they go outside
1146//a class declaration (and [!...!] goes inside a class)
1147//import XXX.*;
1148//class YYY {
1149//[!....stuff from here ....!]
1150//...
1151void parseImport() throws IOException
1152  {
1153  if (dbg) dbgenter(); 
1154
1155  int startline = in.getLine();
1156  int startcol = in.getCol();
1157
1158  while (true)
1159    {
1160    c = in.read();      
1161  
1162    switch (c)
1163      {
1164      case EOF:
1165        unclosed("import", startline, startcol);
1166        if (dbg) dbgexit(); 
1167        return;
1168      
1169      case '\n':
1170        imps.add(new Import(buf));
1171        buf.reset();
1172        break;
1173        
1174      case ']':
1175        imps.add(new Import(buf));
1176        buf.reset();  
1177        if (dbg) dbgexit(); 
1178        return;
1179      
1180      default:
1181        append(c);
1182      }
1183    }
1184  }
1185
1186/*
1187Called when // was read at the top level inside a code block. Appends
1188the contents of a // comment to the buffer (not including the trailing
1189newline)
1190*/
1191void appendCodeSlashComment() throws IOException
1192  {
1193  if (dbg) dbgenter();
1194  
1195  while (true) 
1196    {
1197    c = in.read();
1198    
1199    if (c == EOF)
1200      break;
1201  
1202    //do not append \r, \r\n, or \n, that finishes the // comment
1203    //we need that newline to figure out if the next line is a hash
1204    //line
1205    if (c == '\r') {
1206      in.unread();
1207      break;
1208      }
1209    
1210    if (c == '\n') {
1211      in.unread();
1212      break;  
1213      }
1214
1215    append(c);
1216    }
1217  
1218  if (dbg) dbgread("CodeSLASHComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1219  if (dbg) dbgexit();
1220  }
1221
1222/*
1223Called when /* was read at the top level inside a code block. Appends
1224the contents of a /*comment to the buffer. (not including any trailing
1225newline or spaces)
1226*/
1227void appendCodeStarComment() throws IOException
1228  {
1229  if (dbg) dbgenter(); 
1230  
1231  while (true) 
1232    {
1233    c = in.read();  
1234
1235    if (c == EOF)
1236      break;
1237  
1238    append(c);
1239    
1240    if (c == '*') 
1241      {
1242      if (in.match('/')) {
1243        append('/');
1244        break;
1245        }
1246      }
1247    }
1248
1249  if (dbg) dbgread("CodeSTARComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1250  if (dbg) dbgexit(); 
1251  }
1252
1253/*
1254Called (outside of any comments in the code block) when: 
1255--> parseCode()
1256     ... "
1257         ^ (we are here)
1258*/
1259void appendCodeString() throws IOException
1260  {
1261  if (dbg) dbgenter(); 
1262
1263  int startline = in.getLine();
1264  int startcol = in.getCol();
1265
1266  while (true) 
1267    {
1268    c = in.read();
1269  
1270    if (c == EOF || c == '\r' || c == '\n')
1271      unclosed("string literal", startline, startcol);
1272  
1273    append(c);
1274  
1275    if (c == '\\') {
1276      c = in.read();
1277      if (c == EOF)
1278        unclosed("string literal", startline, startcol);
1279      else {
1280        append(c);
1281        continue;   //so \" does not hit the if below and break
1282        }
1283      }
1284    
1285    if (c == '"')
1286      break;
1287    }
1288
1289  if (dbg) dbgread("appendCodeString Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1290  if (dbg) dbgexit(); 
1291  }
1292
1293
1294/*
1295Called (outside of any comments in the code block) when: 
1296--> parseCode()
1297     ... '
1298         ^ (we are here)
1299*/
1300void appendCodeCharLiteral() throws IOException
1301  {
1302  if (dbg) dbgenter(); 
1303
1304  int startline = in.getLine();
1305  int startcol = in.getCol();
1306
1307  while (true) 
1308    {
1309    c = in.read();
1310  
1311    if (c == EOF || c == '\r' || c == '\n')
1312      unclosed("char literal", startline, startcol);
1313  
1314    append(c);
1315  
1316    if (c == '\\') {
1317      c = in.read();
1318      if (c == EOF)
1319        unclosed("char literal", startline, startcol);
1320      else {
1321        append(c);
1322        continue;   //so \' does not hit the if below and break
1323        }
1324      }
1325    
1326    if (c == '\'')
1327      break;
1328    }
1329
1330  if (dbg) dbgread("appendCodeCharLiteral Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
1331  if (dbg) dbgexit(); 
1332  }
1333
1334
1335/*
1336Reads from the current position till the first nonwhitespace char, EOF or
1337newline is encountered. Reads are into the whitespace buffer. does not
1338consume the character past the non-whitespace character and does
1339NOT read multiple lines of whitespace.
1340*/
1341void readToFirstNonWS() throws IOException 
1342  {
1343  wsbuf.reset();
1344
1345  while (true)
1346    {
1347    c = in.read();
1348  
1349    if (c == '\r' || c == '\n')
1350      break;
1351      
1352    if (c == EOF || ! Character.isWhitespace(c))
1353      break;
1354  
1355    wsbuf.append((char)c);
1356    }
1357    
1358  in.unread();
1359  }
1360
1361//skip till end of whitespace or EOF. does not consume any chars past 
1362//the whitespace.
1363void skipWS() throws IOException
1364  {
1365  int c2 = EOF;
1366  while (true) {
1367    c2 = in.read();
1368    if (c2 == EOF || ! Character.isWhitespace(c2)) {
1369      in.unread();
1370      break;
1371      }
1372    } 
1373  }
1374  
1375//skips to the end of line if the rest of the line is (from the current
1376//position), all whitespace till the end. otherwise, does not change 
1377//current position. consumes trailing newlines (if present) when reading 
1378//whitespace.
1379void skipIfWhitespaceToEnd() throws IOException
1380  {
1381  int count = 0;
1382  
1383  while (true) 
1384    {
1385    c = in.read();
1386      count++;
1387
1388    if (c == '\r') {
1389      in.match('\n');
1390      return;
1391      }
1392      
1393    if (c == '\n' || c == EOF)
1394      return;
1395      
1396    if (! Character.isWhitespace(c))
1397      break;
1398      }
1399
1400  in.unread(count);
1401  }
1402
1403//not used anymore but left here for potential future use. does not
1404//consume the newline (if present)
1405void skipToLineEnd() throws IOException 
1406  {
1407    while (true) 
1408      {
1409      int c = in.read();
1410      if (c == EOF) {
1411        in.unread();
1412      break;
1413        }
1414      if (c == '\n' || c == '\r') { 
1415        in.unread();
1416        break;
1417        }
1418      }
1419    }
1420
1421String quote(final char c) 
1422  {
1423    switch (c)
1424      {
1425      case '\r':
1426            return "\\r";
1427            
1428      case '\n':
1429            return "\\n";
1430 
1431    case '\"':
1432      //can also say: new String(new char[] {'\', '"'})
1433            return "\\\"";    //--> \"
1434 
1435    case '\\':
1436            return "\\\\";
1437    
1438      default:
1439        return String.valueOf(c);
1440      }
1441    }
1442
1443//======= util and debug methods ==========================
1444String methodName(int framenum)
1445  {
1446  StackTraceElement ste[] = new Exception().getStackTrace();
1447  //get method that called us, we are ste[0]
1448  StackTraceElement st = ste[framenum];
1449  String file = st.getFileName();
1450  int line = st.getLineNumber();
1451  String method = st.getMethodName();
1452  String threadname = Thread.currentThread().getName();
1453  return method + "()";   
1454  }
1455
1456void dbgenter() {
1457  System.out.format("%s-->%s\n", StringUtil.repeat('\t', dbgtab++), methodName(2));
1458  }
1459  
1460void dbgexit() {
1461  System.out.format("%s<--%s\n", StringUtil.repeat('\t', --dbgtab), methodName(2));
1462  }
1463
1464void dbgread(String str) {
1465  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1466  }
1467
1468void dbgread(String str, List list) {
1469  System.out.format("%s %s: ", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
1470  for (int n = 0; n < list.size(); n++) {
1471    System.out.print( StringUtil.viewableAscii( (String)list.get(n) ) );
1472    }
1473  System.out.println("");
1474  }
1475
1476void dbgread(char c) {
1477  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(c));
1478  }
1479
1480void dbgread(CharArrayWriter buf) {
1481  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(buf.toString()));
1482  }
1483
1484void unclosed(String blockname, int startline, int startcol) throws IOException
1485  {
1486  throw new IOException(blockname + " tag not closed.\nThis tag was possibly opened in: \nFile:"
1487    + inputFile + ", line:" 
1488    + startline + " column:" + startcol +
1489    ".\nCurrent line:" + in.getLine() + " column:" + in.getCol());  
1490  }
1491
1492void error(String msg, int line, int col) throws IOException
1493  {
1494  throw new IOException("Error in File:" + inputFile + " Line:" + line + " Col:" + col + " " + msg);  
1495  }
1496
1497void error(String msg) throws IOException
1498  {
1499  throw new IOException("Error in File:" + inputFile + " " + msg);  
1500  }
1501
1502//============== Non Parsing methods ================================
1503void o(Object str) {
1504  out.print(str);
1505  }
1506
1507void ol(Object str) {
1508  out.println(str); 
1509  }
1510
1511void ol() {
1512  out.println();
1513  }
1514  
1515/**
1516Returns the src_encoding directive (if any) defined in this page.
1517*/
1518String getSourceEncoding() {
1519  return src_encoding;
1520  }
1521
1522/* 
1523include an external file whose contents will be rendered as part of the page.
1524*/ 
1525void includeFile(CharArrayWriter buf, String option) throws IOException
1526  {
1527  String str;
1528  
1529  if (dbg) dbgread("<new INCLUDE-FILE> "); 
1530  str = removeLeadingTrailingQuote(buf.toString().trim());
1531  
1532  File includeFile = null;
1533  File parentDir = inputFile.getParentFile();
1534  if (parentDir == null) {
1535    parentDir = new File(".");
1536    }
1537
1538  if (str.startsWith("/"))
1539    includeFile = new File(contextRoot, str);
1540  else
1541    includeFile = new File(parentDir, str);
1542        
1543  //System.out.println(">>>>>>>>>> f="+f +";root="+contextRoot);
1544      
1545  if (! includeFile.exists()) {
1546    throw new IOException("Include file does not exist: " + includeFile.getCanonicalPath());
1547    }
1548
1549  if (circularityTrack.contains(includeFile.getAbsolutePath())) {
1550     throw new IOException("Circularity detected when including: " + includeFile.getCanonicalPath() + "\nAlready included the following files: " + circularityTrack);
1551    }
1552
1553  tree.add(new MollyComment(
1554    "//>>>START INCLUDE from: " + includeFile.getAbsolutePath()));
1555    
1556  /*
1557    PageParser pp = new PageParser(contextRoot, includeFile, out, classname, log);
1558    pp.includeMode().parse();  //writes to out
1559  */
1560  
1561  in.insertIntoStream(includeFile);
1562
1563  /* this is printed immediately before the inserted contents can be processed, so don't add this */
1564  /*
1565  tree.add(new MollyComment(
1566    "//>>>END INCLUDE from: " + includeFile.getAbsolutePath()));
1567  */
1568  
1569  circularityTrack.add(includeFile.getAbsolutePath());
1570  }
1571
1572  
1573void writePage() throws IOException
1574  { 
1575  if (! includeMode)
1576    {
1577    if (directives.containsKey(d_src_encoding)) {
1578      this.src_encoding = (String) directives.get(d_src_encoding);
1579      this.src_encoding = removeLeadingTrailingQuote(this.src_encoding);
1580      }
1581  
1582    //create a appropriate PrintWriter based on either the default
1583    //jave encoding or the page specified java encoding
1584    //the java source file will be written out in this encoding
1585  
1586    FileOutputStream  fout = new FileOutputStream(outputFile);
1587    OutputStreamWriter  fw   = (src_encoding != null) ?
1588        new OutputStreamWriter(fout, src_encoding) :
1589        new OutputStreamWriter(fout);
1590        
1591    out = new PrintWriter(new BufferedWriter(fw));
1592    }
1593    
1594  if (! includeMode) 
1595    {
1596    writePackage();
1597    writeImports();
1598    
1599    o ("public class ");
1600    o (classname);
1601    ol(" extends fc.web.page.PageImpl");
1602    ol("{");
1603    }
1604
1605  writeFields();
1606
1607  if (! includeMode) {
1608    writeConstructor();
1609    }
1610    
1611  writeMethods();
1612  
1613  if (! includeMode) {
1614    ol("}");
1615    }
1616  }
1617
1618void writePackage()
1619  {
1620  o ("package ");
1621  o (packagename);
1622  ol(";");
1623  ol();
1624  }
1625  
1626void writeImports() throws IOException
1627  {
1628  ol("import javax.servlet.*;");
1629  ol("import javax.servlet.http.*;");
1630  ol("import java.io.*;");
1631  ol("import java.util.*;");
1632  //write this in case (very rare) that a page overrides the 
1633  //Page.init()/destory methods [we need pageservlet for init(..)]
1634  ol("import fc.web.page.PageServlet;");
1635  for (int n = 0; n < imps.size(); n++) {
1636    ((Element)imps.get(n)).render();
1637    ol();
1638    }
1639  ol();
1640  }
1641
1642void writeFields()
1643  {
1644  }
1645
1646void writeConstructor()
1647  {
1648  }
1649
1650void writeMethods() throws IOException
1651  {
1652  writeDeclaredMethods();
1653  writeIncludedMethods();
1654  writeRenderMethod();
1655  }
1656  
1657void writeDeclaredMethods() throws IOException
1658  {
1659  for (int n = 0; n < decl.size(); n++) {
1660    ((Element)decl.get(n)).render();
1661    }
1662  
1663  if (decl.size() > 0)
1664    ol();
1665  }
1666
1667void writeIncludedMethods() throws IOException
1668  {
1669  for (int n = 0; n < inc_decl.size(); n++) {
1670    ((Element)inc_decl.get(n)).render();
1671    }
1672    
1673  if (inc_decl.size() > 0)
1674    ol();
1675  }
1676
1677void writeRenderMethod() throws IOException
1678  {
1679  if  (! includeMode) {
1680    writeRenderTop();
1681    }
1682  
1683  /* remove leading emptylines if directed */
1684  if (directives.containsKey(d_remove_initial_emptylines)) 
1685    {
1686    if (dbg) System.out.println("[d_remove_initial_emptylines] directive found, removing leading whitepace");
1687    boolean white_space = true;
1688    
1689    //have to use iterator when removing while transversing
1690    Iterator it = tree.iterator();
1691    while (it.hasNext()) 
1692      {
1693      Element e = (Element) it.next();
1694      if (e instanceof Text) 
1695        {
1696        Text t = (Text) e;
1697        if (t.isOnlyWhitespace()) {
1698          it.remove();
1699          }
1700        }
1701      else{
1702        if (! (e instanceof Comment || e instanceof Decl || e instanceof MollyComment)) {
1703          //the initial whitespace mode is not applicable since some other declaration seen
1704          break;
1705          }
1706        }
1707      }
1708    }
1709
1710  /* remove all empty lines if directed */
1711  if (directives.containsKey(d_remove_all_emptylines)) 
1712    {
1713    if (dbg) System.out.println("[d_remove_all_emptylines] directive found, removing leading whitepace");
1714    boolean white_space = true;
1715    
1716    //have to use iterator when removing while transversing
1717    Iterator it = tree.iterator();
1718    while (it.hasNext()) 
1719      {
1720      Element e = (Element) it.next();
1721      if (e instanceof Text) 
1722        {
1723        Text t = (Text) e;
1724        if (t.isOnlyWhitespace()) {
1725          it.remove();
1726          }
1727        }
1728      }
1729    }
1730
1731  
1732  for (int n = 0; n < tree.size(); n++) {
1733    ((Element)tree.get(n)).render();
1734    }
1735    
1736  if (! includeMode) {
1737    writeRenderBottom();
1738    }
1739      
1740  }
1741  
1742void writeRenderTop() throws IOException
1743  {
1744  ol("public void render(HttpServletRequest req, HttpServletResponse res) throws Exception");
1745  ol("\t{");
1746    ol("  /* for people used to typing 'request/response' */");
1747  ol("  final HttpServletRequest  request = req;");
1748  ol("  final HttpServletResponse response = res;");
1749  ol();
1750  //mime+charset
1751  String content_type = "";
1752  if (directives.containsKey(d_mimetype)) 
1753    {
1754    String mtype = (String) directives.get(d_mimetype);
1755    if (!  (mtype.equals("") || mtype.equals(mimetype_none)) ) 
1756      {
1757      mtype = removeLeadingTrailingQuote(mtype);
1758      content_type += mtype;
1759      }
1760    } 
1761  else{
1762    content_type += Page.DEFAULT_MIME_TYPE;
1763    }
1764
1765    
1766  if (directives.containsKey(d_encoding)) {
1767    String encoding = (String) directives.get(d_encoding);
1768    encoding = removeLeadingTrailingQuote(encoding);
1769    /*an empty encoding means that the encoding is specified in the
1770    html header*/
1771    if (! encoding.trim().equals("")) { 
1772      content_type += "; charset=";
1773      content_type += encoding; 
1774      }
1775    }
1776  else{
1777    content_type += "; charset=";
1778    content_type += Page.DEFAULT_ENCODING;
1779    }
1780
1781  o ("  res.setContentType(\""); o (content_type); ol("\");");
1782
1783  //buffer
1784  if (directives.containsKey(d_buffersize)) {
1785    o ("  res.setBufferSize(");
1786    o (directives.get(d_buffersize));
1787    ol(");");
1788    }
1789    
1790  //stream or writer
1791  boolean stream = false;
1792  if (directives.containsKey(d_out)) 
1793    {
1794    String stream_type = ((String) directives.get(d_out)).toLowerCase().intern();
1795
1796    if (stream_type == d_out_stream1 || stream_type == d_out_stream2) {
1797      stream = true;
1798      }
1799    else if (stream_type == d_out_writer) {
1800      stream = false;
1801      }
1802    else{
1803      error("Did not understand directive [directive name=out, value=" + stream_type + "]. Choose between (" +  d_out_stream1 + ") and (" + d_out_writer + ")");
1804      }
1805    }
1806    
1807  if (stream)
1808    ol("  ServletOutputStream out = res.getOutputStream();");
1809  else
1810    ol("  PrintWriter out = res.getWriter();");
1811
1812  }
1813
1814void writeRenderBottom() throws IOException
1815  {
1816  ol();
1817  ol("\t} //~render end");
1818  }
1819
1820
1821/*
1822int tabcount = 1;
1823String tab = "\t";
1824void tabInc() {
1825  tab = StringUtil.repeat('\t', ++tabcount);
1826  }
1827void tabDec() {
1828  tab = StringUtil.repeat('\t', --tabcount);
1829  }
1830*/
1831
1832abstract class Element {
1833  abstract void render() throws IOException;
1834  //text, include etc., implement this as needed. 
1835  void addExp(Exp e) {  
1836    throw new RuntimeException("Internal error: not implemented by this object"); 
1837    }
1838  }
1839    
1840//this should NOT be added to the tree directly but added to Text or Hash
1841//via the addExp() method. This is because exps must be printed inline
1842class Exp extends Element
1843  {
1844  String str;
1845  
1846  Exp(CharArrayWriter buf) {
1847    this.str = buf.toString();
1848    if (dbg) dbgread("<new EXP> "+ str); 
1849    }
1850
1851  void render() {
1852    o("out.print  (");
1853    o(str);
1854    ol(");");
1855    }
1856    
1857  public String toString() {
1858    return "Exp: [" + str + "]";
1859    }
1860  }
1861  
1862class Text extends Element
1863  {
1864  String  offset_space;
1865  final   List list = new ArrayList();
1866
1867  //each text section is parsed by a text node. Within EACH text
1868  //node, we split it's contained text into separate lines and
1869  //generate code to print each line with a "out.println(...)"
1870  //statement. This maintains the same source order as the molly
1871  //page. If we munge together everything and print all of it's
1872  //contents with just one out.println(...)" statement, we would
1873  //get one large line with embedded \n and that would make
1874  //things more difficult to co-relate with the source file.
1875
1876  Text(final String offset, final CharArrayWriter b) 
1877    {
1878    if (offset == null)
1879      offset_space = "\t";
1880    else
1881      offset_space = "\t" + offset;
1882  
1883    final char[] buf = b.toCharArray();
1884
1885    boolean prevWasCR = false;
1886    //jdk default is 32. we say 256. not too large, maybe
1887    //less cache pressure. not too important, gets resized
1888    //as needed anyway.
1889    final CharArrayWriter tmp = new CharArrayWriter(256);
1890    
1891    for (int i=0, j=1; i < buf.length; i++, j++) 
1892      {
1893      char c = buf[i];   
1894      if (j == buf.length) {
1895        tmp.append(quote(c));
1896        list.add(tmp.toString());
1897        tmp.reset();
1898        }
1899      else if (c == '\n') {
1900        tmp.append(quote(c));
1901        if (! prevWasCR) {
1902          list.add(tmp.toString());
1903          tmp.reset();
1904          }
1905        }
1906      else if (c == '\r') {
1907        tmp.append(quote(c));
1908        list.add(tmp.toString());
1909        tmp.reset();
1910        prevWasCR = true;
1911        }
1912      else{
1913        tmp.append(quote(c));
1914        prevWasCR = false;
1915        }
1916      }
1917
1918    if (dbg) {
1919      String classname = getClass().getName();
1920      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
1921      }
1922    }
1923
1924  Text(CharArrayWriter b) 
1925    {
1926    this(null, b);
1927    }
1928    
1929  void addExp(Exp e)
1930    {
1931    list.add(e);
1932    }
1933
1934  void render() 
1935    {
1936    for (int i=0; i<list.size(); i++) 
1937      {
1938      Object obj = list.get(i); //can be String or Exp
1939      if (obj instanceof Exp) {
1940        o(offset_space);
1941        ((Exp)obj).render();
1942        }
1943      else{
1944        o(offset_space);
1945        o("out.print  (\"");
1946        o(obj);
1947        ol("\");"); 
1948        }
1949      }
1950    } //render
1951
1952  //a newline is actuall '\' and '\n' since it's fed to out (..)
1953  //to check for whitepace we need to check for '\', 'n', etc
1954  boolean isOnlyWhitespace() 
1955    {
1956    for (int i = 0; i < list.size(); i++) 
1957      {
1958      Object obj = list.get(i); //can be String or Exp
1959      if (obj instanceof Exp) {
1960        return false;
1961        }
1962      else{
1963        String s = (String) obj;
1964        //dont even ask my why \\\\, fucking ridiculous
1965        if (! s.matches("^(\\\\n|\\\\r|\\\\t| )*$")) {
1966          return false;
1967          }
1968        }
1969      }
1970      
1971    return true;
1972    }
1973    
1974  public String toString() {
1975    StringBuilder buf = new StringBuilder();
1976    buf.append("Text: [");
1977    for (int n = 0; n < list.size(); n++) {
1978      buf.append(StringUtil.viewableAscii(String.valueOf(list.get(n))));
1979      }
1980    buf.append("]");
1981    return buf.toString();
1982    }
1983  
1984  }
1985
1986class Hash extends Text
1987  {
1988  Hash(final String offset, final CharArrayWriter b) 
1989    {
1990    super(offset, b);
1991    }
1992
1993  //same as super.render() except for j == list.size() o/ol() below
1994  void render() 
1995    {
1996    for (int i=0, j=1; i<list.size(); i++, j++) 
1997      {
1998      Object obj = list.get(i); //can be String or Exp
1999      if (obj instanceof Exp) {
2000        o(offset_space);
2001        ((Exp)obj).render();
2002        }
2003      else{
2004        o(offset_space);
2005        o("out.print  (\"");
2006        o(obj);
2007        
2008        if (j == list.size()) 
2009          o ("\");");
2010        else
2011          ol("\");"); 
2012        }
2013      }
2014    } //render
2015
2016  public String toString() {
2017    return "Hash: " + list;
2018    }
2019  }
2020
2021class Heredoc extends Text
2022  {
2023  Heredoc(final CharArrayWriter buf) 
2024    {
2025    super(null, buf);
2026    }
2027
2028  //override, exp cannot be added to heredoc sections
2029  void addExp(Exp e)
2030    {
2031    throw new IllegalStateException("Internal implementation error: this method should not be called for a Heredoc object");
2032    }
2033    
2034  void render() 
2035    {
2036    for (int i=0, j=1; i<list.size(); i++, j++) 
2037      {
2038      Object obj = list.get(i); 
2039      o(offset_space);
2040      o("out.print  (\"");
2041      o(obj);
2042      ol("\");"); 
2043      }
2044    } //render
2045
2046  public String toString() {
2047    return "Heredoc: " + list;
2048    }
2049
2050  }
2051
2052
2053class Code extends Element
2054  {
2055  List list = new ArrayList();
2056  
2057  Code(CharArrayWriter b) 
2058    {
2059    //we split the code section into separate lines and 
2060    //print each line with a out.print(...). This maintains
2061    //the same source order as the molly page. If we munge together
2062    //everything, we would get one large line with embedded \n
2063    //and that would make things more difficult to co-relate.
2064    final char[] buf = b.toCharArray();
2065    CharArrayWriter tmp = new CharArrayWriter();
2066    for (int i=0, j=1; i < buf.length; i++, j++) {
2067      char c = buf[i];   
2068      if (j == buf.length) { //end of buffer
2069        tmp.append(c);
2070        list.add(tmp.toString());
2071        tmp.reset();
2072        }
2073      else if (c == '\n') {
2074        tmp.append(c);
2075        list.add(tmp.toString());
2076        tmp.reset();
2077        }
2078      else
2079        tmp.append(c);
2080      }
2081    if (dbg) {
2082      String classname = getClass().getName();
2083      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
2084      }
2085    }
2086
2087  void render() {
2088    for (int i = 0; i < list.size(); i++) {
2089      o('\t');
2090      o(list.get(i));
2091      }
2092    }
2093    
2094  public String toString() {
2095    return "Code: " + list;
2096    }
2097  }
2098
2099class Comment extends Element
2100  {
2101  String str;
2102  
2103  Comment(CharArrayWriter buf) {
2104    this.str = buf.toString();
2105    if (dbg) dbgread("<new COMMENT> "+ str); 
2106    }
2107
2108  void render() {
2109    //we don't print commented sections
2110    }
2111
2112  public String toString() {
2113    return "Comment: [" + str + "]";
2114    }
2115  }
2116
2117class Decl extends Code
2118  {
2119  Decl(CharArrayWriter buf) {
2120    super(buf);
2121    }
2122
2123  void render() {
2124    for (int i = 0; i < list.size(); i++) {
2125      o (list.get(i));
2126      }
2127    }
2128  }
2129
2130/* base class for Forward and Include */
2131class ForwardIncludeElement extends Element
2132  {
2133  List    parts = new ArrayList();
2134  boolean useBuf = false;
2135  
2136  // the following is for includes with expressions 
2137  // [include foo[=i].html]  
2138  // i could be 1,2,3.. the parser adds the xpression [=i] to this
2139  // object if it's present via the addExp method below
2140  void add(CharArrayWriter buf)
2141    {
2142    parts.add(buf.toString().trim());
2143    if (parts.size() > 1) {
2144      useBuf = true;
2145      }
2146    }
2147
2148  void addExp(Exp e)
2149    {
2150    parts.add(e);
2151    useBuf = true;
2152    }
2153
2154  void render() throws IOException
2155    {
2156    if (parts.size() == 0) {
2157      //log.warn("possible internal error, parts.size()==0 in Forward");
2158      return;
2159      }
2160
2161    ol("\t{ //this code block gives 'rd' its own namespace");
2162  
2163    if (! useBuf) {
2164      o ("\tfinal RequestDispatcher rd = req.getRequestDispatcher(\"");
2165      //only 1 string
2166      o (removeLeadingTrailingQuote(parts.get(0).toString())); 
2167      ol("\");");
2168      }
2169    else{
2170      ol("\tfinal StringBuilder buf = new StringBuilder();");
2171      for (int n = 0; n < parts.size(); n++) {
2172        Object obj = parts.get(n);
2173        if ( n == 0 || (n + 1) == parts.size() ) {
2174          obj = removeLeadingTrailingQuote(obj.toString());
2175          }
2176        if (obj instanceof String) {
2177          o ("\tbuf.append(\"");
2178          o (obj);
2179          ol("\");");
2180          }
2181        else{
2182          o ("\tbuf.append(");
2183          o ( ((Exp)obj).str );
2184          ol(");");
2185          }
2186        } //for
2187      ol("\tfinal RequestDispatcher rd = req.getRequestDispatcher(buf.toString());");
2188      } //else
2189    }
2190
2191
2192  public String toString() {
2193    return "Forward: " + parts;
2194    }
2195  }
2196
2197/* a request dispatcher based include. */
2198class Include extends ForwardIncludeElement
2199  {
2200  Include() {
2201    if (dbg) dbgread("<new INCLUDE> "); 
2202    }
2203    
2204  void render() throws IOException
2205    {
2206    super.render();
2207    ol("\trd.include(req, res);");
2208    ol("\t}   //end rd block");
2209    }
2210
2211  /* uses parent toString */
2212  }
2213
2214/* a request dispatcher based forward */
2215class Forward extends ForwardIncludeElement
2216  {
2217  Forward() {
2218    if (dbg) dbgread("<new FORWARD>"); 
2219    }
2220
2221  void render() throws IOException
2222    {
2223    super.render();
2224    ol("\t//WARNING: any uncommitted page content before this forward will be discarded.");
2225    ol("\t//If the response has already been committed an exception will be thrown. ");
2226
2227    ol("\trd.forward(req, res);");
2228
2229    ol("\t//NOTE: You should 'return' right after this line. There should be no content in your ");
2230    ol("\t//page after the forward statement");
2231    ol("\t}   //end rd block");
2232    }
2233
2234  /* uses parent toString */
2235  }
2236
2237
2238/* a molly mechanism to include an external file containing code and method
2239   declarations. These are typically commom utility methods and global
2240   vars. The included file is not parsed by the molly parser... the contents
2241   are treated as if they were written directly inside a [!....!] block.
2242*/ 
2243class IncludeDecl extends Element
2244  {
2245  String str;
2246  String opt;
2247  
2248  IncludeDecl(CharArrayWriter buf) {
2249    if (dbg) dbgread("<new INCLUDE-DECL> "); 
2250    str = removeLeadingTrailingQuote(buf.toString().trim());
2251    }
2252  
2253  void setOption(String opt) {
2254    this.opt = opt;
2255    }
2256  
2257  void render() throws IOException
2258    {
2259    File f = null;
2260    File parentDir = inputFile.getParentFile();
2261    if (parentDir == null) {
2262      parentDir = new File(".");
2263      }
2264
2265    final int strlen = str.length();
2266    
2267    if (str.startsWith("\"") || str.startsWith("'")) 
2268      {
2269      if (strlen == 1) //just " or ' 
2270        throw new IOException("Bad include file name: " + str);
2271        
2272      str = str.substring(1, strlen);
2273      }
2274
2275    if (str.endsWith("\"") || str.endsWith("'")) 
2276      {
2277      if (strlen == 1) //just " or ' 
2278        throw new IOException("Bad include file name: " + str);
2279        
2280      str = str.substring(0, strlen-1);
2281      }
2282
2283    if (str.startsWith("/"))
2284      f = new File(contextRoot, str);
2285    else
2286      f = new File(parentDir, str);
2287    
2288    /* f = new File(parentDir, str); */
2289    
2290    if (! f.exists()) {
2291      throw new IOException("Include file does not exist: " + f.getCanonicalPath());
2292      }
2293
2294    o("//>>>START INCLUDE DECLARTIONS from: ");
2295    o(f.getAbsolutePath());
2296    ol();
2297        
2298    o(IOUtil.inputStreamToString(new FileInputStream(f)));
2299  
2300    o("//>>>END INCLUDE DECLARATIONS from: ");
2301    o(f.getAbsolutePath());
2302    ol();
2303    
2304    //circularities are tricky, later
2305    //includeMap.put(pageloc, f.getCanonicalPath());
2306    }
2307
2308  public String toString() {
2309    return "IncludeDecl: [" + str + "; options: " + opt + "]";
2310    }
2311  }
2312
2313class Import extends Code
2314  {
2315  Import(CharArrayWriter buf) {
2316    super(buf);
2317    }
2318
2319  void render() {
2320    for (int i = 0; i < list.size(); i++) {
2321      o (list.get(i));
2322      }
2323    }
2324  }
2325
2326class MollyComment extends Element
2327  {
2328  String str;
2329  
2330  MollyComment(String str) {
2331    this.str = str;
2332    if (dbg) dbgread("<new MollyComment> "+ str); 
2333    }
2334
2335  void render() {
2336    ol(str);
2337    }
2338    
2339  public String toString() {
2340    return "MollyComment: [" + str + "]";
2341    }
2342  }
2343  
2344/**
2345removes starting and trailing single/double quotes. used by the
2346include/forward render methods only, NOT used while parsing.
2347*/
2348private static String removeLeadingTrailingQuote(String str)
2349  {
2350  if (str == null)
2351    return str;
2352
2353  if ( str.startsWith("\"") || str.startsWith("'") )  {
2354    str = str.substring(1, str.length());
2355    }
2356
2357  if ( str.endsWith("\"") || str.endsWith("'") ) {
2358    str = str.substring(0, str.length()-1); 
2359    }
2360
2361  return str;
2362  }
2363
2364//===============================================
2365
2366public static void main (String args[]) throws IOException
2367  {
2368  Args myargs = new Args(args);
2369  myargs.setUsage("java " + myargs.getMainClassName() 
2370    + "\n"
2371      + "Required params:\n"
2372    + "     -classname output_class_name\n" 
2373    + "     -in        input_page_file\n"
2374    + "\nOptional params:\n" 
2375    + "     -encoding    <page_encoding>\n"
2376    + "     -contextRoot <webapp root-directory or any other directory>\n"
2377    + "        this directory is used as the starting directory for absolute (starting\n"
2378    + "        with a \"/\") include/forward directives in a page>. If not specified\n"
2379    + "        defaults to the same directory as the page file\n"
2380    + "     -out <output_file_name>\n"
2381    + "        the output file is optional and defaults to the standard out if not specified."
2382    );
2383  //String encoding = myargs.get("encoding", Page.DEFAULT_ENCODING);
2384
2385  File input     = new File(myargs.getRequired("in"));
2386  File contextRoot = null;
2387  
2388  if (myargs.flagExists("contextRoot"))
2389    contextRoot = new File(myargs.get("contextRoot"));
2390  else
2391    contextRoot = input;
2392
2393  PrintWriter output;
2394  
2395  if (myargs.get("out") != null)
2396    output = new PrintWriter(new FileWriter(myargs.get("out")));
2397  else
2398    output = new PrintWriter(new OutputStreamWriter(System.out));
2399    
2400  PageParser parser = new PageParser(contextRoot, input, output, myargs.getRequired("classname"), Log.getDefault());
2401  parser.parse();
2402  }
2403
2404}