// Copyright (c) 2001 Hursh Jain (http://www.mollypages.org) 
// The Molly framework is freely distributable under the terms of an
// MIT-style license. For details, see the molly pages web site at:
// http://www.mollypages.org/. Use, modify, have fun !

package fc.util.pagetemplate;

import java.io.*;
import java.util.*;
import java.util.regex.*;
import fc.io.*;
import fc.util.*;

/*
NOTES

Code blocks of the form 
 [...] 
cause problems with java arrays

String[] or foo[4] etc.., craps out. So we need to use 
 [[...]] 
for the molly code blocks

The order of switch'es in a case statement in various methods is not always arbitrary
(the order matters in this sort of recursive descent parsing)

Read www.mollypages.org/page/grammar/index.mp for a intro to parsing
*/

/**
Parses a template page and writes out the corresponding java file to the specified
output. The parser and scanner is combined into one class here for simplicity (a
seperate scanner is overkill for a simple LL(1) grammar such as page templates).

@author hursh jain
*/
public final class TemplateParser
{
private static final boolean dbg    = false;
private static final int     EOF    = -1;
private              int     dbgtab = 0;

String           classname;
String         packagename = TemplatePage.PACKAGE_NAME;
TemplateReader   in;
PrintWriter      out;
Log              log;
File             inputFile;
File             outputFile;
boolean          includeMode = false;

//Read data
//we use these since stringbuffer/builders do not have a clear/reset function
CharArrayWriter buf = new CharArrayWriter(4096);
CharArrayWriter wsbuf = new CharArrayWriter(32);  // ^(whitespace)* 
int c = EOF;

//PageData
List  decl       = new ArrayList();     //declarations
List  inc_decl       = new ArrayList();     //external included declarations
List  imps           = new ArrayList();     //imports
List  tree           = new ArrayList();     //code, exp, text etc.
Map   directives     = new HashMap();       //page options
Set   circularityTrack = new HashSet();   //track pages already included to stop circular refs

/** The name of the  ("remove-initial-whitespace") directive */
public static String d_remove_initial_emptylines = "remove-initial-whitespace";
/** The name of the  ("remove-all-emptylines") directive */
public static String d_remove_all_emptylines = "remove-all-emptylines";

/* 
This constructor for internal use.

The parser can be invoked recursively to parse included files as
well..that's what the includeMode() does (and this construtor is invoked
when including). When including, we already have a output writer
created, we use that writer (instead of creating a new one based on
src_encoding as we do for in normal page parsing mode).

@param  contextRoot absolute path to the webapp context root directory
@param  input   absolute path to the input page file
@param  input   absolute path to the output file (to be written to).
@param  classname classname to give to the generated java class.
*/
private TemplateParser(File input, PrintWriter outputWriter, String classname, Log log) 
throws IOException
  {
  this.inputFile = input; 
  this.in  = new TemplateReader(input);
  this.out = outputWriter;
  this.classname = classname;
  this.log = log;

  circularityTrack.add(input.getAbsolutePath());
  }


/**
Creates a new page parser.

@param  contextRoot absolute path to the webapp context root directory
@param  input   absolute path to the input page file
@param  output    absolute path to the output file (to be written to).
@param  classname classname to give to the generated java class.
@log  log     destination for internal logging output.
*/
public TemplateParser(File input, File output, String classname, Log log) 
throws IOException
  {
  this.inputFile = input; 
  this.in  = new TemplateReader(input);
  this.outputFile = output;
  this.classname = classname;
  this.log = log;

  circularityTrack.add(input.getAbsolutePath());
  }

void append(final int c)
  {
  Argcheck.istrue(c >= 0, "Internal error: recieved c=" + c);
  buf.append((char)c);
  }

void append(final char c)
  {
  buf.append(c);
  }

void append(final String str)
  {
  buf.append(str);
  }

/* not used anymore */
TemplateParser includeMode()
  {
  includeMode = true;
  return this;
  }

/**
Parses the page. If the parse is successful, the java source will be
generated.

@throws IOException   a parse failure occurred. The java source file
            may or may not be properly generated or written
            in this case.
*/
public void parse() throws IOException
  {
  parseText();  
  if (! includeMode)  {
    writePage();
    out.close();
    }
  else{
    out.flush();
    }
  in.close();
  }

//util method for use in the case '[' branch of parseText below.
private Text newTextNode()
  {
  Text text = new Text(buf);
  tree.add(text);
  buf.reset();
  return text;
  }
  
void parseText() throws IOException
  {
  if (dbg) dbgenter(); 
      
  while (true)
    { 
    c = in.read();
    
    if (c == EOF) {
      tree.add(new Text(buf));
      buf.reset();
      break;
      }
      
    switch (c)
      { 
      //Escape start tags
      case '\\':
        /*  we don't need to do this: previously, expressions
        were [...] but now they are [=...], previously we needed
        to escape \[[ entirely (since if we escaped \[ the second
        [ would start an expression
        */
        /*        
        if (in.match("[["))  
          append("[[");
        */
        //escape only \[... otherwise leave \ alone
        if (in.match("["))
          append("[");
        else
          append(c);
        break;

      case '[':
        /* suppose we have
        \[[
        escape handling above will capture \[
        then the second '[' drops down here. Good so far.
        But we must not create a new text object here by
        default...only if we see another [[ or [= or [include or
        whatever. 
        */
        /*
        But creating a text object at the top is easier
        then repeating this code at every if..else branch below
        but this creates superfluous line breaks.
        
        hello[haha]world
        -->prints as-->
        hello  (text node 1)
        [haha] (text node 2)
        world  (text node 3)
        --> we want
        hello[haha]world (text node 1)
        */
          
        if (in.match('[')) { 
          newTextNode();
          parseCode(); 
          }
        else if (in.match('=')) {
          Text text = newTextNode();
          parseExpression(text);
          }
        else if (in.match('!')) {
          newTextNode();
          parseDeclaration();
          }
        else if (in.match("/*")) {
          newTextNode();
          parseComment(); 
          }
        else if (in.matchIgnoreCase("page")) {
          newTextNode();
          parseDirective();
          }
        else if (in.matchIgnoreCase("include-file")) {
          newTextNode();
          parseIncludeFile();
          }
        else if (in.matchIgnoreCase("include-decl")) {
          newTextNode();
          parseIncludeDecl();
          }
        else if (in.matchIgnoreCase("import")) {
          newTextNode();
          parseImport();
          }
        else  {
          //System.out.println("c1=" + (char)c);
          append(c);
          }
        break;  
  
      default:
        //System.out.println("c2=" + (char)c);
        append(c);
        
      } //switch    
    } //while
    
  if (dbg) dbgexit(); 
  }
  
void parseCode() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();
  
  while (true)
    {
    c = in.read();  
  
    switch (c) /* the order of case tags is important. */
      {
      case EOF:
        unclosed("code", startline, startcol);
        if (dbg) dbgexit(); 
        return;

      case '/':   //Top level:  // and /* comments
        append(c);
        c = in.read();
        append(c);
        if (c == '/') 
          appendCodeSlashComment();
        else if (c == '*') 
          appendCodeStarComment();
          break;        
    
      case '"':     //strings outside of any comment
        append(c);
        appendCodeString();  
        break;
        
      case '\'':
        append(c);
        appendCodeCharLiteral();
        break;
        
      case ']':
        if (in.match(']')) {
          tree.add(new Code(buf));
          buf.reset();
          if (dbg) dbgexit(); 
          return;
          }
        else {
          append(c);
          }
        break;
      
      /* 
      a hash by itself on a line starts a hash section.
      whitespace before the # on that line is used as an
      printing 'out' statements for that hash.
      
      for (int n = 0; n < ; n++) {
      ....# foo #
      | }
      |=> 4 spaces 
      so nice if generated code looked like:
      
      for (int n = 0; n < ; n++) {
          out.print(" foo ");
          }
      */
      case '\n':
      case '\r':
        append(c);       //the \n or \r just read
        readToFirstNonWS();  //won't read past more newlines 
        //is '#' is first non-ws on this line ?
        c = in.read();
        if (c == '#') {           
          tree.add(new Code(buf));
          buf.reset();
          //whitespace provides indentation offset
          parseHash(wsbuf.toString()); 
          }
        else{
          append(wsbuf.toString());  //wsbuf contains codetext
          //let other cases also handle first non-ws or EOF
          in.unread();    
          }
        break;
      
      /* in this case, hash does not start on a new line, like:
         for (...) { #
      */
      case '#':
        tree.add(new Code(buf));
        buf.reset();
        parseHash(null);
        break;  
      
      default:
        append(c);
      } //switch    
    } //while
  }

void parseHash(String offset) throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true)
    {
    c = in.read();  
  
    switch (c)
      {
      case EOF: 
        unclosed("hash", startline, startcol);
        if (dbg) dbgexit(); 
        return;

      //special case: very common and would be a drag to escape
      //this every time:
      //  # <table bgcolor="#ffffff">....   #
      //Now, all of:
      //  bgcolor="#xxx"  
      //  bgcolor='#xxx'
      //  bgcolor="\#xxx" 
      //will work the same and give: bgcolor="#xxx"
      //1)
      //However to get a:
      //  bgcolor=#xxx    (no quoted around #xxx)
      //we still have to say:
      //  bgcolor=\#xxx   
      //2)
      //Of course, since we special case this, then:
      //  #"bar"#
      // that ending # is lost and we end up with
      //  #"bar"  with no closing hash
      //So we need to make sure that we write:
      //  #"bar" #
      // instead

      case '\'':
      case '"':
        append(c);
        if (in.match('#')) 
          append('#');
        break;
        
      case '\\':
        if (in.match('[')) 
          append('[');      
        else if (in.match('#'))
          append('#');
        else
          append(c);
        break;
        
      case '[':
        if (in.match('=')) {
          Hash hash = new Hash(offset, buf);
          tree.add(hash);
          buf.reset();
          parseExpression(hash);
          }
        else{
          append(c);
          }
        break;

      /*
      this case is not needed but is a bit of a optimization
      for (int n = 0; n < 1; n++) {
        #
        foo
      ....#...NL
        }
      avoids printing the dots (spaces) and NL in this case
      (the newline after foo is still printed)
      */
      case '\n':
      case '\r':
        append(c);
        readToFirstNonWS(); 
        c = in.read();
        //'#' is first non-ws on the line
        if (c == '#') {
          tree.add(new Hash(offset, buf));
          buf.reset();
          //skipIfWhitespaceToEnd();
          if (dbg) dbgexit(); 
          return;
          }
        else {
          append(wsbuf.toString());
          in.unread(); //let other cases also handle first non-ws   
          }
        break;

      case '#':
        tree.add(new Hash(offset, buf));  
        //skipIfWhitespaceToEnd();
        buf.reset();
        if (dbg) dbgexit(); 
        return;
        
      default:
        append(c);
      }  //switch 
    } //while
  }

/**
[page <<<FOO]
...as-is..no parse, no interpolation..
FOO
*/
void parseHeredoc(StringBuilder directives_buf) throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();
      
  int i = directives_buf.indexOf("<<<"); /* "<<<".length = 3 */
  CharSequence subseq = directives_buf.substring(
            i+3, 
            /*directives_buf does not have a ending ']' */
            directives_buf.length() 
            );
    
  final String      heredoc     = subseq.toString().trim();
  final int         heredoc_len = heredoc.length();
  final CharArrayWriter heredoc_buf = new CharArrayWriter(2048);

  /* 
  the ending heredoc after newline speeds things up a bit
  which is why is traditionally used i guess, otherwise
  we have to try a full match every first match. this 
  implementation doesn't care where the ending heredoc
  appears (can be anywhere)...simplifies the implementation.
  */
  
  while (true)
    { 
    c = in.read();
    
    if (c == EOF) {
      unclosed("heredoc: <<<"+heredoc, startline, startcol);
      break;
      }
      
    if (c == heredoc.charAt(0))
      {
      boolean matched = true;
      if (heredoc_len > 1) {
        matched = in.match(heredoc.substring(1));
        }
      if (matched) {  
        tree.add(new Heredoc(heredoc_buf));
        break;
        }
      }
    
    //default action
    heredoc_buf.append((char)c);  
    } //while
    
  if (dbg) dbgexit(); 
  }

/*
Text is the parent node for the expression. A new expression is parsed,
created and added to the text object by this method
*/
void parseExpression(Element parent) throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("expression", startline, startcol);
        if (dbg) dbgexit(); 
        return;

      case '\\':
        if (in.match(']')) 
          append(']');    
        else
          append(c);
        break;

      case ']':
        if (buf.toString().trim().length() == 0)
          error("Empty expression not allowed", startline, startcol);
        parent.addExp(new Exp(buf));
        buf.reset();  
        if (dbg) dbgexit(); 
        return;
        
      default:
        append(c);
      }
    }
  }

void parseComment() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("comment", startline, startcol);
        if (dbg) dbgexit(); 
        return;
        
      case '*':
        if (in.match("/]"))
          {
          tree.add(new Comment(buf));
          buf.reset();  
          if (dbg) dbgexit(); 
          return;
          }
        else
          append(c);  
        break;
      
      default:
        append(c);
      }
    }
  }

void parseDeclaration() throws IOException
  {
  if (dbg) dbgenter(); 
  int startline = in.getLine();
  int startcol = in.getCol();

  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("declaration", startline, startcol);
        if (dbg) dbgexit(); 
        return;
      
      case '!':
        if (in.match(']')) {
          decl.add(new Decl(buf));
          buf.reset();  
          if (dbg) dbgexit(); 
          return;
          }
        else{
          append(c);
          }
        break;

      //top level // and /* comments, ']' (close decl tag)
      //is ignored within them
      case '/':   
        append(c);
        c = in.read();
        append(c);
        if (c == '/') 
          appendCodeSlashComment();
        else if (c == '*') 
          appendCodeStarComment();
          break;        
    
      //close tags are ignored within them
      case '"':     //strings outside of any comment
        append(c);
        appendCodeString();  
        break;
        
      case '\'':
        append(c);
        appendCodeCharLiteral();
        break;
            
      default:
        append(c);
      }
    }

  }

void parseDirective() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  StringBuilder directives_buf = new StringBuilder(1024);

  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("directive", startline, startcol);
        if (dbg) dbgexit(); 
        return;
        
      case ']':
        if (directives_buf.indexOf("<<<") >= 0)  {
          parseHeredoc(directives_buf); 
          }
        else{/* other directives used at page-generation time */
          addDirectives(directives_buf);
          }
          
        if (dbg) dbgexit(); 
        return;
      
      default:
        directives_buf.append((char)c);
      }
    }

  }

//[a-zA-Z_\-0-9] == ( \w | - )
static final Pattern directive_pat = Pattern.compile(
  //foo = "bar baz" (embd. spaces)
  "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*\"((?:.|\r|\n)+?)\""  
  + "|"
  //foo = "bar$@#$" (no spaces) OR foo = bar (quotes optional)
  + "\\s*([a-zA-Z_\\-0-9]+)\\s*=\\s*(\\S+)" 
  );
  
    
void addDirectives(StringBuilder directives_buf) throws TemplateParseException
  {
  if (dbg) {
    dbgenter(); 
    System.out.println("-------directives section--------");
    System.out.println(directives_buf.toString());
    System.out.println("-------end directives-------");
    }
  
  String name, value;
  try {
    Matcher m = directive_pat.matcher(directives_buf);
    while (m.find()) 
      {
      if (dbg) System.out.println(">>>>[0]->" + m.group() 
        + "; [1]->" + m.group(1)  
        + " [2]->" + m.group(2)  
        + " [3]->" + m.group(3)  
        + " [4]->" + m.group(4));
        
      name = m.group(1) != null ? m.group(1).toLowerCase() :
                    m.group(3).toLowerCase();
      value = m.group(2) != null ? m.group(2).toLowerCase() :
                     m.group(4).toLowerCase();

      if (name.equals(d_remove_initial_emptylines)) {
        directives.put(name, value.replace("\"|'",""));       
        } 
      else if (name.equals(d_remove_all_emptylines)) {
        directives.put(name, value.replace("\"|'",""));       
        } 
      //else if .... other directives here as needed....
      else 
        throw new Exception("Do not understand directive: " + m.group());
      }
    if (dbg) System.out.println("Added directives: " + directives);
    }
  catch (Exception e) {
    throw new TemplateParseException("File: " + inputFile.getAbsolutePath() 
                  + ";\n" + e.toString());
    }

  if (dbg) dbgexit(); 
  }

void parseIncludeFile() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();
  String option = null;
  
  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("include-file", startline, startcol);
        if (dbg) dbgexit(); 
        return;
        
      case '[':
        if (in.match('=')) {
  //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\"
  //in [include-file... section starting at:", startline, startcol);
  //instead of warn, we will error out. failing early is better.
  //this does preclude having '[=' in the file name, but it's a good
  //tradeoff
          error("Expressions cannot exist in file includes. The offending static-include section starts at:", startline, startcol);
          }
        append(c);
        break;
      
      case ']':
        includeFile(buf, option); /* not added in the tree, just included in the stream */
        buf.reset();  
        if (dbg) dbgexit(); 
        return;
      
      case 'o':
        if (! in.match("ption"))
          append(c);
        else{
          skipWS();
          if (! in.match("=")) {
            error("bad option parameter in file include: ", startline, startcol);
            }
          skipWS();
          
          int c2;
          StringBuilder optionbuf = new StringBuilder();
          while (true) {
            c2 = in.read();
            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
              in.unread();
              break;
              }
            optionbuf.append((char)c2);
            }
          
          option = optionbuf.toString();
          //System.out.println(option);
          } //else
        break;
  
      default:
        append(c);
      }
    }
  }

void parseIncludeDecl() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();
  String option = null;
  
  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("include-decl", startline, startcol);
        if (dbg) dbgexit(); 
        return;
        
      case '[':
        if (in.match('=')) {
    //log.warn("Expressions cannot exist in file includes. Ignoring \"[=\" in [include-static... section starting at:", startline, startcol);
    //we will throw an exception. failing early is better. this
    //does preclude having '[=' in the file name, but it's a good tradeoff
          error("Expressions cannot exist in include-decl. The offending static-include section starts at:", startline, startcol);
          }
        append(c);
        break;
      
      case ']':
        IncludeDecl i = new IncludeDecl(buf);
        if (option != null)
          i.setOption(option);
        inc_decl.add(i);
        buf.reset();  
        if (dbg) dbgexit(); 
        return;
      
      case 'o':
        if (! in.match("ption"))
          append(c);
        else{
          skipWS();
          if (! in.match("=")) {
            error("bad option parameter in include-code: ", startline, startcol);
            }
          skipWS();
          
          int c2;
          StringBuilder optionbuf = new StringBuilder();
          while (true) {
            c2 = in.read();
            if (c2 == ']' || c2 == EOF || Character.isWhitespace(c2)) {   
              in.unread();
              break;
              }
            optionbuf.append((char)c2);
            }
          
          option = optionbuf.toString();
          //System.out.println(option);
          } //else
        break;
  
      default:
        append(c);
      }
    }
  }


//we need to parse imports seperately because they go outside
//a class declaration (and [!...!] goes inside a class)
//import XXX.*;
//class YYY {
//[!....stuff from here ....!]
//...
void parseImport() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true)
    {
    c = in.read();      
  
    switch (c)
      {
      case EOF:
        unclosed("import", startline, startcol);
        if (dbg) dbgexit(); 
        return;
      
      case '\n':
        imps.add(new Import(buf));
        buf.reset();
        break;
        
      case ']':
        imps.add(new Import(buf));
        buf.reset();  
        if (dbg) dbgexit(); 
        return;
      
      default:
        append(c);
      }
    }
  }

/*
Called when // was read at the top level inside a code block. Appends
the contents of a // comment to the buffer (not including the trailing
newline)
*/
void appendCodeSlashComment() throws IOException
  {
  if (dbg) dbgenter();
  
  while (true) 
    {
    c = in.read();
    
    if (c == EOF)
      break;
  
    //do not append \r, \r\n, or \n, that finishes the // comment
    //we need that newline to figure out if the next line is a hash
    //line
    if (c == '\r') {
      in.unread();
      break;
      }
    
    if (c == '\n') {
      in.unread();
      break;  
      }

    append(c);
    }
  
  if (dbg) dbgread("CodeSLASHComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
  if (dbg) dbgexit();
  }

/*
Called when /* was read at the top level inside a code block. Appends
the contents of a /*comment to the buffer. (not including any trailing
newline or spaces)
*/
void appendCodeStarComment() throws IOException
  {
  if (dbg) dbgenter(); 
  
  while (true) 
    {
    c = in.read();  

    if (c == EOF)
      break;
  
    append(c);
    
    if (c == '*') 
      {
      if (in.match('/')) {
        append('/');
        break;
        }
      }
    }

  if (dbg) dbgread("CodeSTARComment Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
  if (dbg) dbgexit(); 
  }

/*
Called (outside of any comments in the code block) when: 
--> parseCode()
     ... "
         ^ (we are here)
*/
void appendCodeString() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true) 
    {
    c = in.read();
  
    if (c == EOF || c == '\r' || c == '\n')
      unclosed("string literal", startline, startcol);
  
    append(c);
  
    if (c == '\\') {
      c = in.read();
      if (c == EOF)
        unclosed("string literal", startline, startcol);
      else {
        append(c);
        continue;   //so \" does not hit the if below and break
        }
      }
    
    if (c == '"')
      break;
    }

  if (dbg) dbgread("appendCodeString Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
  if (dbg) dbgexit(); 
  }


/*
Called (outside of any comments in the code block) when: 
--> parseCode()
     ... '
         ^ (we are here)
*/
void appendCodeCharLiteral() throws IOException
  {
  if (dbg) dbgenter(); 

  int startline = in.getLine();
  int startcol = in.getCol();

  while (true) 
    {
    c = in.read();
  
    if (c == EOF || c == '\r' || c == '\n')
      unclosed("char literal", startline, startcol);
  
    append(c);
  
    if (c == '\\') {
      c = in.read();
      if (c == EOF)
        unclosed("char literal", startline, startcol);
      else {
        append(c);
        continue;   //so \' does not hit the if below and break
        }
      }
    
    if (c == '\'')
      break;
    }

  if (dbg) dbgread("appendCodeCharLiteral Finished: Buffer=" + StringUtil.viewableAscii(buf.toString()));
  if (dbg) dbgexit(); 
  }


/*
Reads from the current position till the first nonwhitespace char, EOF or
newline is encountered. Reads are into the whitespace buffer. does not
consume the character past the non-whitespace character and does
NOT read multiple lines of whitespace.
*/
void readToFirstNonWS() throws IOException 
  {
  wsbuf.reset();

  while (true)
    {
    c = in.read();
  
    if (c == '\r' || c == '\n')
      break;
      
    if (c == EOF || ! Character.isWhitespace(c))
      break;
  
    wsbuf.append((char)c);
    }
    
  in.unread();
  }

//skip till end of whitespace or EOF. does not consume any chars past 
//the whitespace.
void skipWS() throws IOException
  {
  int c2 = EOF;
  while (true) {
    c2 = in.read();
    if (c2 == EOF || ! Character.isWhitespace(c2)) {
      in.unread();
      break;
      }
    } 
  }
  
//skips to the end of line if the rest of the line is (from the current
//position), all whitespace till the end. otherwise, does not change 
//current position. consumes trailing newlines (if present) when reading 
//whitespace.
void skipIfWhitespaceToEnd() throws IOException
  {
  int count = 0;
  
  while (true) 
    {
    c = in.read();
      count++;

    if (c == '\r') {
      in.match('\n');
      return;
      }
      
    if (c == '\n' || c == EOF)
      return;
      
    if (! Character.isWhitespace(c))
      break;
      }

  in.unread(count);
  }

//not used anymore but left here for potential future use. does not
//consume the newline (if present)
void skipToLineEnd() throws IOException 
  {
    while (true) 
      {
      int c = in.read();
      if (c == EOF) {
        in.unread();
      break;
        }
      if (c == '\n' || c == '\r') { 
        in.unread();
        break;
        }
      }
    }

String quote(final char c) 
  {
    switch (c)
      {
      case '\r':
            return "\\r";
            
      case '\n':
            return "\\n";
 
    case '\"':
      //can also say: new String(new char[] {'\', '"'})
            return "\\\"";    //--> \"
 
    case '\\':
            return "\\\\";
    
      default:
        return String.valueOf(c);
      }
    }

//======= util and debug methods ==========================
String methodName(int framenum)
  {
  StackTraceElement ste[] = new Exception().getStackTrace();
  //get method that called us, we are ste[0]
  StackTraceElement st = ste[framenum];
  String file = st.getFileName();
  int line = st.getLineNumber();
  String method = st.getMethodName();
  String threadname = Thread.currentThread().getName();
  return method + "()";   
  }

void dbgenter() {
  System.out.format("%s-->%s\n", StringUtil.repeat('\t', dbgtab++), methodName(2));
  }
  
void dbgexit() {
  System.out.format("%s<--%s\n", StringUtil.repeat('\t', --dbgtab), methodName(2));
  }

void dbgread(String str) {
  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
  }

void dbgread(String str, List list) {
  System.out.format("%s %s: ", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(str));
  for (int n = 0; n < list.size(); n++) {
    System.out.print( StringUtil.viewableAscii( (String)list.get(n) ) );
    }
  System.out.println("");
  }

void dbgread(char c) {
  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(c));
  }

void dbgread(CharArrayWriter buf) {
  System.out.format("%s %s\n", StringUtil.repeat('\t', dbgtab), StringUtil.viewableAscii(buf.toString()));
  }

void unclosed(String blockname, int startline, int startcol) throws IOException
  {
  throw new IOException(blockname + " tag not closed.\nThis tag was possibly opened in: \nFile:"
    + inputFile + ", line:" 
    + startline + " column:" + startcol +
    ".\nCurrent line:" + in.getLine() + " column:" + in.getCol());  
  }

void error(String msg, int line, int col) throws IOException
  {
  throw new IOException("Error in File:" + inputFile + " Line:" + line + " Col:" + col + " " + msg);  
  }

void error(String msg) throws IOException
  {
  throw new IOException("Error in File:" + inputFile + " " + msg);  
  }

//============== Non Parsing methods ================================
void o(Object str) {
  out.print(str);
  }

void ol(Object str) {
  out.println(str); 
  }

void ol() {
  out.println();
  }
  

/* 
include an external file whose contents will be rendered as part of the page.
*/ 
void includeFile(CharArrayWriter buf, String option) throws IOException
  {
  String str;
  
  if (dbg) dbgread("<new INCLUDE-FILE> "); 
  str = removeLeadingTrailingQuote(buf.toString().trim());
  
  File includeFile = null;
  File parentDir = inputFile.getParentFile();
  if (parentDir == null) {
    parentDir = new File(".");
    }

  if (str.startsWith("/"))
    includeFile = new File(str);
  else
    includeFile = new File(parentDir, str);
        
  //System.out.println(">>>>>>>>>> f="+f +";root="+contextRoot);
      
  if (! includeFile.exists()) {
    throw new IOException("Include file does not exist: " + includeFile.getCanonicalPath());
    }

  if (circularityTrack.contains(includeFile.getAbsolutePath())) {
     throw new IOException("Circularity detected when including: " + includeFile.getCanonicalPath() + "\nAlready included the following files: " + circularityTrack);
    }

  tree.add(new MollyComment(
    "//>>>START INCLUDE from: " + includeFile.getAbsolutePath()));
    
  /*
    TemplateParser pp = new TemplateParser(contextRoot, includeFile, out, classname, log);
    pp.includeMode().parse();  //writes to out
  */
  
  in.insertIntoStream(includeFile);

  /* this is printed immediately before the inserted contents can be processed, so don't add this */
  /*
  tree.add(new MollyComment(
    "//>>>END INCLUDE from: " + includeFile.getAbsolutePath()));
  */
  
  circularityTrack.add(includeFile.getAbsolutePath());
  }

  
void writePage() throws IOException
  { 
  if (! includeMode)
    {
    //create a appropriate PrintWriter based on either the default
    //jave encoding or the page specified java encoding
    //the java source file will be written out in this encoding
  
    FileOutputStream  fout = new FileOutputStream(outputFile);
    OutputStreamWriter  fw   = new OutputStreamWriter(fout, TemplatePage.DEFAULT_ENCODING);
        
    out = new PrintWriter(new BufferedWriter(fw));
    }
    
  if (! includeMode) 
    {
    writePackage();
    writeImports();
    
    o ("public class ");
    o (classname);
    ol(" extends fc.util.pagetemplate.TemplatePage");
    ol("{");
    }

  writeFields();

  if (! includeMode) {
    writeConstructor();
    }
    
  writeMethods();
  
  if (! includeMode) {
    ol("}");
    }
  }

void writePackage()
  {
  o ("package ");
  o (packagename);
  ol(";");
  ol();
  }
  
void writeImports() throws IOException
  {
  ol("import java.io.*;");
  ol("import java.util.*;");
  ol("import java.sql.*;");
  for (int n = 0; n < imps.size(); n++) {
    ((Element)imps.get(n)).render();
    ol();
    }
  ol();
  }

void writeFields()
  {
  }

void writeConstructor()
  {
  }

void writeMethods() throws IOException
  {
  writeDeclaredMethods();
  writeIncludedMethods();
  writeRenderMethod();
  }
  
void writeDeclaredMethods() throws IOException
  {
  for (int n = 0; n < decl.size(); n++) {
    ((Element)decl.get(n)).render();
    }
  
  if (decl.size() > 0)
    ol();
  }

void writeIncludedMethods() throws IOException
  {
  for (int n = 0; n < inc_decl.size(); n++) {
    ((Element)inc_decl.get(n)).render();
    }
    
  if (inc_decl.size() > 0)
    ol();
  }

void writeRenderMethod() throws IOException
  {
  if  (! includeMode) {
    writeRenderTop();
    }
  
  /* remove leading emptylines if directed */
  if (directives.containsKey(d_remove_initial_emptylines)) 
    {
    if (dbg) System.out.println("[d_remove_initial_emptylines] directive found, removing leading whitepace");
    boolean white_space = true;
    
    //have to use iterator when removing while transversing
    Iterator it = tree.iterator();
    while (it.hasNext()) 
      {
      Element e = (Element) it.next();
      if (e instanceof Text) 
        {
        Text t = (Text) e;
        if (t.isOnlyWhitespace()) {
          it.remove();
          }
        }
      else{
        if (! (e instanceof Comment || e instanceof Decl || e instanceof MollyComment)) {
          //the initial whitespace mode is not applicable since some other declaration seen
          break;
          }
        }
      }
    }

  /* remove all empty lines if directed */
  if (directives.containsKey(d_remove_all_emptylines)) 
    {
    if (dbg) System.out.println("[d_remove_all_emptylines] directive found, removing leading whitepace");
    boolean white_space = true;
    
    //have to use iterator when removing while transversing
    Iterator it = tree.iterator();
    while (it.hasNext()) 
      {
      Element e = (Element) it.next();
      if (e instanceof Text) 
        {
        Text t = (Text) e;
        if (t.isOnlyWhitespace()) {
          it.remove();
          }
        }
      }
    }

  
  for (int n = 0; n < tree.size(); n++) {
    ((Element)tree.get(n)).render();
    }
    
  if (! includeMode) {
    writeRenderBottom();
    }
      
  }
  
void writeRenderTop() throws IOException
  {
  ol("public void render(PrintWriter out) throws Exception");
  ol("\t{");
  ol();
  }

void writeRenderBottom() throws IOException
  {
  ol();
  o("\t");
  ol("out.flush();");
  ol("out.close();");
  ol("\t} //~render end");
  }


/*
int tabcount = 1;
String tab = "\t";
void tabInc() {
  tab = StringUtil.repeat('\t', ++tabcount);
  }
void tabDec() {
  tab = StringUtil.repeat('\t', --tabcount);
  }
*/

abstract class Element {
  abstract void render() throws IOException;
  //text, include etc., implement this as needed. 
  void addExp(Exp e) {  
    throw new RuntimeException("Internal error: not implemented by this object"); 
    }
  }
    
//this should NOT be added to the tree directly but added to Text or Hash
//via the addExp() method. This is because exps must be printed inline
class Exp extends Element
  {
  String str;
  
  Exp(CharArrayWriter buf) {
    this.str = buf.toString();
    if (dbg) dbgread("<new EXP> "+ str); 
    }

  void render() {
    o("out.print  (");
    o(str);
    ol(");");
    }
    
  public String toString() {
    return "Exp: [" + str + "]";
    }
  }
  
class Text extends Element
  {
  String  offset_space;
  final   List list = new ArrayList();

  //each text section is parsed by a text node. Within EACH text
  //node, we split it's contained text into separate lines and
  //generate code to print each line with a "out.println(...)"
  //statement. This maintains the same source order as the molly
  //page. If we munge together everything and print all of it's
  //contents with just one out.println(...)" statement, we would
  //get one large line with embedded \n and that would make
  //things more difficult to co-relate with the source file.

  Text(final String offset, final CharArrayWriter b) 
    {
    if (offset == null)
      offset_space = "\t";
    else
      offset_space = "\t" + offset;
  
    final char[] buf = b.toCharArray();

    boolean prevWasCR = false;
    //jdk default is 32. we say 256. not too large, maybe
    //less cache pressure. not too important, gets resized
    //as needed anyway.
    final CharArrayWriter tmp = new CharArrayWriter(256);
    
    for (int i=0, j=1; i < buf.length; i++, j++) 
      {
      char c = buf[i];   
      if (j == buf.length) {
        tmp.append(quote(c));
        list.add(tmp.toString());
        tmp.reset();
        }
      else if (c == '\n') {
        tmp.append(quote(c));
        if (! prevWasCR) {
          list.add(tmp.toString());
          tmp.reset();
          }
        }
      else if (c == '\r') {
        tmp.append(quote(c));
        list.add(tmp.toString());
        tmp.reset();
        prevWasCR = true;
        }
      else{
        tmp.append(quote(c));
        prevWasCR = false;
        }
      }

    if (dbg) {
      String classname = getClass().getName();
      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
      }
    }

  Text(CharArrayWriter b) 
    {
    this(null, b);
    }
    
  void addExp(Exp e)
    {
    list.add(e);
    }

  void render() 
    {
    for (int i=0; i<list.size(); i++) 
      {
      Object obj = list.get(i); //can be String or Exp
      if (obj instanceof Exp) {
        o(offset_space);
        ((Exp)obj).render();
        }
      else{
        o(offset_space);
        o("out.print  (\"");
        o(obj);
        ol("\");"); 
        }
      }
    } //render

  //a newline is actuall '\' and '\n' since it's fed to out (..)
  //to check for whitepace we need to check for '\', 'n', etc
  boolean isOnlyWhitespace() 
    {
    for (int i = 0; i < list.size(); i++) 
      {
      Object obj = list.get(i); //can be String or Exp
      if (obj instanceof Exp) {
        return false;
        }
      else{
        String s = (String) obj;
        //dont even ask my why \\\\, fucking ridiculous
        if (! s.matches("^(\\\\n|\\\\r|\\\\t| )*$")) {
          return false;
          }
        }
      }
      
    return true;
    }
    
  public String toString() {
    StringBuilder buf = new StringBuilder();
    buf.append("Text: [");
    for (int n = 0; n < list.size(); n++) {
      buf.append(StringUtil.viewableAscii(String.valueOf(list.get(n))));
      }
    buf.append("]");
    return buf.toString();
    }
  
  }

class Hash extends Text
  {
  Hash(final String offset, final CharArrayWriter b) 
    {
    super(offset, b);
    }

  //same as super.render() except for j == list.size() o/ol() below
  void render() 
    {
    for (int i=0, j=1; i<list.size(); i++, j++) 
      {
      Object obj = list.get(i); //can be String or Exp
      if (obj instanceof Exp) {
        o(offset_space);
        ((Exp)obj).render();
        }
      else{
        o(offset_space);
        o("out.print  (\"");
        o(obj);
        
        if (j == list.size()) 
          o ("\");");
        else
          ol("\");"); 
        }
      }
    } //render

  public String toString() {
    return "Hash: " + list;
    }
  }

class Heredoc extends Text
  {
  Heredoc(final CharArrayWriter buf) 
    {
    super(null, buf);
    }

  //override, exp cannot be added to heredoc sections
  void addExp(Exp e)
    {
    throw new IllegalStateException("Internal implementation error: this method should not be called for a Heredoc object");
    }
    
  void render() 
    {
    for (int i=0, j=1; i<list.size(); i++, j++) 
      {
      Object obj = list.get(i); 
      o(offset_space);
      o("out.print  (\"");
      o(obj);
      ol("\");"); 
      }
    } //render

  public String toString() {
    return "Heredoc: " + list;
    }

  }


class Code extends Element
  {
  List list = new ArrayList();
  
  Code(CharArrayWriter b) 
    {
    //we split the code section into separate lines and 
    //print each line with a out.print(...). This maintains
    //the same source order as the molly page. If we munge together
    //everything, we would get one large line with embedded \n
    //and that would make things more difficult to co-relate.
    final char[] buf = b.toCharArray();
    CharArrayWriter tmp = new CharArrayWriter();
    for (int i=0, j=1; i < buf.length; i++, j++) {
      char c = buf[i];   
      if (j == buf.length) { //end of buffer
        tmp.append(c);
        list.add(tmp.toString());
        tmp.reset();
        }
      else if (c == '\n') {
        tmp.append(c);
        list.add(tmp.toString());
        tmp.reset();
        }
      else
        tmp.append(c);
      }
    if (dbg) {
      String classname = getClass().getName();
      dbgread("<new " + classname.substring(classname.indexOf("$")+1,classname.length()) + ">",list); 
      }
    }

  void render() {
    for (int i = 0; i < list.size(); i++) {
      o('\t');
      o(list.get(i));
      }
    }
    
  public String toString() {
    return "Code: " + list;
    }
  }

class Comment extends Element
  {
  String str;
  
  Comment(CharArrayWriter buf) {
    this.str = buf.toString();
    if (dbg) dbgread("<new COMMENT> "+ str); 
    }

  void render() {
    //we don't print commented sections
    }

  public String toString() {
    return "Comment: [" + str + "]";
    }
  }

class Decl extends Code
  {
  Decl(CharArrayWriter buf) {
    super(buf);
    }

  void render() {
    for (int i = 0; i < list.size(); i++) {
      o (list.get(i));
      }
    }
  }



/* a molly mechanism to include an external file containing code and method
   declarations. These are typically commom utility methods and global
   vars. The included file is not parsed by the template parser... the contents
   are treated as if they were written directly inside a [!....!] block.
*/ 
class IncludeDecl extends Element
  {
  String str;
  String opt;
  
  IncludeDecl(CharArrayWriter buf) {
    if (dbg) dbgread("<new INCLUDE-DECL> "); 
    str = removeLeadingTrailingQuote(buf.toString().trim());
    }
  
  void setOption(String opt) {
    this.opt = opt;
    }
  
  void render() throws IOException
    {
    File f = null;
    File parentDir = inputFile.getParentFile();
    if (parentDir == null) {
      parentDir = new File(".");
      }

    final int strlen = str.length();
    
    if (str.startsWith("\"") || str.startsWith("'")) 
      {
      if (strlen == 1) //just " or ' 
        throw new IOException("Bad include file name: " + str);
        
      str = str.substring(1, strlen);
      }

    if (str.endsWith("\"") || str.endsWith("'")) 
      {
      if (strlen == 1) //just " or ' 
        throw new IOException("Bad include file name: " + str);
        
      str = str.substring(0, strlen-1);
      }

    if (str.startsWith("/"))
      f = new File(str);
    else
      f = new File(parentDir, str);
        
    if (! f.exists()) {
      throw new IOException("Include file does not exist: " + f.getCanonicalPath());
      }

    o("//>>>START INCLUDE DECLARTIONS from: ");
    o(f.getAbsolutePath());
    ol();
        
    o(IOUtil.inputStreamToString(new FileInputStream(f)));
  
    o("//>>>END INCLUDE DECLARATIONS from: ");
    o(f.getAbsolutePath());
    ol();
    
    //circularities are tricky, later
    //includeMap.put(pageloc, f.getCanonicalPath());
    }

  public String toString() {
    return "IncludeDecl: [" + str + "; options: " + opt + "]";
    }
  }

class Import extends Code
  {
  Import(CharArrayWriter buf) {
    super(buf);
    }

  void render() {
    for (int i = 0; i < list.size(); i++) {
      o (list.get(i));
      }
    }
  }

class MollyComment extends Element
  {
  String str;
  
  MollyComment(String str) {
    this.str = str;
    if (dbg) dbgread("<new MollyComment> "+ str); 
    }

  void render() {
    ol(str);
    }
    
  public String toString() {
    return "MollyComment: [" + str + "]";
    }
  }
  
/**
removes starting and trailing single/double quotes. used by the
include/forward render methods only, NOT used while parsing.
*/
private static String removeLeadingTrailingQuote(String str)
  {
  if (str == null)
    return str;

  if ( str.startsWith("\"") || str.startsWith("'") )  {
    str = str.substring(1, str.length());
    }

  if ( str.endsWith("\"") || str.endsWith("'") ) {
    str = str.substring(0, str.length()-1); 
    }

  return str;
  }

//===============================================

public static void main (String args[]) throws IOException
  {
  Args myargs = new Args(args);
  myargs.setUsage("java " + myargs.getMainClassName() 
    + "\n"
      + "Required params:\n"
    + "     -classname output_class_name\n" 
    + "     -in        input_page_file\n"
    + "\nOptional params:\n" 
    + "     -encoding    <page_encoding>\n"
    + "     -out <output_file_name>\n"
    + "        the output file is optional and defaults to the standard out if not specified."
    );
  //String encoding = myargs.get("encoding", Page.DEFAULT_ENCODING);

  File input     = new File(myargs.getRequired("in"));

  PrintWriter output;
  
  if (myargs.get("out") != null)
    output = new PrintWriter(new FileWriter(myargs.get("out")));
  else
    output = new PrintWriter(new OutputStreamWriter(System.out));
    
  TemplateParser parser = new TemplateParser(input, output, myargs.getRequired("classname"), Log.getDefault());
  parser.parse();
  }

}