#!/usr/bin/perl
# GenSite--Generate a static HTML-only web site from templates

# $Id: GenSite.pl 1.12 2003/12/22 06:27:43 JP Exp root $
# $Log: GenSite.pl $
# Revision 1.12  2003/12/22 06:27:43  JP
# Added TO DO notes, changed www4 to www for production!
#
# Revision 1.11  2003/12/01 07:26:35  JP
# Moved &CheckLinks after GenSite variable interpolation.
# Made find (-f) case insensive.
# Moved body margin to CSS from HTML
#
# Revision 1.10  2003/11/23 07:14:33  JP
# Now calculate level once and save it
# Fix broken link count off by 1
#
# Revision 1.9  2003/11/23 04:31:12  JP
# Add s///s to CheckLinks find URL regex
# Renamed $anchor to $url in Checklinks to make more sense
# Renamed $text to $label in Checklinks to make more sense
# Bugfix to correctly omit CVS/RCS directories
# Bugfix for -b in usage and options
#
# Revision 1.8  2003/11/17 07:49:13  JP
# Changed $TemplateDir to www.pages after moving template into CVS
# Changed EM to JPATjpsdomainDOTcom
# Added subrouting name to more error messages.
#
# Revision 1.7  2003/11/13 08:03:04  JP
# Allow $WebSiteDir and $TemplateDir to be the same for easy admin
# Allow -f without -r, and display the results
# Add s///s to -f (so-called 'single line mode', but really lets . match \n)
# Started to add another debug level
#
# Revision 1.6  2003/11/12 07:51:55  JP
# Added search and replace function
# A little misc cleanup and corrections
#
# Revision 1.5  2003/11/10 07:36:47  JP
# Added link count feedback at end of run time
# Added better link checking run time feedback
# Added "global" flag for page interpolation substitutions (bug fix)
#
# Revision 1.4  2003/11/09 10:24:57  JP
# Added 'meta name=GENERATOR'
# Added ability to suppress missing meta Keyword or Desc warnings by using a dash in the template
# Added ability to only process one file with -i (for template testing)
# Added simple local link (file) and remote (URL) checking
#
# Revision 1.3  2003/11/06 07:42:18  JP
# Re-implementednterpolation safely
# Allow pages to be generated but UNLISTED in the nav menu
# Fix level counting
#
# Revision 1.2  2003/11/05 07:23:58  JP
# Allow <a href> in NavOrder.txt to refer to external sites in Nav
# Suppress the warning about entries in NavOrder without pages if trailing -
# Added the "more comments" section
# Added the interpolation code, but that needs work
#
# Revision 1.1  2003/11/04 07:18:02  JP
# Made some fatal errors only warnings
# Made HTML Meta stuff optional
# Added a simple check for a body in the template
# Added more comments
# Fixed Nav Menu links for headings with no page
#
# Revision 1.0  2003/11/03 07:51:42  JP
# Initial Revision
#

##########################################################################
##### TO DOs
#
# * Only copy CSS file if necessary
# * Finish adding another debug level (now 0-3)
# * Change variables to better/more meaningful names
# * Change to a generic open and slurp file subroutine?
# * WARN when external link not <i> and target=_blank, except if $Sitexxxx and $HTMLxxxx
# * Add local bookmark checks
# * Debug why some sites don't work with RLC.
#	LWP User-Agent setting???
# 	Add RCL-Ignore.txt???
# * Correct handling for &amp; in URLs
#
##########################################################################


$ver = '$Revision: 1.12 $'; # JP Vossen <jp@jpsdomain.org>
##########################################################################
(($myname = $0) =~ s/^.*(\/|\\)//ig); # remove up to last "\" or "/"
$Greeting  = ("$myname $ver Copyright 2003 JP Vossen (http://www.jpsdomain.org/)\n");
$Greeting .= ("    Licensed under the GNU GENERAL PUBLIC LICENSE:\n");
$Greeting .= ("    See http://www.gnu.org/copyleft/details.\n");


if (("@ARGV" =~ /\?/) || ("@ARGV" =~ / -h/) || "@ARGV" =~ / --help/) {
    print STDERR ("\n$Greeting\n\n");
    print STDERR <<"EoN";    # Usage notes
Usage: $myname [OPTIONS]

   -w {website}  = Web site destination directory (www).
   -i {page}     = Single template to process.
   -t {template} = Template source directory (www.pages).
   -s {sitename} = Name of the site (jpsdomain.org).
   -u {site URL} = Pubic URL of site (http://www.jpsdomain.org/).
   -c {CSS file} = The CSS file (jpsdomain.css).
   -b {BG file}  = The body cell background image file.
   -S {suffix}   = HTML page suffic (html).
   -T {suffix}   = Template file suffic (page).
   -e {name}     = Contact E-Mail Name (jp{at}jpsdo{at}jpsdomain{dot}org).
   -I {on | off} = Set page body variable interpolation on or off.
   -l {local | remote | both | off } = Do some simple link checking (both).
   -f {term} -r {term} = Global ind and replace a string in the templates.
   -D {0-3}      = Print debug messages to STDERR (0=off, 2=most verbose).
   -q = Be quiet about it.

Generate a static HTML-only web site from template pages.

Runs on Windows and UNIX. Can regenerate a 'live' site if it has access to
the live destination directory, or can run in offline mode to regenerate
files for upload to a hosting site. 

ONLY deals with HTML files and will only overwrite destination HTML files
for which there is a source template.  In other words, binary or other
files such as images are NOT dealt with, and you can have HTML pages in
the destination which are outside the scope of GenSite.pl.

The Navigation menu on the left can be sorted alphabetically, or in a
defined order. If you do not want a defined order, edit this script and
uncomment '\$NavOrderPage = undef;'. If NavOrderPage (-n) is defined but
the file does not exist or is empty, it will be created in alphabetical
order as a template. Edit the file as needed and re-run.

The Navigation menu is limited to two levels, a top level and a sub-level.
Top level items are bold, sub-level items are not bold, and are prefixed
with '-'. The level is indicated in the template file by prefixing the Nav
Name with a dash (-).

If page body variable interpolation (which is implemented as a simple
substitution) is enabled it allows the use of the following variables in
the body of templated pages: $SiteName, $SiteURL, $EmailName,
$EmailAddress and $SiteCopyright. It will slow down program execution a
little bit, and will get confused if any of the above are listed in your
pages.

Search and replace is the only routine that touches the template files, so
be CAREFUL with it.  Make backups!  You have been warned!

BUGS: I got a little carried away with using dashes in the templates files
to mean special things.
EoN
    die ("\n");
} # end of usage

##########################################################################
##### More Comments
# This is a very simple program created by me, for me.  It works for my
# needs for my site, but I have specific needs and constraints. For more
# full featured Perl-based template web site creation tools, check out:
# 	http://perl.apache.org/embperl/
# 	http://www.masonhq.com/
#
# Or, use an entire web site framework like:
# 	http://mamboserver.com/
# 	http://slashcode.com/
#
# My main constraint is that the ISP where most of the site is hosted does
# not support SSI, CGI, or anything else that would facilitate a modern
# site. I have various reasons for staying there, not the least of which is
# inertia. So I needed a simple solution that would create a navigable site
# without any those technologies. Also, after the last redesign JavaScript
# fiasco, I wanted totally PURE HTML. Since the site is mostly static, that
# was fine, except for being able to actually maintain it. So I wrote this.
#
# The other trivia is that the site (www-t and www) directories are hosted
# on Linux, running Samba.  But I access them mostly from a Windows
# workstation, which is where I run this script from as well. When the site
# is regenerated, I can manually kick off an rsync from the Linux box to
# the ISP.  Or even better, I can edit the templates and do nothing. Cron on
# the Linux box will re-gen the site and upload it as needed.  Nice!
#
# Debug Levels (NOT FINISHED YET!!!)
# 0 Off
# 1 More detail
# 2 Find or find and replace output
# 3 ?????
# 4 ?????
##########################################################################


use Getopt::Std;      # Use Perl5 built-in program argument handler
getopts('w:i:t:s:u:c:b:N:S:T:i:l:f:r:D:q'); # Define possible args.

use File::Find;  # Used to find the files in the templates dir
use File::Path;  # Used to make directories
use File::Copy;  # Used to copy files

# Set defaults
# The following two can be the same directory, for easy admin, or
# different dirs, for easy use with CVS. Do what ever is easier and make
# more sense to you.
$WebSiteDir     = $opt_w || 'www';       # Destination dir for gen templates
$TemplateDir    = $opt_t || 'www.pages'; # Source dir for templates
$SinglePage     = $opt_i;                # Single template to process
$SiteName       = $opt_s || 'jpsdomain.org';
$SiteURL        = $opt_u || 'http://www.jpsdomain.org/';
$HTMLSuffix     = $opt_S || 'html';     # Could be 'htm' also
$TemplateSuffix = $opt_T || 'page';     # Template file extension
$CSSPage        = $opt_c || "jpsdomain.css";       # Name of CSS file
$BodyCellBG     = $opt_b || "images/blutxtr1.jpg"; # Name of BG image for 'body' cell
$Google         = $opt_g || "images/Google_Logo_40gry.gif"; # Google logo
$NavOrderPage   = $opt_n || "NavOrder.txt";  # Name of Nav Menu order file
# $NavOrderPage = undef;  # Set this for an alphabetical Nav menu
$PageBodyInterp = $opt_I || "on";
$LinkCheck      = $opt_l || "off";  # local, remote, both, off
$EmailName      = $opt_e || "JPATjpsdomainDOTcom";
$EmailAddress   = $opt_E || "$EmailName";
$SiteCopyright  = "Copyright &#169; 1995-2003, JP Vossen. All rights reserved.";

@Templates   = ();  # Global define for list of template files
@Directories = ();  # Global define for list of site directories
@NavKeys     = ();  # Global define for Nav keys/sort order
$PageBody    = '';  # Global define for the body of the page
%RemoteLinkStatus = '';
%LocalLinkStatus  = '';
$BrokenLocalLinkCount  = 1; # Start at 1 because it's easier to read
$BrokenRemoteLinkCount = 1; # in the debug output when correcting errors
$RemoteLinkCount = 0;       # Have to subtract 1 in the counts though
$LocalLinkCount  = 0;
##########################################################################
##### MAIN
if (! $opt_q) { print ("\n$Greeting\n"); }

# Do a sanity check for find and replace
if ((defined $opt_r) and (! $opt_f)) { die ("$myname: Can't have -r without -f!\n"); }

# If we are just doing a single page, make sure it exists
if (defined($SinglePage)) {
    if ($opt_D > 1) { warn ("$myname:main: SinglePage = '$SinglePage' before s!\!/!.\n"); }
    $SinglePage =~ s!\\!\/!g;   # Replace DOS '\' with UNIX/Perl '/'
    if ($opt_D > 1) { warn ("$myname:main: SinglePage = '$SinglePage' after s!\!/!.\n"); }
    if (-f $SinglePage) {
        if (! $opt_q) { print ("Looking for single page '$SinglePage'...\n"); }
    } elsif (-f "$SinglePage.$TemplateSuffix") {
        if (! $opt_q) { print ("Looking for page '$SinglePage' + '.$TemplateSuffix'...\n"); }
        $SinglePage = "$SinglePage.$TemplateSuffix";
    } else {
        die ("$myname: '$SinglePage' is not readable!\n");
    } # end of file checks
} # end of make sure single page exists

if (! $opt_f) {  # If doing find and replace, we don't do ANYTHING else...
    # Only load LWP if we have to--it's big
    if ($LinkCheck eq "remote") { use LWP::Simple qw(get head); }
} # If we do a find and replace, we don't do ANYTHING else...

if (! $opt_q) { print ("Building template list from '$TemplateDir/*.$TemplateSuffix' files...\n"); }
find(\&BuildTemplateList, $TemplateDir); # Build a list of *.page files in the templates dirs

if (! $opt_f) {  # If doing find and replace, we don't do ANYTHING else...
    if (! $opt_q) { print ("Reading templates from '$TemplateDir/*.$TemplateSuffix' files...\n"); }
    foreach $template (@Templates) {
        &ReadTemplateFile(Details); # Read the details lines from each template
    } # end of reading template details

    if (! $opt_q) { print ("Building the master navigation menu...\n"); }
    &BuildMasterNav;        # Build the master navigation table

    if (! $opt_q) { print ("Building directory structure in '$WebSiteDir'...\n"); }
    &MakeSiteDirs;          # Make sure site sub-dirs exist

    if ($WebSiteDir ne $TemplateDir) { # Only need to copy if the dirs are DIFFERENT
        if (! $opt_q) { print ("Copying CSS file '$TemplateDir/$CSSPage' to '$WebSiteDir/$CSSPage'...\n"); }
        copy ("$TemplateDir/$CSSPage", "$WebSiteDir/$CSSPage") or die ("$myname:WriteHTMLFile could not copy $TemplateDir/$CSSPage to $WebSiteDir/$CSSPage: $!\n");
    } # end of check to see if we copy the CSS file

    if (! $opt_q) {
        print ("Page body interpolation is: $PageBodyInterp.\n");
        print ("Link checks are: $LinkCheck.\n");
        if (($LinkCheck eq "remote") or ($LinkCheck eq "both")) {
            print ("Remote Link checks require internet connectivity and can be VERY SLOW!\n");
        } # end of whine about remote link checks
        print ("Writing the new '$WebSiteDir/*.$HTMLSuffix' files...\n");
    } # end of some user feedback
} # If we do a find and replace, we don't do ANYTHING else...

# Are we writing a single page, or the entire site?
if (defined($SinglePage)) {
    if (! $opt_q) { print ("Processing single page '$SinglePage'...\n"); }
    @Templates = $SinglePage;  # JUST do this page
} # Single page or entire site?

if (! $opt_f) {  # If doing find and replace, we don't do ANYTHING else...
    foreach $template (@Templates) {
        &ReadTemplateFile(''); # Read the body of template file
        &WriteHTMLFile;        # Write the generated web site page
    } # end of write the new site pages

    if (! $opt_q) {
        print ("\n\a$myname built ",@Templates+0," pages in ",time()-$^T," seconds.\n");
        unless ($LinkCheck eq "off") { print ("\tChecked\tTotal\tGood\tBad\n"); }
        if (($LinkCheck eq "remote") or ($LinkCheck eq "both")) {
            print ("\tRemote\t$RemoteLinkCount\t", $RemoteLinkCount - $BrokenRemoteLinkCount - 1,"\t", $BrokenRemoteLinkCount - 1,"\n");
        } # end of remote link summary
        if (($LinkCheck eq "local") or ($LinkCheck eq "both")) {
            print ("\tLocal\t$LocalLinkCount\t", $LocalLinkCount - $BrokenLocalLinkCount - 1,"\t", $BrokenLocalLinkCount - 1,"\n");
        } # end of local link summary
    } # end of ending feedback
} else {

    # Not that we have, somewhat clumsily, only done the things necessary
    # to prep for the find and replace, actually DO the work
    &FindandReplace;

    if (! $opt_q) { print ("\n\a$myname processed ",@Templates+0," pages in ",time()-$^T," seconds.\n"); }
} # If we do a find and replace, we don't do ANYTHING else...


##### End of MAIN
##########################################################################

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub FindandReplace {

    my $termasfound;
    my $search  = $opt_f;           # Assign better variables
    if (defined $opt_r) {
        if (! $opt_q) { print ("Doing find and replace:\n"); }
        if ($opt_D > 0) { warn ("\n$myname:FindandReplace: 1 search = ~$search~, replace = ~$replace~\n"); }
    } else {
        if (! $opt_q) { print ("Doing find:\n"); }
        if ($opt_D > 0) { warn ("\n$myname:FindandReplace: 1 search = ~$search~\n"); }
    } # end of if we are doing a replace

    $search = eval "qq{$search}";        # Interpolate

    if (defined $opt_r) {
        $replace = $opt_r;
        $replace = eval "qq{$replace}";  # Interpolate
        if ($opt_D > 0) { warn ("\n$myname:FindandReplace: 2 search = ~$search~, replace = ~$replace~\n"); }
    } else {
        if ($opt_D > 0) { warn ("\n$myname:FindandReplace: 2 search = ~$search~\n"); }
    } # end of if we are doing a replace

    my $irs = $/;           # Save the default input record separator
    undef($/);              # Undefine input record separator

    foreach $template (@Templates) {
        $count = 0;  # Just to be sure...

        # Open and read an entire template file
        if ($opt_D > 0) { warn ("\n$myname:FindandReplace: Reading template '$template'\n"); }
        open (TEMPLATE, "$template") or die ("$myname:FindandReplace: error opening '$template' for input: $!\n");
        $Page = <TEMPLATE>; # Grab the entire template
        close TEMPLATE;

        if (defined $opt_r) {   # we are doing a replace
            # Do the replacement and count them, if any
            $count = ($Page =~ s/$search/$replace/gs) || 0;

            if ($count > 0) {  # Don't (re-)write if we don't have to
                # Write the new template file
                if ($opt_D > 0) { warn ("\n$myname:FindandReplace: Writing template '$template'\n"); }
                open (TEMPLATE, ">$template") or die ("$myname:FindandReplace: error opening '$template' for output: $!\n");
                print TEMPLATE ("$Page");
                close TEMPLATE;
            } # end of don't (re-)write if we don't have to
    
            # Gloat if we made it this far...
            print ("Made $count replacements in $template.\n");

        } else {
            $count = ($Page =~ s/($search)/$1/gsi) || 0;
            if ($count > 0) { $termasfound = $1; }
            if ($opt_D > 0) { warn ("\n$myname:FindandReplace: term as found = ~$1~\n"); }
            print ("Found $count instances in $template.\n");
       } # end of actually do something
    } # end of find and replace
    print ("Search term as found: ~$termasfound~\n");
    $/ = $irs;              # Restore the default input record separator
} # end of sub FindandReplace


#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub BuildTemplateList {
    # Build a list of *.page files in the templates dirs
    # Already got the name from the File::Find module

    if ($opt_D > 1) { warn ("$myname:BuildTemplateList: Saw '$File::Find::name'\n"); }

    if ($File::Find::name =~ m/\.$TemplateSuffix$/) {
        if ($opt_D > 1) { warn ("$myname:BuildTemplateList: Pushing template '$File::Find::name'\n"); }
        push(@Templates, $File::Find::name)
    } # end of get only template file names

    # Keep track of directories (-d test) [I don't know why
    # 'if (-d $File::Find::name)' doesn't work, but it doesn't].
    if (-d) {
        $subdir = $File::Find::name;
        $subdir =~ s/$TemplateDir/$WebSiteDir/;
        if ($subdir !~ m/CVS$|RCS$/) {
            if ($opt_D > 1) { warn ("$myname:BuildTemplateList: Pushing DIR '$subdir'\n"); }
            push(@Directories, $subdir);
        } # end of should we keep this directory?
    } # end of get only template file names

} # end of sub BuildTemplateList

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub ReadTemplateFile {
    # Read a template file
    # Note: we read each of these files twice, once just to get the
    # Navigation line and once to process the file. We could read each file
    # only once, but then we would be memory constrained which could be a
    # limit for a large site.  I chose to use the slower but unlimited
    # method.  On the other hand, the navigation bar will get out of hand
    # for a large site, so the is not really scalable.  But this script
    # could be extended to support navigation sub-menus or something...

    # Open the file
    if ($opt_D > 1) { warn ("\n$myname:ReadTemplateFile: Opening template '$template'\n"); }
    open (TEMPLATE, "$template") or die ("$myname:ReadTemplateFile: error opening '$template' for input: $!\n");

    if (@_[0] eq 'Details') {  #  Read the template page details

        # Nav Name is mandatory
        chomp($PageDetails{Nav}{$template} = <TEMPLATE>);
        $PageDetails{Nav}{$template} =~ s/^Nav Name: (.*)$/$1/ or die ("$myname:ReadTemplateFile: Bad or missing 'Nav Name' in '$template'\n");
        $NavNameHash{$PageDetails{Nav}{$template}} = $template; # Build a 'Nav Name' to 'file name' lookup hash
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: Nav Name: '$PageDetails{Nav}{$template}'\n"); }

        # Meta Keywords is optional
        chomp($PageDetails{Key}{$template} = <TEMPLATE>);
        $PageDetails{Key}{$template} =~ s/^Meta Keywords: (.+)$/$1/ or warn ("$myname:ReadTemplateFile: Bad or missing 'Meta Keywords' in '$template'\n");
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: Meta Keywords: '$PageDetails{Key}{$template}'\n"); }

        # Meta Description is optional
        chomp($PageDetails{Des}{$template} = <TEMPLATE>);
        $PageDetails{Des}{$template} =~ s/^Meta Description: (.+)$/$1/ or warn ("$myname:ReadTemplateFile: Bad or missing 'Meta Description' in '$template'\n");
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: Meta Description: '$PageDetails{Des}{$template}'\n"); }

        # Title is mandatory
        chomp($PageDetails{Til}{$template} = <TEMPLATE>);
        $PageDetails{Til}{$template} =~ s/^Title: (.*)$/$1/ or die ("$myname:ReadTemplateFile: Bad or missing 'Title' in '$template'\n");
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: Title: '$PageDetails{Til}{$template}'\n"); }

        # Last Modified is mandatory
        chomp($PageDetails{Mod}{$template} = <TEMPLATE>);
        $PageDetails{Mod}{$template} =~ s/^Last Modified: (.*)$/$1/ or die ("$myname:ReadTemplateFile: Bad or missing 'Last Modified:' in '$template'\n");
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: Last Modified: '$PageDetails{Mod}{$template}'\n"); }

        # The body separator is mandatory
        my $trash = <TEMPLATE>;
        $trash =~ m/^$/ or die ("$myname:ReadTemplateFile: Bad or missing body separator in '$template'\n");

        # The body is mandatory
        $trash = <TEMPLATE>;
        $trash =~ m/\w+/ or die ("$myname:ReadTemplateFile: Bad or missing body in '$template'\n");

        # Build the relative URL (used in the web site itself
        $PageDetails{Url}{$template} = $template;
        $PageDetails{Url}{$template} =~ s/$TemplateSuffix$/$HTMLSuffix/;
        $PageDetails{Url}{$template} =~ s!$TemplateDir/!!;
        if ($opt_D > 1) { warn ("$myname:ReadTemplateFile: R-URL: '$PageDetails{Url}{$template}'\n"); }


    } else {
        if ($opt_D > 0) { warn ("$myname:ReadTemplateFile: Reading body\n"); }

        my $trash = <TEMPLATE>;  # Skip Nav Name
        my $trash = <TEMPLATE>;  # Skip Meta Keywords
        my $trash = <TEMPLATE>;  # Skip Meta Description
        my $trash = <TEMPLATE>;  # Skip Title
        my $trash = <TEMPLATE>;  # Skip Last Modified

        my $trash = <TEMPLATE>;  # Skip body separator

        my $irs = $/;           # Save the default input record separator
        undef($/);              # Undefine input record separator
        $PageBody = <TEMPLATE>; # Grab the entire rest of the template (body)
        $/ = $irs;              # Restore the default input record separator


    } # end of figuring out what to read
    close TEMPLATE;

} # end of sub ReadTemplateFile

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub BuildMasterNav {
    # Build the master navigation table (@NavKeys)
    if ($opt_D > 0) { warn ("\n$myname:BuildMasterNav'\n"); }

    # Figure out the sort order of the navigation menu
    # The key is the 'Nav Name' field from the template!

    if (defined($NavOrderPage) and -s "$TemplateDir/$NavOrderPage") {
        # We are using a defined order and the file exists and has non-zero size
        if ($opt_D > 0) { warn ("$myname:BuildMasterNav: We are using a defined order and '$NavOrderPage' exists and has non-zero size\n"); }

        open (NAVSORTORDER, "$TemplateDir/$NavOrderPage") or die ("$myname:BuildMasterNav: error opening '$TemplateDir/$NavOrderPage' for input: $!\n");
        while ($aline = <NAVSORTORDER>) {
            chomp($aline);
            if ($aline =~ m/(^#)|(^\s*$)/) { next; }  # Skip comments and whitespace
            push(@NavKeys, $aline);  # Add to the Nav Key list
            $NavKeyHash{$aline}++;   # Add to a hash for Sanity Checks (below)
            if ($opt_D > 1) { warn ("$myname:BuildMasterNav: Static sort '$aline'\n"); }
        } # end of while input
        close NAVSORTORDER;

        # Do some sort order Sanity Checks 1
        foreach $navname (sort keys %NavNameHash) {

            # Make sure no new pages were created but NOT added to the NavOrder file!
            if ((! defined $NavKeyHash{$navname}) and ($navname !~ m/UNLISTED/)) {
                print ("'$navname' not listed in $TemplateDir/$NavOrderPage!\n");
                if ($opt_D > 0) { warn ("$myname:BuildMasterNav: $TemplateDir/$NavOrderPage is missing an entry for '$navname' which means it won't be listed in the navigation menu. If this is not intended edit the file and add the reference then re-run.\n"); }
            } # end of sanity check 1

            # Make sure there are no duplicate Nav Names in the NavOrder file!
            if ($NavKeyHash{$navname} > 1) {
                die ("$myname:BuildMasterNav: Duplicate entry for '$navname'!  Edit the file and correct the sort order then re-run.\n");
            } # end of sanity check 2

        } # end of sort order Sanity Checks A

        # Make sure no pages were deleted but NOT removed from the NavOrder file!
        # Allow a trailing dash to suppress this warning in case you want
        # nav menu items (i.e. catagory headers) with no associated page.
        # Also, don't warn if there is an <h ref>
        foreach $navname (keys %NavKeyHash) {
            if ((! defined $NavNameHash{$navname})
              and ($navname !~ m/-$/) and ($navname !~ m/<a.*href="/)) {
                print ("'$navname' does not exist but is in $TemplateDir/$NavOrderPage!\n");
                if ($opt_D > 0) { warn ("$myname:BuildMasterNav: $TemplateDir/$NavOrderPage has an entry for '$navname' but there is no page with that name! If this is not intended, edit the file and remove the extra reference then re-run.\n"); }
            } # end of sanity check 1
        } # end of sort order Sanity Checks B
    } # end of defined order, file exists and is non-zero


    if (! defined($NavOrderPage) or  ! -s "$TemplateDir/$NavOrderPage") {
        # We are NOT using a defined order, so build an alphabetical list
        if ($opt_D > 0) { warn ("$myname:BuildMasterNav: We are NOT using a defined order, so build an alphabetical list\n"); }
        foreach $navname (sort keys %NavNameHash) {
            push(@NavKeys, $navname);
            if ($opt_D > 1) { warn ("$myname:BuildMasterNav: Alphabetical sort '$navname'\n"); }
        } # end of while input
    } # end of NOT using defined order OR file no good


    if (defined($NavOrderPage) and ! -s "$TemplateDir/$NavOrderPage") {
        # We are using a defined order and the file does NOT exist or has zero size
        if ($opt_D > 0) { warn ("$myname:BuildMasterNav: We are using a defined order and '$TemplateDir/$NavOrderPage' does NOT exist or has zero size.  Creating it.\n"); }

        open (NAVSORTORDER, ">$TemplateDir/$NavOrderPage") or die ("$myname:BuildMasterNav: error opening '$TemplateDir/$NavOrderPage' for output: $!\n");

        print NAVSORTORDER ("# $TemplateDir/$NavOrderPage\n");
        print NAVSORTORDER ("# Generated by $myname on ",scalar localtime(time), "\n");
        print NAVSORTORDER ("#\n");
        foreach $key (@NavKeys) {
            print NAVSORTORDER ("$key\n");
        } # end of write each key

    } # end of defined order, file NOT exist or is zero
} # end of sub BuildMasterNav

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub MakeSiteDirs {
    # Make sure site sub-dirs exist
    if ($opt_D > 0) { warn ("\n$myname:MakeSiteDirs\n"); }

    # File::Path::mkpath(List of Dirs, Verbose(0|1), File Mode)
    mkpath(\@Directories, 1, 0744);

} # end of sub MakeSiteDirs

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub WriteHTMLFile {
    # Write the generated web site page
    if ($opt_D > 0) { warn ("\n$myname:WriteHTMLFile: template '$template'\n"); }

    # Force interpolation of the body of the page. That means you can use
    # the Perl variables defined above in your page body. This is probably
    # only useful for $SiteName, $SiteURL, $EmailName, $EmailAddress and
    # maybe $SiteCopyright, but it's kind of cool.
    # UPDATE: OK this works, but turns out to be a Bad Idea! It exposes
    # some nasty security risks, and can easily screw up a page that either
    # actually contains other Perl code, or has stuff that looks like Perl
    # code.
    # $PageBody = eval "qq{$PageBody}";

    # THIS is a much better idea, but slower and a little clunky
    if ($PageBodyInterp =~ m/on/i) {
        $PageBody =~ s/\$SiteName/$SiteName/g;
        $PageBody =~ s/\$SiteURL/$SiteURL/g;
        $PageBody =~ s/\$EmailName/$EmailName/g;
        $PageBody =~ s/\$EmailAddress/$EmailAddress/g;
        $PageBody =~ s/\$SiteCopyright/$SiteCopyright/g;
    } # end of page body "interpolation"

    &CheckLinks unless ($LinkCheck eq "off");  # Simple link checks

    # Change the file name from the template (.page) to HTML (.html)
    # And from the template dir to the web site dir
    my $newpage = $template;
    $newpage =~ s/$TemplateSuffix$/$HTMLSuffix/;
    $newpage =~ s/$TemplateDir/$WebSiteDir/;

    # Create the relative page URL (for use in actual links)
    my $relativepage = $newpage;
    $relativepage =~ s!$WebSiteDir/!!;

    if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: newpage = '$newpage'; relativepage = '$relativepage'\n"); }

    # Get the depth of directories (for use with relative paths)
    # Note we search and replace / with / to count the number of /
    # The m// operator only returns true or false.  s/// returns the count.
    my $level = ($relativepage =~ s!/!/!g);
    if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: level = '$level'\n"); }
    $currentlevel = '../' x $level;

    # Create some relative links needed in 'HERE' document
    my $relativecss    = $currentlevel . $CSSPage;
    my $relativebcbg   = $currentlevel . $BodyCellBG;
    my $relativegoogle = $currentlevel . $Google;
    if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: relativecss = '$relativecss'; relativebcbg = '$relativebcbg'\n"); }

    # Allow the HTML Meta tags to be optional
    # Note that the 2 variables are placed on the same line before the
    # <title> tag. Thus, if the tags exist, the will be in the code.  If
    # they don't exist, there will NOT be blank lines in the HTML code.
    my $meta_keywords = '';     # Make sure they don't carry over
    my $meta_description = '';  # Make sure they don't carry over
    if ($PageDetails{Key}{$template} !~ m/^Meta Keywords:|^-/) {
        $meta_keywords = "<meta name=KEYWORDS content=\"$PageDetails{Key}{$template}\">\n";
        if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: <meta name=KEYWORDS content=\"$PageDetails{Key}{$template}\">\n"); }
    } # end of tag for Meta Keywords if it exists
    if ($PageDetails{Des}{$template} !~ m/^Meta Description:|^-/) {
        $meta_description = "<meta name=DESCRIPTION content=\"$PageDetails{Des}{$template}\">\n";
        if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: <meta name=DESCRIPTION content=\"$PageDetails{Des}{$template}\">\n"); }
    } # end of tag for Meta Description if it exists

    if ($opt_D > 0) { warn ("\n$myname:WriteHTMLFile: newpage '$newpage'\n"); }
    open (NEWPAGE, ">$newpage") or die ("$myname:WriteHTMLFile: error opening '$newpage' for output: $!\n");

#<body topmargin=0 leftmargin=0 marginheight=0 marginwidth=0>

# Write the first part of the HTML page
print NEWPAGE <<"EOP1";
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=iso-8859-1">
<link rel="stylesheet" href="$relativecss" type="text/css">
<meta name=GENERATOR content="$myname $ver">
${meta_keywords}${meta_description}<title>$SiteName: $PageDetails{Til}{$template}</title>
</head>
<body>


<!-- TITLE table. -->
<table border="0" width="100%" cellpadding="2" cellspacing="0">
  <tr>
    <td class="title">$PageDetails{Til}{$template}</td>
  </tr>
</table>


<!-- Navigation menu and body table. -->
<table border="0" width="100%" cellpadding="2" cellspacing="0">
  <tr>
    <td class="nav" valign="top">
EOP1


    ########################
    # Do the navigation menu
    foreach $navname (@NavKeys) {

        # Get the page title by first getting the template file name out
        # of NavNameHash, then using the PageDetails indexed hash
        $pagetitle = $PageDetails{Til}{$NavNameHash{$navname}};

        # Get the relative URL we already put in the hash, like above
        $relativeurl = $PageDetails{Url}{$NavNameHash{$navname}};

        $navname1 = $navname;  # Avoid modifying the @NavKeys array
        if ($navname1 =~ s/^-//) {
            # Nav Menu sub-catagory (most common)
            if ($navname eq $PageDetails{Nav}{$template}) {
                # Menu item for current page
                print NEWPAGE ("      - <b><span class=\"current\">$navname1</span></b><br>\n");
            } elsif ($navname1 =~ m/<a.*href="/) {
                # URL to external link in sub-catagory
                print NEWPAGE ("      - $navname1<br>\n");
            } else {
                # Sub item
                print NEWPAGE ("      - <a href=\"${currentlevel}$relativeurl\" title=\"$pagetitle\">$navname1</a><br>\n");
            } # end of are we current page
        } elsif (! defined $NavNameHash{$navname}) {
            $navname1 =~ s/-$//; # Remove the trailing dash (warning suppressor)
            # Menu item entry (catagory header) but no link!
            print NEWPAGE ("      <b>$navname1</b><br>\n");
        } else {
            # Nav Menu main catagory (least common, default)
            if ($navname eq $PageDetails{Nav}{$template}) {
                # Menu item for current page
                print NEWPAGE ("      <b><span class=\"current\">$navname1</span></b><br>\n");
            } elsif ($navname1 =~ m/<a.*href="/) {
                # URL to external link in catagory header
                print NEWPAGE ("      $navname1<br>\n");
            } else {
                # Main item
                print NEWPAGE ("      <a href=\"${currentlevel}$relativeurl\" title=\"$pagetitle\"><b>$navname1</b></a><br>\n");
            } # end of are we current page
        } # end of nav level check

        if ($opt_D > 1) { warn ("$myname:WriteHTMLFile: NAV: navname = '$navname'; relativeurl = '$relativeurl'; pagetitle = '$pagetitle'\n"); }

    } # end of foreach page in the nav menu
    ########################


# Write the second part of the HTML page
print NEWPAGE <<"EOP2";
      <hr>
      <a href="mailto:$EmailAddress" title="$EmailName">Email me</a><br>
      <a href="${currentlevel}jp/cnt-jp.$HTMLSuffix" title="$EmailName">Email Form</a>
      <hr>
    </td>
    <td class="bodycell" valign="top" style="background-image: url('$relativebcbg')">
    <!-- BEGIN real BODY of the page. -->

$PageBody

    <!-- END real BODY of the page. -->
    </td>
  </tr>
</table>


<!-- FOOTER table. -->
<table border="0" width="100%" cellpadding="2" cellspacing="0" class="footer">
  <tr>
    <td>
      <a href="$SiteURL$relativepage" title="This page">
      $SiteURL$relativepage</a><br>
      <cite>$SiteCopyright</cite><br>
      Last Modified: $PageDetails{Mod}{$template}
    </td>
      <td align="center">
      <!-- http://www.google.com/searchcode.html#both -->
      <form method="GET" action="http://www.google.com/search">
        <input type="text" name="q" size="40" maxlength="255" value="">
        <input type="submit" name="btnG" value="Google Search">
        <input type="hidden" name="domains" value="$SiteName"><br>
        <input type="radio" name="sitesearch" value=""> WWW
        <input type="radio" name="sitesearch" value="$SiteName" checked> $SiteName
      </form>
    </td>
    <td align="center">
      <a href="http://www.google.com/"><img src="$relativegoogle" border="0" alt="Google" width="128" height="53"></a>
    </td>
    <!-- SiteSearch Google -->
  </tr>
</table>

</body>
</html>

EOP2
   close NEWPAGE;

} # end of sub WriteHTMLFile

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
sub CheckLinks {
    # Do some simple link rot checks.
    if ($opt_D > 0) { warn ("\n$myname:CheckLinks: checking links in '$template'\n"); }

    # Get a local, relative PATH to use with file tests
    ($relativepath = $template) =~ s!$TemplateDir!$WebSiteDir!;
    if ($relativepath !~ s!^(.*)/.*$!$1!) {
        die ("$myname:CheckLinks: could not get relative path from $template!\n");
    } # end of get relative path

    # Extract the links from the page body
    while ($PageBody =~ m!(<a .*?">(.*?)</a>)!gs) {
        $url = $1;
        $label   = $2;
        $href = $target = $name = '';

        if ($url =~ m!href="(.*?)"!)   { $href   = $1; }
        if ($url =~ m!target="(.*?)"!) { $target = $1; }
        if ($url =~ m!name="(.*?)"!)   { $name   = $1; }

        if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template url:\t$url\n"); }
        if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template label:\t$label\n"); }
        if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template href:\t$href\n"); }
        if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template target:\t$target\n"); }
        if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template name:\t$name\n"); }

        if ($href =~ m!^mailto:!i) { next; }  # Ignore mailto's
        if ($href =~ m!^#!)        { next; }  # Ignore bookmarks
        if ($name !~ m!^$!)        { next; }  # Ignore bookmark names

        $href =~ s!$HTMLSuffix#.*$!$HTMLSuffix!; # Remove bookmarks in links

        # Do some simple checking based on whether the link is "local"
        if ($href =~ m!:!) { # If there is a ':' we assume the link is NOT "local"
            $RemoteLinkCount++;

            # We have to be efficent here and cache the links we check so
            # we can save bandwidth and run faster
            if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template REMOTE href:\t$href\n"); }
            if (($LinkCheck eq "remote") or ($LinkCheck eq "both")) {
                if (! $opt_q) { print ("Checking REMOTE link: '$href':"); }

                # If we do not already have this in the cache, check it
                if (! exists $RemoteLinkStatus{$href}) { # not cached
                    if (! $opt_q) { print ("\tnot cached:"); }
                    if (! head($href)) {
                        $RemoteLinkStatus{$href} = 'Bad';
                    } else {
                        $RemoteLinkStatus{$href} = 'Good';
                    } # end of remote link check
                } else {   # already cached
                    if (! $opt_q) { print ("\tcached:"); }
                } # end of check the cache

                # NOW, we have it cached, what's the deal?
                if ($RemoteLinkStatus{$href} eq 'Bad') {
                    if (! $opt_q) { print ("\tBad\n"); }
                    warn ($BrokenRemoteLinkCount++, " BROKEN remote link in '$template':\n");
                    warn ("\tlabel:\t$label\n\thref:\t$href\n\n");
                } else { # it's good
                    if (! $opt_q) { print ("\tGood\n"); }
                } # end of wha't the status
            } # end of are we doing REMOTE link checking?
        } else {   # We assume it is "local"
            $LocalLinkCount++;

            if ($opt_D > 0) { warn ("\n$myname:CheckLinks: $template LOCAL href:\t$href\n"); }
            if (($LinkCheck eq "local") or ($LinkCheck eq "both")) {

                $rhref = "$relativepath/$href";
                # If we do not already have this in the cache, check it
                if (! exists $LocalLinkStatus{$rhref}) {
                    if (! -s "$rhref") {
                        $LocalLinkStatus{$rhref} = 'Bad';
                    } else {
                        $LocalLinkStatus{$rhref} = 'Good';
                    } # end of local  file test
                } # end of check the cache

                # NOW, we have it cached, what's the deal?
                if ($LocalLinkStatus{$rhref} eq 'Bad') {
                    warn ($BrokenLocalLinkCount++, " BROKEN local link in '$template':\n");
                    warn ("\tlabel:\t$label\n\thref:\t$href\n");
                    warn ("\twants:\t$rhref\n\n");
                } # end of wha't the status
            } # end of are we doing local link checking?
        } # end of local link or not?
    } # end of while extract links from page body
} # end of sub CheckLinks

#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++