/* catm.c
   Concatenate, merge, and expand files.

   Usage:
         catm [-f name][-s char][-e end-of-page-text] listfile file2 file3
      or
         catm [-f name][-s char][-e end-of-page-text] listfile -o file-once -r file3 file4
      or
         catm [-f name][-s char][-e end-of-page-text] listfile -o file-once -r file3 fil4 -o file2-once

   "catm" will read the first file ("listfile") and repeatedly copy the 
      remaining files to standard output, performing substitutions based
      on information read from the first.  "listfile" is the field values file,
      and will contain the names of "fields" and sets of values for
      them.  The files repeatedly copied will be searched for references to
      the field names in <> brackets, and such references to a field will
      be replaced  by the current value of this field as determined from 
      "listfile".

      Redirection of this output to a file or a pipe is most common.
      
      The "-f" option, which must precede the "listfile", indicates that the
      field-names are defined in a separate file.  This allows the "listfile"
      to contain only field values, without needing the field names to be
      defined at the top with the first set of fields.  This file can also
      contain the "END=" directive described below, further
      enabling field content to be completely separated from "catm" processing
      instructions.  Note that "END" is a reserved field name.

      The "-s" option allows the field separator character to be changed from
      the Tab character.  This only has effect if the field file is organized
      with multiple field values per line.
      
      The "-e" option allows the value of an "END=" directive to be specified 
      on the command-line.  If field names are given as the first line of 
      "listfile", then this option removes any need for a separate file 
      containing this directive.  If the END text contains blanks, it will 
      need to be given in quotes.  The default end text could be given with
      -e "%NP%%PS%\nQSdict begin Init ToTop end PE\n"

   While copying the files:
      If the text string "%include file-name%" is encountered, the file named 
         will be included in the output at that point.  If the text string 
         "%DATE%" is encountered, the current date will be
         substituted.  (Include processing).
      If the text string "<text>" is encountered, then it will be replaced
         by the current value associated with the name "text" found from the
         first file (listfile) in the above examples.  (Field processing).
      If the text %DM,name:string:% is encountered, it will be taken as
         an defining a macro.  Wherever %M,name% is subsequently found, 
         the text in "string" will be substituted.  (Macro processing)

   COMMAND ARGUMENT PROCESSING:
      The list of arguments is interpreted as filenames, except for the
         flags "-o" meaning output once, and "-r" meaning output repeatedly.
      The first file is read and interpreted as a set of field definitions
         that will be applied to other files (the ones to be repeatedly output).
      If a "-o" is encountered after the first file, it indicates that the
         following files are to be output directly with only "include"
         processing and macro processing being performed.
      If a "-r" is encountered after the first file, it indicates that the
         following files are to be repeatedly output, in turn using each set 
         of field values from the first file.  On each occasion, full
         "include" and macro processing is performed.
      If neither "-o" nor "-r" are present, all files after the first are
         repeatedly output with field, macro and include processing.

   FIELD PROCESSING
      Files for repeated processing are read in turn and copied to standard 
         output.
         If a string <text> is encountered in the file, where "text" is
            a field-name defined in the first file or in a "-f" field-name 
            file, the value of the field is substituted for the string.
      This process of substituting and copying all the remaining
         files is repeated for each of the groups of field definitions in 
         the first file.
      The simplest way to set up the fields file is to provide on the first
         line the names of the fields, Tab separated.  Subsequent lines will
         then contain the values of the fields for the next version of the
         document being generated.
      If the "-f" option is used, then the file specified after "-f" will be
         read seeking the field names, rather than expecting them at the head of
         the field values file.
      An alternative layout of the field values file is as follows:
         END=text           ->Defines what should separate the documents
                              generated by each group of fields.
                              If omitted, the default text will be
                              %NP%%PS%\nQSdict begin Init ToTop end PE\n
         field1=text         ->Occurrence of <field1> in a file will be
         field2=text           replaced by the substitute text given here
         . . .                after the '='.  Within the text, 
                              \n means newline, \t means tab, \ before 
                              end of line means include the next line with
                              this substitute text.
         NEXT               ->Ends the field definitions for the first
                              document to be generated.  The file arguments
                              will be processed using these fields, then
                              the fields will be emptied before the next
                              set of fields is read.
         field1=text
         . . .              ->End-of-file is equivalent to a 'NEXT'
      Under this scheme, values for the fields must be in the same order for each document,  
         though some may be omitted.  The occurrence of the last field will 
         be sufficient to cause that document to be generated, without the 
         need for an explicit NEXT.  After the first NEXT, lines that
         lack a fieldn= prelude are taken as specifying a value for the next 
         field name.
      If the "-f" option is used to define field names, the "-f" file will
         contain the field names, separated by tabs or new-line characters, and
         possibly containing the END= directive.  The file of fields will
         contain values for the fields, tab separated, in 
         the same order as the field names.
      When a reference to a <fieldname> is found in a document to be 
         output, the current replacement text is treated as if it is part of 
         the input document.  As this replacement text is output, it is 
         scanned for any macro references, include directives, or other 
         field references which are processed as they are encountered.

   INCLUDE PROCESSING
      If the string "%include name%" is found at the beginning of a line, 
         the name is interpreted as a file name;
         reading of the original file is suspended, and the new file is
         inserted in the output stream.  
      A file being included can itself contain "%include newname%" instructions, 
         and such new files will be inserted appropriately.  Calls to include 
         files can be nested to any depth.  (Including a file recursively will
         cause an infinite loop.)  
      A file-name is interpreted relative to the file from which it was called.  
         Names beginning with '..' are interpreted as being in a higher level
         directory.
      A string "%DATE%" within a file will be replaced by the current local date.

   MACRO PROCESSING
      Within a file that is to be copied to output, macros can be defined 
      using the form "%DM,name:string:%", and such strings will not 
      immediately be copied to the output.  These set up text substitutions
      that will apply to the remainder of the files as they are copied to
      output.
      Such macros can be invoked with "%M,name%", so that this text will be
      replaced by the string in the definition.  Invocation is recursive,
      by which I mean that the replacement text is not simply copied directly
      to standard output, but instead is processed as the input stream.  
      Within this replacement text may be other macro invocations 
      ("%M,othername%"), new macro definitions, or "%include.." directives.  
      (A macro cannot be redefined within its own body; our simple syntax 
      will not allow colons to appear in the 'string' part of a definition).

   EXAMPLES:  
      The field values (fields) file might look like
      
      NAME \t PERS \t ADDR1 \t ADDR2 \t ADDR3
      Mr H. Bloggs  \t Henry     \t 360 Hindmarsh Dr  \t Phillip, ACT 2616 \t Australia
      Ms R. McManus \t Rosemary  \t 19 McWilliam Cres \t Florey, WA 6517   \t 
      Mr GA Spence  \t Mr Spence \t 40 Plenty Ave     \t Lower Hutt        \t New Zealand 
      Mrs P Thomas  \t Penni     \t Computer Science  \t ADFA         \t Campbell, ACT 2600

      and the third file (letter) might look like
         %include ../pscode/letter.hd.qs%
         %DATE%
      
         <NAME>
         <ADDR1>
         <ADDR2>
         <ADDR3>

         Dear <PERS>

         According to my records, you have not completed the Essential...
   
      The output pages could be generated with Qs layout using
      
         catm fields -o Qs -r letter | lpr -Plaser
         
      so that the Quikscript file (Qs) will be copied once at the start,
      followed by four copies of the letter, each tailored to a name and
      address given in the fields file.
      
      Alternatively, the field values file (fields) might look like
         NAME=Henry Bloggs
         PERS=Henry
         ADDR1=360 Hindmarsh Dr
         ADDR2=Phillip, ACT 2616
         ADDR3=Australia
         NEXT
         Rosemary McManus
         Rosemary
         19 McWilliam Cres
         Florey, ACT 2617

         GA Spence
         Mr Spence
         40 Plenty Ave
         Lower Hutt
         New Zealand 
         Penni Thomas
         Penni
         Dept Computer Science
         ADFA
         Campbell, ACT 2600

       with the same results.
       
   Author: Graham Freeman
   Copyright: Graham Freeman, November 1994, October 2004, December 2006.
*/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>

#define Fnamelen 512
typedef struct
{
   long int pos;
   char     *name;
} filepos;
typedef struct dmac
{
   char *name;
   char *subst;
   struct dmac *next;
} macro;
typedef struct dicte
{
   char         *word;
   char         *subst;
   struct dicte *next;
} dictentry;


void copyout( filepos *file );
void processline(filepos *file, char line[], int *bcnt);
void defmac(filepos *file, char line[], int *bcnt);
void incproc(filepos *file, char line[], int *bcnt);
void invmac(filepos *file, char line[], int *bcnt);
void dateout(void);
short int getfields( FILE *mac );
dictentry *findfield( char *name, dictentry *dictpos);
void invfield(filepos *file, char line[], int *bcnt);
void getfieldnames( FILE *fp );
int isNullValued();
void fixBackslash(char *bf);
char * readLine( FILE *in );
void debug( dictentry *dict );


   /* the program assumes the keywords are unique */
char  *kword[]  = {"%include ", "%DATE%", "%DM,", "%M,", "<"};
void  (*kwfn[])() = {incproc, dateout, defmac, invmac, invfield};
char  *incl,
      *date,
      datefmt[] = "%d %B %Y";
         /* This format gives Day Month(word) Year (unambiguously) */
char fieldSeparator = '\t';
int tabbed = 0;

#define delim '%'
#define macdelim ':'
char  macdstr[2]= {macdelim,'\0'};
#define Macrostart '<'
#define Macroend   '>'
char  defaultendtext[] = "%NP%%PS%\nQSdict begin Init ToTop end PE\n",
             /* default text output after each generated document */
      *newendtext      = NULL,
             /* requested text after each generated document */
      NEXTdoctext[]    = "NEXT",
             /* Text separating sets of field values for separate 
               documents */
      endfield[]       = "END";
             /* special field name for end-of-document processing */

FILE  *fp, *mac, *output;
dictentry *dict = NULL,     /* dictionary for merge expansions */
          *dictptr;         /* pointer to entry in the dictionary */
macro *maclist = NULL;

#if defined(__MSDOS__) || defined(WIN32)
char  filesep = '\\';
#else
char  filesep = '/';
#endif


/*****************************************************************************/
int main(
   int      argc,
   char     *argv[] )
{
   filepos  *file;
   int      i, j, argrep2, fieldind;
   char     sepch = '\t';
   char     *fieldsfile = NULL;

   incl     = kword[0];
   date     = kword[1];
   output = stdout;
   if (argc < 3)
   {
      fprintf(stderr,
         "catm expects at least two arguments, \n"
         "    the first a file of field names with a set of field text for \n"
         "    each document, the second a file to be copied repeatedly to \n"
         "    output, with field substitution based on the definitions in\n"
         "    the first file.\n\n"
         "   If there are more arguments, the files will be\n"
         "    concatenated during output.\n\n"
         "   Before the first file-name, '-f name' can be given, indicating\n"
         "    that the field names are in a separate file of the given name.\n"
         "    Also allowed is '-s char' by which the separator character\n"
         "    between fields is chosen.\n"
         "    Another option here is '-e text' which specifies what is to be\n"
         "    output at the end of each letter or document.\n"
         "   After the first file-name:\n"
         "    Flag '-o' indicates that the following files are to be output\n"
         "     once, without field processing.\n"
         "    Flag '-r' implies that the following files are to be\n"
         "     repeatedly processed with the sets of field definitions \n"
         "     found in the first file.\n\n"
         "   Field processing involves replacing occurrences of strings\n"
         "    <field>, where the word in <> has been defined in the\n"
         "    file of field definitions, with the current definition\n"
         "    from the field definition file.\n\n"
         "   If any of the files to be copied contains the string \n"
         "       %%include 'name'%%,\n"
         "    the file 'name' will be copied and inserted at that point.\n"
         "    The file path is interpreted relative to the file from which\n"
         "    it was called.  The file-name may begin with '..%c' or with "
         "    '~'.\n"
         "   Also processed are %%DATE%%, %%DM,name:subst:%% and %%M,name%%\n"
         "    DM defines a macro, which can be invoked with M.\n"
         "     eg. %%DM,P1:%%P,10,-10,-.5%%:%% will cause any future %%M,P1%%\n"
         "     to be replaced with %%P,10,-10,-.5%% in the output file.\n\n"
         ,filesep);
      exit(1);
   }
   file = (filepos *) malloc(sizeof(filepos));
   for (i=1; i<argc; i++)
   {
      if (argv[i][0] == '>')
      {
         char *mode = "w";
         j = 1;
         if (argv[i][j] == '>')
         {
            mode = "a";
            ++j;
         }
         if (strlen(&argv[i][j]) == 0)
         {
            ++i;
            j = 0;
         }
         if ( (output = fopen(&argv[i][j], mode)) == NULL)
         {
            fprintf(stderr, "catm: unable to open output file %s\n",
               &argv[i][j]);
            exit(1);
         }
         break;
      }
   }
   argrep2 = argc;
   for (i=1; i<argc; ++i)
   {
      if (argv[i][0] == '>')
      {
         j = 1;
         if (argv[i][j] == '>') ++j;
         if (strlen(&argv[i][j]) == 0) ++i;
         continue;
      }
      if (strcmp(argv[i],"-f") == 0)
      {
         fieldsfile = argv[++i];
      }
      else if (strcmp(argv[i],"-s") == 0)
      {
         sepch = argv[++i][0];
      }
      else if (strcmp(argv[i],"-e") == 0)
      {
         newendtext = argv[++i];
      }
      else 
      {
         if (fieldsfile != NULL)
         {
                                             /* can the file be opened? */
#if defined(__MSDOS__) || defined(WIN32)
            if ((fp = fopen(fieldsfile, "rb")) == NULL)  
#else
            if ((fp = fopen(fieldsfile, "r")) == NULL)  
#endif
            {
               fprintf(stderr,"Unable to open file %s.  catm abandoned\n",
                  argv[i]);
               exit(1);
            }
            if (sepch != '\t') fieldSeparator = sepch;
            getfieldnames(fp);
            fclose(fp);
         }
         break;
      }
   }
   fieldind = i;
   for (i=fieldind+1; i<argc; i++)
   {
      if (argv[i][0] == '>')
      {
         j = 1;
         if (argv[i][j] == '>') ++j;
         if (strlen(&argv[i][j]) == 0) ++i;
         continue;
      }
      
      if (argv[i][0] == '-')
      {
         if (strcmp(argv[i], "-r") == 0)      /* Repeat */
         {
            argrep2 = argc;
            for (j=i+1; j<argc; ++j)
            {
               if (argv[j][0] == '-' || argv[j][0] == '>')
               {
                  argrep2 = j;
                  break;
               }
                                             /* can the file be opened? */
#if defined(__MSDOS__) || defined(WIN32)
               if ((fp = fopen(argv[j], "rb")) == NULL)  
#else
               if ((fp = fopen(argv[j], "r")) == NULL)
#endif
               {
                  fprintf(stderr,"Unable to open file %s.  catm abandoned\n",
                     argv[j]);
                  exit(1);
               }
               fclose(fp);
            }
         }
         else if (strcmp(argv[i], "-o") == 0)    /* Once Only */
         {
            for (++i;i < argc;++i)
            {
               if (argv[i][0] == '-' || argv[i][0] == '>') 
               {
                  --i; 
                  break;
               }
               file->name = argv[i];
               file->pos  = 0;
#if defined(__MSDOS__) || defined(WIN32)
               if ((fp = fopen(file->name, "rb")) == NULL)
#else
               if ((fp = fopen(file->name, "r")) == NULL)
#endif
               {
                  fprintf(stderr,
                     "Unable to open file \"%s\".  catm abandoned\n",
                     file->name);
                  exit(1);
               }
               copyout(file);
               fclose(fp);
            }
         }
         else
         {
            fprintf(stderr,"Unrecognized option \"%s\".  catm abandoned\n",
               argv[i]);
            exit(1);
         }
         continue;
      }
      else
      {
#if defined(__MSDOS__) || defined(WIN32)
         if ((mac = fopen(argv[fieldind], "rb")) == NULL)
#else
         if ((mac = fopen(argv[fieldind], "r")) == NULL)
#endif
         {
            fprintf(stderr,
               "Unable to open definition file \"%s\".  catm abandoned\n",
               argv[fieldind]);
            exit(1);
         }
         while (getfields(mac))
         {
            int j = i;
            if ( isNullValued() ) continue;
            for (; j<argrep2; ++j)
            {
               file->name = argv[j];
               file->pos  = 0;
#if defined(__MSDOS__) || defined(WIN32)
               if ((fp = fopen(file->name, "rb")) == NULL)
#else
               if ((fp = fopen(file->name, "r")) == NULL)
#endif
               {
                  fprintf(stderr,"Unable to open file %s.  Catm abandoned\n",
                     file->name);
                  exit(1);
               }
               copyout(file);
               fclose(fp);
            }
            if (newendtext == NULL)
               fputs(defaultendtext,output);
            else 
               fputs(newendtext, output);
         }
         fclose(mac);
         i = argrep2 - 1;
      }
   }
   return(0);
}

/*****************************************************************************/
void copyout( filepos *file )
   /* Copy the given file to output, checking for strings such as '<field>' or
      '%include', which will need to be expanded.  All special processing is
      performed recursively.  The global "fp" is the file actually read. */
{
   int   size = 1024;
   char *line = (char*)malloc(size);
   
   while (fgets(line,size,fp) != NULL)
   {
      int len = strlen(line);
      int bcnt = 0;
      char *success ;
      while (len == size-1 && line[len-1] != '\r' && line[len-1] != '\n')
      {
         line = (char*) realloc( line, size*2);
         success = fgets(&line[size-1], size+1, fp);
         if (!success) break;
         size *= 2;
         len = strlen(line);
      }
      processline(file,line,&bcnt);
   }
   free(line);
}

/*****************************************************************************/
void processline(filepos *file, char line[], int *bcnt)
   /* Process the text in 'line', searching for any field references, macros,
      etc., handling them with possibly recursive calls. */
{
   char                ch;
   unsigned long int   match = 0;
   int                 ipos = 0;
   static int          lastch;
   int                 lastmatch = 0;

   while ((ch = line[*bcnt]) != '\0')
   {
      ++ *bcnt;
         /* if a match is underway, continue it */
      if (match != 0)
      {
         int wd = -1;
         for (wd = 0; wd < sizeof(kword)/sizeof(char *); ++wd)
         {
            if ((1<<wd & match) != 0)
            {
               lastmatch = wd;
               if (ch != kword[wd][ipos]) match ^= 1<<wd;
               else if (ipos+1 == strlen(kword[wd]))
               {
				  kwfn[wd](file,line,bcnt);
                  ipos = 0;
                  match = 0;
                  lastmatch = -1;
                  break;
               }
            }
         }
         if (match == 0)
         {
               /* match must be aborted; output previous characters */
            int i;
            if (lastmatch < 0) {lastmatch = 0; continue;}
            for (i=0; i<ipos; ++i) fputc(kword[lastmatch][i],output);
            lastmatch = 0;
            ipos = 0;
         }
         else ++ipos;
      }
      if (match == 0)
      {
            /* first check whether the character matches a first kw char 
               and if not, output it */
         int wd;
         for (wd = 0; wd < sizeof(kword)/sizeof(char *); ++wd)
         {
            if (ch == kword[wd][0])
            {
               match |= 1<<wd;
               if (strlen(kword[wd]) == 1)
               {
                  kwfn[wd](file,line,bcnt);
                  match = 0;
                  lastmatch = -1;
                  break;
               }
            }
         }
         if (match == 0)
         {
            if (lastmatch < 0) {lastmatch = 0; continue;}
            fputc(ch,output);
            lastch = ch;
         }
         else ipos = 1;
      }
   }
   if (match != 0)
   {
      int i;
      for (i=0; i<ipos; ++i) fputc(kword[lastmatch][i],output);
      ipos = 0;
   }
}

/*****************************************************************************/
void incproc(filepos *file, char line[], int *bcnt)
      /* process the remainder of this line as the name
         of a file to be included in output.
         Read the file and output it, processing it recursively for any special
         fields, macros, etc. */
{
   int        npos = 0, ch;
   filepos    *newfile;
   char       *newname;

   newname = malloc(Fnamelen);
   npos = 0;
   while ((ch = line[*bcnt]) != '\0')
   {
      ++ *bcnt;
      newname[npos] = ch;
      if (ch == '\n' || npos+1 >= Fnamelen)   /* % not found */
      {
         fputs(incl,output);
         newname[++npos] = '\0';
         fputs(newname,output);
         free(newname);
         return;
      }
      else if (ch != delim) npos++;
      else
      {
         newname[npos] = '\0';
         newfile = (filepos *) malloc(sizeof(filepos));
         /* Build new file name from path of previous file */
         {
            char *firstch;
            firstch = newname + strspn(newname," ");
            if (*firstch == filesep )
               newfile->name = firstch;
            else
            {
               int  i, m, n;
               char dotdotsl[4] = "../";
               char *pt;
               m = strlen(firstch);
               for (i=0; i<=m; i++)          /*copy to end of newname*/
                  newname[Fnamelen-1-i] = firstch[m-i];
               firstch = &newname[Fnamelen-1-m];
               n = 0;
               newname[0] = '\0';
               pt = strrchr(file->name,filesep);
               if (pt != NULL && 
                   *firstch != '~' && *firstch != filesep)
               {
                  /* Process leading '../' strings */
                     /*copy name of calling file to front of newname*/
                  n = pt - file->name + 1;
                  strncpy( newname, file->name, n);
                  newname[n] = '\0';
                  dotdotsl[2] = filesep;
                  while (strncmp(firstch,dotdotsl,3) == 0)
                  {
                     char *slashpt;
                     firstch += 3;
                     newname[n-1] = '\0';
                     slashpt = strrchr(newname,filesep);
                     n = (slashpt==NULL)? 0: slashpt - newname + 1;
                     newname[n] = '\0';
                     if (slashpt == NULL) break;
                  }
               }
               if (*firstch == '~' && firstch[1] == filesep)
               {
                  char *env;
                  env = getenv("HOME");
                  strcpy(newname,env);
                  n = strlen(newname);
                  newname[n++] = filesep;
                  newname[n] = '\0';
                  firstch = &firstch[2];
               }
               /* Now fix any $ strings */
               while ((pt = strchr(firstch,'$')) != NULL)
               {
                  char *env;
                  m = pt - firstch;
                  strncpy( &newname[n], firstch, m);
                  n += m;
                  firstch = pt;
                  pt = strchr(firstch,filesep);
                  if (pt != NULL) *pt = '\0';
                  env = getenv(&firstch[1]);
                  if (env == NULL) env = firstch;
                  if (env[0] == filesep) n = 0;
                  strcpy( &newname[n], env);
                  n += strlen(env);
                  if (pt != NULL)
                  {
                     newname[n++] = filesep;
                     newname[n] = '\0';
                     firstch = pt+1;
                  }
                  else
                     *firstch = '\0';
               }
               strcpy( &newname[n], firstch);
               newfile->name = newname;
            }
         }
         file->pos = ftell(fp);
         newfile->pos = 0;
         fclose(fp);
#if defined(__MSDOS__) || defined(WIN32)
         if ((fp = fopen(newfile->name, "rb")) == NULL)
#else
         if ((fp = fopen(newfile->name, "r")) == NULL)
#endif
         {
            fprintf(stderr,
               "Unable to open file \"%s\".  catm abandoned\n",
               newfile->name);
            exit(1);
         }
         copyout(newfile);
         free(newfile);
         fclose(fp);
#if defined(__MSDOS__) || defined(WIN32)
         fp = fopen(file->name, "rb");
#else
         fp = fopen(file->name, "r");
#endif
         fseek(fp,file->pos,0);
         break;
      }
   }
   free(newname);
}

/*****************************************************************************/
void defmac(filepos *file, char line[], int *bpos)
   /* Define a new macro.
      A macro definition begins with '%DM,name:'.  Record the definition if the
      remaining syntax is matched, ie. ':%' at the end.
      Note that the 'file' argument is unused.  It is present only so that the
      processing functions have a uniform calling sequence. */
{
   int ipos = 0, colons[2];
   char macbuf[1024], ch;
   int macstate = 0, mpos = 0;

   while ((ch = line[*bpos+mpos]) != '\0')
   {
      ++ mpos;
      if (ch == macdelim)
      {  /* colons */
         if (macstate > 1) break;
         colons[macstate] = ipos;
         ++macstate;
      }
      else if (macstate == 2 && ch == delim)
      {  /* % after colons */
         int i,j,k;
         macro *thismac;
         macbuf[ipos] = '\0';
         if (ipos != colons[1]+1) break;
         i = strspn(macbuf," ");
         j = strcspn(&macbuf[i],macdstr);
         if (j == 0) break;

         *bpos += mpos;
         thismac = (macro *) malloc(sizeof(macro));
         thismac->name = (char *) malloc(j+1);
         strncpy(thismac->name,&macbuf[i],j);
         thismac->name[j] = '\0';
         k = colons[1] - colons[0];
         thismac->subst = (char *) malloc(k);
         strncpy(thismac->subst,&macbuf[1+colons[0]],k-1);
         thismac->subst[k-1] = '\0';
         if (maclist == NULL)
         {  /* Place item at start of new list */
            maclist = thismac;
            thismac->next = NULL;
            return;
         }
         else
         {  /* Place in sorted position in list */
            macro *pt;
            macro *prev = NULL;
            for (pt=maclist; pt!=NULL; pt=pt->next)
            {
               int cmp = strcmp(pt->name,thismac->name);
               if (cmp == 0)
               {  /* the macro already exists; replace it */
                  free(pt->subst);
                  pt->subst = thismac->subst;
                  free(thismac->name);
                  free(thismac);
                  return;
               }
               if (cmp > 0)
               {
                  if (prev == NULL)
                  {  /* Place it at the front of the list */
                     thismac->next = maclist;
                     maclist = thismac;
                  }
                  else
                  {
                     thismac->next = prev->next;
                     prev->next = thismac;
                  }
                  return;
               }
               if (pt->next == NULL)
               {
                  pt->next = thismac;
                  thismac->next = NULL;
                  return;
               }
               prev = pt;
            }
         }
      }
      macbuf[ipos] = ch;
      ++ipos;
      if (ch == '\n') break;
   }
      /* improper macro; simply output keyword and reprocess the rest */
   {
      int i;
      for (i=0; i<strlen(kword[2]); ++i) fputc(kword[2][i],output);
   }
}

/*****************************************************************************/
void invmac(filepos *file, char line[], int *bcnt)
   /* Invoke a macro
      A macro starts with '%M,', and will have previously been defined with
      '%DM,name:...'.  Find the definition and use the replacement text,
      processing it recursively for any other replacement. */
{
#define Maclen 50
   int ipos = 0;
   char macname[Maclen], ch;
   int macstate = 0;

   while ((ch = line[*bcnt]) != '\0' && ipos < Maclen)
   {
      ++ *bcnt;
      if (ch == delim)
      {
         macro *pt;
         macname[ipos] = '\0';
         for (pt=maclist; pt != NULL; pt = pt->next)
         {
            int cmp = strcmp(macname,pt->name);
            if (cmp == 0)
            {  /* process the substitute text for any further macros etc */
               int i=0;
               processline(file,pt->subst,&i);
               return;
            }
            if (cmp < 0) break;
         }
      }
      macname[ipos] = ch;
      ++ipos;
      if (ch == '\n' || ch == delim) break;
   }
      /* improper macro; simply output */
   {
      int i;
      for (i=0; i<strlen(kword[3]); ++i) fputc(kword[3][i],output);
      for (i=0; i<ipos; ++i) fputc(macname[i],output);
   }
}

/*****************************************************************************/
void dateout(void)
     /* output current date */
{
   char    datestr[60];
   time_t  now;
   int     cnt;

   now = time(NULL);
   cnt = strftime(datestr, sizeof(datestr), datefmt, localtime(&now));
   fputs(datestr,output);
}

/*****************************************************************************/
short int getfields( FILE *mac )
   /* Read a set of field values from "mac".
      This uses the global dictionary, "dict" that will contain the field names,
       where the values will be stored.
      If the dictionary does not exist, it must be created from information
       found in the file "mac".  The first set of field values must be preceded
       by the name and '='.  Lines will be processed until "NEXT" is 
       encountered, indicating that all field names have been found.
      Return 1 if anything has been read, else 0.
      The dictionary will be emptied of field values before more of
       the "mac" file is read on each call. */
{
   int ch, eofld;
   if (dict == NULL) 
   {
      getfieldnames( mac );
      return(1);
   }
   else
   {
      /* Not the first time.  Use the existing dictionary, substituting
         new text for the field names. */
      char      buf[1025];
      int       i;
      int       numfields;
      static int tabdelims = 0;
      /* First clear out previous definitions */
      for (dictptr = dict; dictptr != NULL; dictptr = dictptr->next)
      {
         free(dictptr->subst);
         dictptr->subst = NULL;
      }
      
      dictptr = dict;
      numfields = 0;
      do
      {
         for ( i=0; 
               (ch=fgetc(mac)) != '=' && ch != '\n' && ch != fieldSeparator && ch > 0; 
               ++i )
            buf[i] = ch;
         if (i == 0) return(0);
         eofld = ch;
         buf[i++] = '\0';
         if (eofld == '\n' || eofld == fieldSeparator || eofld < 0)
                                            /* No '=' on the line */
         {                                  /* if NEXT, exit loop */
            if (eofld != fieldSeparator && strcmp(buf,NEXTdoctext) == 0)
            {
               if (dictptr == dict) continue;
               break;
            }
            else                   /* otherwise use text for next macro */
            {
               int n = i, j;
               if (eofld == fieldSeparator) tabdelims = 1;
               if (dictptr != NULL && strcmp(dictptr->word,endfield) == 0) 
                     dictptr = dictptr->next;
               for (i=0,j=0; i<n; ++i,++j)
               {
                  if ((ch=buf[i]) == '\\')
                  {
                     ch = buf[++i];
                     if      (ch == 'n')  ch = '\n';
                     else if (ch == 't')  ch = '\t';
                     else if (ch == '\n') 
                     { 
                        fprintf(stderr,"catm: Unable to handle \\ at end of"
                              " line in a field definition.  Aborted\n");
                        exit(1);
                     }
                  }
                  buf[j] = ch;
                  if ( j >= sizeof(buf) )
                  {
                     fprintf(stderr,"catm: Internal buffer size exceeded.  "
                            "Job aborted\n");
                     exit(1);
                  }
               }
               buf[j++] = '\0';
               dictptr->subst = malloc(j);
               strcpy(dictptr->subst,buf);
               if (j > 1) numfields++;
               dictptr = dictptr->next;
               if (dictptr != NULL && strcmp(dictptr->word,endfield) == 0) 
                     dictptr = dictptr->next;
               if (dictptr == NULL) break;
               if (tabdelims && eofld == '\n') break;
            }
         }
         else if (ch == '=')       /* line with '=' character */
         {
            while (dictptr != NULL && strcmp(dictptr->word,buf) != 0) 
               dictptr = dictptr->next;
            if (dictptr == NULL) 
            {
               fprintf(stderr,"catm: unrecognized keyword: %s\n",buf);
               exit(1);
            }
            numfields++;
            if (strcmp(dictptr->word,endfield) == 0)
            {
               free(dictptr->subst);
               dictptr->subst = NULL;
            }
            for (i=0; (ch=fgetc(mac)) != '\n' && ch > 0; ++i)
            {
               if (ch == '\\')
               {
                  ch = fgetc(mac);
                  if      (ch == 'n')  ch = '\n';
                  else if (ch == 't')  ch = '\t';
                  else if (ch == '\n') {--i; continue;}
                  else                 buf[i++] = '\\';
               }
               buf[i] = ch;
            }
            buf[i++] = '\0';
            dictptr->subst = malloc(i);
            strcpy(dictptr->subst,buf);
            dictptr = dictptr->next;
            if (dictptr != NULL && strcmp(dictptr->word,endfield) == 0) 
                  dictptr = dictptr->next;
            if (dictptr == NULL) break;
         }
         if (ch < 0) break;
      } while (1);
      if (eofld == fieldSeparator) do {ch = fgetc(mac);} while (ch > 0 && ch != '\n');
      if (ch >= 0 || numfields > 0) return(1);
      return(0);
   }
}

/*****************************************************************************/
dictentry *findfield(
                  char      *name,
                  dictentry *dictpos)
   /* Find the dictionary entry for entry "name".
      Return a pointer to the dictionary entry, if found,
      or NULL. */
{
   while (dictpos != NULL && dictpos->word != NULL)
      if (strcmp(dictpos->word,name) == 0)  return(dictpos);
      else     dictpos = dictpos->next;
   return(NULL);
}

/*****************************************************************************/
void invfield(filepos *file, char line[], int *bcnt)
{
      /* invoke-field processing
         A '<' has been found.  Search for the matching '>', and with the
         enclosed word, find its replacement text in the dictionary.  The
         replacement text must be scanned recursively for any other 
         replacements before output. */
   int   ipos;
   char  newname[Fnamelen], ch;
   for (ipos=0; (ch = line[*bcnt+ipos]) != '\0' && ipos < Fnamelen-1 &&
               ch != ' ' && ch != '\n' && ch != Macroend; ++ipos)
      newname[ipos] = ch;
   newname[ipos++] = '\0';
   if (ch == Macroend)
   {
      if ((dictptr = findfield(newname,dict)) == NULL)
      {
         /* Macro name not found */
         fputc(Macrostart,output);
      }
      else
      {
         /* Output macro substitution */
         if (dictptr->subst != NULL)
         {  /* process the substitute text for any further macros etc */
            int i=0;
            processline(file,dictptr->subst,&i);
         }
         *bcnt += ipos;
      }
   }
   else
   {
      /* No matching end character */
      fputc(Macrostart,output);
   }
}

/*****************************************************************************/
void getfieldnames( FILE *mac )
   /* Extract from file "mac" the names of fields that will be take values as
      the field file is read, and which will be substituted for in files that
      are to be repeatedly copied to output.  Place the names in dictionary
      "dict".  Look also for any "END=" directive. */
{
   char     *line = NULL, 
            *bf;
   int       i;
   /* Create the dictionary from the first set of field definitions. */
   dictentry *previous;
   dict = (dictentry *) malloc(sizeof(dictentry));
   dict->next = NULL;
   dict->subst = NULL;
   dict->word = NULL;
   dictptr = dict;
   previous = NULL;
   while (1)
   {
      char * line = readLine(mac);
      int skip;
      if (line == NULL) 
      {
         close(mac);
         break;
      }
        /* remove leading and trailing blanks */
      bf = line + strspn(line," ");
      skip = bf - line;
      for ( i=strlen(bf)-1; i>=0; --i ) 
         if (bf[i] == ' ') bf[i] = '\0'; 
         else  break;
      if (strcmp(line,NEXTdoctext) == 0)   // NEXT found
      {
         free(line);
         break;
      }
      
      while ( bf[strlen(bf)-1] == '\\' )     // trailing \ before newline
      {
         char *next1 = readLine(mac);
         if (next1 == NULL)
         {
            close(mac);
            break;
         }
         bf[strlen(bf)-1] = '\0';
         line = (char*) realloc(line,strlen(line)+strlen(next1)+1);
         strcat(line,next1);
         free(next1);
         bf = line+skip;
      }
      
      fixBackslash(bf);
      
      if (strncmp(bf,"END=",4) == 0)
      {
         if (newendtext != NULL) free(newendtext);
         newendtext = malloc(strlen(bf)-4);
         strcpy(newendtext,&bf[4]);
         free(line);
         continue;
      }
      
      while (1)      // for each field on the line
      {
         char * eqpos = strchr(bf,'=');
         char * tabpos = strchr(bf,fieldSeparator);
         if (!tabbed && eqpos != NULL && (eqpos < tabpos || tabpos == NULL))
         {            // '=' sep
            char * name = malloc(eqpos-bf);
            strncpy(name, bf, eqpos-bf);
            name[eqpos-bf-1] = '\0';
            bf = eqpos+1;
            dictptr->word = name;
            dictptr->subst = malloc(strlen(bf)+1);
            strcpy(dictptr->subst,bf);
            previous = dictptr;
            dictptr->next = (dictentry *) malloc(sizeof(dictentry));
            dictptr = dictptr->next;
            dictptr->subst = NULL;
            dictptr->word = NULL;
            dictptr->next = NULL;
            free(line);
            break;
         }
         else         // fieldSeparator, or last field name
         {
            if (tabpos != NULL) 
            {
               *tabpos = '\0';
               tabbed = 1;
            }
            dictptr->word = malloc(strlen(bf)+1);
            strcpy(dictptr->word,bf);
            dictptr->subst = NULL;
            previous = dictptr;
            dictptr->next = (dictentry *) malloc(sizeof(dictentry));
            dictptr = dictptr->next;
            dictptr->subst = NULL;
            dictptr->word = NULL;
            dictptr->next = NULL;
            if (tabpos == NULL) { free(line); break; }
            bf = tabpos + 1;
         }
      }
      
      if (tabbed) break;
   }
   
   free(dictptr);
   dictptr = previous;
   if (previous == NULL) dict = NULL;
   else dictptr->next = NULL;
}

/*****************************************************************************/
int isNullValued()
{
   for (dictptr = dict; dictptr != NULL; dictptr = dictptr->next)
   {
      if ( dictptr->subst != NULL) return 0;
   }
   return 1;
}

/*****************************************************************************/
void fixBackslash(char *bf)
{
   int i,j;
   for (i=0,j=0; i<strlen(bf); ++i,++j)
   {
      char ch = bf[i];
      if (ch != '\\') 
      {
         bf[j] = ch;
         continue;
      }
      ch = bf[++i];
      if      (ch == 'n') bf[j] = '\n';
      else if (ch == 't') bf[j] = '\t';
      else
      {
         bf[j++] = '\\';
         bf[j] = ch;
      }
   }
   bf[j] = '\0';
}

/*****************************************************************************/
char * readLine( FILE *in )
{
   char *buf = (char *) malloc(1024);
   int limit = 1024;
   int count = 0, ch;
   while ((ch = fgetc(in)) != '\n' && ch > 0)
   {
      if (count >= limit-1)  
      {
         limit += 1024;
         buf = (char *) realloc(buf,limit);
      }
      buf[count++] = ch;
   }
   if (ch < 0 && count == 0)
   {
      free(buf);
      return(NULL);
   }
   if (count > 0 && buf[count-1] == '\r') buf[count-1] = '\0';
   buf[count] = '\0';
   return buf;
}

/*****************************************************************************/
void debug( dictentry *dict )
{
   dictentry *d = dict;
   while (d != NULL)
   {
      fprintf(stderr,"Item %d  ",d);
      if (d->word != NULL)
      {
         fprintf(stderr,"word %d ",d->word);
         fprintf(stderr,"%s  ",d->word);
      }
      else
         fprintf(stderr,"word 0  ");
      if (d->subst != NULL)
         fprintf(stderr,"subst %s  ",d->subst);
      else
         fprintf(stderr,"subst 0  ");
      if (d->next != NULL) 
         fprintf(stderr,"next %d\n",d->next);
      else
         fprintf(stderr,"next 0\n");
      d = d->next;
   }
}