/*
**  Copyright (c) 2001-2002 The OSSP Project <http://www.ossp.org/>
**  Copyright (c) 2001-2002 Cable & Wireless Deutschland <http://www.cw.com/de/>
**
**  This file is part of OSSP lmtp2nntp, an LMTP speaking local
**  mailer which forwards mails as Usenet news articles via NNTP.
**  It can be found at http://www.ossp.org/pkg/lmtp2nntp/.
**
**  This program is free software; you can redistribute it and/or
**  modify it under the terms of the GNU General Public  License
**  as published by the Free Software Foundation; either version
**  2.0 of the License, or (at your option) any later version.
**
**  This program is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
**  General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this file; if not, write to the Free Software
**  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
**  USA, or contact the OSSP project <ossp@ossp.org>.
**
**  msg.c: mail message manipulation library
*/

#include <stdlib.h>
#include <stdio.h>

#include "lmtp2nntp_msg.h"
#include "lmtp2nntp_argz.h"
#include "fixme.h" //FIMXE logbook only
#include "tai.h"

#include "str.h"

/* third party */
#include "l2.h"

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#if defined(HAVE_DMALLOC_H) && defined(DMALLOC)
#include "dmalloc.h"
#endif

msg_t *msg_create(void)
{
    msg_t *msg;

    if ((msg = (msg_t *)malloc(sizeof(msg_t))) == NULL)
        return NULL;

    msg->azEnvgroups = NULL;
    msg->asEnvgroups = 0;
    msg->cpMsg = NULL;
    msg->azHeaders = NULL;
    msg->asHeaders = 0;
    msg->hdFirst = NULL;
    msg->cpFid = NULL;
    msg->cpBody = NULL;
    msg->cpMsgid = NULL;
    msg->mail_from = NULL;
    msg->azRcpt = NULL;
    msg->asRcpt = 0;
    msg->azNewsgroups = NULL;
    msg->asNewsgroups = 0;
    msg->l2 = NULL; /* this is a copy only */

    return msg;
}

void msg_destroy(msg_t *msg)
{
    if (msg == NULL)
        return;

    if (msg->azEnvgroups != NULL)
        free(msg->azEnvgroups);
    if (msg->cpMsg != NULL)
        free(msg->cpMsg);
    if (msg->azHeaders != NULL)
        free(msg->azHeaders);
    if (msg->cpFid != NULL)
        free(msg->cpFid);
    if (msg->cpBody != NULL)
        free(msg->cpBody);
    if (msg->cpMsgid != NULL)
        free(msg->cpMsgid);
    if (msg->mail_from != NULL)
        free(msg->mail_from);
    if (msg->azRcpt != NULL)
        free(msg->azRcpt);
    if (msg->azNewsgroups != NULL)
        free(msg->azNewsgroups);
    msg->l2 = NULL; /* this is a copy only, the "parent" needs to clean this up */

    free(msg);
    return;
}

msg_rc_t msg_split(msg_t *msg)
{
    char        *cpName;
    char        *cpValue;
    char        *cpRem;     /* Remainder */
    char        *cp;
    char        *cpHeaders;

    /* INPUTS
     *
     * msg->cpMsg
     * must contain the wholly RFC0822 formatted message with native
     * (unescaped) dots at the beginning of a line, the 'From ' envelope,
     * headers, double newline, body, NUL, no trailing dot;
     *
     * OUTPUTS
     *
     * msg->cpMsg
     * free()d and set to NULL
     * 
     * msg->azHeaders, msg->asHeaders contains the headers in argz format, one
     * logical NUL-terminated line per header which might be wrapped into
     * multiple '\n'-ended physical lines. The "From " envelope, "Received:",
     * "Path:", "To:" and "Cc:" headers are removed silently. The
     * "Newsgroups:" and "Message-ID" headers are removed and their values are
     * stored in separate structures (see below).
     *
     * msg->cpBody
     * contains the unmodified body of the message, NUL-terminated, no
     * trailing dot.
     *
     * msg->cpMsgid
     * contains the message id including surrounding angle brackets.
     *
     * msg->azNewsgroups, asNewsgroups
     * is a argz-type array of strings containing the Newsgroups based on the
     * header information.
     */

    logbook(msg->l2, L2_LEVEL_DEBUG, "split message into header and body");
    if (str_parse(msg->cpMsg, "m/((?:.*?)\\n)\\n(.*)$/s", &cpHeaders, &msg->cpBody) <= 0)
        return MSG_ERR_SPLITHEADBODY;

    free(msg->cpMsg);
    msg->cpMsg = NULL;

    logbook(msg->l2, L2_LEVEL_DEBUG, "replace envelope From w/o colon by X-F: pseudotag");
    /* This eliminates the special case of having one header, which is really
     * an embedded envelope, not ending with a colon while all others do.
     * After splitting headers into name and value pairs this envelope ist
     * stripped off.
     */
    if (strncasecmp(cpHeaders, "From", 4) == 0)
        memcpy(cpHeaders, "X-F:", 4);

    logbook(msg->l2, L2_LEVEL_DEBUG, "unwrap header lines");
    /* poor man's s///g simulator as current str library doesn't support global substitution */
    while (str_parse(cpHeaders, "s/(.*?)\\n[ \\t]+(.*)/$1 $2/s", &cpRem) > 0) {
        free(cpHeaders);
        cpHeaders = cpRem;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "split header lines into names and values");
    while (str_parse(cpHeaders, "m/^[> \\t]*([\\x21-\\x7e]+?:)[ \\t]*([^\\n]*?)[ \\t]*\\n(.*)/s", &cpName, &cpValue, &cpRem) > 0) {
        free(cpHeaders);
        cpHeaders = cpRem;
        argz_add(&msg->azHeaders, &msg->asHeaders, cpName);
        argz_add(&msg->azHeaders, &msg->asHeaders, cpValue);
        free(cpName);
        free(cpValue);
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "check for headers we care about and do whatever neccessary");
    msg->cpMsgid = NULL;
    msg->azNewsgroups = NULL;
    msg->asNewsgroups = 0;
    cp = msg->azHeaders;
    while (cp != NULL) {
        logbook(msg->l2, L2_LEVEL_DEBUG, "processing header \"%s\"", cp);
        if (strcasecmp("X-F:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Path:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Received:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            if ((msg->cpFid == NULL) &&
                (str_parse(cp, "m/\\sid\\s+<?([\\w\\d]{1,30})/i", &msg->cpFid) > 0))
                    logbook(msg->l2, L2_LEVEL_DEBUG, "found foreign-ID \"%s\" for logging", msg->cpFid);
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("To:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Cc:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Message-ID:", cp) == 0) {
            if (msg->cpMsgid != NULL)
                return MSG_ERR_SPLITIDMULTI;
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            if ((cp == NULL) || (strlen(cp) == 0))                         /* get  value */
                return MSG_ERR_SPLITIDEMPTY;
            if ((msg->cpMsgid = strdup(cp)) == NULL)
                return MSG_ERR_MEM;
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Newsgroups:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            if (argz_add(&msg->azNewsgroups, &msg->asNewsgroups, cp) != 0) /* get  value */
                return MSG_ERR_MEM;
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
            break;
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next name  */
            break;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "checking Message-ID");
    if (msg->cpMsgid == NULL)
        return MSG_ERR_SPLITIDNONE;

    logbook(msg->l2, L2_LEVEL_DEBUG, "checking Newsgroups");
    if (msg->azNewsgroups != NULL) {
        argz_stringify(msg->azNewsgroups, msg->asNewsgroups, ',');
        if (argz_create_sep(msg->azNewsgroups, ',', &msg->azNewsgroups, &msg->asNewsgroups) != 0)
            return MSG_ERR_MEM;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "adding mandatory Path: header");
    argz_add(&msg->azHeaders, &msg->asHeaders, "Path:");
    argz_add(&msg->azHeaders, &msg->asHeaders, "lmtp2nntp!not-for-mail");

    logbook(msg->l2, L2_LEVEL_DEBUG, "split complete");
    return MSG_OK;
}

msg_rc_t msg_join(msg_t *msg)
{
    char        *cp;
    char        *cpRem;
    char       **aHeaders;
    int          i;
    int          o;
    char        *cpCut;
    char        *cpWrap;
    char         c;
    char         cOld;
    int          n;
    char        *cpHeaders;
    char        *azNewheaders;
    size_t       asNewheaders;

    logbook(msg->l2, L2_LEVEL_DEBUG, "verify Newsgroups");
    if (msg->azNewsgroups == NULL)
        return MSG_ERR_JOINGROUPNONE;
    argz_stringify(msg->azNewsgroups, msg->asNewsgroups, ',');
    if (strlen(msg->azNewsgroups) == 0)
        return MSG_ERR_JOINGROUPEMPTY;
    argz_add(&msg->azHeaders, &msg->asHeaders, "Newsgroups:");
    argz_add(&msg->azHeaders, &msg->asHeaders, msg->azNewsgroups);

    logbook(msg->l2, L2_LEVEL_DEBUG, "verify Message-ID");
    if (msg->cpMsgid == NULL)
        return MSG_ERR_JOINIDNONE;
    if (strlen(msg->cpMsgid) == 0)
        return MSG_ERR_JOINIDEMPTY;
    argz_add(&msg->azHeaders, &msg->asHeaders, "Message-ID:");
    argz_add(&msg->azHeaders, &msg->asHeaders, msg->cpMsgid);

    logbook(msg->l2, L2_LEVEL_DEBUG, "merge name/value pairs into single string");
    argz_add(&msg->azHeaders, &msg->asHeaders, ""); /* append empty string */
    if ((aHeaders = (char **)malloc((argz_count(msg->azHeaders, msg->asHeaders) + 1) * sizeof(char *))) == NULL)
        return MSG_ERR_MEM;
    argz_extract(msg->azHeaders, msg->asHeaders, aHeaders);
    /* replace the trailing NUL, which is *(cp-1) of the predecessor, with a
     * space at every second string. Break action when terminating NULL string
     * is detected */
    i=0;
    while(1) {
        if ((cp = aHeaders[++i]) == NULL)
            break;
        *(cp-1) = ' ';
        if ((cp = aHeaders[++i]) == NULL)
            break;
    }
    free(aHeaders);

    logbook(msg->l2, L2_LEVEL_DEBUG, "fold headers");
    /* A logical line is split into one or more physical '\n'-terminated
     * lines. The physical line is never longer than WRAPAT characters. This
     * includes the folded data and the header name + colon + space for the
     * first line and WRAPUSING string prefix for all other lines. Leading and
     * trailing blanks of folded lines are removed while blanks inside the
     * line are preserved.  The header is never left alone in a physical line.
     * Fragments exceeding WRAPAT characters without having a blank as a
     * splitting point are forcibly cut at a non-blank character.
     */
    azNewheaders = NULL;
    asNewheaders = 0;
    cp = NULL;
    while ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) != NULL) {
        if (strlen(cp) > WRAPAT) {
            cpRem = cp;
            cpWrap = NULL;
            for (o = 0; (cpRem[o] != ':') && (cpRem[o] != NUL); o++); /* offset name so at least one char of value remains in first line */
            o += 2; /* skip ": " */
            while ((strlen(cpRem) + (cpWrap == NULL ? 0 : strlen(WRAPUSING))) > WRAPAT) {
                for (i = WRAPAT - 1 - (cpWrap == NULL ? 0 :
                    strlen(WRAPUSING)); (i >= o) && !isspace((int)cpRem[i]); i--);
                if (i < o)
                    i = WRAPAT - 1 - (cpWrap == NULL ? 0 : strlen(WRAPUSING) - 1); /* sorry, forced cut at non-blank */
                cpCut = cpRem;
                cpRem += i;
                for (; (isspace((int)*cpRem) && (*cpRem != NUL)); cpRem++); /* skip next lines leading blanks */
                for (; (i >= o) && isspace((int)cpCut[i-1]); i--); /* chop off this lines trailing blanks */
                if (i >= o) { /* only keep line fragment if some non-blanks inside */
                    if (cpWrap == NULL) {
                        if ((cpWrap = (char *)malloc(i+strlen(WRAPUSING)+1)) == NULL)
                            return MSG_ERR_MEM;
                        *cpWrap = NUL;
                        o = 1;
                    }
                    else {
                        if ((cpWrap = (char *)realloc(cpWrap, strlen(cpWrap)+i+strlen(WRAPUSING)+1)) == NULL)
                            return MSG_ERR_MEM;
                        strcat(cpWrap, WRAPUSING);
                    }
                    strncat(cpWrap, cpCut, i);
                }
            }
            if (strlen(cpRem) > 0) {
                if ((cpWrap = (char *)realloc(cpWrap, strlen(cpWrap)+strlen(cpRem)+strlen(WRAPUSING)+1)) == NULL)
                        return MSG_ERR_MEM;
                strcat(cpWrap, WRAPUSING);
                strcat(cpWrap, cpRem);
            }
            argz_add(&azNewheaders, &asNewheaders, cpWrap);
            logbook(msg->l2, L2_LEVEL_DEBUG, "a folded header \"%{text}D\"", cpWrap, strlen(cpWrap));
            free(cpWrap);
        }
        else {
            argz_add(&azNewheaders, &asNewheaders, cp);
            logbook(msg->l2, L2_LEVEL_DEBUG, "verbatim header \"%{text}D\"", cp, strlen(cp));
        }
    }
    free(msg->azHeaders);
    msg->azHeaders = azNewheaders;
    msg->asHeaders = asNewheaders;

    logbook(msg->l2, L2_LEVEL_DEBUG, "strigify headers");
    argz_stringify(msg->azHeaders, msg->asHeaders, '\n');
    cpHeaders = msg->azHeaders;

    /********************************************************************
     * header + CRLF + body + '.' + CRLF + NUL, replacing NL with CRLF *
     ********************************************************************/

    logbook(msg->l2, L2_LEVEL_DEBUG, "assemble header and body");
    n = 0;
    /* count size of headers, reserve space for NL to CRLF conversion */
    for (i = 0; ((c = cpHeaders[i]) != NUL); i++) {
        if (c == '\n')
            n++;
        n++;
    }
    /* if headers don't end with NL, reserve space for CRLF */
    if (i >= 0 && cpHeaders[i - 1] != '\n')
        n+=2;
    /* reserve space for CRLF between headers and body */
    n+=2;
    /* count size of body, reserve space for NL-DOT escape and NL to CRLF conversion */
    cOld = '\n';
    for (i = 0; ((c = msg->cpBody[i]) != NUL); i++) {
        if (c == '\n')
            n++;
        if (c == '.' && cOld == '\n')
            n++;
        n++;
        cOld = c;
    }
    /* if body doesn't end with NL, reserve space for CRLF */
    if (i >= 0 && msg->cpBody[i - 1] != '\n')
        n+=2;
    /* reserve space for terminating '.'-CRLF-NUL at the end of the message */
    n+=4;

    if ((msg->cpMsg = (char *)malloc(n)) == NULL)
        return MSG_ERR_MEM;

    n = 0;
    /* copy headers, do NL to CRLF conversion */
    for (i = 0; ((c = cpHeaders[i]) != NUL); i++) {
        if (c == '\n')
            msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = c;
    }
    /* if headers don't end with NL, append CRLF */
    if (i >= 0 && cpHeaders[i - 1] != '\n') {
        msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = '\n';
    }
    /* add CRLF between headers and body */
    msg->cpMsg[n++] = '\r';
    msg->cpMsg[n++] = '\n';
    /* copy body, do NL-DOT escape and NL to CRLF conversion */
    cOld = '\n';
    for (i = 0; ((c = msg->cpBody[i]) != NUL); i++) {
        if (c == '\n')
            msg->cpMsg[n++] = '\r';
        if (c == '.' && cOld == '\n')
            msg->cpMsg[n++] = '.';
        msg->cpMsg[n++] = c;
        cOld = c;
    }
    /* if body doesn't end with NL, append CRLF */
    if (i >= 0 && msg->cpBody[i - 1] != '\n') {
        msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = '\n';
    }
    /* add terminating '.'-CRLF-NUL at the end of the message */
    msg->cpMsg[n++] =  '.';
    msg->cpMsg[n++] = '\r';
    msg->cpMsg[n++] = '\n';
    msg->cpMsg[n]   = NUL;

    logbook(msg->l2, L2_LEVEL_DEBUG, "join complete");
    return MSG_OK;
}

char *msg_error(msg_rc_t rc)
{
    char *str;
                                              str = "MSG: no description";
    if      (rc == MSG_OK                   ) str = "MSG: no error";
    else if (rc == MSG_ERR_MEM              ) str = "MSG: memory";
    else if (rc == MSG_ERR_SPLITHEADBODY    ) str = "MSG: split into header and body failed";
    else if (rc == MSG_ERR_SPLITLEN         ) str = "MSG: header is too short";
    else if (rc == MSG_ERR_SPLITMISSINGFROM ) str = "MSG: header is missing 'From ' envelope";
    else if (rc == MSG_ERR_SPLITIDNONE      ) str = "MSG: header is missing 'Message-ID'";
    else if (rc == MSG_ERR_SPLITIDEMPTY     ) str = "MSG: header has empty 'Message-ID'";
    else if (rc == MSG_ERR_SPLITIDMULTI     ) str = "MSG: header has multiple 'Message-ID's";
    else if (rc == MSG_ERR_JOINGROUPNONE    ) str = "MSG: join with no 'Newsgroup'";
    else if (rc == MSG_ERR_JOINGROUPEMPTY   ) str = "MSG: join with empty 'Newsgroup'";
    else if (rc == MSG_ERR_JOINIDNONE       ) str = "MSG: join with no 'Message-ID'";
    else if (rc == MSG_ERR_JOINIDEMPTY      ) str = "MSG: join with empty 'Message-ID'";
    return str;
}

    //FIXME below is the header rewriting engine which must be cleaned up and integrated

static void headerdestroy(headerdata_t *hdC)
{
    int i;

    if (hdC->ndata > 1) {
        for (i = 0; i < hdC->ndata; i++) {
            if (hdC->data.m[i] == NULL)
                break;
            free(hdC->data.m[i]);
        }
        free (hdC->data.m);
    } else
    if (hdC->ndata == 1)
        if (hdC->data.s != NULL)
            free(hdC->data.s);
    if (hdC->name != NULL)
        free(hdC->name);
    if (hdC->prev != NULL && hdC->prev->next == hdC)
        throw(0,0,0);
    if (hdC->next != NULL && hdC->next->prev == hdC)
        throw(0,0,0);
    free(hdC);
}

static void headerdelete(headerdata_t *hdC)
{
    if (hdC->prev != NULL)
        hdC->prev->next = hdC->next;
    hdC->next = NULL;
    if (hdC->next != NULL)
        hdC->next->prev = hdC->prev;
    hdC->prev = NULL;
    headerdestroy(hdC);
}

static void headerreplace(headerdata_t *hdC, headerdata_t *hdNew)
{
    hdNew->prev = hdC->prev;
    hdC->prev = NULL;
    hdNew->next = hdC->next;
    hdC->next = NULL;
    if (hdNew->prev != NULL)
        hdNew->prev->next = hdNew;
    if (hdNew->next != NULL)
        hdNew->next->prev = hdNew;
    headerdestroy(hdC);
}

static headerdata_t *headercreate(void)
{
    ex_t ex;
    volatile headerdata_t *hdNew = NULL;
    try {
        hdNew = mallocex(sizeof(headerdata_t));
        hdNew->prev  = NULL;
        hdNew->next  = NULL;
        hdNew->name  = NULL;
        hdNew->ndata = 0;
    }
    catch (ex) {
        if (hdNew != NULL)
            free((headerdata_t *)hdNew);
        rethrow;
    }
    return (headerdata_t *)hdNew;
}

struct regex_ctx_st; //FIXME go into a header!
typedef struct regex_ctx_st regex_ctx_t;
struct regex_ctx_st {
    int            nMatch;
    const char   **acpMatch;
    l2_env_t      *l2_env;
    l2_channel_t  *l2;
};

static var_rc_t regex_lookup(
    var_t *var, void *_ctx, 
    const char *var_ptr, size_t var_len, int var_idx,
    const char **val_ptr, size_t *val_len, size_t *val_size)
{
    regex_ctx_t *ctx = (regex_ctx_t *)_ctx;
    var_rc_t rc;
    char *cp;
    int i;

    logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d)", var_ptr, var_len);
    rc = VAR_ERR_UNDEFINED_VARIABLE;
    i = atoi(var_ptr); /* works with both '}' and '\0' termination */
    if (i < ctx->nMatch) {
        *val_ptr = ctx->acpMatch[i];
        *val_len = strlen(ctx->acpMatch[i]);
        *val_size = 0;
        rc = VAR_OK;
    }
    if (rc == VAR_OK)
        logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d) ok: result is \"%s\" (%d)", var_ptr, var_len, *val_ptr, *val_len);
    else
        logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d) failed: %s (%d)", var_ptr, var_len, var_strerror(var, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
    return rc;
}

static void canonifydate(const char *pVal, const char *pArg)
{
    /* RFC0822
     5.  DATE AND TIME SPECIFICATION
     5.1.  SYNTAX

     date-time   =  [ day "," ] date time          ; "dd mm yy" or "hh:mm:ss zzz"
     day         =  "Mon" / "Tue" / "Wed" / "Thu" / "Fri" / "Sat" / "Sun"
     date        =  1*2DIGIT month 2DIGIT          ; day month year e.g. 20 Jun 82
     month       =  "Jan" / "Feb"/ "Mar" / "Apr"/ "May" / "Jun"/ "Jul" / "Aug"/ "Sep" / "Oct"/ "Nov" / "Dec"
     time        =  hour zone                      ; ANSI and Military
     hour        =  2DIGIT ":" 2DIGIT [":" 2DIGIT] ; 00:00:00 - 23:59:59
     zone        =  "UT"  / "GMT"                  ; Universal Time, North American : UT
                 /  "EST" / "EDT"                  ;  Eastern:  - 5/ - 4
                 /  "CST" / "CDT"                  ;  Central:  - 6/ - 5
                 /  "MST" / "MDT"                  ;  Mountain: - 7/ - 6
                 /  "PST" / "PDT"                  ;  Pacific:  - 8/ - 7
                 /  1ALPHA                         ; Military: Z = UT; A:-1; (J not used);  M:-12; N:+1; Y:+12
                 / ( ("+" / "-") 4DIGIT )          ; Local differential  hours+min. (HHMM)
    */
    tai_t *tm;
    tai_rc_t rv;
    char out[32];
    int i;
    time_t tim;
    const struct tm *sttm;
    int ok;
    char *fmt[] = {
        "%a, %d %b %Y %H:%M:%S %z", /* RFC0822 but four digit year */
        "%a, %d %b %y %H:%M:%S %z", /* RFC0822 strict */
        "%a, %d %b %Y %H:%M %z",    /* RFC0822 but four digit year */
        "%a, %d %b %y %H:%M %z",    /* RFC0822 strict */
        "%d %b %Y %H:%M:%S %z",     /* RFC0822 but four digit year */
        "%d %b %y %H:%M:%S %z",     /* RFC0822 strict */
        "%d %b %Y %H:%M %z",        /* RFC0822 but four digit year */
        "%d %b %y %H:%M %z",        /* RFC0822 strict */
        "%a %b %d %H:%M:%S %Y",     /* strange Mon Jan 27 12:34:56 2001 */
        NULL };

    printf("DEBUG: pVal=\"%41s\", pArg=\"%s\" ", pVal, pArg);
    tai_create(&tm);
    tim = time(NULL);
    sttm = localtime(&tim);
    ok = 0;
    for (i = 0; !ok && (fmt[i] != NULL); i++) {
        //printf("DEBUG: date=%s, fmt[%d]=%30s ", pArg, i, fmt[i]);
        if ((rv = tai_parse(tm, pVal, strlen(pVal), fmt[i])) != TAI_OK) 
            ;//printf("FAILED tai_parse() returned %d", rv);
        else
            ok = 1;
#if 0
        else {
            if ((rv = tai_format(tm, out, sizeof(out), fmt[i])) != TAI_OK) 
                printf("#%d: tai_format() returned %d", i, rv);
            if (strcmp(pArg, out) != 0)
                printf("#%d: output \"%s\", expected \"%s\" (input)", i, out, pArg);
        }
#endif
        //printf("\n");
    }
    if (ok) {
        rv = tai_format(tm, out, sizeof(out), "%a, %d %b %Y %H:%M:%S %z");
        printf("OK[%d], %s (%d)\n", ok, out, rv);
    }
    else
        printf("FAILED\n");
    tai_destroy(tm);
}

static var_rc_t operate_cb(
    var_t *var, void *ctx,
    const char  *op_ptr, size_t op_len,
    const char  *arg_ptr, size_t arg_len,
    const char  *val_ptr, size_t val_len,
    char **out_ptr, size_t *out_len, size_t *out_size)
{
    int i;

    if (val_ptr == NULL) {
        *out_ptr = "";
        *out_len = 0;
        *out_size = 0;
        return VAR_OK;
    }
    if (op_len == 6 && strncmp(op_ptr, "return", 6) == 0) {
        *out_ptr = malloc(arg_len);
        *out_len = arg_len;
        *out_size = arg_len;
        memcpy(*out_ptr, arg_ptr, arg_len);
        return VAR_OK;
    }
    else if (op_len == 5 && strncmp(op_ptr, "upper", 5) == 0) {
        *out_ptr = malloc(val_len);
        *out_len = val_len;
        *out_size = val_len;
        for (i = 0; i < val_len; i++)
            (*out_ptr)[i] = (char)toupper((int)(val_ptr[i]));
        return VAR_OK;
    }
    else if (op_len == 12 && strncmp(op_ptr, "canonifydate", 12) == 0) {
        *out_ptr = malloc(val_len);
        *out_len = val_len;
        *out_size = val_len;
        canonifydate(val_ptr, arg_ptr);
        for (i = 0; i < val_len; i++)
            (*out_ptr)[i] = (char)tolower((int)(val_ptr[i]));
        return VAR_OK;
    }
    else 
        return VAR_ERR_UNDEFINED_OPERATION;
}

void msg_headermatrixbuildup(msg_t *msg)
{
    ex_t ex;
    volatile headerdata_t *hdNew = NULL;
    try {
        headerdata_t *hdI, *hdP;
        char *cp;

        cp = NULL;
        while ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) != NULL) { /* for each message header */

            /*FIXME we want O(1) here */
            for (hdP = NULL, hdI = msg->hdFirst; hdI != NULL; hdP = hdI, hdI = hdI->next) { /* for each matrix header */
                if (hdI->name == NULL || strlen(hdI->name) == 0 || hdI->ndata == 0)
                    continue;
                if (strcasecmp(cp, hdI->name) == 0)
                    break;
            }

            if (hdI == NULL) {
                hdNew = headercreate(); /* not found, create new */
                hdNew->name = strdupex(cp);
                hdI = (headerdata_t *)hdNew; hdNew = NULL; /* ex cleanup */
                if (hdP == NULL)
                    msg->hdFirst = hdI; /* no previous? this is the first */
                else {
                    hdP->next = hdI;
                    hdI->prev = hdP;
                }
            }
            cp = argz_next(msg->azHeaders, msg->asHeaders, cp);
            if (hdI->ndata == 0) {
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently empty", hdI->name);
                hdI->data.s = strdupex(cp);
                hdI->ndata = 1;
            }
            else if(hdI->ndata == 1) {
                char *cpOld;
                cpOld = hdI->data.s;
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently single valued", hdI->name);
                hdI->data.m = (char **)mallocex(3 * sizeof(char *));
                hdI->data.m[0] = strdupex(cpOld); //FIXME
                hdI->data.m[1] = strdupex(cp);
                hdI->data.m[2] = NULL;
                hdI->ndata = 2;
            }
            else {
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently multi valued %d", hdI->name, hdI->ndata);
                hdI->data.m = (char **)reallocex(hdI->data.m, (hdI->ndata + 2) * sizeof(char *));
                hdI->data.m[hdI->ndata++] = strdupex(cp);
                hdI->data.m[hdI->ndata] = NULL;
            }
        }
    }
    cleanup {
        if (hdNew != NULL)
            free((headerdata_t *)hdNew);
    }
    catch(ex) {
        rethrow;
    }
}

void msg_headermatrixteardwn(msg_t *msg)
{
    ex_t ex;
    try {
        headerdata_t *hdI;

        if (msg->azHeaders != NULL)
            free(msg->azHeaders);
        msg->azHeaders = NULL;
        msg->asHeaders = 0;

        for (hdI = msg->hdFirst; hdI != NULL; hdI = hdI->next) { /* for each matrix header */
            logbook(msg->l2, L2_LEVEL_DEBUG, "FIXME trace loop hdI=%.8lx, hI->name=\"%s\"", hdI, hdI->name);
            if (hdI->name == NULL || strlen(hdI->name) == 0 || hdI->ndata == 0)
                continue;
            if (hdI->ndata == 0) {
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, no data", hdI->name);
            }
            else if(hdI->ndata == 1) { /* header data is single valued */
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, data=%s", hdI->name, hdI->data.s);
                argz_add(&msg->azHeaders, &msg->asHeaders, hdI->name);
                argz_add(&msg->azHeaders, &msg->asHeaders, hdI->data.s);
            }
            else { /* header data is multi valued */
                int i;
                for (i = 0; i < hdI->ndata; i++) {
                    logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s[%d], data=%s", hdI->name, i, hdI->data.m[i]);
                    argz_add(&msg->azHeaders, &msg->asHeaders, hdI->name);
                    argz_add(&msg->azHeaders, &msg->asHeaders, hdI->data.m[i]);
                }
            }
        }
    }
    catch(ex) {
        rethrow;
    }
}

void headerrewrite(lmtp2nntp_t *ctx)
{
    headerrule_t *hrI;
    headerdata_t *hdI, *hdNew;
    regex_ctx_t *regex_ctx;
#define OVECSIZE 30
    int            ovec[OVECSIZE];
    var_rc_t rc; char *cp; //FIXME what a bad name, it's not the returncode of this function

    /* short circuit in case no headerrules were set up */
    if (ctx->option_firstheaderrule == NULL)
        return;

    { //FIXME debug code block
        int i;
        headerrule_t *hrD;
        headerdata_t *hdD;

        logbook(ctx->l2, L2_LEVEL_DEBUG, "FIXME trace ---------- headerrewrite() ----------");
        for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
            logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
        for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
            if (hdD->ndata == 0)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
            if (hdD->ndata == 1)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
            if (hdD->ndata > 1)
                for (i = 0; i < hdD->ndata; i++)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
        }
    }
    
    regex_ctx = (regex_ctx_t *)mallocex(sizeof(regex_ctx_t));
    regex_ctx->nMatch = 0;
    regex_ctx->acpMatch = NULL;
    regex_ctx->l2_env = ctx->l2_env;
    regex_ctx->l2 = ctx->l2;
    if ((rc = var_config(ctx->config_varregex, VAR_CONFIG_CB_VALUE, regex_lookup, regex_ctx)) != VAR_OK) {
        logbook(ctx->l2, L2_LEVEL_ERROR, "configure regex callback failed with %s (%d)", var_strerror(ctx->config_varregex, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
        throw(0,0,0);
    }
    if ((rc = var_config(ctx->config_varctx, VAR_CONFIG_CB_OPERATION, operate_cb, NULL)) != VAR_OK) { //FIXME isn't main a better place to do this?
        logbook(ctx->l2, L2_LEVEL_ERROR, "configure operate callback failed with %s (%d)", var_strerror(ctx->config_varctx, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
        throw(0,0,0);
    }
    for (hrI = ctx->option_firstheaderrule; hrI != NULL; hrI = hrI->next) { /* for each rule */
        { //FIXME debug code block
            int i;
            headerrule_t *hrD;
            headerdata_t *hdD;

            logbook(ctx->l2, L2_LEVEL_DEBUG, "FIXME trace ---------- headerrewrite() ---------- MIDDLE");
            for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
            for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
                //logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD=%.8lx, hdD->name=%.8lx, hdD->data.s=%.8lx", (long)hdD, (long)&hdD->name, (long)&hdD->data.s);
                if (hdD->ndata == 0)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
                if (hdD->ndata == 1)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
                if (hdD->ndata > 1)
                    for (i = 0; i < hdD->ndata; i++)
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
            }
        }
        if (hrI->regex != NULL) {
            logbook(ctx->l2, L2_LEVEL_DEBUG, "rule has regex %s", hrI->regex);
            for (hdI = ctx->msg->hdFirst; hdI != NULL; hdI = hdI->next) { /* for each header */
                if (hdI->name == NULL || strlen(hdI->name) == 0 || hdI->ndata == 0)
                    continue;
                regex_ctx->nMatch = pcre_exec(hrI->pcreRegex, hrI->pcreExtra, hdI->name, strlen(hdI->name), 0, 0, ovec, OVECSIZE);
                if (regex_ctx->nMatch >= 1) {
                    int i;
                    char *cp;
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "regex matches, %d references", regex_ctx->nMatch);
                    pcre_get_substring_list(hdI->name, ovec, regex_ctx->nMatch, &regex_ctx->acpMatch);
                    if (regex_ctx->acpMatch != NULL)
                        for (i = 0; i < regex_ctx->nMatch; i++)
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "regex reference[%d]=\'%s\'", i, regex_ctx->acpMatch[i] == NULL ? "(UNDEFINED)" : regex_ctx->acpMatch[i]);
                    hdNew = headercreate();
                    /* expanding regex references into header name */
                    {
                        var_rc_t var_rc;
                        char *res_ptr;
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding regex references in headername '%s'", hrI->header);
                        if ((var_rc = var_expand(ctx->config_varregex, hrI->header, strlen(hrI->header), &res_ptr, NULL, FALSE)) != VAR_OK) {
                            logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->header, var_strerror(ctx->config_varregex, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
                        }
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                        if (strlen(res_ptr) == 0) {
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - emtpy headername");
                            hdNew->name = NULL; //FIXME rename ->header to ->name
                            /*FIXME clean up data.s and data.m */
                            hdNew->ndata = 0;
                        }
                        else {
                            hdNew->name = res_ptr; //FIXME rename ->header to ->name
                        }
                    }
                    if (hrI->val == NULL) {
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue before expansion");
                        /*FIXME clean up data.s and data.m */
                        hdNew->ndata = 0;
                    }
                    else {
                        /* expanding regex references into header value */
                        {
                            var_rc_t var_rc;
                            char *res_ptr;
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding regex references in header value '%s'", hrI->val);
                            if ((var_rc = var_expand(ctx->config_varregex, hrI->val, strlen(hrI->val), &res_ptr, NULL, FALSE)) != VAR_OK) {
                                logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->val, var_strerror(ctx->config_varregex, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
                            }
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                            cp = res_ptr;
                        }
                        /* expanding variables into header value */
                        if (hrI->val != NULL) {
                            var_rc_t var_rc;
                            char *res_ptr;
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding variables in header value '%s'", hrI->val);
                            if ((var_rc = var_expand(ctx->config_varctx, cp, strlen(cp), &res_ptr, NULL, FALSE)) != VAR_OK) {
                                logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", cp, var_strerror(ctx->config_varctx, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
                            }
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                            if (strlen(res_ptr) == 0) {
                                logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue after expansion");
                                /*FIXME clean up data.s and data.m */
                                hdNew->ndata = 0;
                            }
                            else {
                                hdNew->data.s = res_ptr;
                                hdNew->ndata = 1;
                            }
                        }
                    }
                    /*FIXME clean up data.m */
                    headerreplace(hdI, hdNew);
                    if (hdNew->prev == NULL)
                        ctx->msg->hdFirst = hdNew;
                    hdI = hdNew;
                }
            }
        }
        else {
            logbook(ctx->l2, L2_LEVEL_DEBUG, "rule has no regex but static header %s", hrI->header);
            hdNew = headercreate();
            hdNew->name = strdupex(hrI->header); //FIXME rename ->header to ->name
            if (hrI->val == NULL) {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted");
                /*FIXME clean up data.s and data.m */
                hdNew->ndata = 0;
            }
            else {
                /*FIXME clean up data.m */
                /* expanding variables into header value */
                var_rc_t var_rc;
                char *res_ptr;
                logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding variables in header value '%s'", hrI->val);
                if ((var_rc = var_expand(ctx->config_varctx, hrI->val, strlen(hrI->val), &res_ptr, NULL, FALSE)) != VAR_OK) {
                    logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->val, var_strerror(ctx->config_varctx, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
                }
                logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                if (strlen(res_ptr) == 0) {
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue after expansion");
                    /*FIXME clean up data.s and data.m */
                    hdNew->ndata = 0;
                }
                else {
                    hdNew->data.s = res_ptr;
                    hdNew->ndata = 1;
                }
            }
            for (hdI = ctx->msg->hdFirst; hdI != NULL; hdI = hdI->next) { /* for each header */
                if (hdI->name == NULL || strlen(hdI->name) == 0)
                    continue;
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hrI->header=%s, hdI->name=%s", hrI->header, hdI->name);
                if (strcasecmp(hrI->header, hdI->name) == 0)
                    break;
            }
            if (hdI != NULL) {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "replacing header %s", hrI->header);
                headerreplace(hdI, hdNew);
                if (hdNew->prev == NULL) {
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "FIXME trace #1");
                    ctx->msg->hdFirst = hdNew;
                }
            }
            else {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "appending header %s", hrI->header);
                for (hdI = ctx->msg->hdFirst; hdI->next != NULL; hdI = hdI->next);
                hdI->next = hdNew;
                hdNew->prev = hdI;
            }
        }
    }

    { //FIXME debug code block
        int i;
        headerrule_t *hrD;
        headerdata_t *hdD;

        logbook(ctx->l2, L2_LEVEL_DEBUG, "FIXME trace ---------- headerrewrite() ---------- FINISH");
        for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
            logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
        for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
            if (hdD->ndata == 0)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
            if (hdD->ndata == 1)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
            if (hdD->ndata > 1)
                for (i = 0; i < hdD->ndata; i++)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
        }
    }
}