/* REMOVE ME - THIS IS TO TRIGGER CI RUN */
/* omkafka.c
 * This output plugin make rsyslog talk to Apache Kafka.
 *
 * Copyright 2014-2017 by Adiscon GmbH.
 *
 * This file is part of rsyslog.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *       -or-
 *       see COPYING.ASL20 in the source distribution
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include "config.h"
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/uio.h>
#include "compat_queue.h"
#include <sys/types.h>
#include <math.h>
#ifdef HAVE_SYS_STAT_H
    #include <sys/stat.h>
#endif
#include <unistd.h>
#include <librdkafka/rdkafka.h>

#ifndef RD_KAFKA_VERSION
    #define RD_KAFKA_VERSION 0
#endif

/* rd_kafka_produceva() and rd_kafka_vu_t landed in librdkafka v1.5.0. */
#if RD_KAFKA_VERSION >= 0x01050000
    #define OMKAFKA_HAS_PRODUCEVA 1
#else
    #define OMKAFKA_HAS_PRODUCEVA 0
#endif

/* rd_kafka_producev() helpers were introduced with librdkafka v0.9.4. */
#if defined(RD_KAFKA_V_RKT) && defined(RD_KAFKA_V_PARTITION) && defined(RD_KAFKA_V_VALUE) &&    \
    defined(RD_KAFKA_V_MSGFLAGS) && defined(RD_KAFKA_V_TIMESTAMP) && defined(RD_KAFKA_V_KEY) && \
    defined(RD_KAFKA_V_END)
    #define OMKAFKA_HAS_PRODUCEV 1
#else
    #define OMKAFKA_HAS_PRODUCEV 0
#endif

/* Static record header helpers first appeared in librdkafka v0.11.0. */
#if defined(RD_KAFKA_V_HEADERS) && (OMKAFKA_HAS_PRODUCEVA || OMKAFKA_HAS_PRODUCEV)
    #define OMKAFKA_HAS_HEADERS 1
#else
    #define OMKAFKA_HAS_HEADERS 0
#endif

#define OMKAFKA_COPY_HEADERS_OR_ABORT(hdrs_var, pData)                                  \
    do {                                                                                \
        (hdrs_var) = rd_kafka_headers_copy((pData)->kafka_headers);                     \
        if ((hdrs_var) == NULL) {                                                       \
            LogError(0, RS_RET_OUT_OF_MEMORY, "omkafka: failed to copy kafka headers"); \
            ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);                                       \
        }                                                                               \
    } while (0)

#include "rsyslog.h"
#include "syslogd-types.h"
#include "srUtils.h"
#include "template.h"
#include "module-template.h"
#include "errmsg.h"
#include "atomic.h"
#include "statsobj.h"
#include "unicode-helper.h"
#include "datetime.h"

MODULE_TYPE_OUTPUT;
MODULE_TYPE_NOKEEP;
MODULE_CNFNAME("omkafka")

/* internal structures
 */
DEF_OMOD_STATIC_DATA;
DEFobjCurrIf(datetime) DEFobjCurrIf(strm) DEFobjCurrIf(statsobj)

    statsobj_t *kafkaStats;
STATSCOUNTER_DEF(ctrQueueSize, mutCtrQueueSize);
STATSCOUNTER_DEF(ctrTopicSubmit, mutCtrTopicSubmit);
STATSCOUNTER_DEF(ctrKafkaFail, mutCtrKafkaFail);
STATSCOUNTER_DEF(ctrCacheMiss, mutCtrCacheMiss);
STATSCOUNTER_DEF(ctrCacheEvict, mutCtrCacheEvict);
STATSCOUNTER_DEF(ctrCacheSkip, mutCtrCacheSkip);
STATSCOUNTER_DEF(ctrKafkaAck, mutCtrKafkaAck);
STATSCOUNTER_DEF(ctrKafkaMsgTooLarge, mutCtrKafkaMsgTooLarge);
STATSCOUNTER_DEF(ctrKafkaUnknownTopic, mutCtrKafkaUnknownTopic);
STATSCOUNTER_DEF(ctrKafkaQueueFull, mutCtrKafkaQueueFull);
STATSCOUNTER_DEF(ctrKafkaUnknownPartition, mutCtrKafkaUnknownPartition);
STATSCOUNTER_DEF(ctrKafkaOtherErrors, mutCtrKafkaOtherErrors);
STATSCOUNTER_DEF(ctrKafkaRespTimedOut, mutCtrKafkaRespTimedOut);
STATSCOUNTER_DEF(ctrKafkaRespTransport, mutCtrKafkaRespTransport);
STATSCOUNTER_DEF(ctrKafkaRespBrokerDown, mutCtrKafkaRespBrokerDown);
STATSCOUNTER_DEF(ctrKafkaRespAuth, mutCtrKafkaRespAuth);
STATSCOUNTER_DEF(ctrKafkaRespSSL, mutCtrKafkaRespSSL);
STATSCOUNTER_DEF(ctrKafkaRespOther, mutCtrKafkaRespOther);

#define MAX_ERRMSG 1024 /* max size of error messages that we support */

#ifndef SLIST_INIT
    #define SLIST_INIT(head)          \
        do {                          \
            (head)->slh_first = NULL; \
        } while (/*CONSTCOND*/ 0)
#endif

#ifndef SLIST_ENTRY
    #define SLIST_ENTRY(type)                         \
        struct {                                      \
            struct type *sle_next; /* next element */ \
        }
#endif

#ifndef SLIST_HEAD
    #define SLIST_HEAD(name, type)                      \
        struct name {                                   \
            struct type *slh_first; /* first element */ \
        }
#endif

#ifndef SLIST_INSERT_HEAD
    #define SLIST_INSERT_HEAD(head, elm, field)        \
        do {                                           \
            (elm)->field.sle_next = (head)->slh_first; \
            (head)->slh_first = (elm);                 \
        } while (/*CONSTCOND*/ 0)
#endif

#ifndef SLIST_REMOVE_HEAD
    #define SLIST_REMOVE_HEAD(head, field)                         \
        do {                                                       \
            (head)->slh_first = (head)->slh_first->field.sle_next; \
        } while (/*CONSTCOND*/ 0)
#endif

#ifndef SLIST_FIRST
    #define SLIST_FIRST(head) ((head)->slh_first)
#endif

#ifndef SLIST_NEXT
    #define SLIST_NEXT(elm, field) ((elm)->field.sle_next)
#endif

#ifndef SLIST_EMPTY
    #define SLIST_EMPTY(head) ((head)->slh_first == NULL)
#endif

#ifndef SLIST_REMOVE
    #define SLIST_REMOVE(head, elm, type, field)                                         \
        do {                                                                             \
            if ((head)->slh_first == (elm)) {                                            \
                SLIST_REMOVE_HEAD((head), field);                                        \
            } else {                                                                     \
                struct type *curelm = (head)->slh_first;                                 \
                while (curelm->field.sle_next != (elm)) curelm = curelm->field.sle_next; \
                curelm->field.sle_next = curelm->field.sle_next->field.sle_next;         \
            }                                                                            \
        } while (/*CONSTCOND*/ 0)
#endif

#define NO_FIXED_PARTITION -1 /* signifies that no fixed partition config exists */

struct kafka_params {
    const char *name;
    const char *val;
};

#ifndef O_LARGEFILE
    #define O_LARGEFILE 0
#endif

/* flags for writeKafka: shall we resubmit a failed message? */
#define RESUBMIT 1
#define NO_RESUBMIT 0

#ifdef HAVE_ATOMIC_BUILTINS64
static uint64 clockTopicAccess = 0;
#else
static unsigned clockTopicAccess = 0;
#endif
/* and the "tick" function */
#ifndef HAVE_ATOMIC_BUILTINS
static pthread_mutex_t mutClock;
#endif
static uint64 getClockTopicAccess(void) {
#ifdef HAVE_ATOMIC_BUILTINS64
    return ATOMIC_INC_AND_FETCH_uint64(&clockTopicAccess, &mutClock);
#else
    return ATOMIC_INC_AND_FETCH_unsigned(&clockTopicAccess, &mutClock);
#endif
}

/* Needed for Kafka timestamp librdkafka > 0.9.4 */
#define KAFKA_TimeStamp "\"%timestamp:::date-unixtimestamp%\""

static int closeTimeout = 1000;
static pthread_mutex_t closeTimeoutMut = PTHREAD_MUTEX_INITIALIZER;

#define OMKAFKA_POLL_THREAD_SLEEP_USEC 100000

/* stats callback window metrics */
static uint64 rtt_avg_usec;
static uint64 throttle_avg_msec;
static uint64 int_latency_avg_usec;

/* dynamic topic cache */
struct s_dynaTopicCacheEntry {
    uchar *pName;
    rd_kafka_topic_t *pTopic;
    uint64 clkTickAccessed;
    pthread_rwlock_t lock;
};
typedef struct s_dynaTopicCacheEntry dynaTopicCacheEntry;

/* Struct for Failed Messages Listitems */
struct s_failedmsg_entry {
    uchar *key;
    uchar *payload;
    uchar *topicname;
    SLIST_ENTRY(s_failedmsg_entry) entries; /*	List. */
};
typedef struct s_failedmsg_entry failedmsg_entry;

typedef struct _instanceData {
    uchar *topic;
    sbool dynaKey;
    sbool dynaTopic;
    dynaTopicCacheEntry **dynCache;
    pthread_mutex_t mutDynCache;
    rd_kafka_topic_t *pTopic;
    int iCurrElt;
    int iCurrCacheSize;
    int bReportErrs;
    int iDynaTopicCacheSize;
    uchar *tplName; /* assigned output template */
    char *brokers;
    sbool autoPartition;
    int fixedPartition;
    int nPartitions;
    uint32_t currPartition;
    DEF_ATOMIC_HELPER_MUT(mutCurrPartition);
    int nConfParams;
    struct kafka_params *confParams;
    int nTopicConfParams;
    struct kafka_params *topicConfParams;
    int nHeaders;
    struct kafka_params *headers;
#if OMKAFKA_HAS_HEADERS
    rd_kafka_headers_t *kafka_headers;
#endif
    uchar *errorFile;
    uchar *key;
    int bReopenOnHup;
    int bResubmitOnFailure; /* Resubmit failed messages into kafka queue*/
    int bKeepFailedMessages; /* Keep Failed messages in memory,
                             only works if bResubmitOnFailure is enabled */
    uchar *failedMsgFile; /* file in which failed messages are being stored on
                          shutdown and loaded on startup */

    int fdErrFile; /* error file fd or -1 if not open */
    pthread_mutex_t mutErrFile;
    uchar *statsFile;
    int fdStatsFile; /* stats file fd or -1 if not open */
    pthread_mutex_t mutStatsFile;
    int bIsOpen;
    int bIsSuspended; /* when broker fail, we need to suspend the action */
    pthread_rwlock_t rkLock;
    pthread_mutex_t mut_doAction; /* make sure one wrkr instance max in parallel */
    pthread_t pollThread;
    int pollThreadRunning;
    int stopPollThread;
    DEF_ATOMIC_HELPER_MUT(mutStopPollThread);
    rd_kafka_t *rk;
    int closeTimeout;
    SLIST_HEAD(failedmsg_listhead, s_failedmsg_entry) failedmsg_head;

    uchar *statsName;
    statsobj_t *stats;
    STATSCOUNTER_DEF(ctrTopicSubmit, mutCtrTopicSubmit);
    STATSCOUNTER_DEF(ctrKafkaFail, mutCtrKafkaFail);
    STATSCOUNTER_DEF(ctrKafkaAck, mutCtrKafkaAck);
} instanceData;

typedef struct wrkrInstanceData {
    instanceData *pData;
} wrkrInstanceData_t;

static void *pollCallbackThread(void *arg) {
    instanceData *const pData = (instanceData *)arg;

    while (!ATOMIC_LOAD_32BIT_RELAXED(&pData->stopPollThread, &pData->mutStopPollThread)) {
        if (pthread_mutex_trylock(&pData->mut_doAction) == 0) {
            pthread_rwlock_rdlock(&pData->rkLock);
            if (pData->bIsOpen && pData->rk != NULL) {
                const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
                DBGPRINTF("omkafka: callback-poller outqueue length: %d, callbacks called %d\n",
                          rd_kafka_outq_len(pData->rk), callbacksCalled);
            }
            pthread_rwlock_unlock(&pData->rkLock);
            pthread_mutex_unlock(&pData->mut_doAction);
        }
        srSleep(0, OMKAFKA_POLL_THREAD_SLEEP_USEC);
    }

    return NULL;
}

static rsRetVal startPollCallbackThread(instanceData *const pData) {
    DEFiRet;

    if (pData->pollThreadRunning) {
        FINALIZE;
    }

    ATOMIC_STORE_32BIT_RELAXED(&pData->stopPollThread, &pData->mutStopPollThread, 0);
    const int r = pthread_create(&pData->pollThread, NULL, pollCallbackThread, pData);
    if (r != 0) {
        LogError(r, RS_RET_ERR, "omkafka: unable to create librdkafka callback poller thread");
        ABORT_FINALIZE(RS_RET_ERR);
    }
    pData->pollThreadRunning = 1;

finalize_it:
    RETiRet;
}

static void stopPollCallbackThread(instanceData *const pData) {
    if (!pData->pollThreadRunning) {
        return;
    }

    ATOMIC_STORE_32BIT_RELAXED(&pData->stopPollThread, &pData->mutStopPollThread, 1);
    const int r = pthread_join(pData->pollThread, NULL);
    if (r != 0) {
        LogError(r, RS_RET_ERR, "omkafka: unable to join librdkafka callback poller thread");
    }
    pData->pollThreadRunning = 0;
}

#define INST_STATSCOUNTER_INC(inst, ctr, mut) \
    do {                                      \
        if (inst->stats) {                    \
            STATSCOUNTER_INC(ctr, mut);       \
        }                                     \
    } while (0);

/* tables for interfacing with the v6 config system */
/* action (instance) parameters */
static struct cnfparamdescr actpdescr[] = {
    {"topic", eCmdHdlrString, CNFPARAM_REQUIRED},
    {"dynatopic", eCmdHdlrBinary, 0},
    {"dynatopic.cachesize", eCmdHdlrInt, 0},
    {"dynakey", eCmdHdlrBinary, 0},
    {"partitions.auto", eCmdHdlrBinary, 0}, /* use librdkafka's automatic partitioning function */
    {"partitions.number", eCmdHdlrPositiveInt, 0},
    {"partitions.usefixed", eCmdHdlrNonNegInt, 0}, /* expert parameter, "nails" partition */
    {"broker", eCmdHdlrArray, 0},
    {"confparam", eCmdHdlrArray, 0},
    {"topicconfparam", eCmdHdlrArray, 0},
    {"kafkaheader", eCmdHdlrArray, 0},
    {"errorfile", eCmdHdlrGetWord, 0},
    {"statsfile", eCmdHdlrGetWord, 0},
    {"key", eCmdHdlrGetWord, 0},
    {"template", eCmdHdlrGetWord, 0},
    {"closetimeout", eCmdHdlrPositiveInt, 0},
    {"reopenonhup", eCmdHdlrBinary, 0},
    {"resubmitonfailure", eCmdHdlrBinary, 0}, /* Resubmit message into kafaj queue on failure */
    {"keepfailedmessages", eCmdHdlrBinary, 0},
    {"failedmsgfile", eCmdHdlrGetWord, 0},
    {"statsname", eCmdHdlrGetWord, 0}};
static struct cnfparamblk actpblk = {CNFPARAMBLK_VERSION, sizeof(actpdescr) / sizeof(struct cnfparamdescr), actpdescr};

BEGINinitConfVars /* (re)set config variables to default values */
    CODESTARTinitConfVars;
ENDinitConfVars

static uint32_t getPartition(instanceData *const __restrict__ pData) {
    if (pData->autoPartition) {
        return RD_KAFKA_PARTITION_UA;
    } else {
        return (pData->fixedPartition == NO_FIXED_PARTITION)
                   ? ATOMIC_INC_AND_FETCH_unsigned(&pData->currPartition, &pData->mutCurrPartition) % pData->nPartitions
                   : (unsigned)pData->fixedPartition;
    }
}

/* must always be called with appropriate locks taken */
static void free_topic(rd_kafka_topic_t **topic) {
    if (*topic != NULL) {
        DBGPRINTF("omkafka: closing topic %s\n", rd_kafka_topic_name(*topic));
        rd_kafka_topic_destroy(*topic);
        *topic = NULL;
    }
}

static void ATTR_NONNULL(1) failedmsg_entry_destruct(failedmsg_entry *const __restrict__ fmsgEntry) {
    free(fmsgEntry->key);
    free(fmsgEntry->payload);
    free(fmsgEntry->topicname);
    free(fmsgEntry);
}

/* note: we need the length of message as we need to deal with
 * non-NUL terminated strings under some circumstances.
 */
static failedmsg_entry *ATTR_NONNULL(3, 5) failedmsg_entry_construct(const char *const key,
                                                                     const size_t keylen,
                                                                     const char *const msg,
                                                                     const size_t msglen,
                                                                     const char *const topicname) {
    failedmsg_entry *etry = NULL;

    if ((etry = malloc(sizeof(struct s_failedmsg_entry))) == NULL) {
        return NULL;
    }

    if (key) {
        if ((etry->key = (uchar *)malloc(keylen + 1)) == NULL) {
            free(etry);
            return NULL;
        }
        memcpy(etry->key, key, keylen);
        etry->key[keylen] = '\0';
    } else {
        etry->key = NULL;
    }

    if ((etry->payload = (uchar *)malloc(msglen + 1)) == NULL) {
        free(etry->key);
        free(etry);
        return NULL;
    }
    memcpy(etry->payload, msg, msglen);
    etry->payload[msglen] = '\0';
    if ((etry->topicname = (uchar *)strdup(topicname)) == NULL) {
        free(etry->key);
        free(etry->payload);
        free(etry);
        return NULL;
    }
    return etry;
}

/* destroy topic item */
/* must be called with write(rkLock) */
static void closeTopic(instanceData *__restrict__ const pData) {
    free_topic(&pData->pTopic);
}

/* these dynaTopic* functions are only slightly modified versions of those found in omfile.c.
 * check the sources in omfile.c for more descriptive comments about each of these functions.
 * i will only put the bare descriptions in this one. 2015-01-09 - Tait Clarridge
 */

/* delete a cache entry from the dynamic topic cache */
/* must be called with lock(mutDynCache) */
static rsRetVal dynaTopicDelCacheEntry(instanceData *__restrict__ const pData, const int iEntry, const int bFreeEntry) {
    dynaTopicCacheEntry **pCache = pData->dynCache;
    DEFiRet;
    assert(pCache != NULL);

    if (pCache[iEntry] == NULL) FINALIZE;
    pthread_rwlock_wrlock(&pCache[iEntry]->lock);

    DBGPRINTF("Removing entry %d for topic '%s' from dynaCache.\n", iEntry,
              pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName);

    if (pCache[iEntry]->pName != NULL) {
        free(pCache[iEntry]->pName);
        pCache[iEntry]->pName = NULL;
    }

    pthread_rwlock_unlock(&pCache[iEntry]->lock);

    if (bFreeEntry) {
        pthread_rwlock_destroy(&pCache[iEntry]->lock);
        free(pCache[iEntry]);
        pCache[iEntry] = NULL;
    }

finalize_it:
    RETiRet;
}

/* clear the entire dynamic topic cache */
static void dynaTopicFreeCacheEntries(instanceData *__restrict__ const pData) {
    register int i;
    assert(pData != NULL);

    pthread_mutex_lock(&pData->mutDynCache);
    for (i = 0; i < pData->iCurrCacheSize; ++i) {
        dynaTopicDelCacheEntry(pData, i, 1);
    }
    pData->iCurrElt = -1; /* invalidate current element */
    pthread_mutex_unlock(&pData->mutDynCache);
}

/* create the topic object */
/* must be called with _atleast_ read(rkLock) */
static rsRetVal createTopic(instanceData *__restrict__ const pData,
                            const uchar *__restrict__ const newTopicName,
                            rd_kafka_topic_t **topic) {
    /* Get a new topic conf */
    rd_kafka_topic_conf_t *const topicconf = rd_kafka_topic_conf_new();
    char errstr[MAX_ERRMSG];
    rd_kafka_topic_t *rkt = NULL;
    DEFiRet;

    *topic = NULL;

    if (topicconf == NULL) {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: error creating kafka topic conf obj: %s\n",
                 rd_kafka_err2str(rd_kafka_last_error()));
        ABORT_FINALIZE(RS_RET_KAFKA_ERROR);
    }
    for (int i = 0; i < pData->nTopicConfParams; ++i) {
        DBGPRINTF("omkafka: setting custom topic configuration parameter: %s:%s\n", pData->topicConfParams[i].name,
                  pData->topicConfParams[i].val);
        if (rd_kafka_topic_conf_set(topicconf, pData->topicConfParams[i].name, pData->topicConfParams[i].val, errstr,
                                    sizeof(errstr)) != RD_KAFKA_CONF_OK) {
            if (pData->bReportErrs) {
                LogError(0, RS_RET_PARAM_ERROR,
                         "error in kafka "
                         "topic conf parameter '%s=%s': %s",
                         pData->topicConfParams[i].name, pData->topicConfParams[i].val, errstr);
            } else {
                DBGPRINTF("omkafka: setting custom topic configuration parameter '%s=%s': %s",
                          pData->topicConfParams[i].name, pData->topicConfParams[i].val, errstr);
            }
            ABORT_FINALIZE(RS_RET_PARAM_ERROR);
        }
    }
    rkt = rd_kafka_topic_new(pData->rk, (char *)newTopicName, topicconf);
    if (rkt == NULL) {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: error creating kafka topic: %s\n",
                 rd_kafka_err2str(rd_kafka_last_error()));
        ABORT_FINALIZE(RS_RET_KAFKA_ERROR);
    }
    *topic = rkt;
finalize_it:
    RETiRet;
}

/* create the topic object */
/* must be called with write(rkLock) */
static rsRetVal prepareTopic(instanceData *__restrict__ const pData, const uchar *__restrict__ const newTopicName) {
    DEFiRet;
    iRet = createTopic(pData, newTopicName, &pData->pTopic);
    if (iRet != RS_RET_OK) {
        if (pData->pTopic != NULL) {
            closeTopic(pData);
        }
    }
    RETiRet;
}

/* check dynamic topic cache for existence of the already created topic.
 * if it does not exist, create a new one, or if we are currently using it
 * as of the last message, keep using it.
 *
 * must be called with read(rkLock)
 * must be called with mutDynCache locked
 */
static rsRetVal ATTR_NONNULL() prepareDynTopic(instanceData *__restrict__ const pData,
                                               const uchar *__restrict__ const newTopicName,
                                               rd_kafka_topic_t **topic,
                                               pthread_rwlock_t **lock) {
    uint64 ctOldest;
    int iOldest;
    int i;
    int iFirstFree;
    rsRetVal localRet;
    dynaTopicCacheEntry **pCache;
    dynaTopicCacheEntry *entry = NULL;
    rd_kafka_topic_t *tmpTopic = NULL;
    DEFiRet;
    assert(pData != NULL);
    assert(newTopicName != NULL);

    pCache = pData->dynCache;
    /* first check, if we still have the current topic */
    if ((pData->iCurrElt != -1) && !ustrcmp(newTopicName, pCache[pData->iCurrElt]->pName)) {
        /* great, we are all set */
        pCache[pData->iCurrElt]->clkTickAccessed = getClockTopicAccess();
        entry = pCache[pData->iCurrElt];
        STATSCOUNTER_INC(ctrCacheSkip, mutCtrCacheSkip);
        FINALIZE;
    }

    /* ok, no luck. Now let's search the table if we find a matching spot.
     * While doing so, we also prepare for creation of a new one.
     */
    pData->iCurrElt = -1;
    iFirstFree = -1;
    iOldest = 0;
    ctOldest = getClockTopicAccess();
    for (i = 0; i < pData->iCurrCacheSize; ++i) {
        if (pCache[i] == NULL || pCache[i]->pName == NULL) {
            if (iFirstFree == -1) iFirstFree = i;
        } else { /*got an element, let's see if it matches */
            if (!ustrcmp(newTopicName, pCache[i]->pName)) {
                /* we found our element! */
                entry = pCache[i];
                pData->iCurrElt = i;
                /* update "timestamp" for LRU */
                pCache[i]->clkTickAccessed = getClockTopicAccess();
                FINALIZE;
            }
            /* did not find it - so lets keep track of the counters for LRU */
            if (pCache[i]->clkTickAccessed < ctOldest) {
                ctOldest = pCache[i]->clkTickAccessed;
                iOldest = i;
            }
        }
    }
    STATSCOUNTER_INC(ctrCacheMiss, mutCtrCacheMiss);

    /* invalidate iCurrElt as we may error-exit out of this function when the currrent
     * iCurrElt has been freed or otherwise become unusable. This is a precaution, and
     * performance-wise it may be better to do that in each of the exits. However, that
     * is error-prone, so I prefer to do it here. -- rgerhards, 2010-03-02
     */
    pData->iCurrElt = -1;

    if (iFirstFree == -1 && (pData->iCurrCacheSize < pData->iDynaTopicCacheSize)) {
        /* there is space left, so set it to that index */
        iFirstFree = pData->iCurrCacheSize++;
    }

    if (iFirstFree == -1) {
        dynaTopicDelCacheEntry(pData, iOldest, 0);
        STATSCOUNTER_INC(ctrCacheEvict, mutCtrCacheEvict);
        iFirstFree = iOldest; /* this one *is* now free ;) */
    } else {
        pCache[iFirstFree] = NULL;
    }
    /* we need to allocate memory for the cache structure */
    if (pCache[iFirstFree] == NULL) {
        CHKmalloc(pCache[iFirstFree] = (dynaTopicCacheEntry *)calloc(1, sizeof(dynaTopicCacheEntry)));
        CHKiRet(pthread_rwlock_init(&pCache[iFirstFree]->lock, NULL));
    }

    /* Ok, we finally can open the topic */
    localRet = createTopic(pData, newTopicName, &tmpTopic);

    if (localRet != RS_RET_OK) {
        LogError(0, localRet,
                 "Could not open dynamic topic '%s' "
                 "[state %d] - discarding message",
                 newTopicName, localRet);
        ABORT_FINALIZE(localRet);
    }

    if ((pCache[iFirstFree]->pName = ustrdup(newTopicName)) == NULL) {
        free_topic(&tmpTopic);
        ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
    }
    pCache[iFirstFree]->pTopic = tmpTopic;
    pCache[iFirstFree]->clkTickAccessed = getClockTopicAccess();
    entry = pCache[iFirstFree];
    pData->iCurrElt = iFirstFree;
    DBGPRINTF("Added new entry %d for topic cache, topic '%s'.\n", iFirstFree, newTopicName);

finalize_it:
    if (iRet == RS_RET_OK) {
        *topic = entry->pTopic;
        *lock = &entry->lock;
    }
    RETiRet;
}

/* write data error request/replies to separate error file
 * Note: we open the file but never close it before exit. If it
 * needs to be closed, HUP must be sent.
 */
static rsRetVal writeLogError(instanceData *const pData, int level, const char *fac, const char *buf) {
    int bLocked = 0;
    struct json_object *json = NULL;
    DEFiRet;

    if (pData->errorFile == NULL) {
        FINALIZE;
    }

    // only log errors
    if (level > LOG_ERR) {
        FINALIZE;
    }

    json = json_object_new_object();
    if (json == NULL) {
        ABORT_FINALIZE(RS_RET_ERR);
    }
    struct json_object *jval;
    jval = json_object_new_int(level);
    json_object_object_add(json, "loglevel", jval);
    jval = json_object_new_string(fac);
    json_object_object_add(json, "logfacility", jval);
    jval = json_object_new_string(buf);
    json_object_object_add(json, "msg", jval);

    struct iovec iov[2];
    iov[0].iov_base = (void *)json_object_get_string(json);
    iov[0].iov_len = strlen(iov[0].iov_base);
    iov[1].iov_base = (char *)"\n";
    iov[1].iov_len = 1;

    /* we must protect the file write do operations due to other wrks & HUP */
    pthread_mutex_lock(&pData->mutErrFile);
    bLocked = 1;
    if (pData->fdErrFile == -1) {
        pData->fdErrFile = open((char *)pData->errorFile, O_WRONLY | O_CREAT | O_APPEND | O_LARGEFILE | O_CLOEXEC,
                                S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
        if (pData->fdErrFile == -1) {
            LogError(errno, RS_RET_ERR, "omkafka: error opening error file %s", pData->errorFile);
            ABORT_FINALIZE(RS_RET_ERR);
        }
    }

    /* Note: we do not do real error-handling on the err file, as this
     * complicates things way to much.
     */
    const ssize_t nwritten = writev(pData->fdErrFile, iov, sizeof(iov) / sizeof(struct iovec));
    if (nwritten != (ssize_t)iov[0].iov_len + 1) {
        LogError(errno, RS_RET_ERR, "omkafka: error writing error file, write returns %lld\n", (long long)nwritten);
    }

finalize_it:
    if (bLocked) pthread_mutex_unlock(&pData->mutErrFile);
    if (json != NULL) json_object_put(json);
    RETiRet;
}

/* write data error request/replies to separate error file
 * Note: we open the file but never close it before exit. If it
 * needs to be closed, HUP must be sent.
 */
static rsRetVal writeDataError(instanceData *const pData,
                               const char *const __restrict__ data,
                               const size_t lenData,
                               const int kafkaErr) {
    int bLocked = 0;
    struct json_object *json = NULL;
    DEFiRet;

    if (pData->errorFile == NULL) {
        FINALIZE;
    }

    json = json_object_new_object();
    if (json == NULL) {
        ABORT_FINALIZE(RS_RET_ERR);
    }
    struct json_object *jval;
    jval = json_object_new_int(kafkaErr);
    json_object_object_add(json, "errcode", jval);
    jval = json_object_new_string(rd_kafka_err2str(kafkaErr));
    json_object_object_add(json, "errmsg", jval);
    jval = json_object_new_string_len(data, lenData);
    json_object_object_add(json, "data", jval);

    struct iovec iov[2];
    iov[0].iov_base = (void *)json_object_get_string(json);
    iov[0].iov_len = strlen(iov[0].iov_base);
    iov[1].iov_base = (char *)"\n";
    iov[1].iov_len = 1;

    /* we must protect the file write do operations due to other wrks & HUP */
    pthread_mutex_lock(&pData->mutErrFile);
    bLocked = 1;
    if (pData->fdErrFile == -1) {
        pData->fdErrFile = open((char *)pData->errorFile, O_WRONLY | O_CREAT | O_APPEND | O_LARGEFILE | O_CLOEXEC,
                                S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
        if (pData->fdErrFile == -1) {
            LogError(errno, RS_RET_ERR, "omkafka: error opening error file %s", pData->errorFile);
            ABORT_FINALIZE(RS_RET_ERR);
        }
    }

    /* Note: we do not do real error-handling on the err file, as this
     * complicates things way to much.
     */
    const ssize_t nwritten = writev(pData->fdErrFile, iov, sizeof(iov) / sizeof(struct iovec));
    if (nwritten != (ssize_t)iov[0].iov_len + 1) {
        LogError(errno, RS_RET_ERR, "omkafka: error writing error file, write returns %lld\n", (long long)nwritten);
    }

finalize_it:
    if (bLocked) pthread_mutex_unlock(&pData->mutErrFile);
    if (json != NULL) json_object_put(json);
    RETiRet;
}

/* write librdkafka stats object to a file
 * Note: we open the file but never close it before exit. If it
 * needs to be closed, HUP must be sent.
 * Assumes pData->statsFile != NULL.
 */
static rsRetVal writeStats(instanceData *const pData, char *statsData, const size_t lenData) {
    int bLocked = 0;
    DEFiRet;

    /* Protect the file write from operations due to other wrks & HUP */
    pthread_mutex_lock(&pData->mutStatsFile);
    bLocked = 1;
    if (pData->fdStatsFile == -1) {
        pData->fdStatsFile = open((char *)pData->statsFile, O_WRONLY | O_CREAT | O_APPEND | O_LARGEFILE | O_CLOEXEC,
                                  S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
        if (pData->fdStatsFile == -1) {
            LogError(errno, RS_RET_ERR, "omkafka: error opening stats file %s", pData->statsFile);
            ABORT_FINALIZE(RS_RET_ERR);
        }
    }

    ssize_t nwritten = write(pData->fdStatsFile, statsData, lenData);
    nwritten += write(pData->fdStatsFile, "\n", 1);
    if (nwritten != (ssize_t)lenData + 1) {
        LogError(errno, RS_RET_ERR, "omkafka: error writing stats file, write returns %lld, expected %lld\n",
                 (long long)nwritten, (long long)(lenData + 1));
    }

finalize_it:
    if (bLocked) pthread_mutex_unlock(&pData->mutStatsFile);
    RETiRet;
}

/* identify and count specific types of kafka failures.
 */
static rsRetVal updateKafkaFailureCounts(rd_kafka_resp_err_t err) {
    DEFiRet;
    if (err == RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE) {
        STATSCOUNTER_INC(ctrKafkaMsgTooLarge, mutCtrKafkaMsgTooLarge);
    } else if (err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) {
        STATSCOUNTER_INC(ctrKafkaUnknownTopic, mutCtrKafkaUnknownTopic);
    } else if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
        STATSCOUNTER_INC(ctrKafkaQueueFull, mutCtrKafkaQueueFull);
    } else if (err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION) {
        STATSCOUNTER_INC(ctrKafkaUnknownPartition, mutCtrKafkaUnknownPartition);
    } else {
        STATSCOUNTER_INC(ctrKafkaOtherErrors, mutCtrKafkaOtherErrors);
    }

    RETiRet;
}

/* must be called with read(rkLock)
 * b_do_resubmit tells if we shall resubmit on error or not. This is needed
 *               when we submit already resubmitted messages.
 */
static rsRetVal ATTR_NONNULL(1, 3) writeKafka(instanceData *const pData,
                                              uchar *const key,
                                              uchar *const msg,
                                              uchar *const msgTimestamp,
                                              uchar *const topic,
                                              const int b_do_resubmit) {
    DEFiRet;
    const int partition = getPartition(pData);
    rd_kafka_topic_t *rkt = NULL;
    pthread_rwlock_t *dynTopicLock = NULL;
    failedmsg_entry *fmsgEntry;
    int topic_mut_locked = 0;
    rd_kafka_resp_err_t msg_kafka_response = RD_KAFKA_RESP_ERR_NO_ERROR;
#if OMKAFKA_HAS_PRODUCEVA || OMKAFKA_HAS_PRODUCEV
    int64_t ttMsgTimestamp;
#else
    int msg_enqueue_status = 0;
#endif

    DBGPRINTF("omkafka: trying to send: key:'%s', msg:'%s', timestamp:'%s'\n", key, msg, msgTimestamp);

    if (pData->dynaTopic) {
        DBGPRINTF("omkafka: topic to insert to: %s\n", topic);
        /* ensure locking happens all inside this function */
        pthread_mutex_lock(&pData->mutDynCache);
        const rsRetVal localRet = prepareDynTopic(pData, topic, &rkt, &dynTopicLock);
        if (localRet == RS_RET_OK) {
            pthread_rwlock_rdlock(dynTopicLock);
            topic_mut_locked = 1;
        }
        pthread_mutex_unlock(&pData->mutDynCache);
        CHKiRet(localRet);
    } else {
        rkt = pData->pTopic;
    }

#if OMKAFKA_HAS_PRODUCEVA || OMKAFKA_HAS_PRODUCEV
    if (msgTimestamp == NULL) {
        /* Resubmitted items don't have a timestamp */
        ttMsgTimestamp = 0;
    } else {
        ttMsgTimestamp = atoi((char *)msgTimestamp); /* Convert timestamp into int */
        ttMsgTimestamp *= 1000; /* Timestamp in Milliseconds for kafka */
    }
#endif

#if OMKAFKA_HAS_PRODUCEVA
    DBGPRINTF("omkafka: rd_kafka_producev timestamp=%s/%" PRId64 "\n", msgTimestamp, ttMsgTimestamp);

    /* Using new Kafka produceva API, build argument list dynamically */
    rd_kafka_vu_t v[8];
    int i = 0;

    #define V_RKT(RKT) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_RKT, .u.rkt = (RKT)})
    #define V_PART(PART) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_PARTITION, .u.i32 = (PART)})
    #define V_VALUE(PTR, LEN) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_VALUE, .u.mem.ptr = (PTR), .u.mem.size = (LEN)})
    #define V_MSGFLAGS(FLG) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_MSGFLAGS, .u.i = (FLG)})
    #define V_TIMESTAMP(TS) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_TIMESTAMP, .u.i64 = (TS)})
    #define V_KEY(KEYPTR, KEYLEN) \
        ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_KEY, .u.mem.ptr = (KEYPTR), .u.mem.size = (KEYLEN)})
    #define V_HEADERS(HDRS) ((rd_kafka_vu_t){.vtype = RD_KAFKA_VTYPE_HEADERS, .u.headers = (HDRS)})

    v[i++] = V_RKT(rkt);
    v[i++] = V_PART(partition);
    v[i++] = V_VALUE(msg, strlen((char *)msg));
    v[i++] = V_MSGFLAGS(RD_KAFKA_MSG_F_COPY);
    v[i++] = V_TIMESTAMP(ttMsgTimestamp);

    if (key == NULL) {
        v[i++] = V_KEY(NULL, 0);
    } else {
        DBGPRINTF("omkafka: rd_kafka_producev key=%s\n", key);
        v[i++] = V_KEY(key, strlen((char *)key));
    }

    #if OMKAFKA_HAS_HEADERS
    rd_kafka_headers_t *hdrs = NULL;
    if (pData->kafka_headers) {
        OMKAFKA_COPY_HEADERS_OR_ABORT(hdrs, pData);
        v[i++] = V_HEADERS(hdrs);
    }
    #endif

    #undef V_RKT
    #undef V_PART
    #undef V_VALUE
    #undef V_MSGFLAGS
    #undef V_TIMESTAMP
    #undef V_KEY
    #undef V_HEADERS

    rd_kafka_error_t *rke = rd_kafka_produceva(pData->rk, v, i);
    if (rke == NULL) {
        msg_kafka_response = RD_KAFKA_RESP_ERR_NO_ERROR;
    } else {
        msg_kafka_response = rd_kafka_error_code(rke);
        rd_kafka_error_destroy(rke);
    }

    if (msg_kafka_response != RD_KAFKA_RESP_ERR_NO_ERROR) {
    #if OMKAFKA_HAS_HEADERS
        if (hdrs) {
            rd_kafka_headers_destroy(hdrs);
        }
    #endif
        updateKafkaFailureCounts(msg_kafka_response);

        /* Put into kafka queue, again if configured! */
        if (pData->bResubmitOnFailure && b_do_resubmit && msg_kafka_response != RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE) {
            DBGPRINTF(
                "omkafka: Failed to produce to topic '%s' (rd_kafka_producev)"
                "partition %d: '%d/%s' - adding MSG '%s' to failed for RETRY!\n",
                rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), msg);
            CHKmalloc(fmsgEntry = failedmsg_entry_construct((char *)key, key ? strlen((char *)key) : 0, (char *)msg,
                                                            strlen((char *)msg), rd_kafka_topic_name(rkt)));
            SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
        } else {
            LogError(0, RS_RET_KAFKA_PRODUCE_ERR,
                     "omkafka: Failed to produce to topic '%s' (rd_kafka_producev)"
                     "partition %d: %d/%s - KEY '%s' -MSG '%s'\n",
                     rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), key,
                     msg);
        }
    }
#elif OMKAFKA_HAS_PRODUCEV
    DBGPRINTF("omkafka: rd_kafka_producev timestamp=%s/%" PRId64 "\n", msgTimestamp, ttMsgTimestamp);

    const size_t msg_len = strlen((char *)msg);
    const size_t key_len = key ? strlen((char *)key) : 0;

    if (key == NULL) {
    #if OMKAFKA_HAS_HEADERS
        if (pData->kafka_headers) {
            rd_kafka_headers_t *hdrs;
            OMKAFKA_COPY_HEADERS_OR_ABORT(hdrs, pData);
            msg_kafka_response = rd_kafka_producev(
                pData->rk, RD_KAFKA_V_RKT(rkt), RD_KAFKA_V_PARTITION(partition), RD_KAFKA_V_VALUE(msg, msg_len),
                RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_TIMESTAMP(ttMsgTimestamp), RD_KAFKA_V_KEY(NULL, 0),
                RD_KAFKA_V_HEADERS(hdrs), RD_KAFKA_V_END);
            if (msg_kafka_response != RD_KAFKA_RESP_ERR_NO_ERROR) {
                rd_kafka_headers_destroy(hdrs);
            }
        } else
    #endif
        {
            msg_kafka_response =
                rd_kafka_producev(pData->rk, RD_KAFKA_V_RKT(rkt), RD_KAFKA_V_PARTITION(partition),
                                  RD_KAFKA_V_VALUE(msg, msg_len), RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
                                  RD_KAFKA_V_TIMESTAMP(ttMsgTimestamp), RD_KAFKA_V_KEY(NULL, 0), RD_KAFKA_V_END);
        }
    } else {
        DBGPRINTF("omkafka: rd_kafka_producev key=%s\n", key);
    #if OMKAFKA_HAS_HEADERS
        if (pData->kafka_headers) {
            rd_kafka_headers_t *hdrs;
            OMKAFKA_COPY_HEADERS_OR_ABORT(hdrs, pData);
            msg_kafka_response = rd_kafka_producev(
                pData->rk, RD_KAFKA_V_RKT(rkt), RD_KAFKA_V_PARTITION(partition), RD_KAFKA_V_VALUE(msg, msg_len),
                RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY), RD_KAFKA_V_TIMESTAMP(ttMsgTimestamp),
                RD_KAFKA_V_KEY(key, key_len), RD_KAFKA_V_HEADERS(hdrs), RD_KAFKA_V_END);
            if (msg_kafka_response != RD_KAFKA_RESP_ERR_NO_ERROR) {
                rd_kafka_headers_destroy(hdrs);
            }
        } else
    #endif
        {
            msg_kafka_response =
                rd_kafka_producev(pData->rk, RD_KAFKA_V_RKT(rkt), RD_KAFKA_V_PARTITION(partition),
                                  RD_KAFKA_V_VALUE(msg, msg_len), RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
                                  RD_KAFKA_V_TIMESTAMP(ttMsgTimestamp), RD_KAFKA_V_KEY(key, key_len), RD_KAFKA_V_END);
        }
    }

    if (msg_kafka_response != RD_KAFKA_RESP_ERR_NO_ERROR) {
        updateKafkaFailureCounts(msg_kafka_response);

        /* Put into kafka queue, again if configured! */
        if (pData->bResubmitOnFailure && b_do_resubmit && msg_kafka_response != RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE) {
            DBGPRINTF(
                "omkafka: Failed to produce to topic '%s' (rd_kafka_producev)"
                "partition %d: '%d/%s' - adding MSG '%s' to failed for RETRY!\n",
                rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), msg);
            CHKmalloc(fmsgEntry = failedmsg_entry_construct((char *)key, key_len, (char *)msg, msg_len,
                                                            rd_kafka_topic_name(rkt)));
            SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
        } else {
            LogError(0, RS_RET_KAFKA_PRODUCE_ERR,
                     "omkafka: Failed to produce to topic '%s' (rd_kafka_producev)"
                     "partition %d: %d/%s - KEY '%s' -MSG '%s'\n",
                     rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), key,
                     msg);
        }
    }

#else

    DBGPRINTF("omkafka: rd_kafka_produce\n");
    /* Using old kafka produce API */
    msg_enqueue_status = rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, msg, strlen((char *)msg), key,
                                          key ? strlen((char *)key) : 0, NULL);
    if (msg_enqueue_status == -1) {
        msg_kafka_response = rd_kafka_last_error();
        updateKafkaFailureCounts(msg_kafka_response);

        /* Put into kafka queue, again if configured! */
        if (pData->bResubmitOnFailure && b_do_resubmit && msg_kafka_response != RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE) {
            DBGPRINTF(
                "omkafka: Failed to produce to topic '%s' (rd_kafka_produce)"
                "partition %d: '%d/%s' - adding MSG '%s' KEY '%s' to failed for RETRY!\n",
                rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), msg,
                key ? (const char *)key : "");
            CHKmalloc(fmsgEntry = failedmsg_entry_construct((char *)key, key ? strlen((char *)key) : 0, (char *)msg,
                                                            strlen((char *)msg), rd_kafka_topic_name(rkt)));
            SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
        } else {
            LogError(0, RS_RET_KAFKA_PRODUCE_ERR,
                     "omkafka: Failed to produce to topic '%s' (rd_kafka_produce) "
                     "partition %d: %d/%s - MSG '%s' KEY '%s'\n",
                     rd_kafka_topic_name(rkt), partition, msg_kafka_response, rd_kafka_err2str(msg_kafka_response), msg,
                     key);
        }
    }
#endif

    const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
    DBGPRINTF("omkafka: writeKafka kafka outqueue length: %d, callbacks called %d\n", rd_kafka_outq_len(pData->rk),
              callbacksCalled);

#if OMKAFKA_HAS_PRODUCEVA || OMKAFKA_HAS_PRODUCEV
    if (msg_kafka_response != RD_KAFKA_RESP_ERR_NO_ERROR) {
#else
    if (msg_enqueue_status == -1) {
#endif
        STATSCOUNTER_INC(ctrKafkaFail, mutCtrKafkaFail);
        INST_STATSCOUNTER_INC(pData, pData->ctrKafkaFail, pData->mutCtrKafkaFail);
        ABORT_FINALIZE(RS_RET_KAFKA_PRODUCE_ERR);
        /* ABORT_FINALIZE isn't absolutely necessary as of now,
           because this is the last line anyway, but its useful to ensure
           correctness in case we add more stuff below this line at some point*/
    }

finalize_it:
    if (topic_mut_locked) {
        pthread_rwlock_unlock(dynTopicLock);
    }
    DBGPRINTF("omkafka: writeKafka returned %d\n", iRet);
    if (iRet != RS_RET_OK) {
        iRet = RS_RET_SUSPENDED;
    }
    STATSCOUNTER_SETMAX_NOMUT(ctrQueueSize, (unsigned)rd_kafka_outq_len(pData->rk));
    STATSCOUNTER_INC(ctrTopicSubmit, mutCtrTopicSubmit);
    INST_STATSCOUNTER_INC(pData, pData->ctrTopicSubmit, pData->mutCtrTopicSubmit);
    RETiRet;
}

static void deliveryCallback(rd_kafka_t __attribute__((unused)) * rk,
                             const rd_kafka_message_t *rkmessage,
                             void *opaque) {
    instanceData *const pData = (instanceData *)opaque;
    failedmsg_entry *fmsgEntry;
    const char *const topicname = (rkmessage->rkt != NULL) ? rd_kafka_topic_name(rkmessage->rkt) : (char *)pData->topic;
    const char *const logTopic = (topicname != NULL) ? topicname : "[unknown]";
    const char *const payload = (rkmessage->payload != NULL) ? rkmessage->payload : "";
    const size_t payloadLen = (rkmessage->payload != NULL) ? rkmessage->len : 0;
    const int payloadLogLen = (payloadLen > 0 && (payload[payloadLen - 1] == '\n' || payload[payloadLen - 1] == '\0'))
                                  ? (int)payloadLen - 1
                                  : (int)payloadLen;
    const char *const key = (rkmessage->key != NULL) ? rkmessage->key : "";
    const size_t keyLen = (rkmessage->key != NULL) ? rkmessage->key_len : 0;
    DEFiRet;

    if (rkmessage->err) {
        updateKafkaFailureCounts(rkmessage->err);

        /* Put into kafka queue, again if configured! */
        if (pData->bResubmitOnFailure && rkmessage->payload != NULL) {
            DBGPRINTF(
                "omkafka: kafka delivery FAIL on Topic '%s', msg '%.*s', key '%.*s' -"
                " adding to FAILED MSGs for RETRY!\n",
                logTopic, payloadLogLen, payload, (int)keyLen, key);
            CHKmalloc(fmsgEntry = failedmsg_entry_construct(rkmessage->key, keyLen, payload, payloadLen, logTopic));
            SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
        } else {
            LogError(0, RS_RET_ERR, "omkafka: kafka delivery FAIL on Topic '%s', msg '%.*s', key '%.*s'\n", logTopic,
                     payloadLogLen, payload, (int)keyLen, key);
            writeDataError(pData, payload, payloadLen, rkmessage->err);
        }
        STATSCOUNTER_INC(ctrKafkaFail, mutCtrKafkaFail);
        INST_STATSCOUNTER_INC(pData, pData->ctrKafkaFail, pData->mutCtrKafkaFail);
    } else {
        DBGPRINTF("omkafka: kafka delivery SUCCESS on msg '%.*s'\n", payloadLogLen, payload);
        STATSCOUNTER_INC(ctrKafkaAck, mutCtrKafkaAck);
        INST_STATSCOUNTER_INC(pData, pData->ctrKafkaAck, pData->mutCtrKafkaAck);
    }
finalize_it:
    if (iRet != RS_RET_OK) {
        DBGPRINTF("omkafka: deliveryCallback returned failure %d\n", iRet);
    }
}

/**
 * This function looks for a json object that corresponds to the
 * passed name and returns it is found. Otherwise returns NULL.
 * It will be used for processing stats callback json object.
 */
static struct fjson_object *get_object(struct fjson_object *fj_obj, const char *name) {
    struct fjson_object_iterator it = fjson_object_iter_begin(fj_obj);
    struct fjson_object_iterator itEnd = fjson_object_iter_end(fj_obj);
    while (!fjson_object_iter_equal(&it, &itEnd)) {
        const char *key = fjson_object_iter_peek_name(&it);
        struct fjson_object *val = fjson_object_iter_peek_value(&it);
        if (!strncmp(key, name, strlen(name))) {
            return val;
        }
        fjson_object_iter_next(&it);
    }

    return NULL;
}

/**
 * This function performs a two level search in stats callback json
 * object. It iterates over broker objects and for each broker object
 * returns desired level2 value (such as avg/min/max) for specified
 * level1 window statistic (such as rtt/throttle/int_latency). Threshold
 * allows skipping values that are too small, so that they don't
 * impact on aggregate averaged value that is returned.
 */
static uint64 jsonExtractWindoStats(struct fjson_object *stats_object,
                                    const char *level1_obj_name,
                                    const char *level2_obj_name,
                                    unsigned long skip_threshold) {
    uint64 level2_val;
    uint64 agg_val = 0;
    uint64 ret_val = 0;
    int active_brokers = 0;

    struct fjson_object *brokers_obj = get_object(stats_object, "brokers");
    if (brokers_obj == NULL) {
        LogMsg(0, NO_ERRCODE, LOG_ERR, "jsonExtractWindowStat: failed to find brokers object");
        return ret_val;
    }

    /* iterate over borkers to get level1 window objects at level2 (min, max, avg, etc.) */
    struct fjson_object_iterator it = fjson_object_iter_begin(brokers_obj);
    struct fjson_object_iterator itEnd = fjson_object_iter_end(brokers_obj);
    while (!fjson_object_iter_equal(&it, &itEnd)) {
        struct fjson_object *val = fjson_object_iter_peek_value(&it);
        struct fjson_object *level1_obj = get_object(val, level1_obj_name);
        if (level1_obj == NULL) return ret_val;

        struct fjson_object *level2_obj = get_object(level1_obj, level2_obj_name);
        if (level2_obj == NULL) return ret_val;

        level2_val = fjson_object_get_int64(level2_obj);
        if (level2_val > skip_threshold) {
            agg_val += level2_val;
            active_brokers++;
        }
        fjson_object_iter_next(&it);
    }
    if (active_brokers > 0) {
        ret_val = agg_val / active_brokers;
    }

    return ret_val;
}

/**
 * librdkafka will call this function after every statistics.interval.ms
 * interval, which is specified in confParam. See the explanation at:
 * https://github.com/edenhill/librdkafka/wiki/Statistics
 *
 * Here we have extracted windows stats: rtt, throttle time, and internal
 * latency averages. These values will be logged as impstats messages.
 */
static int statsCallback(rd_kafka_t __attribute__((unused)) * rk,
                         char *json,
                         size_t __attribute__((unused)) json_len,
                         void __attribute__((unused)) * opaque) {
    instanceData *const pData = (instanceData *)opaque;
    char buf[2048];
    char handler_name[1024] = "unknown";
    int replyq = 0;
    int msg_cnt = 0;
    int msg_size = 0;
    uint64 msg_max = 0;
    uint64 msg_size_max = 0;

    struct fjson_object *stats_object = NULL;
    struct fjson_object *fj_obj = NULL;

    DBGPRINTF("omkafka: librdkafka stats callback: %s\n", json);

    /* prepare fjson object from stats callback for parsing */
    stats_object = fjson_tokener_parse(json);
    if (stats_object == NULL) {
        LogMsg(0, NO_ERRCODE, LOG_ERR, "statsCallback: fjson tokenizer failed:");
        return 0;
    }
    enum fjson_type type = fjson_object_get_type(stats_object);
    if (type != fjson_type_object) {
        LogMsg(0, NO_ERRCODE, LOG_ERR, "statsCallback: json is not of type object; can't process statsCB\n");
        return 0;
    }

    /* top level stats extraction through libfastjson based parsing */
    fj_obj = get_object(stats_object, "name");
    if (fj_obj != NULL) snprintf(handler_name, sizeof(handler_name), "%s", (char *)fjson_object_get_string(fj_obj));
    fj_obj = get_object(stats_object, "replyq");
    replyq = (fj_obj == NULL) ? 0 : fjson_object_get_int(fj_obj);
    fj_obj = get_object(stats_object, "msg_cnt");
    msg_cnt = (fj_obj == NULL) ? 0 : fjson_object_get_int(fj_obj);
    fj_obj = get_object(stats_object, "msg_size");
    msg_size = (fj_obj == NULL) ? 0 : fjson_object_get_int(fj_obj);
    fj_obj = get_object(stats_object, "msg_max");
    msg_max = (fj_obj == NULL) ? 0 : fjson_object_get_int64(fj_obj);
    fj_obj = get_object(stats_object, "msg_size_max");
    msg_size_max = (fj_obj == NULL) ? 0 : fjson_object_get_int64(fj_obj);

    /* window stats extraction to be picked up by impstats counters */
    rtt_avg_usec = jsonExtractWindoStats(stats_object, "rtt", "avg", 100);
    throttle_avg_msec = jsonExtractWindoStats(stats_object, "throttle", "avg", 0);
    int_latency_avg_usec = jsonExtractWindoStats(stats_object, "int_latency", "avg", 0);
    json_object_put(stats_object);

    /* emit a log line to get stats visibility per librdkafka client */
    snprintf(buf, sizeof(buf),
             "statscb_window_stats: handler_name=%s replyq=%d msg_cnt=%d msg_size=%d "
             "msg_max=%lld msg_size_max=%lld rtt_avg_usec=%lld throttle_avg_msec=%lld "
             "int_latency_avg_usec=%lld",
             handler_name, replyq, msg_cnt, msg_size, msg_max, msg_size_max, rtt_avg_usec, throttle_avg_msec,
             int_latency_avg_usec);
    LogMsg(0, NO_ERRCODE, LOG_INFO, "%s\n", buf);

    /* Write the entire json stats object, if requested */
    if (pData->statsFile != NULL) writeStats(pData, json, json_len);

    return 0;
}

static void kafkaLogger(const rd_kafka_t __attribute__((unused)) * rk, int level, const char *fac, const char *buf) {
    // Get InstanceData pointer from opaque
    instanceData *const pData = rd_kafka_opaque(rk);

    DBGPRINTF("omkafka: pData[%p] kafka log message [%d,%s]: %s\n", pData, level, fac, buf);

    // write log messages to error file if pData is not NULL
    if (pData != NULL) {
        writeLogError(pData, level, fac, buf);
    }
}

/* should be called with write(rkLock) */
static void do_rd_kafka_destroy(instanceData *const __restrict__ pData) {
    if (pData->rk == NULL) {
        DBGPRINTF("omkafka: onDestroy can't close, handle wasn't open\n");
        goto done;
    }
    const char *const actName = pData->statsName ? (const char *)pData->statsName : rd_kafka_name(pData->rk);
    const char *const topicName =
        pData->dynaTopic ? (const char *)pData->topic
                         : (pData->pTopic ? rd_kafka_topic_name(pData->pTopic) : (const char *)pData->topic);
    int queuedCount = rd_kafka_outq_len(pData->rk);
    DBGPRINTF("omkafka: onDestroy closing - items left in outqueue: %d\n", queuedCount);

    struct timespec tOut;
    timeoutComp(&tOut, pData->closeTimeout);

    while (timeoutVal(&tOut) > 0) {
        queuedCount = rd_kafka_outq_len(pData->rk);
        if (queuedCount > 0) {
            /* Flush all remaining kafka messages (rd_kafka_poll is called inside) */
            const int flushStatus = rd_kafka_flush(pData->rk, pData->closeTimeout);
            if (flushStatus == RD_KAFKA_RESP_ERR_NO_ERROR) {
                DBGPRINTF(
                    "omkafka: onDestroy flushed remaining '%d' messages "
                    "to kafka topic '%s'\n",
                    queuedCount, topicName);

                /* Trigger callbacks a last time before shutdown */
                const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
                DBGPRINTF(
                    "omkafka: onDestroy kafka outqueue length: %d, "
                    "callbacks called %d\n",
                    rd_kafka_outq_len(pData->rk), callbacksCalled);
            } else /* TODO: Handle unsend messages here! */ {
                /* timeout = RD_KAFKA_RESP_ERR__TIMED_OUT */
                LogError(0, RS_RET_KAFKA_ERROR,
                         "omkafka[%s]: onDestroy "
                         "failed to flush remaining '%d' messages to "
                         "topic '%s' on shutdown: %s",
                         actName, queuedCount, topicName, rd_kafka_err2str(flushStatus));
#if RD_KAFKA_VERSION >= 0x010001ff
                rd_kafka_purge(pData->rk, RD_KAFKA_PURGE_F_QUEUE | RD_KAFKA_PURGE_F_INFLIGHT);
                /* Trigger callbacks a last time before shutdown */
                const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
                DBGPRINTF(
                    "omkafka: onDestroy kafka outqueue length: %d, "
                    "callbacks called %d\n",
                    rd_kafka_outq_len(pData->rk), callbacksCalled);
#endif
            }
        } else {
            break;
        }
    }
    if (queuedCount > 0) {
        LogMsg(0, RS_RET_ERR, LOG_WARNING,
               "omkafka[%s]: queue-drain on shutdown timed out for topic '%s', "
               "items left in outqueue: %d -- this may indicate data loss",
               actName, topicName, rd_kafka_outq_len(pData->rk));
    }
    if (pData->dynaTopic) {
        dynaTopicFreeCacheEntries(pData);
    } else {
        closeTopic(pData);
    }

    /* Final destroy of kafka!*/
    rd_kafka_destroy(pData->rk);

#if RD_KAFKA_VERSION < 0x00090001
    /* Wait for kafka being destroyed in old API */
    if (rd_kafka_wait_destroyed(10000) < 0) {
        LogError(0, RS_RET_ERR, "omkafka: rd_kafka_destroy did not finish after grace timeout (10s)!");
    } else {
        DBGPRINTF("omkafka: rd_kafka_destroy successfully finished\n");
    }
#endif

    pData->rk = NULL;
done:
    return;
}

/* should be called with write(rkLock) */
static void closeKafka(instanceData *const __restrict__ pData) {
    if (pData->bIsOpen) {
        do_rd_kafka_destroy(pData);
        pData->bIsOpen = 0;
    }
}

static void errorCallback(rd_kafka_t __attribute__((unused)) * rk,
                          int __attribute__((unused)) err,
                          const char *reason,
                          void __attribute__((unused)) * opaque) {
    /* Get InstanceData pointer */
    instanceData *const pData = (instanceData *)opaque;

    /* count kafka transport errors that cause action suspension */
    if (err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT) {
        STATSCOUNTER_INC(ctrKafkaRespTimedOut, mutCtrKafkaRespTimedOut);
    } else if (err == RD_KAFKA_RESP_ERR__TRANSPORT || err == RD_KAFKA_RESP_ERR__RESOLVE) {
        STATSCOUNTER_INC(ctrKafkaRespTransport, mutCtrKafkaRespTransport);
    } else if (err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) {
        STATSCOUNTER_INC(ctrKafkaRespBrokerDown, mutCtrKafkaRespBrokerDown);
    } else if (err == RD_KAFKA_RESP_ERR__AUTHENTICATION) {
        STATSCOUNTER_INC(ctrKafkaRespAuth, mutCtrKafkaRespAuth);
    } else if (err == RD_KAFKA_RESP_ERR__SSL) {
        STATSCOUNTER_INC(ctrKafkaRespSSL, mutCtrKafkaRespSSL);
    } else {
        STATSCOUNTER_INC(ctrKafkaRespOther, mutCtrKafkaRespOther);
    }

    /* Suspend action on any error that means brokers are unreachable.
     * RD_KAFKA_RESP_ERR__RESOLVE (-193) was missing from this list, causing
     * host-resolution failures to bypass suspension and silence
     * action.reportSuspension/reportSuspensionContinuation messages. */
    if (err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT || err == RD_KAFKA_RESP_ERR__TRANSPORT ||
        err == RD_KAFKA_RESP_ERR__RESOLVE || err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN ||
        err == RD_KAFKA_RESP_ERR__AUTHENTICATION || err == RD_KAFKA_RESP_ERR__SSL) {
        if (pData != NULL) {
            pData->bIsSuspended = 1;
        }
        LogMsg(0, RS_RET_KAFKA_ERROR, LOG_WARNING, "omkafka: action suspended due to kafka error %d: %s", err,
               rd_kafka_err2str(err));
    } else {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: kafka error message: %d,'%s','%s'", err, rd_kafka_err2str(err),
                 reason);
    }
}


#if 0 /* the stock librdkafka version in Ubuntu 14.04 LTS does NOT support metadata :-( */
/* Note: this is a skeleton, with some code missing--> add it when it is actually implemented. */
static int
getConfiguredPartitions()
{
	struct rd_kafka_metadata *pMetadata;
	if(rd_kafka_metadata(pData->rk, 0, rkt, &pMetadata, 8)
		== RD_KAFKA_RESP_ERR_NO_ERROR) {
		dbgprintf("omkafka: topic '%s' has %d partitions\n",
			  pData->topic, pMetadata->topics[0]->partition_cnt);
		rd_kafka_metadata_destroy(pMetadata);
	} else {
		dbgprintf("omkafka: error reading metadata\n");
		// TODO: handle this gracefull **when** we actually need
		// the metadata -- or remove completely. 2014-12-12 rgerhards
	}
}
#endif

/* should be called with write(rkLock) */
static rsRetVal openKafka(instanceData *const __restrict__ pData) {
    char errstr[MAX_ERRMSG];
    DEFiRet;

    if (pData->bIsOpen) FINALIZE;

    pData->pTopic = NULL;

    /* main conf */
    rd_kafka_conf_t *const conf = rd_kafka_conf_new();
    if (conf == NULL) {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: error creating kafka conf obj: %s\n",
                 rd_kafka_err2str(rd_kafka_last_error()));
        ABORT_FINALIZE(RS_RET_KAFKA_ERROR);
    }

#ifdef DEBUG
    /* enable kafka debug output */
    if (rd_kafka_conf_set(conf, "debug", RD_KAFKA_DEBUG_CONTEXTS, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: error setting kafka debug option: %s\n", errstr);
        /* DO NOT ABORT IN THIS CASE! */
    }
#endif

    for (int i = 0; i < pData->nConfParams; ++i) {
        DBGPRINTF("omkafka: setting custom configuration parameter: %s:%s\n", pData->confParams[i].name,
                  pData->confParams[i].val);
        if (rd_kafka_conf_set(conf, pData->confParams[i].name, pData->confParams[i].val, errstr, sizeof(errstr)) !=
            RD_KAFKA_CONF_OK) {
            if (pData->bReportErrs) {
                LogError(0, RS_RET_PARAM_ERROR,
                         "error setting custom configuration "
                         "parameter '%s=%s': %s",
                         pData->confParams[i].name, pData->confParams[i].val, errstr);
            } else {
                DBGPRINTF("omkafka: error setting custom configuration parameter '%s=%s': %s",
                          pData->confParams[i].name, pData->confParams[i].val, errstr);
            }
            ABORT_FINALIZE(RS_RET_PARAM_ERROR);
        }
    }
    rd_kafka_conf_set_opaque(conf, (void *)pData);
    rd_kafka_conf_set_dr_msg_cb(conf, deliveryCallback);
    rd_kafka_conf_set_error_cb(conf, errorCallback);
    rd_kafka_conf_set_stats_cb(conf, statsCallback);
#if RD_KAFKA_VERSION >= 0x00090001
    rd_kafka_conf_set_log_cb(conf, kafkaLogger);
#endif

    char kafkaErrMsg[1024];
    pData->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, kafkaErrMsg, sizeof(kafkaErrMsg));
    if (pData->rk == NULL) {
        LogError(0, RS_RET_KAFKA_ERROR, "omkafka: error creating kafka handle: %s\n", kafkaErrMsg);
        ABORT_FINALIZE(RS_RET_KAFKA_ERROR);
    }

#if RD_KAFKA_VERSION < 0x00090001
    rd_kafka_conf_set_log_cb(pData->rk, kafkaLogger);
#endif
    DBGPRINTF("omkafka setting brokers: '%s'n", pData->brokers);
    if (rd_kafka_brokers_add(pData->rk, (char *)pData->brokers) == 0) {
        LogError(0, RS_RET_KAFKA_NO_VALID_BROKERS, "omkafka: no valid brokers specified: %s\n", pData->brokers);
        ABORT_FINALIZE(RS_RET_KAFKA_NO_VALID_BROKERS);
    }

    pData->bIsOpen = 1;
    /* callback poll thread is best-effort and must not prevent action open */
    if (startPollCallbackThread(pData) != RS_RET_OK) {
        LogMsg(0, RS_RET_ERR, LOG_WARNING,
               "omkafka: proceeding without callback poll thread; callbacks are only drained on active writes");
    }
finalize_it:
    if (iRet == RS_RET_OK) {
        pData->bReportErrs = 1;
    } else {
        pData->bReportErrs = 0;
        if (pData->rk != NULL) {
            do_rd_kafka_destroy(pData);
        }
    }
    RETiRet;
}

static rsRetVal setupKafkaHandle(instanceData *const __restrict__ pData, int recreate) {
    DEFiRet;
    pthread_rwlock_wrlock(&pData->rkLock);
    if (recreate) {
        closeKafka(pData);
    }
    CHKiRet(openKafka(pData));
    if (!pData->dynaTopic) {
        if (pData->pTopic == NULL) CHKiRet(prepareTopic(pData, pData->topic));
    }
finalize_it:
    if (iRet != RS_RET_OK) {
        if (pData->rk != NULL) {
            closeKafka(pData);
        }

        /* Parameter Error's cannot be resumed, so we need to disable the action */
        if (iRet == RS_RET_PARAM_ERROR) {
            iRet = RS_RET_DISABLE_ACTION;
            LogError(0, iRet,
                     "omkafka: action will be disabled due invalid "
                     "kafka configuration parameters\n");
        }
    }
    pthread_rwlock_unlock(&pData->rkLock);
    RETiRet;
}

static rsRetVal checkFailedMessages(instanceData *const __restrict__ pData) {
    failedmsg_entry *fmsgEntry;
    DEFiRet;

    /* Loop through failed messages, reprocess them first! */
    while (!SLIST_EMPTY(&pData->failedmsg_head)) {
        fmsgEntry = SLIST_FIRST(&pData->failedmsg_head);
        assert(fmsgEntry != NULL);
        /* Put back into kafka! */
        iRet = writeKafka(pData, (uchar *)fmsgEntry->key, (uchar *)fmsgEntry->payload, NULL, fmsgEntry->topicname,
                          NO_RESUBMIT);
        if (iRet != RS_RET_OK) {
            LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING,
                   "omkafka: failed to deliver failed msg '%.*s' with status %d. "
                   "- suspending AGAIN!",
                   (int)(strlen((char *)fmsgEntry->payload) - 1), (char *)fmsgEntry->payload, iRet);
            ABORT_FINALIZE(RS_RET_SUSPENDED);
        } else {
            DBGPRINTF("omkafka: successfully delivered failed msg '%.*s'.\n",
                      (int)(strlen((char *)fmsgEntry->payload) - 1), (char *)fmsgEntry->payload);
            /* Note: we can use SLIST even though it is o(n), because the element
             * in question is always either the root or the next element and
             * SLIST_REMOVE iterates only until the element to be deleted is found.
             * We cannot use SLIST_REMOVE_HEAD() as new elements may have been
             * added in the delivery callback!
             * TODO: sounds like bad logic -- why do we add and remove, just simply
             * keep it in queue?
             */
            SLIST_REMOVE(&pData->failedmsg_head, fmsgEntry, s_failedmsg_entry, entries);
            failedmsg_entry_destruct(fmsgEntry);
        }
    }

finalize_it:
    RETiRet;
}

/* This function persists failed messages into a data file, so they can
 * be resend on next startup.
 * alorbach, 2017-06-02
 */
static rsRetVal ATTR_NONNULL(1) persistFailedMsgs(instanceData *const __restrict__ pData) {
    DEFiRet;
    int fdMsgFile = -1;
    ssize_t nwritten;

    if (SLIST_EMPTY(&pData->failedmsg_head)) {
        DBGPRINTF("omkafka: persistFailedMsgs: We do not need to persist failed messages.\n");
        FINALIZE;
    }

    fdMsgFile = open((char *)pData->failedMsgFile, O_WRONLY | O_CREAT | O_APPEND | O_LARGEFILE | O_CLOEXEC,
                     S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
    if (fdMsgFile == -1) {
        LogError(errno, RS_RET_ERR, "omkafka: persistFailedMsgs error opening failed msg file %s",
                 pData->failedMsgFile);
        ABORT_FINALIZE(RS_RET_ERR);
    }

    while (!SLIST_EMPTY(&pData->failedmsg_head)) {
        failedmsg_entry *fmsgEntry = SLIST_FIRST(&pData->failedmsg_head);
        assert(fmsgEntry != NULL);
        nwritten = write(fdMsgFile, fmsgEntry->topicname, ustrlen(fmsgEntry->topicname));
        if (nwritten != -1) nwritten = write(fdMsgFile, "\t", 1);
        if ((nwritten != -1) && (fmsgEntry->key)) nwritten = write(fdMsgFile, fmsgEntry->key, ustrlen(fmsgEntry->key));
        if (nwritten != -1) nwritten = write(fdMsgFile, "\t", 1);
        if (nwritten != -1) nwritten = write(fdMsgFile, fmsgEntry->payload, ustrlen(fmsgEntry->payload));
        if (nwritten == -1) {
            LogError(errno, RS_RET_ERR, "omkafka: persistFailedMsgs error writing failed msg file");
            ABORT_FINALIZE(RS_RET_ERR);
        } else {
            DBGPRINTF(
                "omkafka: persistFailedMsgs successfully written loaded msg '%.*s' for "
                "topic '%s'\n",
                (int)(strlen((char *)fmsgEntry->payload) - 1), fmsgEntry->payload, fmsgEntry->topicname);
        }
        SLIST_REMOVE_HEAD(&pData->failedmsg_head, entries);
        failedmsg_entry_destruct(fmsgEntry);
    }

finalize_it:
    if (fdMsgFile != -1) {
        close(fdMsgFile);
    }
    if (iRet != RS_RET_OK) {
        LogError(0, iRet,
                 "omkafka: could not persist failed messages "
                 "file %s - failed messages will be lost.",
                 (char *)pData->failedMsgFile);
    }
    RETiRet;
}

/* This function loads failed messages from a data file, so they can
 * be resend after action startup.
 * alorbach, 2017-06-06
 */
static rsRetVal loadFailedMsgs(instanceData *const __restrict__ pData) {
    DEFiRet;
    struct stat stat_buf;
    failedmsg_entry *fmsgEntry;
    strm_t *pstrmFMSG = NULL;
    cstr_t *pCStr = NULL;
    uchar *puStr;
    char *pStrTabPos;
    char *pStrTabPos2;

    assert(pData->failedMsgFile != NULL);

    /* check if the file exists */
    if (stat((char *)pData->failedMsgFile, &stat_buf) == -1) {
        if (errno == ENOENT) {
            DBGPRINTF(
                "omkafka: loadFailedMsgs failed messages file %s wasn't found, "
                "continue startup\n",
                pData->failedMsgFile);
            ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
        } else {
            LogError(errno, RS_RET_IO_ERROR, "omkafka: loadFailedMsgs could not open failed messages file %s",
                     pData->failedMsgFile);
            ABORT_FINALIZE(RS_RET_IO_ERROR);
        }
    } else {
        DBGPRINTF("omkafka: loadFailedMsgs found failed message file %s.\n", pData->failedMsgFile);
    }

    /* File exists, we can load and process it */
    CHKiRet(strm.Construct(&pstrmFMSG));
    CHKiRet(strm.SettOperationsMode(pstrmFMSG, STREAMMODE_READ));
    CHKiRet(strm.SetsType(pstrmFMSG, STREAMTYPE_FILE_SINGLE));
    CHKiRet(strm.SetFName(pstrmFMSG, pData->failedMsgFile, ustrlen(pData->failedMsgFile)));
    CHKiRet(strm.ConstructFinalize(pstrmFMSG));

    while (strm.ReadLine(pstrmFMSG, &pCStr, 0, 0, NULL, 0, NULL) == RS_RET_OK) {
        if (rsCStrLen(pCStr) == 0) {
            /* we do not process empty lines */
            DBGPRINTF("omkafka: loadFailedMsgs msg was empty!");
        } else {
            puStr = rsCStrGetSzStrNoNULL(pCStr);  // topic
            pStrTabPos = index((char *)puStr, '\t');  // key
            pStrTabPos2 = (pStrTabPos == NULL) ? NULL : index((char *)pStrTabPos + 1, '\t');  // msg
            if ((pStrTabPos != NULL) && (pStrTabPos2 != NULL)) {
                *pStrTabPos = '\0'; /* split string into two */
                *pStrTabPos2 = '\0'; /* split string into two */
                DBGPRINTF(
                    "omkafka: loadFailedMsgs successfully loaded msg '%s' for "
                    "topic '%s' key '%s' \n",
                    pStrTabPos2 + 1, (char *)puStr, pStrTabPos + 1);
                if (strlen(pStrTabPos + 1)) {
                    CHKmalloc(fmsgEntry =
                                  failedmsg_entry_construct(pStrTabPos + 1, strlen(pStrTabPos + 1), pStrTabPos2 + 1,
                                                            strlen(pStrTabPos2 + 1), (char *)puStr));
                } else {
                    CHKmalloc(fmsgEntry = failedmsg_entry_construct(NULL, 0, pStrTabPos2 + 1, strlen(pStrTabPos2 + 1),
                                                                    (char *)puStr));
                }
                SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
            } else {
                LogError(0, RS_RET_ERR, "omkafka: loadFailedMsgs droping invalid msg found: %s",
                         (char *)rsCStrGetSzStrNoNULL(pCStr));
            }
        }

        rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */
    }
finalize_it:
    if (pstrmFMSG != NULL) {
        strm.Destruct(&pstrmFMSG);
    }

    if (iRet != RS_RET_OK) {
        /* We ignore FILE NOT FOUND here */
        if (iRet != RS_RET_FILE_NOT_FOUND) {
            LogError(0, iRet,
                     "omkafka: could not load failed messages "
                     "from file %s error %d - failed messages will not be resend.",
                     (char *)pData->failedMsgFile, iRet);
        }
    } else {
        DBGPRINTF("omkafka: loadFailedMsgs unlinking '%s'\n", (char *)pData->failedMsgFile);
        /* Delete file if still exists! */
        const int r = unlink((char *)pData->failedMsgFile);
        if (r != 0 && r != ENOENT) {
            LogError(errno, RS_RET_ERR,
                     "omkafka: loadFailedMsgs failed to remove "
                     "file \"%s\"",
                     (char *)pData->failedMsgFile);
        }
    }

    RETiRet;
}

BEGINdoHUP
    CODESTARTdoHUP;
    /* Keep doHUP in the same outer serialization domain as doAction and
     * the callback poller.  Those paths hold mut_doAction while polling
     * librdkafka, and callbacks from that poll path may take the error or
     * stats file mutexes.  Preserve one order here:
     * mut_doAction -> file mutexes / kafka handle locks. */
    pthread_mutex_lock(&pData->mut_doAction);
    pthread_mutex_lock(&pData->mutErrFile);
    if (pData->fdErrFile != -1) {
        close(pData->fdErrFile);
        pData->fdErrFile = -1;
    }
    pthread_mutex_unlock(&pData->mutErrFile);
    pthread_mutex_lock(&pData->mutStatsFile);
    if (pData->fdStatsFile != -1) {
        close(pData->fdStatsFile);
        pData->fdStatsFile = -1;
    }
    pthread_mutex_unlock(&pData->mutStatsFile);
    if (pData->bReopenOnHup) {
        CHKiRet(setupKafkaHandle(pData, 1));
    } else {
        /* Optional */
        const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
        LogMsg(0, NO_ERRCODE, LOG_INFO,
               "omkafka: doHUP kafka - '%s' outqueue length: %d,"
               "callbacks called %d\n",
               pData->tplName, rd_kafka_outq_len(pData->rk), callbacksCalled);
    }
finalize_it:
    pthread_mutex_unlock(&pData->mut_doAction);
ENDdoHUP

BEGINcreateInstance
    CODESTARTcreateInstance;
    pData->currPartition = 0;
    pData->bIsOpen = 0;
    pData->bIsSuspended = 0;
    pData->fdErrFile = -1;
    pData->fdStatsFile = -1;
    pData->pTopic = NULL;
    pData->bReportErrs = 1;
    pData->bReopenOnHup = 1;
    pData->bResubmitOnFailure = 0;
    pData->bKeepFailedMessages = 0;
    pData->failedMsgFile = NULL;
    SLIST_INIT(&pData->failedmsg_head);
    CHKiRet(pthread_mutex_init(&pData->mut_doAction, NULL));
    CHKiRet(pthread_mutex_init(&pData->mutErrFile, NULL));
    CHKiRet(pthread_mutex_init(&pData->mutStatsFile, NULL));
    CHKiRet(pthread_rwlock_init(&pData->rkLock, NULL));
    CHKiRet(pthread_mutex_init(&pData->mutDynCache, NULL));
    INIT_ATOMIC_HELPER_MUT(pData->mutCurrPartition);
    INIT_ATOMIC_HELPER_MUT(pData->mutStopPollThread);
    pData->pollThreadRunning = 0;
    ATOMIC_STORE_32BIT_RELAXED(&pData->stopPollThread, &pData->mutStopPollThread, 0);
finalize_it:
ENDcreateInstance


BEGINcreateWrkrInstance
    CODESTARTcreateWrkrInstance;
ENDcreateWrkrInstance


BEGINisCompatibleWithFeature
    CODESTARTisCompatibleWithFeature;
ENDisCompatibleWithFeature


BEGINfreeInstance
    CODESTARTfreeInstance;
    /* Helpers for Failed Msg List */
    failedmsg_entry *fmsgEntry1;
    failedmsg_entry *fmsgEntry2;
    stopPollCallbackThread(pData);
    if (pData->fdErrFile != -1) close(pData->fdErrFile);
    if (pData->fdStatsFile != -1) close(pData->fdStatsFile);
    /* Closing Kafka first! */
    pthread_rwlock_wrlock(&pData->rkLock);
    closeKafka(pData);
    if (pData->dynaTopic && pData->dynCache != NULL) {
        free(pData->dynCache);
        pData->dynCache = NULL;
    }
    /* Persist failed messages */
    if (pData->bResubmitOnFailure && pData->bKeepFailedMessages && pData->failedMsgFile != NULL) {
        persistFailedMsgs(pData);
    }
    pthread_rwlock_unlock(&pData->rkLock);

    if (pData->stats) {
        statsobj.Destruct(&pData->stats);
    }

    /* Delete Linked List for failed msgs */
    fmsgEntry1 = SLIST_FIRST(&pData->failedmsg_head);
    while (fmsgEntry1 != NULL) {
        fmsgEntry2 = SLIST_NEXT(fmsgEntry1, entries);
        failedmsg_entry_destruct(fmsgEntry1);
        fmsgEntry1 = fmsgEntry2;
    }
    SLIST_INIT(&pData->failedmsg_head);
    /* Free other mem */
    free(pData->errorFile);
    free(pData->statsFile);
    free(pData->failedMsgFile);
    free(pData->topic);
    free(pData->brokers);
    free(pData->tplName);
    free(pData->statsName);
    for (int i = 0; i < pData->nConfParams; ++i) {
        free((void *)pData->confParams[i].name);
        free((void *)pData->confParams[i].val);
    }
    free(pData->confParams);
    for (int i = 0; i < pData->nTopicConfParams; ++i) {
        free((void *)pData->topicConfParams[i].name);
        free((void *)pData->topicConfParams[i].val);
    }
    free(pData->topicConfParams);
    for (int i = 0; i < pData->nHeaders; ++i) {
        free((void *)pData->headers[i].name);
        free((void *)pData->headers[i].val);
    }
    free(pData->headers);
#if OMKAFKA_HAS_HEADERS
    if (pData->kafka_headers != NULL) rd_kafka_headers_destroy(pData->kafka_headers);
#endif
    DESTROY_ATOMIC_HELPER_MUT(pData->mutCurrPartition);
    pthread_rwlock_destroy(&pData->rkLock);
    pthread_mutex_destroy(&pData->mut_doAction);
    pthread_mutex_destroy(&pData->mutErrFile);
    pthread_mutex_destroy(&pData->mutStatsFile);
    pthread_mutex_destroy(&pData->mutDynCache);
    DESTROY_ATOMIC_HELPER_MUT(pData->mutStopPollThread);
ENDfreeInstance

BEGINfreeWrkrInstance
    CODESTARTfreeWrkrInstance;
ENDfreeWrkrInstance


BEGINdbgPrintInstInfo
    CODESTARTdbgPrintInstInfo;
ENDdbgPrintInstInfo


BEGINtryResume
    int iKafkaRet;
    const struct rd_kafka_metadata *metadata;
    CODESTARTtryResume;
    pthread_mutex_lock(&pWrkrData->pData->mut_doAction); /* see doAction header comment! */
    CHKiRet(setupKafkaHandle(pWrkrData->pData, 0));

    if ((iKafkaRet = rd_kafka_metadata(pWrkrData->pData->rk, 0, NULL, &metadata, 1000)) != RD_KAFKA_RESP_ERR_NO_ERROR) {
        DBGPRINTF("omkafka: tryResume failed, brokers down %d,%s\n", iKafkaRet, rd_kafka_err2str(iKafkaRet));
        ABORT_FINALIZE(RS_RET_SUSPENDED);
    } else {
        DBGPRINTF("omkafka: tryResume success, %d brokers UP\n", metadata->broker_cnt);
        /* Reset suspended state */
        pWrkrData->pData->bIsSuspended = 0;
        /* free mem*/
        rd_kafka_metadata_destroy(metadata);
    }

finalize_it:
    pthread_mutex_unlock(&pWrkrData->pData->mut_doAction); /* see doAction header comment! */
    DBGPRINTF("omkafka: tryResume returned %d\n", iRet);
ENDtryResume


/* IMPORTANT NOTE on multithreading:
 * librdkafka creates background threads itself. So omkafka basically needs to move
 * memory buffers over to librdkafka, which then does the heavy hauling. As such, we
 * think that it is best to run max one wrkr instance of omkafka -- otherwise we just
 * get additional locking (contention) overhead without any real gain. As such,
 * we use a global mutex for doAction which ensures only one worker can be active
 * at any given time. That mutex is also used to guard utility functions (like
 * tryResume) which may also be accessed by multiple workers in parallel.
 * Note: shall this method be changed, the kafka connection/suspension handling needs
 * to be refactored. The current code assumes that all workers share state information
 * including librdkafka handles.
 */
BEGINdoAction
    CODESTARTdoAction;
    failedmsg_entry *fmsgEntry;
    instanceData *const pData = pWrkrData->pData;
    int need_unlock = 0;
    int dynaTopicID = 0;
    int dynaKeyID = 0;

    if (pData->dynaKey) {
        dynaKeyID = 2;
        if (pData->dynaTopic) {
            dynaTopicID = 3;
        }
    } else {
        if (pData->dynaTopic) {
            dynaTopicID = 2;
        }
    }
    pthread_mutex_lock(&pData->mut_doAction);
    if (!pData->bIsOpen) CHKiRet(setupKafkaHandle(pData, 0));

    /* Lock here to prevent msg loss */
    pthread_rwlock_rdlock(&pData->rkLock);
    need_unlock = 1;

    /* We need to trigger callbacks first in order to suspend the Action properly on failure */
    const int callbacksCalled = rd_kafka_poll(pData->rk, 0); /* call callbacks */
    DBGPRINTF("omkafka: doAction kafka outqueue length: %d, callbacks called %d\n", rd_kafka_outq_len(pData->rk),
              callbacksCalled);

    /* Reprocess failed messages! */
    if (pData->bResubmitOnFailure) {
        iRet = checkFailedMessages(pData);
        if (iRet != RS_RET_OK) {
            DBGPRINTF("omkafka: doAction failed to submit FAILED messages with status %d\n", iRet);

            if (pData->bResubmitOnFailure) {
                if (pData->dynaKey || pData->key) {
                    DBGPRINTF(
                        "omkafka: also adding MSG '%.*s' for topic '%s' key '%s' "
                        "to failed for RETRY!\n",
                        (int)(strlen((char *)ppString[0]) - 1), ppString[0],
                        pData->dynaTopic ? ppString[dynaTopicID] : pData->topic,
                        pData->dynaKey ? ppString[dynaKeyID] : pData->key);
                } else {
                    DBGPRINTF(
                        "omkafka: also adding MSG '%.*s' for topic '%s' "
                        "to failed for RETRY!\n",
                        (int)(strlen((char *)ppString[0]) - 1), ppString[0],
                        pData->dynaTopic ? ppString[dynaTopicID] : pData->topic);
                }
                CHKmalloc(fmsgEntry = failedmsg_entry_construct(
                              (char *)(pData->dynaKey ? ppString[dynaKeyID] : pData->key),
                              pData->dynaKey || pData->key
                                  ? strlen((char *)(pData->dynaKey ? ppString[dynaKeyID] : pData->key))
                                  : 0,
                              (char *)ppString[0], strlen((char *)ppString[0]),
                              (char *)(pData->dynaTopic ? ppString[dynaTopicID] : pData->topic)));
                SLIST_INSERT_HEAD(&pData->failedmsg_head, fmsgEntry, entries);
            }
            ABORT_FINALIZE(iRet);
        }
    }

    /* support dynamic topic */
    iRet = writeKafka(pData, pData->dynaKey ? ppString[dynaKeyID] : pData->key, ppString[0], ppString[1],
                      pData->dynaTopic ? ppString[dynaTopicID] : pData->topic, RESUBMIT);

finalize_it:
    if (need_unlock) {
        pthread_rwlock_unlock(&pData->rkLock);
    }

    if (iRet != RS_RET_OK) {
        DBGPRINTF("omkafka: doAction failed with status %d\n", iRet);
    }

    /* Suspend Action if broker problems were reported in error callback */
    if (pData->bIsSuspended) {
        DBGPRINTF("omkafka: doAction broker failure detected, suspending action\n");
        iRet = RS_RET_SUSPENDED;
    }
    pthread_mutex_unlock(&pData->mut_doAction); /* must be after last pData access! */
ENDdoAction


static void setInstParamDefaults(instanceData *pData) {
    pData->topic = NULL;
    pData->pTopic = NULL;
    pData->dynaKey = 0;
    pData->dynaTopic = 0;
    pData->iDynaTopicCacheSize = 50;
    pData->brokers = NULL;
    pData->autoPartition = 0;
    pData->fixedPartition = NO_FIXED_PARTITION;
    pData->nPartitions = 1;
    pData->nConfParams = 0;
    pData->confParams = NULL;
    pData->nTopicConfParams = 0;
    pData->topicConfParams = NULL;
    pData->nHeaders = 0;
    pData->headers = NULL;
#if OMKAFKA_HAS_HEADERS
    pData->kafka_headers = NULL;
#endif
    pData->errorFile = NULL;
    pData->statsFile = NULL;
    pData->failedMsgFile = NULL;
    pData->key = NULL;
    pData->closeTimeout = 2000;
}

static rsRetVal processKafkaParam(char *const param, const char **const name, const char **const paramval) {
    DEFiRet;
    char *val = strstr(param, "=");
    if (val == NULL) {
        LogError(0, RS_RET_PARAM_ERROR,
                 "missing equal sign in "
                 "parameter '%s'",
                 param);
        ABORT_FINALIZE(RS_RET_PARAM_ERROR);
    }
    *val = '\0'; /* terminates name */
    ++val; /* now points to begin of value */
    CHKmalloc(*name = strdup(param));
    CHKmalloc(*paramval = strdup(val));
finalize_it:
    RETiRet;
}

BEGINnewActInst
    struct cnfparamvals *pvals;
    int i;
    int iNumTpls;
    CODESTARTnewActInst;
    if ((pvals = nvlstGetParams(lst, &actpblk, NULL)) == NULL) {
        ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
    }

    CHKiRet(createInstance(&pData));
    setInstParamDefaults(pData);

    for (i = 0; i < actpblk.nParams; ++i) {
        if (!pvals[i].bUsed) continue;
        if (!strcmp(actpblk.descr[i].name, "topic")) {
            CHKmalloc(pData->topic = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "dynakey")) {
            pData->dynaKey = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "dynatopic")) {
            pData->dynaTopic = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "dynatopic.cachesize")) {
            pData->iDynaTopicCacheSize = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "closetimeout")) {
            pData->closeTimeout = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "partitions.auto")) {
            pData->autoPartition = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "partitions.number")) {
            pData->nPartitions = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "partitions.usefixed")) {
            pData->fixedPartition = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "broker")) {
            es_str_t *es = es_newStr(128);
            int bNeedComma = 0;
            for (int j = 0; j < pvals[i].val.d.ar->nmemb; ++j) {
                if (bNeedComma) es_addChar(&es, ',');
                es_addStr(&es, pvals[i].val.d.ar->arr[j]);
                bNeedComma = 1;
            }
            pData->brokers = es_str2cstr(es, NULL);
            es_deleteStr(es);
        } else if (!strcmp(actpblk.descr[i].name, "confparam")) {
            pData->nConfParams = pvals[i].val.d.ar->nmemb;
            CHKmalloc(pData->confParams = malloc(sizeof(struct kafka_params) * pvals[i].val.d.ar->nmemb));
            for (int j = 0; j < pvals[i].val.d.ar->nmemb; ++j) {
                char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL);
                CHKiRet(processKafkaParam(cstr, &pData->confParams[j].name, &pData->confParams[j].val));
                free(cstr);
            }
        } else if (!strcmp(actpblk.descr[i].name, "topicconfparam")) {
            pData->nTopicConfParams = pvals[i].val.d.ar->nmemb;
            CHKmalloc(pData->topicConfParams = malloc(sizeof(struct kafka_params) * pvals[i].val.d.ar->nmemb));
            for (int j = 0; j < pvals[i].val.d.ar->nmemb; ++j) {
                char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL);
                CHKiRet(processKafkaParam(cstr, &pData->topicConfParams[j].name, &pData->topicConfParams[j].val));
                free(cstr);
            }
        } else if (!strcmp(actpblk.descr[i].name, "kafkaheader")) {
#if OMKAFKA_HAS_HEADERS
            pData->nHeaders = pvals[i].val.d.ar->nmemb;
            CHKmalloc(pData->headers = malloc(sizeof(struct kafka_params) * pvals[i].val.d.ar->nmemb));
            for (int j = 0; j < pvals[i].val.d.ar->nmemb; ++j) {
                char *cstr = es_str2cstr(pvals[i].val.d.ar->arr[j], NULL);
                CHKiRet(processKafkaParam(cstr, &pData->headers[j].name, &pData->headers[j].val));
                free(cstr);
            }
#else
            LogError(0, RS_RET_CONFIG_ERROR,
                     "omkafka: kafkaheader parameter requires librdkafka with record header support (v0.11 or newer)");
            ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
#endif
        } else if (!strcmp(actpblk.descr[i].name, "errorfile")) {
            CHKmalloc(pData->errorFile = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "statsfile")) {
            CHKmalloc(pData->statsFile = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "key")) {
            CHKmalloc(pData->key = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "template")) {
            CHKmalloc(pData->tplName = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "reopenonhup")) {
            pData->bReopenOnHup = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "resubmitonfailure")) {
            pData->bResubmitOnFailure = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "keepfailedmessages")) {
            pData->bKeepFailedMessages = pvals[i].val.d.n;
        } else if (!strcmp(actpblk.descr[i].name, "failedmsgfile")) {
            CHKmalloc(pData->failedMsgFile = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else if (!strcmp(actpblk.descr[i].name, "statsname")) {
            CHKmalloc(pData->statsName = (uchar *)es_str2cstr(pvals[i].val.d.estr, NULL));
        } else {
            LogError(0, RS_RET_INTERNAL_ERROR, "omkafka: program error, non-handled param '%s'\n",
                     actpblk.descr[i].name);
        }
    }

#if OMKAFKA_HAS_HEADERS
    if (pData->nHeaders > 0) {
        pData->kafka_headers = rd_kafka_headers_new(pData->nHeaders);
        if (pData->kafka_headers == NULL) {
            LogError(0, RS_RET_OUT_OF_MEMORY, "omkafka: failed to create kafka headers");
            ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
        }
        for (int j = 0; j < pData->nHeaders; ++j) {
            if (rd_kafka_header_add(pData->kafka_headers, pData->headers[j].name, -1, pData->headers[j].val, -1) !=
                RD_KAFKA_RESP_ERR_NO_ERROR) {
                LogError(0, RS_RET_KAFKA_ERROR, "omkafka: failed to add kafka header '%s'", pData->headers[j].name);
                rd_kafka_headers_destroy(pData->kafka_headers);
                pData->kafka_headers = NULL;
                ABORT_FINALIZE(RS_RET_KAFKA_ERROR);
            }
        }
    }
#endif

    if (pData->brokers == NULL) {
        CHKmalloc(pData->brokers = strdup("localhost:9092"));
        LogMsg(0, NO_ERRCODE, LOG_INFO,
               "imkafka: \"broker\" parameter not specified "
               "using default of localhost:9092 -- this may not be what you want!");
    }

    if (pData->dynaKey && pData->key == NULL) {
        LogError(0, RS_RET_CONFIG_ERROR,
                 "omkafka: requested dynamic key, but no "
                 "name for key template given - action definition invalid");
        ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
    }

    if (pData->dynaTopic && pData->topic == NULL) {
        LogError(0, RS_RET_CONFIG_ERROR,
                 "omkafka: requested dynamic topic, but no "
                 "name for topic template given - action definition invalid");
        ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
    }

    iNumTpls = 2;
    if (pData->dynaKey) ++iNumTpls;
    if (pData->dynaTopic) ++iNumTpls;
    CODE_STD_STRING_REQUESTnewActInst(iNumTpls);
    CHKiRet(OMSRsetEntry(*ppOMSR, 0,
                         (uchar *)strdup((pData->tplName == NULL) ? "RSYSLOG_FileFormat" : (char *)pData->tplName),
                         OMSR_NO_RQD_TPL_OPTS));

    CHKiRet(OMSRsetEntry(*ppOMSR, 1, (uchar *)strdup(" KAFKA_TimeStamp"), OMSR_NO_RQD_TPL_OPTS));
    if (pData->dynaKey) CHKiRet(OMSRsetEntry(*ppOMSR, 2, ustrdup(pData->key), OMSR_NO_RQD_TPL_OPTS));

    if (pData->dynaTopic) {
        CHKiRet(OMSRsetEntry(*ppOMSR, pData->dynaKey ? 3 : 2, ustrdup(pData->topic), OMSR_NO_RQD_TPL_OPTS));
        CHKmalloc(pData->dynCache =
                      (dynaTopicCacheEntry **)calloc(pData->iDynaTopicCacheSize, sizeof(dynaTopicCacheEntry *)));
        pData->iCurrElt = -1;
    }

    pthread_mutex_lock(&closeTimeoutMut);
    if (closeTimeout < pData->closeTimeout) {
        closeTimeout = pData->closeTimeout;
    }
    pthread_mutex_unlock(&closeTimeoutMut);

    /* Load failed messages here (If enabled), do NOT check for IRET!*/
    if (pData->bKeepFailedMessages && pData->failedMsgFile != NULL) {
        loadFailedMsgs(pData);
    }

    if (pData->statsName) {
        CHKiRet(statsobj.Construct(&pData->stats));
        CHKiRet(statsobj.SetName(pData->stats, (uchar *)pData->statsName));
        CHKiRet(statsobj.SetOrigin(pData->stats, (uchar *)"omkafka"));

        /* Track following stats */
        STATSCOUNTER_INIT(pData->ctrTopicSubmit, pData->mutCtrTopicSubmit);
        CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"submitted", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                    &pData->ctrTopicSubmit));
        STATSCOUNTER_INIT(pData->ctrKafkaFail, pData->mutCtrKafkaFail);
        CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"failures", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                    &pData->ctrKafkaFail));
        STATSCOUNTER_INIT(pData->ctrKafkaAck, pData->mutCtrKafkaAck);
        CHKiRet(statsobj.AddCounter(pData->stats, (uchar *)"acked", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                    &pData->ctrKafkaAck));
        CHKiRet(statsobj.ConstructFinalize(pData->stats));
    }

    CODE_STD_FINALIZERnewActInst;
    cnfparamvalsDestruct(pvals, &actpblk);
ENDnewActInst


BEGINmodExit
    CODESTARTmodExit;
    statsobj.Destruct(&kafkaStats);
    CHKiRet(objRelease(statsobj, CORE_COMPONENT));
    DESTROY_ATOMIC_HELPER_MUT(mutClock);

    pthread_mutex_lock(&closeTimeoutMut);
    int timeout = closeTimeout;
    pthread_mutex_unlock(&closeTimeoutMut);
    pthread_mutex_destroy(&closeTimeoutMut);
    if (rd_kafka_wait_destroyed(timeout) != 0) {
        LogMsg(0, RS_RET_OK, LOG_WARNING,
               "omkafka: could not terminate librdkafka gracefully, "
               "%d threads still remain.\n",
               rd_kafka_thread_cnt());
    }
finalize_it:
ENDmodExit


NO_LEGACY_CONF_parseSelectorAct BEGINqueryEtryPt CODESTARTqueryEtryPt;
CODEqueryEtryPt_STD_OMOD_QUERIES;
CODEqueryEtryPt_STD_OMOD8_QUERIES;
CODEqueryEtryPt_STD_CONF2_CNFNAME_QUERIES;
CODEqueryEtryPt_STD_CONF2_OMOD_QUERIES;
CODEqueryEtryPt_doHUP
ENDqueryEtryPt


BEGINmodInit()
    CODESTARTmodInit;
    uchar *pTmp;
    INITLegCnfVars;
    *ipIFVersProvided = CURR_MOD_IF_VERSION;
    CODEmodInit_QueryRegCFSLineHdlr dbgprintf("just because librdkafka needs it, sqrt of 4 is %f\n", sqrt(4.0));
    CHKiRet(objUse(datetime, CORE_COMPONENT));
    CHKiRet(objUse(strm, CORE_COMPONENT));
    CHKiRet(objUse(statsobj, CORE_COMPONENT));

    INIT_ATOMIC_HELPER_MUT(mutClock);

    DBGPRINTF("omkafka %s using librdkafka version %s, 0x%x\n", VERSION, rd_kafka_version_str(), rd_kafka_version());
    CHKiRet(statsobj.Construct(&kafkaStats));
    CHKiRet(statsobj.SetName(kafkaStats, (uchar *)"omkafka"));
    CHKiRet(statsobj.SetOrigin(kafkaStats, (uchar *)"omkafka"));
    STATSCOUNTER_INIT(ctrTopicSubmit, mutCtrTopicSubmit);
    CHKiRet(
        statsobj.AddCounter(kafkaStats, (uchar *)"submitted", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrTopicSubmit));
    STATSCOUNTER_INIT(ctrQueueSize, mutCtrQueueSize);
    CHKiRet(
        statsobj.AddCounter(kafkaStats, (uchar *)"maxoutqsize", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrQueueSize));
    STATSCOUNTER_INIT(ctrKafkaFail, mutCtrKafkaFail);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrKafkaFail));
    STATSCOUNTER_INIT(ctrCacheSkip, mutCtrCacheSkip);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"topicdynacache.skipped", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrCacheSkip));
    STATSCOUNTER_INIT(ctrCacheMiss, mutCtrCacheMiss);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"topicdynacache.miss", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrCacheMiss));
    STATSCOUNTER_INIT(ctrCacheEvict, mutCtrCacheEvict);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"topicdynacache.evicted", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrCacheEvict));
    STATSCOUNTER_INIT(ctrKafkaAck, mutCtrKafkaAck);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"acked", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrKafkaAck));
    STATSCOUNTER_INIT(ctrKafkaMsgTooLarge, mutCtrKafkaMsgTooLarge);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures_msg_too_large", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaMsgTooLarge));
    STATSCOUNTER_INIT(ctrKafkaUnknownTopic, mutCtrKafkaUnknownTopic);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures_unknown_topic", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaUnknownTopic));
    STATSCOUNTER_INIT(ctrKafkaQueueFull, mutCtrKafkaQueueFull);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures_queue_full", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaQueueFull));
    STATSCOUNTER_INIT(ctrKafkaUnknownPartition, mutCtrKafkaUnknownPartition);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures_unknown_partition", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaUnknownPartition));
    STATSCOUNTER_INIT(ctrKafkaOtherErrors, mutCtrKafkaOtherErrors);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"failures_other", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaOtherErrors));
    STATSCOUNTER_INIT(ctrKafkaRespTimedOut, mutCtrKafkaRespTimedOut);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"errors_timed_out", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaRespTimedOut));
    STATSCOUNTER_INIT(ctrKafkaRespTransport, mutCtrKafkaRespTransport);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"errors_transport", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaRespTransport));
    STATSCOUNTER_INIT(ctrKafkaRespBrokerDown, mutCtrKafkaRespBrokerDown);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"errors_broker_down", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaRespBrokerDown));
    STATSCOUNTER_INIT(ctrKafkaRespAuth, mutCtrKafkaRespAuth);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"errors_auth", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaRespAuth));
    STATSCOUNTER_INIT(ctrKafkaRespSSL, mutCtrKafkaRespSSL);
    CHKiRet(
        statsobj.AddCounter(kafkaStats, (uchar *)"errors_ssl", ctrType_IntCtr, CTR_FLAG_RESETTABLE, &ctrKafkaRespSSL));
    STATSCOUNTER_INIT(ctrKafkaRespOther, mutCtrKafkaRespOther);
    CHKiRet(statsobj.AddCounter(kafkaStats, (uchar *)"errors_other", ctrType_IntCtr, CTR_FLAG_RESETTABLE,
                                &ctrKafkaRespOther));
    CHKiRet(statsobj.AddCounter(kafkaStats, UCHAR_CONSTANT("rtt_avg_usec"), ctrType_Int, CTR_FLAG_NONE, &rtt_avg_usec));
    CHKiRet(statsobj.AddCounter(kafkaStats, UCHAR_CONSTANT("throttle_avg_msec"), ctrType_Int, CTR_FLAG_NONE,
                                &throttle_avg_msec));
    CHKiRet(statsobj.AddCounter(kafkaStats, UCHAR_CONSTANT("int_latency_avg_usec"), ctrType_Int, CTR_FLAG_NONE,
                                &int_latency_avg_usec));
    CHKiRet(statsobj.ConstructFinalize(kafkaStats));

    DBGPRINTF("omkafka: Add KAFKA_TimeStamp to template system ONCE\n");
    pTmp = (uchar *)KAFKA_TimeStamp;
    tplAddLine(ourConf, " KAFKA_TimeStamp", &pTmp);
ENDmodInit
