/*
 * librdkafka - Apache Kafka C library
 *
 * Copyright (c) 2012-2015, Magnus Edenhill
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "test.h"

/* Typical include path would be <librdkafka/rdkafka.h>, but this program
 * is built from within the librdkafka source tree and thus differs. */
#include "rdkafka.h"  /* for Kafka driver */

static int prod_msg_remains = 0;
static int fails = 0;

/**
 * Delivery reported callback.
 * Called for each message once to signal its delivery status.
 */
static void dr_cb (rd_kafka_t *rk, void *payload, size_t len,
		   rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {

	if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
		TEST_FAIL("Message delivery failed: %s\n",
			  rd_kafka_err2str(err));

	if (prod_msg_remains == 0)
		TEST_FAIL("Too many messages delivered (prod_msg_remains %i)",
			  prod_msg_remains);

	prod_msg_remains--;
}


/**
 * Produces 'msgcnt' messages split over 'partition_cnt' partitions.
 */
static void produce_messages (uint64_t testid, const char *topic,
                              int partition_cnt, int msg_base, int msgcnt) {
	int r;
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	char errstr[512];
	int i;
	int32_t partition;
	int msgid = msg_base;

	test_conf_init(&conf, &topic_conf, 20);

	rd_kafka_conf_set_dr_cb(conf, dr_cb);

        /* Make sure all replicas are in-sync after producing
         * so that consume test wont fail. */
        rd_kafka_topic_conf_set(topic_conf, "request.required.acks", "-1",
                                errstr, sizeof(errstr));

	/* Create kafka instance */
	rk = test_create_handle(RD_KAFKA_PRODUCER, conf);

	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("Failed to create topic: %s\n",
                          rd_kafka_err2str(rd_kafka_last_error()));

        /* Produce messages */
	prod_msg_remains = msgcnt;
	for (partition = 0 ; partition < partition_cnt ; partition++) {
		int batch_cnt = msgcnt / partition_cnt;

		for (i = 0 ; i < batch_cnt ; i++) {
                        char key[128];
                        char buf[128];
			rd_snprintf(key, sizeof(key),
				 "testid=%"PRIu64", partition=%i, msg=%i",
				 testid, (int)partition, msgid);
                        rd_snprintf(buf, sizeof(buf),
                                 "data: testid=%"PRIu64", partition=%i, msg=%i",
				 testid, (int)partition, msgid);

                        r = rd_kafka_produce(rkt, partition,
                                             RD_KAFKA_MSG_F_COPY,
                                             buf, strlen(buf),
                                             key, strlen(key),
                                             NULL);
                        if (r == -1)
                                TEST_FAIL("Failed to produce message %i "
                                          "to partition %i: %s",
                                          msgid, (int)partition,
                                          rd_kafka_err2str(rd_kafka_last_error()));
			msgid++;
		}
        }


	/* Wait for messages to be delivered */
	while (rd_kafka_outq_len(rk) > 0)
		rd_kafka_poll(rk, 100);

	if (fails)
		TEST_FAIL("%i failures, see previous errors", fails);

	if (prod_msg_remains != 0)
		TEST_FAIL("Still waiting for %i messages to be produced",
			  prod_msg_remains);

	/* Destroy topic */
	rd_kafka_topic_destroy(rkt);

	/* Destroy rdkafka instance */
	TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
	rd_kafka_destroy(rk);
}



static int *cons_msgs;
static int  cons_msgs_size;
static int  cons_msgs_cnt;
static int  cons_msg_next;
static int  cons_msg_stop = -1;
static int64_t cons_last_offset = -1;  /* last offset received */

static void verify_consumed_msg_reset (int msgcnt) {
	if (cons_msgs) {
		free(cons_msgs);
		cons_msgs = NULL;
	}

	if (msgcnt) {
		int i;

		cons_msgs = malloc(sizeof(*cons_msgs) * msgcnt);
		for (i = 0 ; i < msgcnt ; i++)
			cons_msgs[i] = -1;
	}

	cons_msgs_size = msgcnt;
	cons_msgs_cnt = 0;
        cons_msg_next = 0;
        cons_msg_stop = -1;
        cons_last_offset = -1;

        TEST_SAY("Reset consumed_msg stats, making room for %d new messages\n",
                 msgcnt);
}


static int int_cmp (const void *_a, const void *_b) {
	int a = *(int *)_a;
	int b = *(int *)_b;
        /* Sort -1 (non-received msgs) at the end */
	return (a == -1 ? 100000000 : a) - (b == -1 ? 10000000 : b);
}

static void verify_consumed_msg_check0 (const char *func, int line,
                                        const char *desc,
                                        int expected_cnt) {
	int i;
	int fails = 0;
        int not_recvd = 0;

        TEST_SAY("%s: received %d/%d/%d messages\n",
                 desc, cons_msgs_cnt, expected_cnt, cons_msgs_size);
        if (expected_cnt > cons_msgs_size)
                TEST_FAIL("expected_cnt %d > cons_msgs_size %d\n",
                          expected_cnt, cons_msgs_size);

	if (cons_msgs_cnt < expected_cnt) {
		TEST_SAY("%s: Missing %i messages in consumer\n",
			 desc,expected_cnt - cons_msgs_cnt);
		fails++;
	}

	qsort(cons_msgs, cons_msgs_size, sizeof(*cons_msgs), int_cmp);

	for (i = 0 ; i < expected_cnt ; i++) {
		if (cons_msgs[i] != i) {
                        if (cons_msgs[i] == -1) {
                                not_recvd++;
                                TEST_SAY("%s: msg %d/%d not received\n",
                                         desc, i, expected_cnt);
                        } else
                                TEST_SAY("%s: Consumed message #%i is wrong, "
                                         "expected #%i\n",
                                         desc, cons_msgs[i], i);
			fails++;
		}
	}

        if (not_recvd)
                TEST_SAY("%s: %d messages not received at all\n",
                         desc, not_recvd);

	if (fails)
		TEST_FAIL("%s: See above error(s)", desc);
        else
                TEST_SAY("%s: message range check: %d/%d messages consumed: "
                         "succeeded\n", desc, cons_msgs_cnt, expected_cnt);

}


#define verify_consumed_msg_check(desc,expected_cnt)                        \
	verify_consumed_msg_check0(__FUNCTION__,__LINE__, desc, expected_cnt)



static void verify_consumed_msg0 (const char *func, int line,
				  uint64_t testid, int32_t partition,
				  int msgnum,
				  rd_kafka_message_t *rkmessage) {
	uint64_t in_testid;
	int in_part;
	int in_msgnum;
	char buf[128];

	if (rkmessage->key_len +1 >= sizeof(buf))
		TEST_FAIL("Incoming message key too large (%i): "
			  "not sourced by this test",
			  (int)rkmessage->key_len);

	rd_snprintf(buf, sizeof(buf), "%.*s",
		 (int)rkmessage->key_len, (char *)rkmessage->key);

	if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i",
		   &in_testid, &in_part, &in_msgnum) != 3)
		TEST_FAIL("Incorrect key format: %s", buf);

        if (test_level > 2) {
		TEST_SAY("%s:%i: Our testid %"PRIu64", part %i (%i), "
			 "msg %i/%i, key's: \"%s\"\n",
			 func, line,
			 testid, (int)partition, (int)rkmessage->partition,
			 msgnum, cons_msgs_size, buf);
	}

	if (testid != in_testid ||
	    (partition != -1 && partition != in_part) ||
	    (msgnum != -1 && msgnum != in_msgnum) ||
	    (in_msgnum < 0 || in_msgnum > cons_msgs_size))
		goto fail_match;

	if (cons_msgs_cnt == cons_msgs_size) {
		TEST_SAY("Too many messages in cons_msgs (%i) while reading "
			 "message key \"%s\"\n",
			 cons_msgs_cnt, buf);
		verify_consumed_msg_check("?", cons_msgs_size);
		TEST_FAIL("See above error(s)");
	}

	cons_msgs[cons_msgs_cnt++] = in_msgnum;
        cons_last_offset = rkmessage->offset;

	return;

 fail_match:
	TEST_FAIL("%s:%i: Our testid %"PRIu64", part %i, msg %i/%i did "
		  "not match message's key: \"%s\"\n",
		  func, line,
		  testid, (int)partition, msgnum, cons_msgs_size, buf);
}

#define verify_consumed_msg(testid,part,msgnum,rkmessage) \
	verify_consumed_msg0(__FUNCTION__,__LINE__,testid,part,msgnum,rkmessage)


static void consume_cb (rd_kafka_message_t *rkmessage, void *opaque) {
        int64_t testid = *(int64_t *)opaque;

	if (test_level > 2)
		TEST_SAY("Consumed message #%d? at offset %"PRId64": %s\n",
			 cons_msg_next, rkmessage->offset,
			 rd_kafka_err2str(rkmessage->err));

        if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
                TEST_SAY("EOF at offset %"PRId64"\n", rkmessage->offset);
                return;
        }

        if (rkmessage->err)
                TEST_FAIL("Consume message from partition %i "
                          "has error: %s",
                          (int)rkmessage->partition,
                          rd_kafka_err2str(rkmessage->err));

        verify_consumed_msg(testid, rkmessage->partition,
                            cons_msg_next, rkmessage);

        if (cons_msg_next == cons_msg_stop) {
                rd_kafka_yield(NULL/*FIXME*/);
        }

        cons_msg_next++;
}

static void consume_messages_callback_multi (const char *desc,
                                             uint64_t testid, const char *topic,
                                             int32_t partition,
                                             const char *offset_store_method,
                                             int msg_base,
                                             int msg_cnt,
                                             int64_t initial_offset,
                                             int iterations) {
	rd_kafka_t *rk;
	rd_kafka_topic_t *rkt;
	rd_kafka_conf_t *conf;
	rd_kafka_topic_conf_t *topic_conf;
	int i;

        TEST_SAY("%s: Consume messages %d+%d from %s [%"PRId32"] "
                 "from offset %"PRId64" in %d iterations\n",
                 desc, msg_base, msg_cnt, topic, partition,
                 initial_offset, iterations);

	test_conf_init(&conf, &topic_conf, 20);

        test_topic_conf_set(topic_conf, "offset.store.method",
                            offset_store_method);

        if (!strcmp(offset_store_method, "broker")) {
                /* Broker based offset storage requires a group.id */
                test_conf_set(conf, "group.id", topic);
        }

        test_conf_set(conf, "enable.partition.eof", "true");

	/* Create kafka instance */
	rk = test_create_handle(RD_KAFKA_CONSUMER, conf);

        rd_kafka_topic_conf_set(topic_conf, "auto.offset.reset", "smallest",
                                NULL, 0);

	rkt = rd_kafka_topic_new(rk, topic, topic_conf);
	if (!rkt)
		TEST_FAIL("%s: Failed to create topic: %s\n",
                          desc, rd_kafka_err2str(rd_kafka_last_error()));

	cons_msg_stop = cons_msg_next + msg_cnt - 1;

        /* Consume the same batch of messages multiple times to
         * make sure back-to-back start&stops work. */
        for (i = 0 ; i < iterations ; i++) {
                int cnta;
                test_timing_t t_stop;

                TEST_SAY("%s: Iteration #%i: Consuming from "
                         "partition %i at offset %"PRId64", "
                         "msgs range %d..%d\n",
                         desc, i, partition, initial_offset,
                         cons_msg_next, cons_msg_stop);

                /* Consume messages */
                if (rd_kafka_consume_start(rkt, partition, initial_offset) == -1)
                        TEST_FAIL("%s: consume_start(%i) failed: %s",
                                  desc, (int)partition,
                                  rd_kafka_err2str(rd_kafka_last_error()));


                /* Stop consuming messages when this number of messages
                 * is reached. */
                cnta = cons_msg_next;
                do {
                        rd_kafka_consume_callback(rkt, partition, 1000,
                                                  consume_cb, &testid);
                } while (cons_msg_next < cons_msg_stop);

                TEST_SAY("%s: Iteration #%i: consumed %i messages\n",
                         desc, i, cons_msg_next - cnta);

                TIMING_START(&t_stop, "rd_kafka_consume_stop()");
                rd_kafka_consume_stop(rkt, partition);
                TIMING_STOP(&t_stop);

                /* Advance next offset so we dont reconsume
                 * messages on the next run. */
                if (initial_offset != RD_KAFKA_OFFSET_STORED) {
                        initial_offset = cons_last_offset+1;
			cons_msg_stop = cons_msg_next + msg_cnt - 1;
		}
        }

	/* Destroy topic */
	rd_kafka_topic_destroy(rkt);

	/* Destroy rdkafka instance */
	TEST_SAY("%s: Destroying kafka instance %s\n", desc, rd_kafka_name(rk));
	rd_kafka_destroy(rk);
}



static void test_produce_consume (const char *offset_store_method) {
	int msgcnt = 100;
        int partition_cnt = 1;
	int i;
	uint64_t testid;
	int msg_base = 0;
        const char *topic;

	/* Generate a testid so we can differentiate messages
	 * from other tests */
	testid = test_id_generate();

        /* Read test.conf to configure topic name */
        test_conf_init(NULL, NULL, 20);
        topic = test_mk_topic_name("0014", 1/*random*/);

	TEST_SAY("Topic %s, testid %"PRIu64", offset.store.method=%s\n",
                 topic, testid, offset_store_method);

	/* Produce messages */
	produce_messages(testid, topic, partition_cnt, msg_base, msgcnt);

        /* 100% of messages */
        verify_consumed_msg_reset(msgcnt);

	/* Consume 50% of messages with callbacks: stored offsets with no prior
         * offset stored. */
	for (i = 0 ; i < partition_cnt ; i++)
		consume_messages_callback_multi("STORED.1/2", testid, topic, i,
                                                offset_store_method,
                                                msg_base,
                                                (msgcnt / partition_cnt) / 2,
                                                RD_KAFKA_OFFSET_STORED,
                                                1);
        verify_consumed_msg_check("STORED.1/2", msgcnt / 2);

        /* Consume the rest using the now stored offset */
        for (i = 0 ; i < partition_cnt ; i++)
		consume_messages_callback_multi("STORED.2/2", testid, topic, i,
                                                offset_store_method,
                                                msg_base,
                                                (msgcnt / partition_cnt) / 2,
                                                RD_KAFKA_OFFSET_STORED,
                                                1);
        verify_consumed_msg_check("STORED.2/2", msgcnt);


	/* Consume messages with callbacks: logical offsets */
	verify_consumed_msg_reset(msgcnt);
	for (i = 0 ; i < partition_cnt ; i++) {
                int p_msg_cnt = msgcnt / partition_cnt;
                int64_t initial_offset = RD_KAFKA_OFFSET_TAIL(p_msg_cnt);
                const int iterations = 4;
		consume_messages_callback_multi("TAIL+", testid, topic, i,
                                                offset_store_method,
                                                /* start here (msgid) */
                                                msg_base,
                                                /* consume this many messages
                                                 * per iteration. */
                                                p_msg_cnt / iterations,
                                                /* start here (offset) */
                                                initial_offset,
                                                iterations);
        }

        verify_consumed_msg_check("TAIL+", msgcnt);

        verify_consumed_msg_reset(0);

	return;
}




int main_0014_reconsume_191 (int argc, char **argv) {
	if (test_broker_version >= TEST_BRKVER(0,8,2,0))
		test_produce_consume("broker");
        test_produce_consume("file");
	return 0;
}
