ASR_BASE

Change-Id: Icf3719cc0afe3eeb3edc7fa80a2eb5199ca9dda1
diff --git a/package/kernel/mfp/files/fp_database_hash.c b/package/kernel/mfp/files/fp_database_hash.c
new file mode 100644
index 0000000..be35203
--- /dev/null
+++ b/package/kernel/mfp/files/fp_database_hash.c
@@ -0,0 +1,1454 @@
+/*
+ *	Fast path database hash implementation
+ *
+ *	This program is free software; you can redistribute it and/or
+ *	modify it under the terms of the GNU FP_ERR( Public License
+ *	as published by the Free Software Foundation; either version
+ *	2 of the License, or (at your option) any later version.
+ *
+ *	Notes:
+ *	Implementation according to Documentation/RCU/rcuref.txt
+ */
+
+#define pr_fmt(fmt) "mfp" " database:%s:%d: " fmt, __func__, __LINE__
+
+#include "fp_common.h"
+#include "fp_database.h"
+#include "fp_device.h"
+#include "fp_core.h"
+#ifdef CONFIG_ASR_TOE
+#include "../linux/drivers/marvell/toev2/toe.h"
+#endif
+
+#define FP_ZONE			(NF_CT_DEFAULT_ZONE_ID)
+#define GUARD_TIMEOUT_SEC		(10)
+
+static u32 hash_rnd __read_mostly;
+
+static inline const char *state_to_string(enum entry_state state)
+{
+	return entry_state_names[state];
+}
+
+static inline int __fpdb_dump_entry(char *buf, struct fpdb_entry *el)
+{
+	int len = sprintf(buf, "fpdb dump entry (0x%p):\n", el);
+
+	len += fp_dump_tuple(buf + len, &el->in_tuple);
+	len += sprintf(buf + len, "\n");
+	len += fp_dump_tuple(buf + len, &el->out_tuple);
+	if (el->hh.hh_len) {
+		struct ethhdr *eth = (struct ethhdr *)(((u8 *) el->hh.hh_data) +
+					(HH_DATA_OFF(sizeof(*eth))));
+		len += sprintf(buf + len, "\nMAC header: src=%pM dst=%pM type=%04x\n",
+			       eth->h_source, eth->h_dest, eth->h_proto);
+	} else {
+		len += sprintf(buf + len, "\nMAC header was not set\n");
+	}
+	len += sprintf(buf + len, "Interfaces: in %p: %s, out %p: %s\n",
+			el->in_dev,el->in_dev->dev->name, el->out_dev,el->out_dev->dev->name);
+	len += sprintf(buf + len, "State: %s hits=%d pointer=%p\n",
+		       state_to_string(el->state), el->hit_counter, el);
+	len += sprintf(buf + len, "ct info: ct=%p timeout: %x rc=%d\n",
+		       el->ct, el->ct->timeout, atomic_read(&el->rc));
+
+	if (debug_level & DBG_INFO)
+		len += sprintf(buf + len, "DEBUG: (NAT=%s) (route: in=%d out=%d)\n",
+			   NF_CT_NAT(el->ct) ? "YES" : "NO",
+			   el->debug.in_route_type, el->debug.out_route_type);
+
+	return len;
+}
+
+void fpdb_dump_entry(char *msg, struct fpdb_entry *el)
+{
+	char buf[MAX_DEBUG_PRINT_SIZE];
+	int len = 0;
+
+	BUG_ON(!el);
+
+	if (msg)
+		len = sprintf(buf, "%s", msg);
+
+	len += __fpdb_dump_entry(buf + len, el);
+
+	pr_err("%s", buf);
+}
+
+void fpdb_dump_tuple(char *msg, struct nf_conntrack_tuple *t)
+{
+	char buf[MAX_DEBUG_PRINT_SIZE];
+	int len = 0;
+
+	BUG_ON(!t);
+
+	if (msg)
+		len = sprintf(buf, "%s", msg);
+
+	len += sprintf(buf + len, "fpdb dump tuple:\n");
+	len += fp_dump_tuple(buf + len, t);
+
+	pr_err("%s\n", buf);
+}
+
+static int fpdb_print_entry(struct fpdb_entry *el, void *data)
+{
+	char in[256], out[256];
+	unsigned int state, use;
+	int *first_entry = data;
+
+	if (atomic_inc_not_zero(&el->rc)) {
+		if (*first_entry == true) {
+			pr_err("l2  l3  l4  timeout\thash\thits\tstate  in_dev  out_dev  tuple_in  tuple_out ct block use refcnt\n");
+			*first_entry = false;
+		}
+		__fp_dump_tuple(in, &el->in_tuple, 0);
+		__fp_dump_tuple(out, &el->out_tuple, 0);
+		state = el->ct->proto.tcp.state;
+		use = atomic_read(&el->ct->ct_general.use);
+		pr_err("%s  %s  %s  %d\t%d\t%d\t%s  %s  %s  %s  %s  %p  %d  %d  %d\n",
+				  el->hh.hh_len ? "eth" : "NA",
+				  el->in_tuple.src.l3num == AF_INET6 ?
+				  "ipv6" : "ipv4",
+				  el->in_tuple.dst.protonum == IPPROTO_UDP ?
+				  "udp" : "tcp",
+				  jiffies_to_msecs(el->ct->timeout - jiffies) / 1000U,
+				  el->bucket, el->hit_counter,
+				  el->in_tuple.dst.protonum == IPPROTO_UDP ?
+				  "N/A" : tcp_conntrack_names[state],
+				  el->in_dev->dev->name,
+				  el->out_dev->dev->name,
+				  in, out, el->ct, el->block, use, atomic_read(&el->rc));
+		fpdb_put(el);
+	}
+	return 0;
+}
+
+void fpdb_dump_db(void)
+{
+	int first_entry = true;
+	fpdb_iterate(fpdb_print_entry, &first_entry);
+}
+
+/****************************************************************************
+ *			Fast Path Database prototypes
+ ****************************************************************************/
+
+struct fpdb_htable {
+	struct hlist_head *h;
+	unsigned int size;
+	int vmalloced;
+};
+
+struct fp_database {
+	struct fpdb_stats stats;
+	volatile u32 num_entries;
+	struct fpdb_htable htable;
+	spinlock_t lock;
+	struct nf_ct_ext_type *nfct_ext;
+	struct kmem_cache *db_cache;
+};
+
+struct timeout_entry {
+	struct list_head list;
+	struct timer_list *timeout;
+};
+/****************************************************************************
+ *			Fast Path Database globals
+ ****************************************************************************/
+
+static struct fp_database *db;
+/* TODO - do we need something else here??
+   Or is there only one "net" in ESHEL? */
+struct net *net = &init_net;
+
+#ifdef CONFIG_ASR_TOE
+extern int fp_cm_genl_send_tuple(struct nf_conntrack_tuple *tuple, struct fpdb_entry *el,
+									 int add, int len);
+static inline bool get_remote_mac_addr(struct fpdb_entry *el, char *mac)
+{
+	struct neighbour *neigh;
+	struct neigh_table *tbl;
+	struct nf_conntrack_tuple *tuple;
+	struct net_device *br;
+
+	if (el->in_dev->br != el->out_dev->br)
+		return false;
+
+	tuple = &el->in_tuple;
+	br = el->out_dev->br;
+	tbl = (tuple->src.l3num == AF_INET6) ? &nd_tbl : &arp_tbl;
+
+	neigh = neigh_lookup(tbl, tuple->dst.u3.all, br);
+	if (neigh) {
+		memcpy(mac, neigh->ha, ETH_ALEN);
+		neigh_release(neigh);
+		return true;
+	}
+
+	return false;
+}
+
+#endif
+
+static void guard_timer_timeout(struct timer_list *t)
+{
+	struct fpdb_entry *el = from_timer(el, &t, guard_timer);
+
+	pr_err("Entry was hold and could not be removed for %d sec. [%px][rc=%d] state=%d\n",
+	       GUARD_TIMEOUT_SEC, el, atomic_read(&el->rc), el->state);
+
+	/* BUG_ON(debug_level & DBG_WARN_AS_ERR);*/
+	if (atomic_read(&el->rc) > 0) {
+		FP_ERR_DUMP_ENTRY(NULL, el);
+		pr_err("Extend the timer when rc is not 0!\n");
+		mod_timer(el->guard_timer, jiffies + GUARD_TIMEOUT_SEC * HZ);
+	}
+}
+
+#ifdef FP_USE_SRAM_POOL_OPT
+static void *local_nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
+{
+	struct hlist_nulls_head *hash;
+	unsigned int nr_slots, i;
+	size_t sz;
+
+	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
+	nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
+	sz = nr_slots * sizeof(struct hlist_nulls_head);
+	hash = (void *)sram_pool_alloc(sz);
+	if (hash && nulls)
+		for (i = 0; i < nr_slots; i++)
+			INIT_HLIST_NULLS_HEAD(&hash[i], i);
+	return hash;
+}
+
+static void local_nf_ct_free_hashtable(void *hash, unsigned int size)
+{
+	sram_pool_free((unsigned long)hash, size * sizeof(struct hlist_nulls_head));
+}
+#endif
+
+static inline int fpdb_alloc_hashtable(struct fpdb_htable *htable)
+{
+	/* Currently use the same size used by others.. */
+	htable->size = nf_conntrack_htable_size;
+#ifdef FP_USE_SRAM_POOL_OPT
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	htable->h = local_nf_ct_alloc_hashtable(&htable->size, &htable->vmalloced, 0);
+#else
+	htable->h = local_nf_ct_alloc_hashtable(&htable->size, 0);
+#endif
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	htable->h = nf_ct_alloc_hashtable(&htable->size, &htable->vmalloced, 0);
+#else
+	htable->h = nf_ct_alloc_hashtable(&htable->size, 0);
+#endif
+#endif
+	if (!htable->h)
+		return -ENOMEM;
+
+	pr_debug("allocated fpdb hashtable (size = %d)\n", htable->size);
+
+	return 0;
+}
+
+static inline void fpdb_free_hashtable(struct fpdb_htable *htable)
+{
+#ifdef FP_USE_SRAM_POOL_OPT
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	local_nf_ct_free_hashtable(htable->h, htable->vmalloced, htable->size);
+#else
+	local_nf_ct_free_hashtable(htable->h, htable->size);
+#endif
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	nf_ct_free_hashtable(htable->h, htable->vmalloced, htable->size);
+#else
+	kvfree(htable->h);
+	htable->h = NULL;
+#endif
+#endif
+}
+
+/****************************************************************************
+ *			Fast Path Database API
+ ****************************************************************************/
+
+/**
+ * Allocates and initializes a new database entry
+ *
+ * @param flags  kmalloc flags
+ *
+ * @return new allocated and initialized database entry
+ */
+struct fpdb_entry *fpdb_alloc(gfp_t flags)
+{
+	struct fpdb_entry *el;
+
+#ifdef FP_USE_SRAM_POOL_OPT
+	el = (struct fpdb_entry *)sram_pool_alloc(sizeof(struct fpdb_entry));
+#else
+	el = kmem_cache_zalloc(db->db_cache, flags);
+#endif
+	if (!el) {
+		pr_err("no memory\n");
+		return NULL;
+	}
+
+	spin_lock_init(&el->lock);
+	INIT_HLIST_NODE(&el->hlist);
+	INIT_LIST_HEAD(&el->debug.trace.list);
+
+	el->state = ENTRY_INITIALIZED;
+#ifdef CONFIG_ASR_TOE
+	el->nl_flag = 0;
+#endif
+	return el;
+}
+
+
+/**
+ * Free a database entry
+ *
+ * @param flags  fpdb_entry * e
+ *
+ * @return void
+ */
+void fpdb_free(struct fpdb_entry * el)
+{
+	fpdev_put(el->out_dev);
+	fpdev_put(el->in_dev);
+
+#ifdef FP_USE_SRAM_POOL_OPT
+	sram_pool_free((unsigned long)el, sizeof(struct fpdb_entry));
+#else
+	kmem_cache_free(db->db_cache, el);
+#endif
+	return;
+}
+
+
+/**
+ * jenkins hash function using the source tuple
+ *
+ * @return hash key
+ */
+static inline unsigned int
+fpdb_hash_by_src(const struct nf_conntrack_tuple *tuple)
+{
+	unsigned int hash_src, hash_dst, hash;
+
+	BUG_ON(!tuple);
+
+	hash_src = jhash_3words((__force u32) tuple->src.u3.ip,
+			    (__force u32) tuple->src.u.all ^ FP_ZONE,
+			    tuple->src.l3num, hash_rnd);
+	hash_dst = jhash_3words((__force u32) tuple->dst.u3.ip,
+			    (__force u32) tuple->dst.u.all ^ FP_ZONE,
+			    tuple->dst.protonum, hash_rnd);
+	hash = jhash_2words(hash_src, hash_dst, hash_rnd);
+
+	return ((u64)hash * db->htable.size) >> 32;
+}
+
+/**
+ * rcu callback
+ *
+ * @param head
+ */
+static void fpdb_rcu_free(struct rcu_head *head)
+{
+	struct fpdb_entry *el = container_of(rcu_dereference(head),
+		struct fpdb_entry, rcu);
+
+	if (el == NULL) {
+		pr_err("fpdb_rcu_free el = NULL!\n");
+		return;
+	}
+
+	BUG_ON(!el || atomic_read(&el->rc) || el->state != ENTRY_DYING);
+
+	FP_DEBUG_DUMP_ENTRY("fpdb_rcu_free: entry was deleted\n", el);
+
+	if (el->guard_timer) {
+		del_timer_sync(el->guard_timer);
+		kfree(el->guard_timer);
+		el->guard_timer = NULL;
+	}
+
+	spin_lock_bh(&db->lock);
+	db->num_entries--;
+	spin_unlock_bh(&db->lock);
+
+	fpdev_put(el->out_dev);
+	fpdev_put(el->in_dev);
+
+#ifdef FP_USE_SRAM_POOL_OPT
+	sram_pool_free((unsigned long)el, sizeof(struct fpdb_entry));
+#else
+	kmem_cache_free(db->db_cache, el);
+#endif
+}
+
+
+
+/**
+ * decrement an entry's reference count and delete if 0
+ *
+ * @param el  pointer to a previously allocated fpdb_entry
+ */
+void fpdb_put(struct fpdb_entry *el)
+{
+	if (atomic_dec_and_test(&el->rc))
+		call_rcu(&el->rcu, fpdb_rcu_free);
+}
+
+
+#define FP_SMALL_MEM_LIMIT	(64 * 1024 * 1204)
+/**
+ * Adds a previously allocated entry to the database
+ * and updates its reference count to 1.
+ *
+ * @attention el must be allocated first with fpdb_alloc()
+ *      Initial Implementation - Hash by input tuple only
+ * @param el  pointer to a previously allocated fpdb_entry
+ *
+ */
+void fpdb_add(struct fpdb_entry *el)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
+	int pages = totalram_pages();
+#else
+	int pages = totalram_pages;
+#endif
+	unsigned int hash;
+	u32 max_num;
+
+#ifdef CONFIG_ASR_TOE
+	char mac[ETH_ALEN];
+#endif
+
+	spin_lock_bh(&el->lock);
+	spin_lock_bh(&db->lock);
+	BUG_ON(!el || !el->out_dev);
+	BUG_ON(el->state != ENTRY_INITIALIZED);
+
+	hash = fpdb_hash_by_src(&el->in_tuple);
+
+	atomic_set(&el->rc, 1);
+	el->state = ENTRY_ALIVE;
+	el->bucket = hash;
+	el->tstamp = jiffies;
+	if (!el->tstamp)
+		el->tstamp = 1;
+
+	BUG_ON(in_irq());
+	WARN_ON_ONCE(irqs_disabled());
+	hlist_add_head_rcu(&el->hlist, &db->htable.h[hash]);
+	db->num_entries++;
+
+#ifdef CONFIG_ASR_TOE
+	if (get_remote_mac_addr(el, mac))
+		mfp_toe_add_dmac(el->out_dev->dev, mac);
+#endif
+
+	spin_unlock_bh(&db->lock);
+	spin_unlock_bh(&el->lock);
+
+	/* Normally Conntrack MAX is HashSize*8. So here is not suit to only check double*/
+	/*we will modify the code to check 6 times of hash size --Yhuang 20160617*/
+
+	if (pages <= (FP_SMALL_MEM_LIMIT >> PAGE_SHIFT))
+		max_num = 2 * db->htable.size;
+	else
+		max_num = 6 * db->htable.size;
+
+	if (unlikely(db->num_entries > max_num)) {
+		pr_err_ratelimited("%s: database overloaded (%d entries, max=%d)\n",
+			   __func__, db->num_entries, max_num);
+		/*
+		if (debug_level & DBG_WARN_AS_ERR) {
+			fpdb_dump_db();
+			BUG();
+		}
+		*/
+		fpdb_flush();
+	} else if (unlikely(db->num_entries > ((max_num * 3) / 4))) {
+		fpdb_del_least_used_entry(max_num);
+	}
+
+	if (db->stats.max_entries < db->num_entries)
+		db->stats.max_entries = db->num_entries;
+	FP_DEBUG_DUMP_ENTRY("fpdb_add: entry was added\n", el);
+}
+
+/**
+ * Query the database for an entry matching the input tuple
+ * and increment the reference count for that entry if found.
+ *
+ * @attention The user MUST call fpdb_put() as soon as the entry
+ *            is not used!
+ *
+ * @param tuple  pointer to a nf_conntrack_tuple
+ *
+ * @return pointer to the matching entry, NULL if not found
+ */
+struct fpdb_entry *fpdb_get(struct nf_conntrack_tuple *tuple)
+{
+	unsigned int hash, iterations = 0;
+	struct fpdb_entry *el;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	struct hlist_node *h;
+#endif
+
+	BUG_ON(!tuple);
+
+	db->stats.lookups++;
+
+	hash = fpdb_hash_by_src(tuple);
+
+	rcu_read_lock_bh();
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	hlist_for_each_entry_rcu(el, h, &db->htable.h[hash], hlist) {
+#else
+	hlist_for_each_entry_rcu(el, &db->htable.h[hash], hlist) {
+#endif
+		if (el && nf_ct_tuple_equal(&el->in_tuple, tuple)) {
+			if (!atomic_inc_not_zero(&el->rc))
+				goto not_found;
+			rcu_read_unlock_bh();
+			if (!iterations)
+				db->stats.hits++;
+			el->hit_counter++;
+			FP_DEBUG_DUMP_ENTRY("fpdb_get: entry was found:\n", el);
+			return el;
+		}
+		iterations++;
+		db->stats.iterations++; /* Total Iterations*/
+	}
+
+not_found:
+	rcu_read_unlock_bh();
+	FP_DEBUG_DUMP_TUPLE("fpdb_get: entry was not found:\n", tuple);
+
+	return NULL;
+}
+
+#ifdef CONFIG_ASR_TOE
+static int fpdb_del_toe_tuple(struct fpdb_entry *el)
+{
+	struct toe_tuple_buff toe_tuple;
+	struct fp_net_device *dst, *src;
+	u32 nat_ip = 0;
+	u16 nat_port = 0;
+	u8 proto = 0, in_pkt = 0, out_pkt = 0, fwd = 0, nat = 0;
+	u8 rx_tx;
+
+	BUG_ON(!el);
+
+	if (!el->nl_flag)
+		return 0;
+
+	src = rcu_dereference_bh(el->in_dev);
+	dst = rcu_dereference_bh(el->out_dev);
+	if (!strncasecmp(src->dev->name, "ccinet", 6))
+		in_pkt = PDU_PKT;
+	else if (!strncasecmp(src->dev->name, "usbnet", 6))
+		in_pkt = USB_PKT;
+	else if (!strncasecmp(src->dev->name, "wlan", 4))
+		in_pkt = WIFI_PKT;
+	else if (!strncasecmp(src->dev->name, "eth", 3))
+		in_pkt = ETH_PKT;
+	else
+		in_pkt = AP_PKT;
+
+	if (!strncasecmp(dst->dev->name, "ccinet", 6))
+		out_pkt = PDU_PKT;
+	else if (!strncasecmp(dst->dev->name, "usbnet", 6))
+		out_pkt = USB_PKT;
+	else if (!strncasecmp(dst->dev->name, "wlan", 4))
+		out_pkt = WIFI_PKT;
+	else if (!strncasecmp(dst->dev->name, "eth", 3))
+		out_pkt = ETH_PKT;
+	else
+		out_pkt = AP_PKT;
+
+	fwd = (in_pkt != AP_PKT) && (out_pkt != AP_PKT);
+	if (fwd && (el->out_tuple.src.l3num == AF_INET)) {
+		if (in_pkt == PDU_PKT && (out_pkt == USB_PKT || out_pkt == WIFI_PKT || out_pkt == ETH_PKT)) {
+			nat = 1;
+			nat_ip = ntohl(el->out_tuple.src.u3.ip);
+			nat_port = ntohs(el->out_tuple.src.u.all);
+		} else if ((in_pkt == USB_PKT || in_pkt == WIFI_PKT || in_pkt == ETH_PKT) && out_pkt == PDU_PKT) {
+			nat = 1;
+			nat_ip = ntohl(el->out_tuple.dst.u3.ip);
+			nat_port = ntohs(el->out_tuple.dst.u.all);
+		} else
+			/* CP TOE WIFI/WIFI TOE CP no need nat */
+			nat = 0;
+	}
+
+	/* rx: cp -> ap, usb, wifi */
+	if (in_pkt == PDU_PKT)
+		rx_tx = 1;
+	/* rx: ap -> usb, ap -> wifi */
+	else if ((in_pkt == AP_PKT) && (out_pkt != PDU_PKT))
+		rx_tx = 1;
+	/*
+	 * tx:
+	 * ap -> cp
+	 * usb/wifi -> ap/cp */
+	else
+		rx_tx = 0;
+
+	if (el->in_tuple.src.l3num == AF_INET6) {
+		memcpy(toe_tuple.src_ip6, el->in_tuple.src.u3.all, sizeof(toe_tuple.src_ip6));
+		memcpy(toe_tuple.dst_ip6, el->in_tuple.dst.u3.all, sizeof(toe_tuple.src_ip6));
+		toe_tuple.ip6 = 1;
+	} else {
+		toe_tuple.src_ip = ntohl(el->in_tuple.src.u3.ip);
+		toe_tuple.dst_ip = ntohl(el->in_tuple.dst.u3.ip);
+		toe_tuple.ip6 = 0;
+		toe_tuple.nat = nat;
+		toe_tuple.nat_port = nat_port;
+		toe_tuple.nat_ip = nat_ip;
+	}
+
+	if (el->in_tuple.dst.protonum == IPPROTO_UDP)
+		proto = TOE_UDP;
+	else if (el->in_tuple.dst.protonum == IPPROTO_TCP)
+		proto = TOE_TCP;
+	else
+		proto = TOE_MAX;
+
+	toe_tuple.src_port = ntohs(el->in_tuple.src.u.all);
+	toe_tuple.dst_port = ntohs(el->in_tuple.dst.u.all);
+	toe_tuple.prot = proto;
+	toe_tuple.fwd = fwd;
+	toe_tuple.rxtx = rx_tx;
+	toe_tuple.out_pkt = out_pkt;
+
+	return toe_del_connection(&toe_tuple);
+}
+#endif
+
+void __fpdb_del(struct fpdb_entry *entry, bool hlist_del)
+{
+	BUG_ON(!entry);
+	if(entry->state != ENTRY_ALIVE)
+		return;
+
+	entry->state = ENTRY_DYING;
+
+#ifdef CONFIG_ASR_TOE
+	if (entry->nl_flag) {
+		fp_cm_genl_send_tuple(&entry->in_tuple, entry, 0, 0);
+		if (fpdb_del_toe_tuple(entry))
+			pr_debug("fpdb_del_toe_tuple failed!!!\r\n");
+		entry->nl_flag = 0;
+	}
+#endif
+
+	BUG_ON(entry->guard_timer);
+	if (hlist_del)
+		hlist_del_rcu(&entry->hlist);
+
+	if (atomic_dec_and_test(&entry->rc)) {
+		/* move start timer here to avoid rc is not zero yhuang 20160624*/
+		entry->guard_timer = kmalloc(sizeof(*entry->guard_timer), GFP_ATOMIC);
+		if (entry->guard_timer) {
+			timer_setup(entry->guard_timer, guard_timer_timeout, 0);
+			mod_timer(entry->guard_timer, jiffies + GUARD_TIMEOUT_SEC * HZ);
+		} else {
+			pr_err("Guard timer allocation failed!");
+		}
+
+		/* prevent out of order so that guard timer can be stopped */
+		mb();
+		call_rcu(&entry->rcu, fpdb_rcu_free);
+	} else {
+		pr_err("__fpdb_del fail. entry:%p, rc=%d, state=%d\n", entry,
+			atomic_read(&entry->rc), entry->state);
+	}
+}
+
+void fpdb_lock_bh(void)
+{
+	return spin_lock_bh(&db->lock);
+}
+
+void fpdb_unlock_bh(void)
+{
+	return spin_unlock_bh(&db->lock);
+}
+
+void fpdb_del(struct fpdb_entry *entry)
+{
+	spin_lock_bh(&db->lock);
+	__fpdb_del(entry, true);
+	spin_unlock_bh(&db->lock);
+}
+
+/**
+ * Replace a previously allocated entry with an prexisting one
+ * to the database.
+ *
+ * @attention nel must be allocated first with fpdb_alloc()
+ *            el - must be already in the database/
+ * @param el  pointer to a previously added fpdb_entry
+ * @param nel  pointer to a newely allocated fpdb_entry
+ * NOTE: must be called from softirq/lock_bh context
+ */
+void fpdb_replace(struct fpdb_entry *el, struct fpdb_entry *nel)
+{
+	unsigned int hash;
+
+	BUG_ON(!el || !el->out_dev);
+	BUG_ON(!nel || !nel->out_dev);
+	BUG_ON(nel->state != ENTRY_INITIALIZED);
+
+	hash = fpdb_hash_by_src(&nel->in_tuple);
+
+	atomic_set(&nel->rc, 1);
+	nel->state = ENTRY_ALIVE;
+	nel->bucket = hash;
+
+	BUG_ON(el->bucket != nel->bucket);
+
+	db->num_entries++;
+	hlist_replace_rcu(&el->hlist, &nel->hlist);
+	__fpdb_del(el, false);
+}
+
+
+static int device_cmp(struct nf_conn *ct, void *dev)
+{
+	struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+	struct fpdb_entry *orig, *reply;
+	struct net_device *net = (struct net_device *)dev;
+
+	if (!fp)
+		return 0;
+
+	orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+	reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+	if (orig && (fpdev_cmp_if(orig->in_dev, net) ||
+		     fpdev_cmp_if(orig->out_dev, net)))
+		return 1;
+	if (reply && (fpdev_cmp_if(reply->in_dev, net) ||
+		      fpdev_cmp_if(reply->out_dev, net)))
+		return 1;
+
+	return 0;
+}
+
+static inline bool
+tuple_cmp_port(const struct nf_conntrack_tuple *t, unsigned int port)
+{
+	return (ntohs(t->dst.u.all) == port || ntohs(t->src.u.all) == port);
+}
+
+static int port_cmp(struct nf_conn *ct, void *ptr)
+{
+	struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+	struct fpdb_entry *orig, *reply;
+	unsigned int port = (unsigned int)(unsigned long)ptr;
+
+	if (!fp)
+		return 0;
+
+	orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+	reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+	if (orig && (tuple_cmp_port(&orig->in_tuple, port) ||
+		     tuple_cmp_port(&orig->out_tuple, port)))
+		return 1;
+	if (reply && (tuple_cmp_port(&reply->in_tuple, port) ||
+		      tuple_cmp_port(&reply->out_tuple, port)))
+		return 1;
+
+	return 0;
+}
+
+/* kill all fastpath related conntracks */
+static int nf_fp_remove(struct nf_conn *ct, void *data)
+{
+	return test_bit(IPS_FASTPATH_BIT, &ct->status);
+}
+
+int fpdb_del_block_entry_by_dev(struct fpdb_entry *el, void *data)
+{
+	struct net_device *dev = (struct net_device *)data;
+	struct nf_conn_fastpath *ct_fp;
+
+	if (fpdev_cmp_if(el->in_dev, dev) ||
+	    fpdev_cmp_if(el->out_dev, dev)) {
+
+		spin_lock_bh(&db->lock);
+		ct_fp = nfct_fastpath(el->ct);
+		if (ct_fp) {
+			if (ct_fp->fpd_el[el->dir] == NULL) {
+				spin_unlock_bh(&db->lock);
+				return 0;
+			}
+
+			ct_fp->fpd_el[el->dir] = NULL;
+		}
+		spin_unlock_bh(&db->lock);
+
+		fpdb_del(el);
+		printk(KERN_DEBUG "delete a block entry related to %s\n", dev->name);
+	}
+
+	return 0;
+}
+
+static int nf_fpdb_del(struct nf_conn *ct, void *del)
+{
+	struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+	struct fpdb_entry *orig, *reply;
+
+	if (!fp)
+		return 0;
+
+	orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+	reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+	if (orig && orig == (struct fpdb_entry *)del) {
+		orig->tstamp = 0;
+		return 1;
+	}
+
+	if (reply && reply == (struct fpdb_entry *)del) {
+		reply->tstamp = 0;
+		return 1;
+	}
+
+	return 0;
+}
+
+static int fpdb_find_lest_used_entry(struct fpdb_entry *el, void *data)
+{
+	struct fpdb_entry **p_el = (struct fpdb_entry **)data;
+
+	if (!*p_el)
+		*p_el = el;
+	else if (el->tstamp && time_before(el->tstamp, (*p_el)->tstamp))
+		*p_el = el;
+
+	return 0;
+}
+
+void fpdb_del_least_used_entry(int max_num)
+{
+	struct fpdb_entry *el = NULL;
+
+	fpdb_iterate(fpdb_find_lest_used_entry, &el);
+
+	if (!el)
+		return;
+
+	pr_info_ratelimited("%s: el=0x%x (%d entries, max=%d)\n",
+		__func__, (unsigned)el, db->num_entries, max_num);
+	nf_ct_iterate_cleanup(&nf_fpdb_del, (void *)el, 0, 0);
+}
+
+/**
+ * Remove all fastpath related connections with the specified network device
+ *
+ * caller should have rtnl locked
+ *
+ * @param dev
+ */
+void fpdb_del_by_dev(struct net_device *dev)
+{
+	nf_ct_iterate_cleanup(&device_cmp, (void *)dev, 0, 0);
+
+	printk(KERN_DEBUG "All entries related to %s deleted\n", dev->name);
+}
+
+/**
+ * Remove all fastpath related connections with the specified port
+ * 
+ * caller should have rtnl locked
+ * 
+ * @param port
+ */
+void fpdb_del_by_port(unsigned int port)
+{
+	nf_ct_iterate_cleanup(&port_cmp, (void *)(unsigned long)port, 0, 0);
+
+	pr_debug("All entries with port=%d deleted\n", port);
+}
+
+/**
+ * flush the entire database by cleaning all fastpath related
+ * conntracks
+ * 
+ * MUST BE CALLED IN PROCESS CONTEXT
+ */
+void fpdb_flush(void)
+{
+	nf_ct_iterate_cleanup(&nf_fp_remove, 0, 0, 0);
+
+	pr_debug("All entries flushed\n");
+}
+
+/**
+ * Iterate through all fpdb entries
+ * MUST BE CALLED IN PROCESS CONTEXT
+ * 
+ * @param iter   callback function called per every entry
+ *               If returns 0, iteration stops.
+ * @param data   private data to be passed to the iter callback
+ */
+void fpdb_iterate(int (*iter)(struct fpdb_entry *e, void *data), void *data)
+{
+	int i;
+	struct fpdb_entry *e;
+
+	for (i = 0; i < db->htable.size; i++) {
+		rcu_read_lock_bh();
+		hlist_for_each_entry_rcu(e, &db->htable.h[i], hlist) {
+			if (iter(e, data))
+				break;
+		}
+		rcu_read_unlock_bh();
+	}
+}
+
+/**
+ * Add the current entry state to the entry's trace buffer when
+ * debug_level mask contains DBG_TRACE_LOG
+ *
+ * @param entry - entry to log
+ * @param tcph - NULL for UDP
+ */
+void fpdb_trace(struct fpdb_entry *entry, struct tcphdr *tcph)
+{
+	if (debug_level & DBG_TRACE_LOG) {
+		struct fpdb_trace *trace = kzalloc(sizeof(struct fpdb_trace), GFP_ATOMIC);
+
+		BUG_ON(!entry);
+
+		trace->timeout = jiffies_to_msecs(entry->ct->timeout - jiffies) / 1000U;
+		trace->ct_status = entry->ct->status;
+		trace->hit_counter = entry->hit_counter;
+
+		if (tcph) {
+			trace->tcp_state = entry->ct->proto.tcp.state;
+			trace->tcph = *tcph;
+		}
+
+		list_add(&trace->list, &entry->debug.trace.list);
+		if (++entry->debug.trace.sz > 5) {
+			/* TODO - change to configurable param */
+			trace = list_entry(entry->debug.trace.list.prev, struct fpdb_trace, list);
+			list_del(entry->debug.trace.list.prev);
+			kfree(trace);
+			entry->debug.trace.sz--;
+		}
+	}
+}
+
+/****************************************************************************
+ *			Fast Path Database private
+ ****************************************************************************/
+
+/* SYS FS and PROC FS */
+
+static void fpdb_get_stats(void)
+{
+	int i, count, max = 0;
+	struct fpdb_entry *el;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	struct hlist_node *h;
+#endif
+
+	memset(db->stats.hist, 0, sizeof(db->stats.hist));
+	db->stats.num_occupied = 0;
+
+	for (i = 0; i < db->htable.size; i++) {
+		count = 0;
+
+		rcu_read_lock_bh();
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+		hlist_for_each_entry_rcu(el, h, &db->htable.h[i], hlist)
+#else
+		hlist_for_each_entry_rcu(el, &db->htable.h[i], hlist)
+#endif
+			count++;
+		rcu_read_unlock_bh();
+
+		if (count)
+			db->stats.num_occupied++;
+
+		if (count < HISTOGRAM_SIZE) {
+			db->stats.hist[count].buckets++;
+			db->stats.hist[count].entries += count;
+		} else {
+			db->stats.hist[HISTOGRAM_SIZE].buckets++;
+			db->stats.hist[HISTOGRAM_SIZE].entries += count;
+		}
+
+		max = (count > max) ? count : max;
+	}
+
+	db->stats.largest_bucket = max;
+}
+
+static ssize_t fpdb_sysfs_flush(struct fastpath_module *m, const char *buf, size_t count)
+{
+	struct net_device *dev;
+
+	if (count > 2) {
+		char *str = kmalloc(sizeof(char)*count, GFP_KERNEL);
+		sprintf(str, "%s", buf);
+		str[count-1] = '\0';
+		dev = dev_get_by_name(&init_net, str);
+		kfree(str);
+
+		if (dev) {
+			fpdb_del_by_dev(dev);
+			dev_put(dev);
+			return count;
+		}
+	}
+
+	fpdb_flush();
+	return count;
+}
+
+static ssize_t fpdb_sysfs_stats_show(struct fastpath_module *m, char *buf)
+{
+	int len, i;
+	u32 sum_pct = 0;
+
+	fpdb_get_stats();
+
+	len = sprintf(buf, "Fast Path Database (HASH) statistics:\n");
+	len += sprintf(buf + len, "Max number of entries: %d ",
+				db->stats.max_entries);
+	len += sprintf(buf + len, "Total lookups: %d, Total hits: %d, "
+		       "hit rate %d%%\n", db->stats.lookups, db->stats.hits,
+		       (100 * db->stats.hits) / (db->stats.lookups ?
+						 db->stats.lookups : 1));
+	len += sprintf(buf + len, "Database Size is %d Buckets\n",
+		       db->htable.size);
+	len += sprintf(buf + len, "Number of occupied buckets: %d\n",
+		       db->stats.num_occupied);
+	len += sprintf(buf + len, "Database contains %d entries\n",
+		       db->num_entries);
+	len += sprintf(buf + len, "Largest bucket contains %d entries\n",
+		       db->stats.largest_bucket);
+	len += sprintf(buf + len, "Load Factor is %d (%d/%d)\n",
+		       db->num_entries /
+		       (db->htable.size ? db->htable.size : 1),
+		       db->num_entries, db->htable.size);
+	len += sprintf(buf + len, "find_entry() iterations/lookups: %d/%d\n",
+		       db->stats.iterations, db->stats.lookups);
+	len += sprintf(buf + len, "Histogram:\n");
+	len += sprintf(buf + len, "Size   buckets   entries   sum-pct\n");
+	for (i = 0; i < HISTOGRAM_SIZE; i++) {
+		if (sum_pct < 100)
+			sum_pct += (100 * db->stats.hist[i].entries) /
+				   (db->num_entries ?
+				    db->num_entries : 1);
+		else
+			sum_pct = 100;
+
+		len += sprintf(buf + len, "%4d%10d%10d%10d\n", i,
+			       db->stats.hist[i].buckets,
+			       db->stats.hist[i].entries, sum_pct);
+	}
+	len += sprintf(buf + len, ">%3d%10d%10d%10d\n",	i - 1,
+		       db->stats.hist[i].buckets,
+		       db->stats.hist[i].entries, 100);
+
+	return len;
+}
+
+static ssize_t fpdb_sysfs_stats_clear(struct fastpath_module *m, const char *buf,
+				   size_t count)
+{
+	pr_debug("reset stats...\n");
+	memset(&db->stats, 0, sizeof(db->stats));
+	return count;
+}
+
+static unsigned int dbg_hash;
+
+static ssize_t fpdb_sysfs_entry_debug_select(struct fastpath_module *m, const char *buf,
+				   size_t count)
+{
+	sscanf(buf, "%u", &dbg_hash);
+	return count;
+}
+
+static ssize_t fpdb_sysfs_entry_debug_show(struct fastpath_module *m, char *buf)
+{
+	struct fpdb_entry *el;
+	int i = 0, len;
+	struct fpdb_trace *itr;
+	struct nf_conn_fastpath *fp_ext;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	struct hlist_node *h;
+#endif
+
+	if (dbg_hash > db->htable.size)
+		return sprintf(buf, "invalid hash (%d)\n", dbg_hash);
+
+	len = sprintf(buf, "debug info for bucket%u:\n", dbg_hash);
+	rcu_read_lock_bh();
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+	hlist_for_each_entry_rcu(el, h, &db->htable.h[dbg_hash], hlist) {
+#else
+	hlist_for_each_entry_rcu(el, &db->htable.h[dbg_hash], hlist) {
+#endif
+		len += __fpdb_dump_entry(buf+len, el);
+		fp_ext = nf_ct_ext_find(el->ct, NF_CT_EXT_FASTPATH);
+		BUG_ON(!fp_ext);
+		len += sprintf(buf+len, "fastpath_ext orig:=%p reply=%p\n",
+			       fp_ext->fpd_el[IP_CT_DIR_ORIGINAL],
+			       fp_ext->fpd_el[IP_CT_DIR_REPLY]);
+		if (el->in_tuple.dst.protonum == IPPROTO_UDP)
+			continue;
+		len += sprintf(buf+len, "%d: trace:\n", i++);
+		len += sprintf(buf+len, "hits timeout tcp_state tcp_flags ct_status\n");
+		list_for_each_entry(itr, &el->debug.trace.list, list)
+			len += sprintf(buf+len, "%d %d %s %c%c%c%c%c%c %lu\n",
+				       itr->hit_counter, itr->timeout,
+				       tcp_conntrack_names[itr->tcp_state],
+				       itr->tcph.urg ? 'U' : '-',
+				       itr->tcph.ack ? 'A' : '-',
+				       itr->tcph.psh ? 'P' : '-',
+				       itr->tcph.rst ? 'R' : '-',
+				       itr->tcph.syn ? 'S' : '-',
+				       itr->tcph.fin ? 'F' : '-',
+				       itr->ct_status);
+	}
+	rcu_read_unlock_bh();
+
+	return len;
+}
+
+
+static FP_ATTR(stats, S_IRUGO|S_IWUSR, fpdb_sysfs_stats_show, fpdb_sysfs_stats_clear);
+static FP_ATTR(flush, S_IWUSR, NULL, fpdb_sysfs_flush);
+static FP_ATTR(bucket, S_IRUGO|S_IWUSR, fpdb_sysfs_entry_debug_show, fpdb_sysfs_entry_debug_select);
+
+static struct attribute *fp_database_attrs[] = {
+	&fp_attr_stats.attr,
+	&fp_attr_flush.attr,
+	&fp_attr_bucket.attr,
+	NULL, /* need to NULL terminate the list of attributes */
+};
+
+#ifdef CONFIG_PROC_FS
+static bool first;
+struct fpdb_iter_state {
+	struct seq_net_private p;
+	unsigned int bucket;
+};
+
+static struct hlist_node *fpdb_entries_get_first(struct seq_file *seq)
+{
+	struct fpdb_iter_state *st = seq->private;
+	struct hlist_node *n;
+
+	for (st->bucket = 0; st->bucket < db->htable.size; st->bucket++) {
+		n = rcu_dereference(db->htable.h[st->bucket].first);
+		if (n) {
+			first = true;
+			return n;
+		}
+	}
+	return NULL;
+}
+
+static struct hlist_node *fpdb_entries_get_next(struct seq_file *seq,
+						struct hlist_node *head)
+{
+	struct fpdb_iter_state *st = seq->private;
+
+	first = false;
+	head = rcu_dereference(head->next);
+
+	while (head == NULL) {
+		if (++st->bucket >= db->htable.size)
+			return NULL;
+		head = rcu_dereference(db->htable.h[st->bucket].first);
+	}
+	return head;
+}
+
+static struct hlist_node *fpdb_entries_get_idx(struct seq_file *seq, loff_t pos)
+{
+	struct hlist_node *head = fpdb_entries_get_first(seq);
+
+	if (head)
+		while (pos && (head = fpdb_entries_get_next(seq, head)))
+			pos--;
+	return pos ? NULL : head;
+}
+
+static void *fpdb_seq_start(struct seq_file *seq, loff_t *pos)
+	__acquires(RCU)
+{
+	rcu_read_lock_bh();
+	return fpdb_entries_get_idx(seq, *pos);
+}
+
+static void *fpdb_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	(*pos)++;
+	return fpdb_entries_get_next(seq, v);
+}
+
+static void fpdb_seq_stop(struct seq_file *seq, void *v)
+	__releases(RCU)
+{
+	rcu_read_unlock_bh();
+}
+
+static int fpdb_seq_show(struct seq_file *s, void *v)
+{
+	struct hlist_node *n = v;
+	struct fpdb_entry *el;
+	char in[256], out[256];
+	unsigned int state, use;
+
+	el = hlist_entry(n, struct fpdb_entry, hlist);
+	if (atomic_inc_not_zero(&el->rc)) {
+		if (first == true) {
+			seq_printf(s, "l2  l3  l4  timeout\thash\thits\tstate  in_dev  out_dev  tuple_in  tuple_out ct block use refcnt\n");
+		}
+		__fp_dump_tuple(in, &el->in_tuple, 0);
+		__fp_dump_tuple(out, &el->out_tuple, 0);
+		state = el->ct->proto.tcp.state;
+		use = atomic_read(&el->ct->ct_general.use);
+		seq_printf(s, "%s  %s  %s  %d\t%d\t%d\t%s  %s  %s  %s  %s  %p  %d  %d  %d"
+#ifdef CONFIG_ASR_TOE
+			" %dKbps"
+#endif
+			"\n",
+				  el->hh.hh_len ? "eth" : "NA",
+				  el->in_tuple.src.l3num == AF_INET6 ?
+				  "ipv6" : "ipv4",
+				  el->in_tuple.dst.protonum == IPPROTO_UDP ?
+				  "udp" : "tcp",
+				  jiffies_to_msecs(el->ct->timeout - jiffies) / 1000U,
+				  el->bucket, el->hit_counter,
+				  el->in_tuple.dst.protonum == IPPROTO_UDP ?
+				  "N/A" : tcp_conntrack_names[state],
+				  el->in_dev->dev->name,
+				  el->out_dev->dev->name,
+				  in, out, el->ct, el->block, use, atomic_read(&el->rc)
+#ifdef CONFIG_ASR_TOE
+				  , el->speed
+#endif
+				  );
+		fpdb_put(el);
+	}
+	return 0;
+}
+
+static const struct seq_operations fpdb_seq_ops = {
+	.start = fpdb_seq_start,
+	.next = fpdb_seq_next,
+	.stop = fpdb_seq_stop,
+	.show = fpdb_seq_show
+};
+
+#endif /* CONFIG_PROC_FS */
+
+static int fpdb_net_init(struct net *net)
+{
+	if (!proc_create_net("fastpath", 0440, net->proc_net, &fpdb_seq_ops,
+			sizeof(struct fpdb_iter_state)))
+
+		return -ENOMEM;
+	return 0;
+}
+
+static void fpdb_net_exit(struct net *net)
+{
+	remove_proc_entry("fastpath", net->proc_net);
+}
+
+static struct pernet_operations fpdb_net_ops = {
+	.init = fpdb_net_init,
+	.exit = fpdb_net_exit,
+};
+
+static void fp_database_release(struct kobject *kobj)
+{
+	struct fastpath_module *module = to_fpmod(kobj);
+	int wait_time = 200;
+
+	fpdb_flush();
+	do {
+		/* wait all fpdb freed, then call kmem_cache_destroy */
+		synchronize_rcu();
+		msleep(10);
+		if (--wait_time <= 0)
+			break;
+	} while (db->num_entries);
+
+	pr_info("%d fpdb entry left\n", db->num_entries);
+	nf_ct_extend_unregister(db->nfct_ext);
+	unregister_pernet_subsys(&fpdb_net_ops);
+	fpdb_free_hashtable(&db->htable);
+	kmem_cache_destroy(db->db_cache);
+#ifdef FP_USE_SRAM_POOL_OPT
+	sram_pool_free((unsigned long)db, sizeof(struct fp_database));
+#else
+	kfree(db);
+#endif
+	kfree(module);
+
+	pr_debug("fp_database released\n");
+}
+
+static struct kobj_type ktype_database = {
+	.sysfs_ops	= &fp_sysfs_ops,
+	.default_attrs	= fp_database_attrs,
+	.release	= fp_database_release,
+};
+
+static void fpdb_destroy_ext(struct nf_conn *ct)
+{
+	struct nf_conn_fastpath *ct_fp;
+	struct fpdb_entry *orig, *reply;
+	BUG_ON(!ct);
+
+	spin_lock_bh(&db->lock);
+	ct_fp = nfct_fastpath(ct);
+	if (ct_fp) {
+		orig = ct_fp->fpd_el[IP_CT_DIR_ORIGINAL];
+		reply = ct_fp->fpd_el[IP_CT_DIR_REPLY];
+	} else {
+		orig = NULL;
+		reply = NULL;
+	}
+
+	if (orig == NULL && reply == NULL) {
+		spin_unlock_bh(&db->lock);
+		return;
+	}
+
+	ct_fp->fpd_el[IP_CT_DIR_ORIGINAL] = NULL;
+	ct_fp->fpd_el[IP_CT_DIR_REPLY] = NULL;
+	if (orig) {
+		FP_DEBUG_DUMP_ENTRY("Delete orig entry:\n", orig);
+		__fpdb_del(orig, true);
+	}
+
+	if (reply) {
+		FP_DEBUG_DUMP_ENTRY("Delete reply entry:\n", reply);
+		__fpdb_del(reply, true);
+	}
+	spin_unlock_bh(&db->lock);
+}
+
+static struct nf_ct_ext_type fpdb_ct_extend = {
+	.len = sizeof(struct nf_conn_fastpath),
+	.align = __alignof__(struct nf_conn_fastpath),
+	.id = NF_CT_EXT_FASTPATH,
+	.destroy = fpdb_destroy_ext,
+};
+
+static int fp_database_probe(struct fastpath_module *module)
+{
+	struct fp_database *priv;
+	int ret;
+
+#ifdef FP_USE_SRAM_POOL_OPT
+	priv = (struct fp_database *)sram_pool_alloc(sizeof(struct fp_database));
+#else
+	priv = kzalloc(sizeof(struct fp_database), GFP_KERNEL);
+#endif
+	if (!priv) {
+		pr_err("no memory\n");
+		return -ENOMEM;
+	}
+	spin_lock_init(&priv->lock);
+	get_random_bytes(&hash_rnd, sizeof(hash_rnd));
+
+	priv->db_cache = kmem_cache_create("fpdb_entry",
+			sizeof(struct fpdb_entry), 0, SLAB_HWCACHE_ALIGN, NULL);
+	if (!priv->db_cache) {
+		pr_err("kmem_cache_create fpdb_entry failed\n");
+		ret = -ENOMEM;
+		goto kfree_priv;
+	}
+
+	ret = fpdb_alloc_hashtable(&priv->htable);
+	if (ret < 0) {
+		pr_err("fpdb_alloc_hashtable failed (ret=%d)\n", ret);
+		goto kfree_cache;
+	}
+
+	ret = register_pernet_subsys(&fpdb_net_ops);
+	if (ret < 0) {
+		pr_err("cannot register pernet operations (ret=%d)\n", ret);
+		goto free_hashtable;
+	}
+
+	priv->nfct_ext = &fpdb_ct_extend;
+	ret = nf_ct_extend_register(priv->nfct_ext);
+	if (ret < 0) {
+		pr_err("nf_ct_extend_register failed (%d)\n", ret);
+		goto unreg_pernet;
+	}
+
+	db = module->priv = priv;
+	snprintf(module->name, sizeof(module->name), "fp_database");
+
+	kobject_init(&module->kobj, &ktype_database);
+	ret = kobject_add(&module->kobj, module->fastpath->kobj, "%s", module->name);
+	if (ret < 0) {
+		pr_err("kobject_add failed (%d)\n", ret);
+		goto nf_ct_extend_unreg;
+	}
+	kobject_uevent(&module->kobj, KOBJ_ADD);
+
+	pr_debug("fp_database probed\n");
+	return 0;
+
+nf_ct_extend_unreg:
+	kobject_put(&module->kobj);
+	nf_ct_extend_unregister(priv->nfct_ext);
+unreg_pernet:
+	unregister_pernet_subsys(&fpdb_net_ops);
+free_hashtable:
+	fpdb_free_hashtable(&priv->htable);
+kfree_cache:
+	kmem_cache_destroy(priv->db_cache);
+kfree_priv:
+#ifdef FP_USE_SRAM_POOL_OPT
+	sram_pool_free((unsigned long)priv, sizeof(struct fp_database));
+#else
+	kfree(priv);
+#endif
+	return ret;
+}
+
+static int fp_database_remove(struct fastpath_module *module)
+{
+	kobject_put(&module->kobj);
+
+	pr_debug("fp_database removed\n");
+	return 0;
+}
+
+struct fastpath_module_ops fp_database_ops = {
+	.probe = fp_database_probe,
+	.remove = fp_database_remove,
+};
+