ASR_BASE
Change-Id: Icf3719cc0afe3eeb3edc7fa80a2eb5199ca9dda1
diff --git a/package/kernel/mfp/files/fp_database_hash.c b/package/kernel/mfp/files/fp_database_hash.c
new file mode 100644
index 0000000..be35203
--- /dev/null
+++ b/package/kernel/mfp/files/fp_database_hash.c
@@ -0,0 +1,1454 @@
+/*
+ * Fast path database hash implementation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU FP_ERR( Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * Notes:
+ * Implementation according to Documentation/RCU/rcuref.txt
+ */
+
+#define pr_fmt(fmt) "mfp" " database:%s:%d: " fmt, __func__, __LINE__
+
+#include "fp_common.h"
+#include "fp_database.h"
+#include "fp_device.h"
+#include "fp_core.h"
+#ifdef CONFIG_ASR_TOE
+#include "../linux/drivers/marvell/toev2/toe.h"
+#endif
+
+#define FP_ZONE (NF_CT_DEFAULT_ZONE_ID)
+#define GUARD_TIMEOUT_SEC (10)
+
+static u32 hash_rnd __read_mostly;
+
+static inline const char *state_to_string(enum entry_state state)
+{
+ return entry_state_names[state];
+}
+
+static inline int __fpdb_dump_entry(char *buf, struct fpdb_entry *el)
+{
+ int len = sprintf(buf, "fpdb dump entry (0x%p):\n", el);
+
+ len += fp_dump_tuple(buf + len, &el->in_tuple);
+ len += sprintf(buf + len, "\n");
+ len += fp_dump_tuple(buf + len, &el->out_tuple);
+ if (el->hh.hh_len) {
+ struct ethhdr *eth = (struct ethhdr *)(((u8 *) el->hh.hh_data) +
+ (HH_DATA_OFF(sizeof(*eth))));
+ len += sprintf(buf + len, "\nMAC header: src=%pM dst=%pM type=%04x\n",
+ eth->h_source, eth->h_dest, eth->h_proto);
+ } else {
+ len += sprintf(buf + len, "\nMAC header was not set\n");
+ }
+ len += sprintf(buf + len, "Interfaces: in %p: %s, out %p: %s\n",
+ el->in_dev,el->in_dev->dev->name, el->out_dev,el->out_dev->dev->name);
+ len += sprintf(buf + len, "State: %s hits=%d pointer=%p\n",
+ state_to_string(el->state), el->hit_counter, el);
+ len += sprintf(buf + len, "ct info: ct=%p timeout: %x rc=%d\n",
+ el->ct, el->ct->timeout, atomic_read(&el->rc));
+
+ if (debug_level & DBG_INFO)
+ len += sprintf(buf + len, "DEBUG: (NAT=%s) (route: in=%d out=%d)\n",
+ NF_CT_NAT(el->ct) ? "YES" : "NO",
+ el->debug.in_route_type, el->debug.out_route_type);
+
+ return len;
+}
+
+void fpdb_dump_entry(char *msg, struct fpdb_entry *el)
+{
+ char buf[MAX_DEBUG_PRINT_SIZE];
+ int len = 0;
+
+ BUG_ON(!el);
+
+ if (msg)
+ len = sprintf(buf, "%s", msg);
+
+ len += __fpdb_dump_entry(buf + len, el);
+
+ pr_err("%s", buf);
+}
+
+void fpdb_dump_tuple(char *msg, struct nf_conntrack_tuple *t)
+{
+ char buf[MAX_DEBUG_PRINT_SIZE];
+ int len = 0;
+
+ BUG_ON(!t);
+
+ if (msg)
+ len = sprintf(buf, "%s", msg);
+
+ len += sprintf(buf + len, "fpdb dump tuple:\n");
+ len += fp_dump_tuple(buf + len, t);
+
+ pr_err("%s\n", buf);
+}
+
+static int fpdb_print_entry(struct fpdb_entry *el, void *data)
+{
+ char in[256], out[256];
+ unsigned int state, use;
+ int *first_entry = data;
+
+ if (atomic_inc_not_zero(&el->rc)) {
+ if (*first_entry == true) {
+ pr_err("l2 l3 l4 timeout\thash\thits\tstate in_dev out_dev tuple_in tuple_out ct block use refcnt\n");
+ *first_entry = false;
+ }
+ __fp_dump_tuple(in, &el->in_tuple, 0);
+ __fp_dump_tuple(out, &el->out_tuple, 0);
+ state = el->ct->proto.tcp.state;
+ use = atomic_read(&el->ct->ct_general.use);
+ pr_err("%s %s %s %d\t%d\t%d\t%s %s %s %s %s %p %d %d %d\n",
+ el->hh.hh_len ? "eth" : "NA",
+ el->in_tuple.src.l3num == AF_INET6 ?
+ "ipv6" : "ipv4",
+ el->in_tuple.dst.protonum == IPPROTO_UDP ?
+ "udp" : "tcp",
+ jiffies_to_msecs(el->ct->timeout - jiffies) / 1000U,
+ el->bucket, el->hit_counter,
+ el->in_tuple.dst.protonum == IPPROTO_UDP ?
+ "N/A" : tcp_conntrack_names[state],
+ el->in_dev->dev->name,
+ el->out_dev->dev->name,
+ in, out, el->ct, el->block, use, atomic_read(&el->rc));
+ fpdb_put(el);
+ }
+ return 0;
+}
+
+void fpdb_dump_db(void)
+{
+ int first_entry = true;
+ fpdb_iterate(fpdb_print_entry, &first_entry);
+}
+
+/****************************************************************************
+ * Fast Path Database prototypes
+ ****************************************************************************/
+
+struct fpdb_htable {
+ struct hlist_head *h;
+ unsigned int size;
+ int vmalloced;
+};
+
+struct fp_database {
+ struct fpdb_stats stats;
+ volatile u32 num_entries;
+ struct fpdb_htable htable;
+ spinlock_t lock;
+ struct nf_ct_ext_type *nfct_ext;
+ struct kmem_cache *db_cache;
+};
+
+struct timeout_entry {
+ struct list_head list;
+ struct timer_list *timeout;
+};
+/****************************************************************************
+ * Fast Path Database globals
+ ****************************************************************************/
+
+static struct fp_database *db;
+/* TODO - do we need something else here??
+ Or is there only one "net" in ESHEL? */
+struct net *net = &init_net;
+
+#ifdef CONFIG_ASR_TOE
+extern int fp_cm_genl_send_tuple(struct nf_conntrack_tuple *tuple, struct fpdb_entry *el,
+ int add, int len);
+static inline bool get_remote_mac_addr(struct fpdb_entry *el, char *mac)
+{
+ struct neighbour *neigh;
+ struct neigh_table *tbl;
+ struct nf_conntrack_tuple *tuple;
+ struct net_device *br;
+
+ if (el->in_dev->br != el->out_dev->br)
+ return false;
+
+ tuple = &el->in_tuple;
+ br = el->out_dev->br;
+ tbl = (tuple->src.l3num == AF_INET6) ? &nd_tbl : &arp_tbl;
+
+ neigh = neigh_lookup(tbl, tuple->dst.u3.all, br);
+ if (neigh) {
+ memcpy(mac, neigh->ha, ETH_ALEN);
+ neigh_release(neigh);
+ return true;
+ }
+
+ return false;
+}
+
+#endif
+
+static void guard_timer_timeout(struct timer_list *t)
+{
+ struct fpdb_entry *el = from_timer(el, &t, guard_timer);
+
+ pr_err("Entry was hold and could not be removed for %d sec. [%px][rc=%d] state=%d\n",
+ GUARD_TIMEOUT_SEC, el, atomic_read(&el->rc), el->state);
+
+ /* BUG_ON(debug_level & DBG_WARN_AS_ERR);*/
+ if (atomic_read(&el->rc) > 0) {
+ FP_ERR_DUMP_ENTRY(NULL, el);
+ pr_err("Extend the timer when rc is not 0!\n");
+ mod_timer(el->guard_timer, jiffies + GUARD_TIMEOUT_SEC * HZ);
+ }
+}
+
+#ifdef FP_USE_SRAM_POOL_OPT
+static void *local_nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
+{
+ struct hlist_nulls_head *hash;
+ unsigned int nr_slots, i;
+ size_t sz;
+
+ BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
+ nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
+ sz = nr_slots * sizeof(struct hlist_nulls_head);
+ hash = (void *)sram_pool_alloc(sz);
+ if (hash && nulls)
+ for (i = 0; i < nr_slots; i++)
+ INIT_HLIST_NULLS_HEAD(&hash[i], i);
+ return hash;
+}
+
+static void local_nf_ct_free_hashtable(void *hash, unsigned int size)
+{
+ sram_pool_free((unsigned long)hash, size * sizeof(struct hlist_nulls_head));
+}
+#endif
+
+static inline int fpdb_alloc_hashtable(struct fpdb_htable *htable)
+{
+ /* Currently use the same size used by others.. */
+ htable->size = nf_conntrack_htable_size;
+#ifdef FP_USE_SRAM_POOL_OPT
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ htable->h = local_nf_ct_alloc_hashtable(&htable->size, &htable->vmalloced, 0);
+#else
+ htable->h = local_nf_ct_alloc_hashtable(&htable->size, 0);
+#endif
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ htable->h = nf_ct_alloc_hashtable(&htable->size, &htable->vmalloced, 0);
+#else
+ htable->h = nf_ct_alloc_hashtable(&htable->size, 0);
+#endif
+#endif
+ if (!htable->h)
+ return -ENOMEM;
+
+ pr_debug("allocated fpdb hashtable (size = %d)\n", htable->size);
+
+ return 0;
+}
+
+static inline void fpdb_free_hashtable(struct fpdb_htable *htable)
+{
+#ifdef FP_USE_SRAM_POOL_OPT
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ local_nf_ct_free_hashtable(htable->h, htable->vmalloced, htable->size);
+#else
+ local_nf_ct_free_hashtable(htable->h, htable->size);
+#endif
+#else
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ nf_ct_free_hashtable(htable->h, htable->vmalloced, htable->size);
+#else
+ kvfree(htable->h);
+ htable->h = NULL;
+#endif
+#endif
+}
+
+/****************************************************************************
+ * Fast Path Database API
+ ****************************************************************************/
+
+/**
+ * Allocates and initializes a new database entry
+ *
+ * @param flags kmalloc flags
+ *
+ * @return new allocated and initialized database entry
+ */
+struct fpdb_entry *fpdb_alloc(gfp_t flags)
+{
+ struct fpdb_entry *el;
+
+#ifdef FP_USE_SRAM_POOL_OPT
+ el = (struct fpdb_entry *)sram_pool_alloc(sizeof(struct fpdb_entry));
+#else
+ el = kmem_cache_zalloc(db->db_cache, flags);
+#endif
+ if (!el) {
+ pr_err("no memory\n");
+ return NULL;
+ }
+
+ spin_lock_init(&el->lock);
+ INIT_HLIST_NODE(&el->hlist);
+ INIT_LIST_HEAD(&el->debug.trace.list);
+
+ el->state = ENTRY_INITIALIZED;
+#ifdef CONFIG_ASR_TOE
+ el->nl_flag = 0;
+#endif
+ return el;
+}
+
+
+/**
+ * Free a database entry
+ *
+ * @param flags fpdb_entry * e
+ *
+ * @return void
+ */
+void fpdb_free(struct fpdb_entry * el)
+{
+ fpdev_put(el->out_dev);
+ fpdev_put(el->in_dev);
+
+#ifdef FP_USE_SRAM_POOL_OPT
+ sram_pool_free((unsigned long)el, sizeof(struct fpdb_entry));
+#else
+ kmem_cache_free(db->db_cache, el);
+#endif
+ return;
+}
+
+
+/**
+ * jenkins hash function using the source tuple
+ *
+ * @return hash key
+ */
+static inline unsigned int
+fpdb_hash_by_src(const struct nf_conntrack_tuple *tuple)
+{
+ unsigned int hash_src, hash_dst, hash;
+
+ BUG_ON(!tuple);
+
+ hash_src = jhash_3words((__force u32) tuple->src.u3.ip,
+ (__force u32) tuple->src.u.all ^ FP_ZONE,
+ tuple->src.l3num, hash_rnd);
+ hash_dst = jhash_3words((__force u32) tuple->dst.u3.ip,
+ (__force u32) tuple->dst.u.all ^ FP_ZONE,
+ tuple->dst.protonum, hash_rnd);
+ hash = jhash_2words(hash_src, hash_dst, hash_rnd);
+
+ return ((u64)hash * db->htable.size) >> 32;
+}
+
+/**
+ * rcu callback
+ *
+ * @param head
+ */
+static void fpdb_rcu_free(struct rcu_head *head)
+{
+ struct fpdb_entry *el = container_of(rcu_dereference(head),
+ struct fpdb_entry, rcu);
+
+ if (el == NULL) {
+ pr_err("fpdb_rcu_free el = NULL!\n");
+ return;
+ }
+
+ BUG_ON(!el || atomic_read(&el->rc) || el->state != ENTRY_DYING);
+
+ FP_DEBUG_DUMP_ENTRY("fpdb_rcu_free: entry was deleted\n", el);
+
+ if (el->guard_timer) {
+ del_timer_sync(el->guard_timer);
+ kfree(el->guard_timer);
+ el->guard_timer = NULL;
+ }
+
+ spin_lock_bh(&db->lock);
+ db->num_entries--;
+ spin_unlock_bh(&db->lock);
+
+ fpdev_put(el->out_dev);
+ fpdev_put(el->in_dev);
+
+#ifdef FP_USE_SRAM_POOL_OPT
+ sram_pool_free((unsigned long)el, sizeof(struct fpdb_entry));
+#else
+ kmem_cache_free(db->db_cache, el);
+#endif
+}
+
+
+
+/**
+ * decrement an entry's reference count and delete if 0
+ *
+ * @param el pointer to a previously allocated fpdb_entry
+ */
+void fpdb_put(struct fpdb_entry *el)
+{
+ if (atomic_dec_and_test(&el->rc))
+ call_rcu(&el->rcu, fpdb_rcu_free);
+}
+
+
+#define FP_SMALL_MEM_LIMIT (64 * 1024 * 1204)
+/**
+ * Adds a previously allocated entry to the database
+ * and updates its reference count to 1.
+ *
+ * @attention el must be allocated first with fpdb_alloc()
+ * Initial Implementation - Hash by input tuple only
+ * @param el pointer to a previously allocated fpdb_entry
+ *
+ */
+void fpdb_add(struct fpdb_entry *el)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
+ int pages = totalram_pages();
+#else
+ int pages = totalram_pages;
+#endif
+ unsigned int hash;
+ u32 max_num;
+
+#ifdef CONFIG_ASR_TOE
+ char mac[ETH_ALEN];
+#endif
+
+ spin_lock_bh(&el->lock);
+ spin_lock_bh(&db->lock);
+ BUG_ON(!el || !el->out_dev);
+ BUG_ON(el->state != ENTRY_INITIALIZED);
+
+ hash = fpdb_hash_by_src(&el->in_tuple);
+
+ atomic_set(&el->rc, 1);
+ el->state = ENTRY_ALIVE;
+ el->bucket = hash;
+ el->tstamp = jiffies;
+ if (!el->tstamp)
+ el->tstamp = 1;
+
+ BUG_ON(in_irq());
+ WARN_ON_ONCE(irqs_disabled());
+ hlist_add_head_rcu(&el->hlist, &db->htable.h[hash]);
+ db->num_entries++;
+
+#ifdef CONFIG_ASR_TOE
+ if (get_remote_mac_addr(el, mac))
+ mfp_toe_add_dmac(el->out_dev->dev, mac);
+#endif
+
+ spin_unlock_bh(&db->lock);
+ spin_unlock_bh(&el->lock);
+
+ /* Normally Conntrack MAX is HashSize*8. So here is not suit to only check double*/
+ /*we will modify the code to check 6 times of hash size --Yhuang 20160617*/
+
+ if (pages <= (FP_SMALL_MEM_LIMIT >> PAGE_SHIFT))
+ max_num = 2 * db->htable.size;
+ else
+ max_num = 6 * db->htable.size;
+
+ if (unlikely(db->num_entries > max_num)) {
+ pr_err_ratelimited("%s: database overloaded (%d entries, max=%d)\n",
+ __func__, db->num_entries, max_num);
+ /*
+ if (debug_level & DBG_WARN_AS_ERR) {
+ fpdb_dump_db();
+ BUG();
+ }
+ */
+ fpdb_flush();
+ } else if (unlikely(db->num_entries > ((max_num * 3) / 4))) {
+ fpdb_del_least_used_entry(max_num);
+ }
+
+ if (db->stats.max_entries < db->num_entries)
+ db->stats.max_entries = db->num_entries;
+ FP_DEBUG_DUMP_ENTRY("fpdb_add: entry was added\n", el);
+}
+
+/**
+ * Query the database for an entry matching the input tuple
+ * and increment the reference count for that entry if found.
+ *
+ * @attention The user MUST call fpdb_put() as soon as the entry
+ * is not used!
+ *
+ * @param tuple pointer to a nf_conntrack_tuple
+ *
+ * @return pointer to the matching entry, NULL if not found
+ */
+struct fpdb_entry *fpdb_get(struct nf_conntrack_tuple *tuple)
+{
+ unsigned int hash, iterations = 0;
+ struct fpdb_entry *el;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ struct hlist_node *h;
+#endif
+
+ BUG_ON(!tuple);
+
+ db->stats.lookups++;
+
+ hash = fpdb_hash_by_src(tuple);
+
+ rcu_read_lock_bh();
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ hlist_for_each_entry_rcu(el, h, &db->htable.h[hash], hlist) {
+#else
+ hlist_for_each_entry_rcu(el, &db->htable.h[hash], hlist) {
+#endif
+ if (el && nf_ct_tuple_equal(&el->in_tuple, tuple)) {
+ if (!atomic_inc_not_zero(&el->rc))
+ goto not_found;
+ rcu_read_unlock_bh();
+ if (!iterations)
+ db->stats.hits++;
+ el->hit_counter++;
+ FP_DEBUG_DUMP_ENTRY("fpdb_get: entry was found:\n", el);
+ return el;
+ }
+ iterations++;
+ db->stats.iterations++; /* Total Iterations*/
+ }
+
+not_found:
+ rcu_read_unlock_bh();
+ FP_DEBUG_DUMP_TUPLE("fpdb_get: entry was not found:\n", tuple);
+
+ return NULL;
+}
+
+#ifdef CONFIG_ASR_TOE
+static int fpdb_del_toe_tuple(struct fpdb_entry *el)
+{
+ struct toe_tuple_buff toe_tuple;
+ struct fp_net_device *dst, *src;
+ u32 nat_ip = 0;
+ u16 nat_port = 0;
+ u8 proto = 0, in_pkt = 0, out_pkt = 0, fwd = 0, nat = 0;
+ u8 rx_tx;
+
+ BUG_ON(!el);
+
+ if (!el->nl_flag)
+ return 0;
+
+ src = rcu_dereference_bh(el->in_dev);
+ dst = rcu_dereference_bh(el->out_dev);
+ if (!strncasecmp(src->dev->name, "ccinet", 6))
+ in_pkt = PDU_PKT;
+ else if (!strncasecmp(src->dev->name, "usbnet", 6))
+ in_pkt = USB_PKT;
+ else if (!strncasecmp(src->dev->name, "wlan", 4))
+ in_pkt = WIFI_PKT;
+ else if (!strncasecmp(src->dev->name, "eth", 3))
+ in_pkt = ETH_PKT;
+ else
+ in_pkt = AP_PKT;
+
+ if (!strncasecmp(dst->dev->name, "ccinet", 6))
+ out_pkt = PDU_PKT;
+ else if (!strncasecmp(dst->dev->name, "usbnet", 6))
+ out_pkt = USB_PKT;
+ else if (!strncasecmp(dst->dev->name, "wlan", 4))
+ out_pkt = WIFI_PKT;
+ else if (!strncasecmp(dst->dev->name, "eth", 3))
+ out_pkt = ETH_PKT;
+ else
+ out_pkt = AP_PKT;
+
+ fwd = (in_pkt != AP_PKT) && (out_pkt != AP_PKT);
+ if (fwd && (el->out_tuple.src.l3num == AF_INET)) {
+ if (in_pkt == PDU_PKT && (out_pkt == USB_PKT || out_pkt == WIFI_PKT || out_pkt == ETH_PKT)) {
+ nat = 1;
+ nat_ip = ntohl(el->out_tuple.src.u3.ip);
+ nat_port = ntohs(el->out_tuple.src.u.all);
+ } else if ((in_pkt == USB_PKT || in_pkt == WIFI_PKT || in_pkt == ETH_PKT) && out_pkt == PDU_PKT) {
+ nat = 1;
+ nat_ip = ntohl(el->out_tuple.dst.u3.ip);
+ nat_port = ntohs(el->out_tuple.dst.u.all);
+ } else
+ /* CP TOE WIFI/WIFI TOE CP no need nat */
+ nat = 0;
+ }
+
+ /* rx: cp -> ap, usb, wifi */
+ if (in_pkt == PDU_PKT)
+ rx_tx = 1;
+ /* rx: ap -> usb, ap -> wifi */
+ else if ((in_pkt == AP_PKT) && (out_pkt != PDU_PKT))
+ rx_tx = 1;
+ /*
+ * tx:
+ * ap -> cp
+ * usb/wifi -> ap/cp */
+ else
+ rx_tx = 0;
+
+ if (el->in_tuple.src.l3num == AF_INET6) {
+ memcpy(toe_tuple.src_ip6, el->in_tuple.src.u3.all, sizeof(toe_tuple.src_ip6));
+ memcpy(toe_tuple.dst_ip6, el->in_tuple.dst.u3.all, sizeof(toe_tuple.src_ip6));
+ toe_tuple.ip6 = 1;
+ } else {
+ toe_tuple.src_ip = ntohl(el->in_tuple.src.u3.ip);
+ toe_tuple.dst_ip = ntohl(el->in_tuple.dst.u3.ip);
+ toe_tuple.ip6 = 0;
+ toe_tuple.nat = nat;
+ toe_tuple.nat_port = nat_port;
+ toe_tuple.nat_ip = nat_ip;
+ }
+
+ if (el->in_tuple.dst.protonum == IPPROTO_UDP)
+ proto = TOE_UDP;
+ else if (el->in_tuple.dst.protonum == IPPROTO_TCP)
+ proto = TOE_TCP;
+ else
+ proto = TOE_MAX;
+
+ toe_tuple.src_port = ntohs(el->in_tuple.src.u.all);
+ toe_tuple.dst_port = ntohs(el->in_tuple.dst.u.all);
+ toe_tuple.prot = proto;
+ toe_tuple.fwd = fwd;
+ toe_tuple.rxtx = rx_tx;
+ toe_tuple.out_pkt = out_pkt;
+
+ return toe_del_connection(&toe_tuple);
+}
+#endif
+
+void __fpdb_del(struct fpdb_entry *entry, bool hlist_del)
+{
+ BUG_ON(!entry);
+ if(entry->state != ENTRY_ALIVE)
+ return;
+
+ entry->state = ENTRY_DYING;
+
+#ifdef CONFIG_ASR_TOE
+ if (entry->nl_flag) {
+ fp_cm_genl_send_tuple(&entry->in_tuple, entry, 0, 0);
+ if (fpdb_del_toe_tuple(entry))
+ pr_debug("fpdb_del_toe_tuple failed!!!\r\n");
+ entry->nl_flag = 0;
+ }
+#endif
+
+ BUG_ON(entry->guard_timer);
+ if (hlist_del)
+ hlist_del_rcu(&entry->hlist);
+
+ if (atomic_dec_and_test(&entry->rc)) {
+ /* move start timer here to avoid rc is not zero yhuang 20160624*/
+ entry->guard_timer = kmalloc(sizeof(*entry->guard_timer), GFP_ATOMIC);
+ if (entry->guard_timer) {
+ timer_setup(entry->guard_timer, guard_timer_timeout, 0);
+ mod_timer(entry->guard_timer, jiffies + GUARD_TIMEOUT_SEC * HZ);
+ } else {
+ pr_err("Guard timer allocation failed!");
+ }
+
+ /* prevent out of order so that guard timer can be stopped */
+ mb();
+ call_rcu(&entry->rcu, fpdb_rcu_free);
+ } else {
+ pr_err("__fpdb_del fail. entry:%p, rc=%d, state=%d\n", entry,
+ atomic_read(&entry->rc), entry->state);
+ }
+}
+
+void fpdb_lock_bh(void)
+{
+ return spin_lock_bh(&db->lock);
+}
+
+void fpdb_unlock_bh(void)
+{
+ return spin_unlock_bh(&db->lock);
+}
+
+void fpdb_del(struct fpdb_entry *entry)
+{
+ spin_lock_bh(&db->lock);
+ __fpdb_del(entry, true);
+ spin_unlock_bh(&db->lock);
+}
+
+/**
+ * Replace a previously allocated entry with an prexisting one
+ * to the database.
+ *
+ * @attention nel must be allocated first with fpdb_alloc()
+ * el - must be already in the database/
+ * @param el pointer to a previously added fpdb_entry
+ * @param nel pointer to a newely allocated fpdb_entry
+ * NOTE: must be called from softirq/lock_bh context
+ */
+void fpdb_replace(struct fpdb_entry *el, struct fpdb_entry *nel)
+{
+ unsigned int hash;
+
+ BUG_ON(!el || !el->out_dev);
+ BUG_ON(!nel || !nel->out_dev);
+ BUG_ON(nel->state != ENTRY_INITIALIZED);
+
+ hash = fpdb_hash_by_src(&nel->in_tuple);
+
+ atomic_set(&nel->rc, 1);
+ nel->state = ENTRY_ALIVE;
+ nel->bucket = hash;
+
+ BUG_ON(el->bucket != nel->bucket);
+
+ db->num_entries++;
+ hlist_replace_rcu(&el->hlist, &nel->hlist);
+ __fpdb_del(el, false);
+}
+
+
+static int device_cmp(struct nf_conn *ct, void *dev)
+{
+ struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+ struct fpdb_entry *orig, *reply;
+ struct net_device *net = (struct net_device *)dev;
+
+ if (!fp)
+ return 0;
+
+ orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+ reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+ if (orig && (fpdev_cmp_if(orig->in_dev, net) ||
+ fpdev_cmp_if(orig->out_dev, net)))
+ return 1;
+ if (reply && (fpdev_cmp_if(reply->in_dev, net) ||
+ fpdev_cmp_if(reply->out_dev, net)))
+ return 1;
+
+ return 0;
+}
+
+static inline bool
+tuple_cmp_port(const struct nf_conntrack_tuple *t, unsigned int port)
+{
+ return (ntohs(t->dst.u.all) == port || ntohs(t->src.u.all) == port);
+}
+
+static int port_cmp(struct nf_conn *ct, void *ptr)
+{
+ struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+ struct fpdb_entry *orig, *reply;
+ unsigned int port = (unsigned int)(unsigned long)ptr;
+
+ if (!fp)
+ return 0;
+
+ orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+ reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+ if (orig && (tuple_cmp_port(&orig->in_tuple, port) ||
+ tuple_cmp_port(&orig->out_tuple, port)))
+ return 1;
+ if (reply && (tuple_cmp_port(&reply->in_tuple, port) ||
+ tuple_cmp_port(&reply->out_tuple, port)))
+ return 1;
+
+ return 0;
+}
+
+/* kill all fastpath related conntracks */
+static int nf_fp_remove(struct nf_conn *ct, void *data)
+{
+ return test_bit(IPS_FASTPATH_BIT, &ct->status);
+}
+
+int fpdb_del_block_entry_by_dev(struct fpdb_entry *el, void *data)
+{
+ struct net_device *dev = (struct net_device *)data;
+ struct nf_conn_fastpath *ct_fp;
+
+ if (fpdev_cmp_if(el->in_dev, dev) ||
+ fpdev_cmp_if(el->out_dev, dev)) {
+
+ spin_lock_bh(&db->lock);
+ ct_fp = nfct_fastpath(el->ct);
+ if (ct_fp) {
+ if (ct_fp->fpd_el[el->dir] == NULL) {
+ spin_unlock_bh(&db->lock);
+ return 0;
+ }
+
+ ct_fp->fpd_el[el->dir] = NULL;
+ }
+ spin_unlock_bh(&db->lock);
+
+ fpdb_del(el);
+ printk(KERN_DEBUG "delete a block entry related to %s\n", dev->name);
+ }
+
+ return 0;
+}
+
+static int nf_fpdb_del(struct nf_conn *ct, void *del)
+{
+ struct nf_conn_fastpath *fp = nfct_fastpath(ct);
+ struct fpdb_entry *orig, *reply;
+
+ if (!fp)
+ return 0;
+
+ orig = fp->fpd_el[IP_CT_DIR_ORIGINAL];
+ reply = fp->fpd_el[IP_CT_DIR_REPLY];
+
+ if (orig && orig == (struct fpdb_entry *)del) {
+ orig->tstamp = 0;
+ return 1;
+ }
+
+ if (reply && reply == (struct fpdb_entry *)del) {
+ reply->tstamp = 0;
+ return 1;
+ }
+
+ return 0;
+}
+
+static int fpdb_find_lest_used_entry(struct fpdb_entry *el, void *data)
+{
+ struct fpdb_entry **p_el = (struct fpdb_entry **)data;
+
+ if (!*p_el)
+ *p_el = el;
+ else if (el->tstamp && time_before(el->tstamp, (*p_el)->tstamp))
+ *p_el = el;
+
+ return 0;
+}
+
+void fpdb_del_least_used_entry(int max_num)
+{
+ struct fpdb_entry *el = NULL;
+
+ fpdb_iterate(fpdb_find_lest_used_entry, &el);
+
+ if (!el)
+ return;
+
+ pr_info_ratelimited("%s: el=0x%x (%d entries, max=%d)\n",
+ __func__, (unsigned)el, db->num_entries, max_num);
+ nf_ct_iterate_cleanup(&nf_fpdb_del, (void *)el, 0, 0);
+}
+
+/**
+ * Remove all fastpath related connections with the specified network device
+ *
+ * caller should have rtnl locked
+ *
+ * @param dev
+ */
+void fpdb_del_by_dev(struct net_device *dev)
+{
+ nf_ct_iterate_cleanup(&device_cmp, (void *)dev, 0, 0);
+
+ printk(KERN_DEBUG "All entries related to %s deleted\n", dev->name);
+}
+
+/**
+ * Remove all fastpath related connections with the specified port
+ *
+ * caller should have rtnl locked
+ *
+ * @param port
+ */
+void fpdb_del_by_port(unsigned int port)
+{
+ nf_ct_iterate_cleanup(&port_cmp, (void *)(unsigned long)port, 0, 0);
+
+ pr_debug("All entries with port=%d deleted\n", port);
+}
+
+/**
+ * flush the entire database by cleaning all fastpath related
+ * conntracks
+ *
+ * MUST BE CALLED IN PROCESS CONTEXT
+ */
+void fpdb_flush(void)
+{
+ nf_ct_iterate_cleanup(&nf_fp_remove, 0, 0, 0);
+
+ pr_debug("All entries flushed\n");
+}
+
+/**
+ * Iterate through all fpdb entries
+ * MUST BE CALLED IN PROCESS CONTEXT
+ *
+ * @param iter callback function called per every entry
+ * If returns 0, iteration stops.
+ * @param data private data to be passed to the iter callback
+ */
+void fpdb_iterate(int (*iter)(struct fpdb_entry *e, void *data), void *data)
+{
+ int i;
+ struct fpdb_entry *e;
+
+ for (i = 0; i < db->htable.size; i++) {
+ rcu_read_lock_bh();
+ hlist_for_each_entry_rcu(e, &db->htable.h[i], hlist) {
+ if (iter(e, data))
+ break;
+ }
+ rcu_read_unlock_bh();
+ }
+}
+
+/**
+ * Add the current entry state to the entry's trace buffer when
+ * debug_level mask contains DBG_TRACE_LOG
+ *
+ * @param entry - entry to log
+ * @param tcph - NULL for UDP
+ */
+void fpdb_trace(struct fpdb_entry *entry, struct tcphdr *tcph)
+{
+ if (debug_level & DBG_TRACE_LOG) {
+ struct fpdb_trace *trace = kzalloc(sizeof(struct fpdb_trace), GFP_ATOMIC);
+
+ BUG_ON(!entry);
+
+ trace->timeout = jiffies_to_msecs(entry->ct->timeout - jiffies) / 1000U;
+ trace->ct_status = entry->ct->status;
+ trace->hit_counter = entry->hit_counter;
+
+ if (tcph) {
+ trace->tcp_state = entry->ct->proto.tcp.state;
+ trace->tcph = *tcph;
+ }
+
+ list_add(&trace->list, &entry->debug.trace.list);
+ if (++entry->debug.trace.sz > 5) {
+ /* TODO - change to configurable param */
+ trace = list_entry(entry->debug.trace.list.prev, struct fpdb_trace, list);
+ list_del(entry->debug.trace.list.prev);
+ kfree(trace);
+ entry->debug.trace.sz--;
+ }
+ }
+}
+
+/****************************************************************************
+ * Fast Path Database private
+ ****************************************************************************/
+
+/* SYS FS and PROC FS */
+
+static void fpdb_get_stats(void)
+{
+ int i, count, max = 0;
+ struct fpdb_entry *el;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ struct hlist_node *h;
+#endif
+
+ memset(db->stats.hist, 0, sizeof(db->stats.hist));
+ db->stats.num_occupied = 0;
+
+ for (i = 0; i < db->htable.size; i++) {
+ count = 0;
+
+ rcu_read_lock_bh();
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ hlist_for_each_entry_rcu(el, h, &db->htable.h[i], hlist)
+#else
+ hlist_for_each_entry_rcu(el, &db->htable.h[i], hlist)
+#endif
+ count++;
+ rcu_read_unlock_bh();
+
+ if (count)
+ db->stats.num_occupied++;
+
+ if (count < HISTOGRAM_SIZE) {
+ db->stats.hist[count].buckets++;
+ db->stats.hist[count].entries += count;
+ } else {
+ db->stats.hist[HISTOGRAM_SIZE].buckets++;
+ db->stats.hist[HISTOGRAM_SIZE].entries += count;
+ }
+
+ max = (count > max) ? count : max;
+ }
+
+ db->stats.largest_bucket = max;
+}
+
+static ssize_t fpdb_sysfs_flush(struct fastpath_module *m, const char *buf, size_t count)
+{
+ struct net_device *dev;
+
+ if (count > 2) {
+ char *str = kmalloc(sizeof(char)*count, GFP_KERNEL);
+ sprintf(str, "%s", buf);
+ str[count-1] = '\0';
+ dev = dev_get_by_name(&init_net, str);
+ kfree(str);
+
+ if (dev) {
+ fpdb_del_by_dev(dev);
+ dev_put(dev);
+ return count;
+ }
+ }
+
+ fpdb_flush();
+ return count;
+}
+
+static ssize_t fpdb_sysfs_stats_show(struct fastpath_module *m, char *buf)
+{
+ int len, i;
+ u32 sum_pct = 0;
+
+ fpdb_get_stats();
+
+ len = sprintf(buf, "Fast Path Database (HASH) statistics:\n");
+ len += sprintf(buf + len, "Max number of entries: %d ",
+ db->stats.max_entries);
+ len += sprintf(buf + len, "Total lookups: %d, Total hits: %d, "
+ "hit rate %d%%\n", db->stats.lookups, db->stats.hits,
+ (100 * db->stats.hits) / (db->stats.lookups ?
+ db->stats.lookups : 1));
+ len += sprintf(buf + len, "Database Size is %d Buckets\n",
+ db->htable.size);
+ len += sprintf(buf + len, "Number of occupied buckets: %d\n",
+ db->stats.num_occupied);
+ len += sprintf(buf + len, "Database contains %d entries\n",
+ db->num_entries);
+ len += sprintf(buf + len, "Largest bucket contains %d entries\n",
+ db->stats.largest_bucket);
+ len += sprintf(buf + len, "Load Factor is %d (%d/%d)\n",
+ db->num_entries /
+ (db->htable.size ? db->htable.size : 1),
+ db->num_entries, db->htable.size);
+ len += sprintf(buf + len, "find_entry() iterations/lookups: %d/%d\n",
+ db->stats.iterations, db->stats.lookups);
+ len += sprintf(buf + len, "Histogram:\n");
+ len += sprintf(buf + len, "Size buckets entries sum-pct\n");
+ for (i = 0; i < HISTOGRAM_SIZE; i++) {
+ if (sum_pct < 100)
+ sum_pct += (100 * db->stats.hist[i].entries) /
+ (db->num_entries ?
+ db->num_entries : 1);
+ else
+ sum_pct = 100;
+
+ len += sprintf(buf + len, "%4d%10d%10d%10d\n", i,
+ db->stats.hist[i].buckets,
+ db->stats.hist[i].entries, sum_pct);
+ }
+ len += sprintf(buf + len, ">%3d%10d%10d%10d\n", i - 1,
+ db->stats.hist[i].buckets,
+ db->stats.hist[i].entries, 100);
+
+ return len;
+}
+
+static ssize_t fpdb_sysfs_stats_clear(struct fastpath_module *m, const char *buf,
+ size_t count)
+{
+ pr_debug("reset stats...\n");
+ memset(&db->stats, 0, sizeof(db->stats));
+ return count;
+}
+
+static unsigned int dbg_hash;
+
+static ssize_t fpdb_sysfs_entry_debug_select(struct fastpath_module *m, const char *buf,
+ size_t count)
+{
+ sscanf(buf, "%u", &dbg_hash);
+ return count;
+}
+
+static ssize_t fpdb_sysfs_entry_debug_show(struct fastpath_module *m, char *buf)
+{
+ struct fpdb_entry *el;
+ int i = 0, len;
+ struct fpdb_trace *itr;
+ struct nf_conn_fastpath *fp_ext;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ struct hlist_node *h;
+#endif
+
+ if (dbg_hash > db->htable.size)
+ return sprintf(buf, "invalid hash (%d)\n", dbg_hash);
+
+ len = sprintf(buf, "debug info for bucket%u:\n", dbg_hash);
+ rcu_read_lock_bh();
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
+ hlist_for_each_entry_rcu(el, h, &db->htable.h[dbg_hash], hlist) {
+#else
+ hlist_for_each_entry_rcu(el, &db->htable.h[dbg_hash], hlist) {
+#endif
+ len += __fpdb_dump_entry(buf+len, el);
+ fp_ext = nf_ct_ext_find(el->ct, NF_CT_EXT_FASTPATH);
+ BUG_ON(!fp_ext);
+ len += sprintf(buf+len, "fastpath_ext orig:=%p reply=%p\n",
+ fp_ext->fpd_el[IP_CT_DIR_ORIGINAL],
+ fp_ext->fpd_el[IP_CT_DIR_REPLY]);
+ if (el->in_tuple.dst.protonum == IPPROTO_UDP)
+ continue;
+ len += sprintf(buf+len, "%d: trace:\n", i++);
+ len += sprintf(buf+len, "hits timeout tcp_state tcp_flags ct_status\n");
+ list_for_each_entry(itr, &el->debug.trace.list, list)
+ len += sprintf(buf+len, "%d %d %s %c%c%c%c%c%c %lu\n",
+ itr->hit_counter, itr->timeout,
+ tcp_conntrack_names[itr->tcp_state],
+ itr->tcph.urg ? 'U' : '-',
+ itr->tcph.ack ? 'A' : '-',
+ itr->tcph.psh ? 'P' : '-',
+ itr->tcph.rst ? 'R' : '-',
+ itr->tcph.syn ? 'S' : '-',
+ itr->tcph.fin ? 'F' : '-',
+ itr->ct_status);
+ }
+ rcu_read_unlock_bh();
+
+ return len;
+}
+
+
+static FP_ATTR(stats, S_IRUGO|S_IWUSR, fpdb_sysfs_stats_show, fpdb_sysfs_stats_clear);
+static FP_ATTR(flush, S_IWUSR, NULL, fpdb_sysfs_flush);
+static FP_ATTR(bucket, S_IRUGO|S_IWUSR, fpdb_sysfs_entry_debug_show, fpdb_sysfs_entry_debug_select);
+
+static struct attribute *fp_database_attrs[] = {
+ &fp_attr_stats.attr,
+ &fp_attr_flush.attr,
+ &fp_attr_bucket.attr,
+ NULL, /* need to NULL terminate the list of attributes */
+};
+
+#ifdef CONFIG_PROC_FS
+static bool first;
+struct fpdb_iter_state {
+ struct seq_net_private p;
+ unsigned int bucket;
+};
+
+static struct hlist_node *fpdb_entries_get_first(struct seq_file *seq)
+{
+ struct fpdb_iter_state *st = seq->private;
+ struct hlist_node *n;
+
+ for (st->bucket = 0; st->bucket < db->htable.size; st->bucket++) {
+ n = rcu_dereference(db->htable.h[st->bucket].first);
+ if (n) {
+ first = true;
+ return n;
+ }
+ }
+ return NULL;
+}
+
+static struct hlist_node *fpdb_entries_get_next(struct seq_file *seq,
+ struct hlist_node *head)
+{
+ struct fpdb_iter_state *st = seq->private;
+
+ first = false;
+ head = rcu_dereference(head->next);
+
+ while (head == NULL) {
+ if (++st->bucket >= db->htable.size)
+ return NULL;
+ head = rcu_dereference(db->htable.h[st->bucket].first);
+ }
+ return head;
+}
+
+static struct hlist_node *fpdb_entries_get_idx(struct seq_file *seq, loff_t pos)
+{
+ struct hlist_node *head = fpdb_entries_get_first(seq);
+
+ if (head)
+ while (pos && (head = fpdb_entries_get_next(seq, head)))
+ pos--;
+ return pos ? NULL : head;
+}
+
+static void *fpdb_seq_start(struct seq_file *seq, loff_t *pos)
+ __acquires(RCU)
+{
+ rcu_read_lock_bh();
+ return fpdb_entries_get_idx(seq, *pos);
+}
+
+static void *fpdb_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ (*pos)++;
+ return fpdb_entries_get_next(seq, v);
+}
+
+static void fpdb_seq_stop(struct seq_file *seq, void *v)
+ __releases(RCU)
+{
+ rcu_read_unlock_bh();
+}
+
+static int fpdb_seq_show(struct seq_file *s, void *v)
+{
+ struct hlist_node *n = v;
+ struct fpdb_entry *el;
+ char in[256], out[256];
+ unsigned int state, use;
+
+ el = hlist_entry(n, struct fpdb_entry, hlist);
+ if (atomic_inc_not_zero(&el->rc)) {
+ if (first == true) {
+ seq_printf(s, "l2 l3 l4 timeout\thash\thits\tstate in_dev out_dev tuple_in tuple_out ct block use refcnt\n");
+ }
+ __fp_dump_tuple(in, &el->in_tuple, 0);
+ __fp_dump_tuple(out, &el->out_tuple, 0);
+ state = el->ct->proto.tcp.state;
+ use = atomic_read(&el->ct->ct_general.use);
+ seq_printf(s, "%s %s %s %d\t%d\t%d\t%s %s %s %s %s %p %d %d %d"
+#ifdef CONFIG_ASR_TOE
+ " %dKbps"
+#endif
+ "\n",
+ el->hh.hh_len ? "eth" : "NA",
+ el->in_tuple.src.l3num == AF_INET6 ?
+ "ipv6" : "ipv4",
+ el->in_tuple.dst.protonum == IPPROTO_UDP ?
+ "udp" : "tcp",
+ jiffies_to_msecs(el->ct->timeout - jiffies) / 1000U,
+ el->bucket, el->hit_counter,
+ el->in_tuple.dst.protonum == IPPROTO_UDP ?
+ "N/A" : tcp_conntrack_names[state],
+ el->in_dev->dev->name,
+ el->out_dev->dev->name,
+ in, out, el->ct, el->block, use, atomic_read(&el->rc)
+#ifdef CONFIG_ASR_TOE
+ , el->speed
+#endif
+ );
+ fpdb_put(el);
+ }
+ return 0;
+}
+
+static const struct seq_operations fpdb_seq_ops = {
+ .start = fpdb_seq_start,
+ .next = fpdb_seq_next,
+ .stop = fpdb_seq_stop,
+ .show = fpdb_seq_show
+};
+
+#endif /* CONFIG_PROC_FS */
+
+static int fpdb_net_init(struct net *net)
+{
+ if (!proc_create_net("fastpath", 0440, net->proc_net, &fpdb_seq_ops,
+ sizeof(struct fpdb_iter_state)))
+
+ return -ENOMEM;
+ return 0;
+}
+
+static void fpdb_net_exit(struct net *net)
+{
+ remove_proc_entry("fastpath", net->proc_net);
+}
+
+static struct pernet_operations fpdb_net_ops = {
+ .init = fpdb_net_init,
+ .exit = fpdb_net_exit,
+};
+
+static void fp_database_release(struct kobject *kobj)
+{
+ struct fastpath_module *module = to_fpmod(kobj);
+ int wait_time = 200;
+
+ fpdb_flush();
+ do {
+ /* wait all fpdb freed, then call kmem_cache_destroy */
+ synchronize_rcu();
+ msleep(10);
+ if (--wait_time <= 0)
+ break;
+ } while (db->num_entries);
+
+ pr_info("%d fpdb entry left\n", db->num_entries);
+ nf_ct_extend_unregister(db->nfct_ext);
+ unregister_pernet_subsys(&fpdb_net_ops);
+ fpdb_free_hashtable(&db->htable);
+ kmem_cache_destroy(db->db_cache);
+#ifdef FP_USE_SRAM_POOL_OPT
+ sram_pool_free((unsigned long)db, sizeof(struct fp_database));
+#else
+ kfree(db);
+#endif
+ kfree(module);
+
+ pr_debug("fp_database released\n");
+}
+
+static struct kobj_type ktype_database = {
+ .sysfs_ops = &fp_sysfs_ops,
+ .default_attrs = fp_database_attrs,
+ .release = fp_database_release,
+};
+
+static void fpdb_destroy_ext(struct nf_conn *ct)
+{
+ struct nf_conn_fastpath *ct_fp;
+ struct fpdb_entry *orig, *reply;
+ BUG_ON(!ct);
+
+ spin_lock_bh(&db->lock);
+ ct_fp = nfct_fastpath(ct);
+ if (ct_fp) {
+ orig = ct_fp->fpd_el[IP_CT_DIR_ORIGINAL];
+ reply = ct_fp->fpd_el[IP_CT_DIR_REPLY];
+ } else {
+ orig = NULL;
+ reply = NULL;
+ }
+
+ if (orig == NULL && reply == NULL) {
+ spin_unlock_bh(&db->lock);
+ return;
+ }
+
+ ct_fp->fpd_el[IP_CT_DIR_ORIGINAL] = NULL;
+ ct_fp->fpd_el[IP_CT_DIR_REPLY] = NULL;
+ if (orig) {
+ FP_DEBUG_DUMP_ENTRY("Delete orig entry:\n", orig);
+ __fpdb_del(orig, true);
+ }
+
+ if (reply) {
+ FP_DEBUG_DUMP_ENTRY("Delete reply entry:\n", reply);
+ __fpdb_del(reply, true);
+ }
+ spin_unlock_bh(&db->lock);
+}
+
+static struct nf_ct_ext_type fpdb_ct_extend = {
+ .len = sizeof(struct nf_conn_fastpath),
+ .align = __alignof__(struct nf_conn_fastpath),
+ .id = NF_CT_EXT_FASTPATH,
+ .destroy = fpdb_destroy_ext,
+};
+
+static int fp_database_probe(struct fastpath_module *module)
+{
+ struct fp_database *priv;
+ int ret;
+
+#ifdef FP_USE_SRAM_POOL_OPT
+ priv = (struct fp_database *)sram_pool_alloc(sizeof(struct fp_database));
+#else
+ priv = kzalloc(sizeof(struct fp_database), GFP_KERNEL);
+#endif
+ if (!priv) {
+ pr_err("no memory\n");
+ return -ENOMEM;
+ }
+ spin_lock_init(&priv->lock);
+ get_random_bytes(&hash_rnd, sizeof(hash_rnd));
+
+ priv->db_cache = kmem_cache_create("fpdb_entry",
+ sizeof(struct fpdb_entry), 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!priv->db_cache) {
+ pr_err("kmem_cache_create fpdb_entry failed\n");
+ ret = -ENOMEM;
+ goto kfree_priv;
+ }
+
+ ret = fpdb_alloc_hashtable(&priv->htable);
+ if (ret < 0) {
+ pr_err("fpdb_alloc_hashtable failed (ret=%d)\n", ret);
+ goto kfree_cache;
+ }
+
+ ret = register_pernet_subsys(&fpdb_net_ops);
+ if (ret < 0) {
+ pr_err("cannot register pernet operations (ret=%d)\n", ret);
+ goto free_hashtable;
+ }
+
+ priv->nfct_ext = &fpdb_ct_extend;
+ ret = nf_ct_extend_register(priv->nfct_ext);
+ if (ret < 0) {
+ pr_err("nf_ct_extend_register failed (%d)\n", ret);
+ goto unreg_pernet;
+ }
+
+ db = module->priv = priv;
+ snprintf(module->name, sizeof(module->name), "fp_database");
+
+ kobject_init(&module->kobj, &ktype_database);
+ ret = kobject_add(&module->kobj, module->fastpath->kobj, "%s", module->name);
+ if (ret < 0) {
+ pr_err("kobject_add failed (%d)\n", ret);
+ goto nf_ct_extend_unreg;
+ }
+ kobject_uevent(&module->kobj, KOBJ_ADD);
+
+ pr_debug("fp_database probed\n");
+ return 0;
+
+nf_ct_extend_unreg:
+ kobject_put(&module->kobj);
+ nf_ct_extend_unregister(priv->nfct_ext);
+unreg_pernet:
+ unregister_pernet_subsys(&fpdb_net_ops);
+free_hashtable:
+ fpdb_free_hashtable(&priv->htable);
+kfree_cache:
+ kmem_cache_destroy(priv->db_cache);
+kfree_priv:
+#ifdef FP_USE_SRAM_POOL_OPT
+ sram_pool_free((unsigned long)priv, sizeof(struct fp_database));
+#else
+ kfree(priv);
+#endif
+ return ret;
+}
+
+static int fp_database_remove(struct fastpath_module *module)
+{
+ kobject_put(&module->kobj);
+
+ pr_debug("fp_database removed\n");
+ return 0;
+}
+
+struct fastpath_module_ops fp_database_ops = {
+ .probe = fp_database_probe,
+ .remove = fp_database_remove,
+};
+