blob: a130630f6685e6bb4a933df60fb320aee6692bb6 [file] [log] [blame]
/*
* Fast path Forward
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#define pr_fmt(fmt) "mfp" " forward: %s:%d: " fmt, __func__, __LINE__
#include <linux/if_vlan.h>
#include <linux/if_ether.h>
#include <linux/tcp.h>
#include <net/ip.h>
#include <net/udp.h>
#include <net/ipv6.h>
#include <net/icmp.h>
#include <linux/relay.h>
#include <linux/debugfs.h>
#include <linux/skbrb.h>
#include "fp_common.h"
#include "fp_classifier.h"
#include "fp_database.h"
#include "fp_device.h"
#include "fp_core.h"
#include "fp_ndisc.h"
/**
* For netif_rx called in interrupt or irq_disabled, direct hook can't use.
* So enable netfilter hook for this situation.
*/
//#define FP_RX_IN_INTR_TO_NETFILTER
/* function prototypes */
static int fp_forward_direct(struct sk_buff *skb);
static int fp_forward_queue(struct sk_buff *skb);
static unsigned int fp_forward_nf_hook(void *priv,
struct sk_buff *skb,
const struct nf_hook_state *state);
static int fp_forward_output(struct sk_buff *skb);
static int fp_forward_netif_rx(struct sk_buff *skb);
#ifdef CONFIG_ASR_TOE
extern int fp_cm_genl_send_tuple(struct nf_conntrack_tuple *tuple, struct fpdb_entry *el,
int add, int len);
#endif
struct rx_hook_struct {
const char *name;
int (*connect)(void);
void (*disconnect)(void);
};
struct tx_hook_struct {
const char *name;
int (*output)(struct sk_buff *);
};
struct fp_forward {
spinlock_t lock;
struct rx_hook_struct *rx_hook;
struct tx_hook_struct *tx_hook;
};
static int (*output)(struct sk_buff *); /* global output function pointer */
static int drop_on_busy = 1; /* drop packets if output dev is busy */
static int bypass_fastpath = 0;
static int reply_ra;
static unsigned int pkt_debug_level;
static struct rchan *fp_chan;
static struct dentry *fp_dir;
/*--------------------------------------*/
/*-------------- RX HOOKS --------------*/
/*--------------------------------------*/
/** netif_rx hook */
static int netif_rx_hook_connect(void)
{
netif_rx_fastpath_register(&fp_forward_netif_rx);
return 0;
}
static void netif_rx_hook_disconnect(void)
{
netif_rx_fastpath_unregister();
}
/** netfilter rx_hook */
static struct nf_hook_ops nf_rx_hook_data[] __read_mostly = {
{
.hook = fp_forward_nf_hook,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_PRE_ROUTING,
.priority = NF_IP_PRI_FIRST,
},
{
.hook = fp_forward_nf_hook,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_PRE_ROUTING,
.priority = NF_IP6_PRI_FIRST,
},
};
static int nf_rx_hook_connect(void)
{
return nf_register_net_hooks(&init_net, nf_rx_hook_data, ARRAY_SIZE(nf_rx_hook_data));
}
static void nf_rx_hook_disconnect(void)
{
nf_unregister_net_hooks(&init_net, nf_rx_hook_data, ARRAY_SIZE(nf_rx_hook_data));
}
#define RX_HOOK_NETIF (0)
#define RX_HOOK_NETFILTER (1)
#define RX_HOOK_NONE (2)
static struct rx_hook_struct rx_hooks[] = {
[RX_HOOK_NETIF].name = "direct (netif_rx)",
[RX_HOOK_NETIF].connect = &netif_rx_hook_connect,
[RX_HOOK_NETIF].disconnect = &netif_rx_hook_disconnect,
[RX_HOOK_NETFILTER].name = "netfilter (NF_INET_PRE_ROUTING)",
[RX_HOOK_NETFILTER].connect = &nf_rx_hook_connect,
[RX_HOOK_NETFILTER].disconnect = &nf_rx_hook_disconnect,
[RX_HOOK_NONE].name = "disconnected",
};
/*--------------------------------------*/
/*-------------- TX HOOKS --------------*/
/*--------------------------------------*/
#define TX_HOOK_NDO_START_XMIT (0)
#define TX_HOOK_DEV_QUEUE_XMIT (1)
#define TX_HOOK_NONE (2)
static struct tx_hook_struct tx_hooks[] = {
[TX_HOOK_NDO_START_XMIT].name = "direct (ndo_start_xmit)",
[TX_HOOK_NDO_START_XMIT].output = &fp_forward_direct,
[TX_HOOK_DEV_QUEUE_XMIT].name = "queue (dev_queue_xmit)",
[TX_HOOK_DEV_QUEUE_XMIT].output = &fp_forward_queue,
[TX_HOOK_NONE].name = "disconnected"
};
static unsigned int fp_forward_rx_hook = FP_FORWARD_RX_HOOK_DEFAULT;
static unsigned int fp_forward_tx_hook = FP_FORWARD_TX_HOOK_DEFAULT;
static void fp_print_pkt(char *buf, u32 buf_len)
{
if (!fp_chan)
return;
relay_write(fp_chan, buf, buf_len);
}
static void fp_dump_input_pkt(struct sk_buff *skb, char *rx_tx)
{
struct iphdr *iph = (struct iphdr *)(skb->data);
u8 version = iph->version;
char buf[512] = {0};
u32 len = 0;
u64 ts_nsec;
unsigned long rem_nsec;
ts_nsec = local_clock();
rem_nsec = do_div(ts_nsec, 1000000000);
if (version == 4) {
if (iph->protocol == IPPROTO_ICMP) {
struct icmphdr *icmph = (struct icmphdr *)(iph + 1);
int type = icmph->type;
if (type == 8) {
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive icmp request, src=%pI4 dst=%pI4 ID=%u SEQ=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(icmph->un.echo.id),
ntohs(icmph->un.echo.sequence));
}else if (type == 0) {
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive icmp reply, src=%pI4 dst=%pI4 ID=%u SEQ=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(icmph->un.echo.id),
ntohs(icmph->un.echo.sequence));
}
} else if (iph->protocol == IPPROTO_UDP) {
struct udphdr *uh = (struct udphdr *)(iph + 1);
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive UDP, src=%pI4 dst=%pI4 ID=%u sp=%u dp=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(iph->id), ntohs(uh->source), ntohs(uh->dest));
} else if (iph->protocol == IPPROTO_TCP) {
struct tcphdr *th = (struct tcphdr *)(iph + 1);
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive TCP, src=%pI4 dst=%pI4 ID=%u sp=%u dp=%u seq=%u ack=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(iph->id), ntohs(th->source), ntohs(th->dest),
ntohl(th->seq), ntohl(th->ack_seq));
} else {
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive pkt type %u, src=%pI4 dst=%pI4 ID=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, iph->protocol, &iph->saddr, &iph->daddr, ntohs(iph->id));
}
} else if (version == 6) {
struct ipv6hdr *ip6h;
__be16 frag_off;
int offset;
u8 nexthdr;
ip6h = (struct ipv6hdr *)(skb->data);
nexthdr = ip6h->nexthdr;
/* not support fragment pkt */
if (nexthdr == NEXTHDR_FRAGMENT)
return;
if (ipv6_ext_hdr(nexthdr)) {
offset = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &nexthdr, &frag_off);
if (offset < 0)
return;
} else
offset = sizeof(struct ipv6hdr);
if (nexthdr == IPPROTO_ICMPV6) {
struct icmp6hdr *icmp6;
if (!pskb_may_pull(skb, ((unsigned char*)ip6h + offset + 6 - skb->data)))
return;
icmp6 = (struct icmp6hdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive icmp6, src=%pI6c dst=%pI6c type=%u code=%u id=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, icmp6->icmp6_type, icmp6->icmp6_type,
ntohs(icmp6->icmp6_identifier));
} else if (nexthdr == IPPROTO_UDP) {
struct udphdr *uh6 = (struct udphdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive UDP6, src=%pI6c dst=%pI6c sp=%u dp=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, ntohs(uh6->source), ntohs(uh6->dest));
} else if (nexthdr == IPPROTO_TCP) {
struct tcphdr *th6 = (struct tcphdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive TCP6, src=%pI6c dst=%pI6c sp=%u dp=%u seq=%u ack=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, ntohs(th6->source), ntohs(th6->dest),
ntohl(th6->seq), ntohl(th6->ack_seq));
} else {
len = scnprintf(buf, 512, "[%5lu.%06lu] --->>%s, receive pkt type %u, src=%pI6c dst=%pI6c\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, nexthdr, &ip6h->saddr, &ip6h->daddr);
}
}else {
return;
}
fp_print_pkt(buf, len);
}
static void fp_dump_output_pkt(struct sk_buff *skb, char *rx_tx)
{
struct iphdr *iph;
u8 version;
char buf[512] = {0};
u32 len = 0;
u64 ts_nsec;
unsigned long rem_nsec;
ts_nsec = local_clock();
rem_nsec = do_div(ts_nsec, 1000000000);
iph = (struct iphdr *)(skb->data);
version = iph->version;
if (version == 4) {
if (iph->protocol == IPPROTO_ICMP) {
struct icmphdr *icmph = (struct icmphdr *)(iph + 1);
int type = icmph->type;
if (type == 8) {
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send icmp request, src=%pI4 dst=%pI4 ID=%u SEQ=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(icmph->un.echo.id),
ntohs(icmph->un.echo.sequence));
}else if (type == 0) {
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send icmp reply, src=%pI4 dst=%pI4 ID=%u SEQ=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(icmph->un.echo.id),
ntohs(icmph->un.echo.sequence));
}
} else if (iph->protocol == IPPROTO_UDP) {
struct udphdr *uh = (struct udphdr *)(iph + 1);
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send UDP, src=%pI4 dst=%pI4 ID=%u sp=%u dp=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(iph->id), ntohs(uh->source), ntohs(uh->dest));
} else if (iph->protocol == IPPROTO_TCP) {
struct tcphdr *th = (struct tcphdr *)(iph + 1);
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send TCP, src=%pI4 dst=%pI4 ID=%u sp=%u dp=%u seq=%u ack=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &iph->saddr, &iph->daddr, ntohs(iph->id), ntohs(th->source), ntohs(th->dest),
ntohl(th->seq), ntohl(th->ack_seq));
} else {
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send pkt type %u, src=%pI4 dst=%pI4 ID=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, iph->protocol, &iph->saddr, &iph->daddr, ntohs(iph->id));
}
} else if (version == 6) {
struct ipv6hdr *ip6h;
__be16 frag_off;
int offset;
u8 nexthdr;
ip6h = (struct ipv6hdr *)(skb->data);
nexthdr = ip6h->nexthdr;
/* not support fragment pkt */
if (nexthdr == NEXTHDR_FRAGMENT)
return;
if (ipv6_ext_hdr(nexthdr)) {
offset = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &nexthdr, &frag_off);
if (offset < 0)
return;
} else
offset = sizeof(struct ipv6hdr);
if (nexthdr == IPPROTO_ICMPV6) {
struct icmp6hdr *icmp6;
if (!pskb_may_pull(skb, ((unsigned char*)ip6h + offset + 6 - skb->data)))
return;
icmp6 = (struct icmp6hdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send icmp6, src=%pI6c dst=%pI6c type=%u code=%u id=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, icmp6->icmp6_type, icmp6->icmp6_type,
ntohs(icmp6->icmp6_identifier));
} else if (nexthdr == IPPROTO_UDP) {
struct udphdr *uh6 = (struct udphdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send UDP6, src=%pI6c dst=%pI6c sp=%u dp=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, ntohs(uh6->source), ntohs(uh6->dest));
} else if (nexthdr == IPPROTO_TCP) {
struct tcphdr *th6 = (struct tcphdr *)((unsigned char*)ip6h + offset);
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send TCP6, src=%pI6c dst=%pI6c sp=%u dp=%u seq=%u ack=%u\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, &ip6h->saddr, &ip6h->daddr, ntohs(th6->source), ntohs(th6->dest),
ntohl(th6->seq), ntohl(th6->ack_seq));
} else {
len = scnprintf(buf, 512, "[%5lu.%06lu] <<---%s, send pkt type %u, src=%pI6c dst=%pI6c\n",
(unsigned long)ts_nsec, rem_nsec / 1000,
rx_tx, nexthdr, &ip6h->saddr, &ip6h->daddr);
}
} else {
return;
}
fp_print_pkt(buf, len);
}
/**
* Forward an skb directly to the output interface if classified as
* fastpath. skb->dev must point to the src net_device (done in
* eth_type_trans or in drivers)
*
* @todo 1. Add an option to enable/disable fastpath for a
* specific net_device from userspace (via
* ifconfig/ethtool)
* @note If the source net_device doesn't have fastpath enabled,
* a packet from it can still traverse through fastpath if
* the output net_device supports it and there was a match
* in the fastpath database.
* @param skb skb to forward through fastpath
*
* @return 1 if skb consumed by fastpath, 0 otherwise (should be
* sent through slowpath)
*/
static int fp_forward_direct(struct sk_buff *skb)
{
int ret, len = skb->len; /* default is slowpath */
struct fp_net_device *dst, *src;
struct fpdb_entry *el;
struct netdev_queue *txq;
struct sk_buff *skb2 = skb;
struct nf_conntrack_tuple tuple;
const struct net_device_ops *ops;
/*
* fastpath direct tx hook should be used only when no packets can
* arrive in irq/irq disable context, since fastpath only protects
* at soft-irq level. Otherwise this could possibly result in a deadlock.
*/
WARN_ONCE(in_irq() || irqs_disabled(),
"fastpath direct tx called from irq, or irq disabled!\n");
el = fpc_classify_start(skb, &tuple);
if (unlikely(!el))
goto slowpath;
rcu_read_lock_bh();
src = rcu_dereference_bh(el->in_dev);
dst = rcu_dereference_bh(el->out_dev);
if (pkt_debug_level == 2 || pkt_debug_level == 3) {
if (!strncasecmp(dst->dev->name, "ccinet", 6))
fp_dump_output_pkt(skb, "F_UL");
else
fp_dump_output_pkt(skb, "F_DL");
}
ops = dst->dev->netdev_ops;
if (fpc_classify_finish(skb, el)) {
rcu_read_unlock_bh();
goto slowpath;
}
skb_reset_mac_header(skb);
txq = netdev_core_pick_tx(dst->dev, skb, NULL);
HARD_TX_LOCK(dst->dev, txq, smp_processor_id());
if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
skb2 = NULL;
dst->stats.queue_stopped++;
}
if (skb2)
skb2->dev = dst->dev;
ret = skb2 ? ops->ndo_start_xmit(skb2, dst->dev) : NETDEV_TX_BUSY;
switch (ret) {
case NETDEV_TX_OK:
/* sent through fastpath */
txq_trans_update(txq);
src->stats.rx_packets++;
src->stats.rx_bytes += len;
dst->stats.tx_packets++;
if (dst->dev->header_ops)
dst->stats.tx_bytes += len + ETH_HLEN;
else
dst->stats.tx_bytes += len;
break;
case NET_XMIT_CN:
src->stats.rx_dropped++;
dst->stats.tx_dropped++;
break;
case NET_XMIT_DROP:
case NETDEV_TX_BUSY:
default:
if (unlikely(skb2)) {
/* shouldn't happen since we check txq before trying to transmit */
src->stats.rx_errors++;
dst->stats.tx_errors++;
printk(KERN_DEBUG "Failed to send through fastpath (ret=%d)\n", ret);
}
if (drop_on_busy) {
src->stats.rx_dropped++;
dst->stats.tx_dropped++;
dev_kfree_skb_any(skb);
}
}
HARD_TX_UNLOCK(dst->dev, txq);
#ifdef CONFIG_ASR_TOE
if ((0 == el->nl_flag) && (ret == NETDEV_TX_OK)) {
//fpdb_dump_entry("fp_cm_genl_send_tuple, entry dump:\n", el);
fp_cm_genl_send_tuple(&tuple, el, 1, len);
}
#endif
rcu_read_unlock_bh();
if (likely(skb2) || drop_on_busy)
return 1;
slowpath:
if (pkt_debug_level == 2 || pkt_debug_level == 3) {
if (!strncasecmp(skb->dev->name, "ccinet", 6))
fp_dump_output_pkt(skb, "S_DL");
else
fp_dump_output_pkt(skb, "S_UL");
}
/* DO NOT do skb copy if the skb is allocated from skbrb
* (skb ring buffer for bridge performace)
*/
if (!IS_SKBRB_SKB(skb) && FP_IS_SKB_P(skb))
pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
return 0;
}
/**
* Forward an skb to the output device's queue if classified as fastpath.
*
* @param skb skb to forward
*
* @return 1 if consumed by fastpath, 0 otherwise (should be sent through slowpath)
*/
static int fp_forward_queue(struct sk_buff *skb)
{
int ret, len = skb->len;
struct fp_net_device *dst, *src;
struct fpdb_entry *el;
struct vlan_hdr *vhdr;
struct nf_conntrack_tuple tuple;
if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
vhdr = (struct vlan_hdr *) skb->data;
skb = skb_vlan_untag(skb);
if (unlikely(!skb))
return 0;
if (skb_vlan_tag_present(skb)) {
if (!vlan_do_receive(&skb)) {
if (unlikely(!skb))
return 1;
}
}
}
el = fpc_classify_start(skb, &tuple);
if (unlikely(!el)) {
/* DO NOT do skb copy if the skb is allocated from skbrb
* (skb ring buffer for bridge performace)
*/
if (!IS_SKBRB_SKB(skb) && FP_IS_SKB_P(skb))
pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
return 0;
}
#ifdef CONFIG_ASR_TOE
if (0 == el->nl_flag) {
//fpdb_dump_entry("fp_cm_genl_send_tuple, entry dump:\n", el);
fp_cm_genl_send_tuple(&tuple, el, 1, len);
}
#endif
src = fpdev_hold(el->in_dev);
dst = fpdev_hold(el->out_dev);
if (fpc_classify_finish(skb, el))
return 0;
skb->dev = dst->dev;
ret = dev_queue_xmit(skb);
switch (ret) {
case NET_XMIT_SUCCESS:
src->stats.rx_bytes += len;
src->stats.rx_packets++;
dst->stats.tx_bytes += len + ETH_HLEN;
dst->stats.tx_packets++;
break;
case NET_XMIT_CN:
src->stats.rx_dropped++;
dst->stats.tx_dropped++;
dst->stats.queue_stopped++;
break;
case NET_XMIT_DROP:
default:
pr_info("unexpected return code from dev_queue_xmit (%d)\n", ret);
src->stats.rx_errors++;
dst->stats.tx_errors++;
}
fpdev_put(dst);
fpdev_put(src);
return 1;
}
static unsigned int fp_forward_nf_hook(void *priv, struct sk_buff *skb,
const struct nf_hook_state *state)
{
WARN_ON_ONCE(irqs_disabled());
if (fp_forward_output(skb))
return NF_STOLEN;
return NF_ACCEPT;
}
static int fp_forward_netif_rx(struct sk_buff *skb)
{
struct iphdr *iph;
struct ipv6hdr *ipv6h;
u32 len;
struct vlan_hdr *vhdr;
int ret;
if (unlikely(bypass_fastpath == 1))
goto slowpath;
#ifdef FP_RX_IN_INTR_TO_NETFILTER
if (in_irq() || irqs_disabled())
goto slowpath;
#endif
if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
vhdr = (struct vlan_hdr *) skb->data;
skb = skb_vlan_untag(skb);
if (unlikely(!skb))
goto slowpath;
if (skb_vlan_tag_present(skb)) {
if (!vlan_do_receive(&skb)) {
if (unlikely(!skb))
return 1;
}
}
}
if (pkt_debug_level == 1 || pkt_debug_level == 3) {
if (!strncasecmp(skb->dev->name, "ccinet", 6))
fp_dump_input_pkt(skb, "DL");
else
fp_dump_input_pkt(skb, "UL");
}
iph = (struct iphdr *)skb->data;
if (likely(iph->version == 4)) {
if (iph->ihl < 5)
goto slowpath_warn;
len = ntohs(iph->tot_len);
if (skb->len < len || len < (iph->ihl * 4))
goto slowpath_warn;
} else if (likely(iph->version == 6)) {
ipv6h = (struct ipv6hdr *)skb->data;
len = ntohs(ipv6h->payload_len);
if (!len && ipv6h->nexthdr == NEXTHDR_HOP)
goto done;
if (len + sizeof(struct ipv6hdr) > skb->len)
goto slowpath_warn;
len = len + sizeof(struct ipv6hdr);
} else {
goto slowpath;
}
/* trim possible padding on skb*/
if (pskb_trim_rcsum(skb, len))
goto slowpath_warn;
done:
ret = fp_forward_output(skb);
if (!ret) {
if (reply_ra && fpnd_is_rs(skb)) {
struct ipv6hdr *ipv6h = (struct ipv6hdr *)skb->data;
printk(KERN_DEBUG "received RS on dev (%s), saddr=%pI6c, daddr=%pI6c\n",
skb->dev->name, &ipv6h->saddr, &ipv6h->daddr);
return fpnd_process_rs(skb);
}
if (fpnd_is_ra(skb)) {
struct ipv6hdr *ipv6h = (struct ipv6hdr *)skb->data;
printk(KERN_DEBUG "received RA on dev (%s), saddr=%pI6c, daddr=%pI6c\n",
skb->dev->name, &ipv6h->saddr, &ipv6h->daddr);
fpnd_process_ra(skb->dev, skb);
}
}
return ret;
slowpath_warn:
pr_debug_ratelimited("bad ip header received\n");
slowpath:
return 0;
}
static inline int fp_forward_output(struct sk_buff *skb)
{
if (unlikely(bypass_fastpath == 1))
return 0;
if (output)
return output(skb);
return 0;
}
static inline void tx_hook_disconnect(struct fp_forward *priv)
{
BUG_ON(!priv);
priv->tx_hook = &tx_hooks[TX_HOOK_NONE];
output = priv->tx_hook->output;
}
static inline void tx_hook_connect(struct fp_forward *priv)
{
BUG_ON(!priv || !priv->tx_hook || !priv->tx_hook->output);
output = priv->tx_hook->output;
}
static inline void rx_hook_disconnect(struct fp_forward *priv)
{
BUG_ON(!priv);
if (priv->rx_hook->disconnect)
priv->rx_hook->disconnect();
priv->rx_hook = &rx_hooks[RX_HOOK_NONE];
}
static inline int rx_hook_connect(struct fp_forward *priv)
{
int ret;
BUG_ON(!priv || !priv->rx_hook || !priv->rx_hook->connect);
ret = priv->rx_hook->connect();
if (ret < 0) {
pr_err("rx_hook connect failed (%d)\n", ret);
priv->rx_hook = &rx_hooks[RX_HOOK_NONE];
return ret;
}
return 0;
}
static inline void fp_forward_disconnect(struct fp_forward *priv)
{
tx_hook_disconnect(priv);
rx_hook_disconnect(priv);
#ifdef FP_RX_IN_INTR_TO_NETFILTER
if (priv->rx_hook != &rx_hooks[RX_HOOK_NETFILTER])
nf_rx_hook_disconnect();
#endif
}
static inline int fp_forward_connect(struct fp_forward *priv)
{
int ret;
tx_hook_connect(priv);
ret = rx_hook_connect(priv);
if (ret < 0) {
pr_err("rx_hook connect failed (%d)\n", ret);
tx_hook_disconnect(priv);
return ret;
}
#ifdef FP_RX_IN_INTR_TO_NETFILTER
if (priv->rx_hook != &rx_hooks[RX_HOOK_NETFILTER]) {
ret = nf_rx_hook_connect();
if (ret < 0) {
pr_err("netfilter rx_hook connect failed (%d)\n", ret);
return ret;
}
//pr_info("=== mfp: also enable netfilter hook for RX\n");
}
#endif
return 0;
}
static ssize_t rx_hook_show(struct fastpath_module *m, char *buf)
{
struct fp_forward *priv = m->priv;
int i, len = sprintf(buf, "fastpath forward rx hooks:\n");
char c;
for (i = 0; i < ARRAY_SIZE(rx_hooks); i++) {
c = (priv->rx_hook == &rx_hooks[i]) ? '*' : ' ';
len += sprintf(buf+len, "%c %s\n", c, rx_hooks[i].name);
}
return len;
}
static ssize_t rx_hook_store(struct fastpath_module *m, const char *buf,
size_t count)
{
struct fp_forward *priv = m->priv;
struct rx_hook_struct *rx_hook;
unsigned int idx;
int ret;
sscanf(buf, "%u", &idx);
if (idx > ARRAY_SIZE(rx_hooks) - 1) {
pr_debug("Invalid rx hook=%d\n", idx);
return -EINVAL;
}
rx_hook = &rx_hooks[idx];
if (rx_hook == priv->rx_hook)
return count; /* no change */
#ifdef FP_RX_IN_INTR_TO_NETFILTER
if (priv->rx_hook != &rx_hooks[RX_HOOK_NETFILTER])
rx_hook_disconnect(priv);
#else
rx_hook_disconnect(priv);
#endif
priv->rx_hook = rx_hook;
#ifdef FP_RX_IN_INTR_TO_NETFILTER
if (rx_hook == &rx_hooks[RX_HOOK_NETFILTER])
return count;
#endif
ret = rx_hook_connect(priv);
if (ret < 0)
return ret;
return count;
}
static ssize_t tx_hook_show(struct fastpath_module *m, char *buf)
{
struct fp_forward *priv = m->priv;
int i, len = sprintf(buf, "fastpath forward tx hooks:\n");
char c;
for (i = 0; i < ARRAY_SIZE(tx_hooks); i++) {
c = (priv->tx_hook == &tx_hooks[i]) ? '*' : ' ';
len += sprintf(buf+len, "%c %s\n", c, tx_hooks[i].name);
}
return len;
}
static ssize_t tx_hook_store(struct fastpath_module *m, const char *buf,
size_t count)
{
struct fp_forward *priv = m->priv;
struct tx_hook_struct *tx_hook;
unsigned int idx;
sscanf(buf, "%u", &idx);
if (idx > ARRAY_SIZE(tx_hooks) - 1) {
pr_debug("Invalid tx hook=%d\n", idx);
return -EINVAL;
}
tx_hook = &tx_hooks[idx];
if (tx_hook == priv->tx_hook)
return count; /* no change */
tx_hook_disconnect(priv);
priv->tx_hook = tx_hook;
tx_hook_connect(priv);
return count;
}
static ssize_t dob_show(struct fastpath_module *m, char *buf)
{
return sprintf(buf, "fastpath forward drop on busy: %d\n", drop_on_busy);
}
static ssize_t dob_store(struct fastpath_module *m, const char *buf,
size_t count)
{
unsigned int dob;
sscanf(buf, "%u", &dob);
if (dob != 0 && dob != 1) {
pr_debug("Invalid value %d - should be 1/0 \n", dob);
return -EINVAL;
}
drop_on_busy = dob;
return count;
}
static ssize_t bypass_show(struct fastpath_module *m, char *buf)
{
return sprintf(buf, "fastpath bypass flag: %d\n", bypass_fastpath);
}
static ssize_t bypass_store(struct fastpath_module *m, const char *buf,
size_t count)
{
unsigned int bypass_fastpath_flag;
sscanf(buf, "%u", &bypass_fastpath_flag);
if (bypass_fastpath_flag != 0 && bypass_fastpath_flag != 1) {
pr_debug("bypass_store: Invalid value %d - should be 1/0 \n",
bypass_fastpath_flag);
return -EINVAL;
}
bypass_fastpath = bypass_fastpath_flag;
return count;
}
static void fp_forward_release(struct kobject *kobj)
{
struct fastpath_module *module = to_fpmod(kobj);
struct fp_forward *priv = module->priv;
fp_forward_disconnect(priv);
pr_debug("fp_forward released\n");
kfree(priv);
kfree(module);
}
static ssize_t reply_ra_show(struct fastpath_module *m, char *buf)
{
return sprintf(buf, "fastpath reply_ra flag: %d\n", reply_ra);
}
static ssize_t reply_ra_store(struct fastpath_module *m, const char *buf,
size_t count)
{
unsigned int reply_ra_flag;
sscanf(buf, "%u", &reply_ra_flag);
if (reply_ra_flag != 0 && reply_ra_flag != 1) {
pr_debug("reply_ra_store: Invalid value %d - should be 1/0 \n",
reply_ra_flag);
return -EINVAL;
}
reply_ra = reply_ra_flag;
return count;
}
static struct dentry *create_buf_file_handler(const char *filename,
struct dentry *parent,
umode_t mode,
struct rchan_buf *buf,
int *is_global)
{
struct dentry *buf_file;
buf_file = debugfs_create_file(filename, mode|S_IRUGO|S_IWUSR, parent, buf,
&relay_file_operations);
*is_global = 1;
return buf_file;
}
static int remove_buf_file_handler(struct dentry *dentry)
{
debugfs_remove(dentry);
return 0;
}
static struct rchan_callbacks fp_relay_callbacks =
{
.create_buf_file = create_buf_file_handler,
.remove_buf_file = remove_buf_file_handler,
};
static ssize_t pkt_debug_level_show(struct fastpath_module *m, char *buf)
{
return sprintf(buf, "%d\n", pkt_debug_level);
}
static ssize_t set_pkt_debug_level(struct fastpath_module *m, const char *buf,
size_t count)
{
unsigned int cmd;
sscanf(buf, "%u", &cmd);
if (cmd > 3) {
pr_debug("Invalid value for pkt_debug_level %d\n", cmd);
return -EINVAL;
}
/* 0: off
1: rx enable
2: tx enable
3: rx+tx enable
*/
pkt_debug_level = cmd;
if (cmd > 0) {
fp_dir = debugfs_create_dir("fastpath", NULL);
if (!fp_dir) {
pr_err("debugfs_create_dir fastpath failed.\n");
return count;
}
fp_chan = relay_open("pkt_debug", fp_dir, 0x200000, 1, &fp_relay_callbacks, NULL);;
if(!fp_chan){
pr_err("relay_open pkt_debug failed.\n");
debugfs_remove(fp_dir);
}
} else if (cmd == 0) {
if (fp_chan) {
relay_close(fp_chan);
fp_chan = NULL;
debugfs_remove(fp_dir);
}
}
return count;
}
static FP_ATTR(rx_hook, S_IRUGO|S_IWUSR, rx_hook_show, rx_hook_store);
static FP_ATTR(tx_hook, S_IRUGO|S_IWUSR, tx_hook_show, tx_hook_store);
static FP_ATTR(drop_on_busy, S_IRUGO|S_IWUSR, dob_show, dob_store);
static FP_ATTR(bypass_fastpath, S_IRUGO|S_IWUSR, bypass_show, bypass_store);
static FP_ATTR(reply_ra, S_IRUGO|S_IWUSR, reply_ra_show, reply_ra_store);
static FP_ATTR(pkt_debug, S_IRUGO|S_IWUSR, pkt_debug_level_show, set_pkt_debug_level);
static struct attribute *fp_forward_attrs[] = {
&fp_attr_rx_hook.attr,
&fp_attr_tx_hook.attr,
&fp_attr_drop_on_busy.attr,
&fp_attr_bypass_fastpath.attr,
&fp_attr_reply_ra.attr,
&fp_attr_pkt_debug.attr,
NULL, /* need to NULL terminate the list of attributes */
};
static struct kobj_type ktype_forward = {
.sysfs_ops = &fp_sysfs_ops,
.default_attrs = fp_forward_attrs,
.release = fp_forward_release,
};
static int fp_forward_probe(struct fastpath_module *module)
{
int ret;
struct fp_forward *priv;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv) {
pr_err("no memeory\n");
return -ENOMEM;
}
module->priv = priv;
snprintf(module->name, sizeof(module->name), "fp_forward");
spin_lock_init(&priv->lock);
if ((fp_forward_rx_hook > ARRAY_SIZE(rx_hooks) - 1) ||
(fp_forward_tx_hook > ARRAY_SIZE(tx_hooks) - 1)) {
pr_err("Invalid hook (rx_hook=%d , tx_hook=%d)\n",
fp_forward_rx_hook, fp_forward_tx_hook);
ret = -EINVAL;
goto priv_kfree;
}
priv->rx_hook = &rx_hooks[fp_forward_rx_hook];
priv->tx_hook = &tx_hooks[fp_forward_tx_hook];
ret = fp_forward_connect(priv);
if (ret < 0) {
pr_err("rx connect failed\n");
goto priv_kfree;
}
kobject_init(&module->kobj, &ktype_forward);
ret = kobject_add(&module->kobj, module->fastpath->kobj, "%s", module->name);
if (ret < 0) {
pr_err("kobject_add failed (%d)\n", ret);
goto fp_forward_disconnect;
}
kobject_uevent(&module->kobj, KOBJ_ADD);
pr_debug("fp_forward probed\n");
return 0;
fp_forward_disconnect:
kobject_put(&module->kobj);
fp_forward_disconnect(priv);
priv_kfree:
kfree(priv);
return ret;
}
static int fp_forward_remove(struct fastpath_module *module)
{
if (fp_chan) {
relay_close(fp_chan);
fp_chan = NULL;
debugfs_remove(fp_dir);
}
kobject_put(&module->kobj);
pr_debug("fp_forward removed\n");
return 0;
}
struct fastpath_module_ops fp_forward_ops = {
.probe = fp_forward_probe,
.remove = fp_forward_remove
};
module_param(fp_forward_rx_hook, uint, 0);
MODULE_PARM_DESC(fp_forward_rx_hook, "fastpath forward rx hook (default="
__MODULE_STRING(FP_FORWARD_RX_HOOK_DEFAULT) ")");
module_param(fp_forward_tx_hook, uint, 0);
MODULE_PARM_DESC(fp_forward_tx_hook, "fastpath forward rx hook (default="
__MODULE_STRING(FP_FORWARD_TX_HOOK_DEFAULT) ")");