| // SPDX-License-Identifier: GPL-2.0-only | |
| // Copyright (c) 2020 Facebook Inc. | |
| #include <linux/ethtool_netlink.h> | |
| #include <linux/netdevice.h> | |
| #include <linux/slab.h> | |
| #include <linux/types.h> | |
| #include <linux/workqueue.h> | |
| #include <net/udp_tunnel.h> | |
| #include <net/vxlan.h> | |
| enum udp_tunnel_nic_table_entry_flags { | |
| UDP_TUNNEL_NIC_ENTRY_ADD = BIT(0), | |
| UDP_TUNNEL_NIC_ENTRY_DEL = BIT(1), | |
| UDP_TUNNEL_NIC_ENTRY_OP_FAIL = BIT(2), | |
| UDP_TUNNEL_NIC_ENTRY_FROZEN = BIT(3), | |
| }; | |
| struct udp_tunnel_nic_table_entry { | |
| __be16 port; | |
| u8 type; | |
| u8 flags; | |
| u16 use_cnt; | |
| #define UDP_TUNNEL_NIC_USE_CNT_MAX U16_MAX | |
| u8 hw_priv; | |
| }; | |
| /** | |
| * struct udp_tunnel_nic - UDP tunnel port offload state | |
| * @work: async work for talking to hardware from process context | |
| * @dev: netdev pointer | |
| * @lock: protects all fields | |
| * @need_sync: at least one port start changed | |
| * @need_replay: space was freed, we need a replay of all ports | |
| * @n_tables: number of tables under @entries | |
| * @missed: bitmap of tables which overflown | |
| * @entries: table of tables of ports currently offloaded | |
| */ | |
| struct udp_tunnel_nic { | |
| struct delayed_work work; | |
| struct net_device *dev; | |
| struct mutex lock; | |
| u8 need_sync:1; | |
| u8 need_replay:1; | |
| unsigned int n_tables; | |
| unsigned long missed; | |
| struct udp_tunnel_nic_table_entry *entries[] __counted_by(n_tables); | |
| }; | |
| /* We ensure all work structs are done using driver state, but not the code. | |
| * We need a workqueue we can flush before module gets removed. | |
| */ | |
| static struct workqueue_struct *udp_tunnel_nic_workqueue; | |
| static const char *udp_tunnel_nic_tunnel_type_name(unsigned int type) | |
| { | |
| switch (type) { | |
| case UDP_TUNNEL_TYPE_VXLAN: | |
| return "vxlan"; | |
| case UDP_TUNNEL_TYPE_GENEVE: | |
| return "geneve"; | |
| case UDP_TUNNEL_TYPE_VXLAN_GPE: | |
| return "vxlan-gpe"; | |
| default: | |
| return "unknown"; | |
| } | |
| } | |
| static bool | |
| udp_tunnel_nic_entry_is_free(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| return entry->use_cnt == 0 && !entry->flags; | |
| } | |
| static bool | |
| udp_tunnel_nic_entry_is_present(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| return entry->use_cnt && !(entry->flags & ~UDP_TUNNEL_NIC_ENTRY_FROZEN); | |
| } | |
| static bool | |
| udp_tunnel_nic_entry_is_frozen(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| return entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN; | |
| } | |
| static void | |
| udp_tunnel_nic_entry_freeze_used(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| if (!udp_tunnel_nic_entry_is_free(entry)) | |
| entry->flags |= UDP_TUNNEL_NIC_ENTRY_FROZEN; | |
| } | |
| static void | |
| udp_tunnel_nic_entry_unfreeze(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_FROZEN; | |
| } | |
| static bool | |
| udp_tunnel_nic_entry_is_queued(struct udp_tunnel_nic_table_entry *entry) | |
| { | |
| return entry->flags & (UDP_TUNNEL_NIC_ENTRY_ADD | | |
| UDP_TUNNEL_NIC_ENTRY_DEL); | |
| } | |
| static void | |
| udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_nic_table_entry *entry, | |
| unsigned int flag) | |
| { | |
| entry->flags |= flag; | |
| utn->need_sync = 1; | |
| } | |
| static void | |
| udp_tunnel_nic_ti_from_entry(struct udp_tunnel_nic_table_entry *entry, | |
| struct udp_tunnel_info *ti) | |
| { | |
| memset(ti, 0, sizeof(*ti)); | |
| ti->port = entry->port; | |
| ti->type = entry->type; | |
| ti->hw_priv = entry->hw_priv; | |
| } | |
| static bool | |
| udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) | |
| if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j])) | |
| return false; | |
| return true; | |
| } | |
| static bool | |
| udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_table_info *table; | |
| unsigned int i, j; | |
| if (!utn->missed) | |
| return false; | |
| for (i = 0; i < utn->n_tables; i++) { | |
| table = &dev->udp_tunnel_nic_info->tables[i]; | |
| if (!test_bit(i, &utn->missed)) | |
| continue; | |
| for (j = 0; j < table->n_entries; j++) | |
| if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j])) | |
| return true; | |
| } | |
| return false; | |
| } | |
| static void | |
| __udp_tunnel_nic_get_port(struct net_device *dev, unsigned int table, | |
| unsigned int idx, struct udp_tunnel_info *ti) | |
| { | |
| struct udp_tunnel_nic_table_entry *entry; | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| entry = &utn->entries[table][idx]; | |
| if (entry->use_cnt) | |
| udp_tunnel_nic_ti_from_entry(entry, ti); | |
| } | |
| static void | |
| __udp_tunnel_nic_set_port_priv(struct net_device *dev, unsigned int table, | |
| unsigned int idx, u8 priv) | |
| { | |
| dev->udp_tunnel_nic->entries[table][idx].hw_priv = priv; | |
| } | |
| static void | |
| udp_tunnel_nic_entry_update_done(struct udp_tunnel_nic_table_entry *entry, | |
| int err) | |
| { | |
| bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL; | |
| WARN_ON_ONCE(entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD && | |
| entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL); | |
| if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD && | |
| (!err || (err == -EEXIST && dodgy))) | |
| entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_ADD; | |
| if (entry->flags & UDP_TUNNEL_NIC_ENTRY_DEL && | |
| (!err || (err == -ENOENT && dodgy))) | |
| entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_DEL; | |
| if (!err) | |
| entry->flags &= ~UDP_TUNNEL_NIC_ENTRY_OP_FAIL; | |
| else | |
| entry->flags |= UDP_TUNNEL_NIC_ENTRY_OP_FAIL; | |
| } | |
| static void | |
| udp_tunnel_nic_device_sync_one(struct net_device *dev, | |
| struct udp_tunnel_nic *utn, | |
| unsigned int table, unsigned int idx) | |
| { | |
| struct udp_tunnel_nic_table_entry *entry; | |
| struct udp_tunnel_info ti; | |
| int err; | |
| entry = &utn->entries[table][idx]; | |
| if (!udp_tunnel_nic_entry_is_queued(entry)) | |
| return; | |
| udp_tunnel_nic_ti_from_entry(entry, &ti); | |
| if (entry->flags & UDP_TUNNEL_NIC_ENTRY_ADD) | |
| err = dev->udp_tunnel_nic_info->set_port(dev, table, idx, &ti); | |
| else | |
| err = dev->udp_tunnel_nic_info->unset_port(dev, table, idx, | |
| &ti); | |
| udp_tunnel_nic_entry_update_done(entry, err); | |
| if (err) | |
| netdev_warn(dev, | |
| "UDP tunnel port sync failed port %d type %s: %d\n", | |
| be16_to_cpu(entry->port), | |
| udp_tunnel_nic_tunnel_type_name(entry->type), | |
| err); | |
| } | |
| static void | |
| udp_tunnel_nic_device_sync_by_port(struct net_device *dev, | |
| struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) | |
| udp_tunnel_nic_device_sync_one(dev, utn, i, j); | |
| } | |
| static void | |
| udp_tunnel_nic_device_sync_by_table(struct net_device *dev, | |
| struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| unsigned int i, j; | |
| int err; | |
| for (i = 0; i < utn->n_tables; i++) { | |
| /* Find something that needs sync in this table */ | |
| for (j = 0; j < info->tables[i].n_entries; j++) | |
| if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j])) | |
| break; | |
| if (j == info->tables[i].n_entries) | |
| continue; | |
| err = info->sync_table(dev, i); | |
| if (err) | |
| netdev_warn(dev, "UDP tunnel port sync failed for table %d: %d\n", | |
| i, err); | |
| for (j = 0; j < info->tables[i].n_entries; j++) { | |
| struct udp_tunnel_nic_table_entry *entry; | |
| entry = &utn->entries[i][j]; | |
| if (udp_tunnel_nic_entry_is_queued(entry)) | |
| udp_tunnel_nic_entry_update_done(entry, err); | |
| } | |
| } | |
| } | |
| static void | |
| __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| if (!utn->need_sync) | |
| return; | |
| if (dev->udp_tunnel_nic_info->sync_table) | |
| udp_tunnel_nic_device_sync_by_table(dev, utn); | |
| else | |
| udp_tunnel_nic_device_sync_by_port(dev, utn); | |
| utn->need_sync = 0; | |
| /* Can't replay directly here, in case we come from the tunnel driver's | |
| * notification - trying to replay may deadlock inside tunnel driver. | |
| */ | |
| utn->need_replay = udp_tunnel_nic_should_replay(dev, utn); | |
| } | |
| static void | |
| udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| if (!utn->need_sync) | |
| return; | |
| queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 0); | |
| } | |
| static bool | |
| udp_tunnel_nic_table_is_capable(const struct udp_tunnel_nic_table_info *table, | |
| struct udp_tunnel_info *ti) | |
| { | |
| return table->tunnel_types & ti->type; | |
| } | |
| static bool | |
| udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| unsigned int i; | |
| /* Special case IPv4-only NICs */ | |
| if (info->flags & UDP_TUNNEL_NIC_INFO_IPV4_ONLY && | |
| ti->sa_family != AF_INET) | |
| return false; | |
| for (i = 0; i < utn->n_tables; i++) | |
| if (udp_tunnel_nic_table_is_capable(&info->tables[i], ti)) | |
| return true; | |
| return false; | |
| } | |
| static int | |
| udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic_table_entry *entry; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) { | |
| entry = &utn->entries[i][j]; | |
| if (!udp_tunnel_nic_entry_is_free(entry) && | |
| entry->port == ti->port && | |
| entry->type != ti->type) { | |
| __set_bit(i, &utn->missed); | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| static void | |
| udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn, | |
| unsigned int table, unsigned int idx, int use_cnt_adj) | |
| { | |
| struct udp_tunnel_nic_table_entry *entry = &utn->entries[table][idx]; | |
| bool dodgy = entry->flags & UDP_TUNNEL_NIC_ENTRY_OP_FAIL; | |
| unsigned int from, to; | |
| WARN_ON(entry->use_cnt + (u32)use_cnt_adj > U16_MAX); | |
| /* If not going from used to unused or vice versa - all done. | |
| * For dodgy entries make sure we try to sync again (queue the entry). | |
| */ | |
| entry->use_cnt += use_cnt_adj; | |
| if (!dodgy && !entry->use_cnt == !(entry->use_cnt - use_cnt_adj)) | |
| return; | |
| /* Cancel the op before it was sent to the device, if possible, | |
| * otherwise we'd need to take special care to issue commands | |
| * in the same order the ports arrived. | |
| */ | |
| if (use_cnt_adj < 0) { | |
| from = UDP_TUNNEL_NIC_ENTRY_ADD; | |
| to = UDP_TUNNEL_NIC_ENTRY_DEL; | |
| } else { | |
| from = UDP_TUNNEL_NIC_ENTRY_DEL; | |
| to = UDP_TUNNEL_NIC_ENTRY_ADD; | |
| } | |
| if (entry->flags & from) { | |
| entry->flags &= ~from; | |
| if (!dodgy) | |
| return; | |
| } | |
| udp_tunnel_nic_entry_queue(utn, entry, to); | |
| } | |
| static bool | |
| udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn, | |
| unsigned int table, unsigned int idx, | |
| struct udp_tunnel_info *ti, int use_cnt_adj) | |
| { | |
| struct udp_tunnel_nic_table_entry *entry = &utn->entries[table][idx]; | |
| if (udp_tunnel_nic_entry_is_free(entry) || | |
| entry->port != ti->port || | |
| entry->type != ti->type) | |
| return false; | |
| if (udp_tunnel_nic_entry_is_frozen(entry)) | |
| return true; | |
| udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj); | |
| return true; | |
| } | |
| /* Try to find existing matching entry and adjust its use count, instead of | |
| * adding a new one. Returns true if entry was found. In case of delete the | |
| * entry may have gotten removed in the process, in which case it will be | |
| * queued for removal. | |
| */ | |
| static bool | |
| udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti, int use_cnt_adj) | |
| { | |
| const struct udp_tunnel_nic_table_info *table; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) { | |
| table = &dev->udp_tunnel_nic_info->tables[i]; | |
| if (!udp_tunnel_nic_table_is_capable(table, ti)) | |
| continue; | |
| for (j = 0; j < table->n_entries; j++) | |
| if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti, | |
| use_cnt_adj)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| static bool | |
| udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti) | |
| { | |
| return udp_tunnel_nic_try_existing(dev, utn, ti, +1); | |
| } | |
| static bool | |
| udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti) | |
| { | |
| return udp_tunnel_nic_try_existing(dev, utn, ti, -1); | |
| } | |
| static bool | |
| udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn, | |
| struct udp_tunnel_info *ti) | |
| { | |
| const struct udp_tunnel_nic_table_info *table; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) { | |
| table = &dev->udp_tunnel_nic_info->tables[i]; | |
| if (!udp_tunnel_nic_table_is_capable(table, ti)) | |
| continue; | |
| for (j = 0; j < table->n_entries; j++) { | |
| struct udp_tunnel_nic_table_entry *entry; | |
| entry = &utn->entries[i][j]; | |
| if (!udp_tunnel_nic_entry_is_free(entry)) | |
| continue; | |
| entry->port = ti->port; | |
| entry->type = ti->type; | |
| entry->use_cnt = 1; | |
| udp_tunnel_nic_entry_queue(utn, entry, | |
| UDP_TUNNEL_NIC_ENTRY_ADD); | |
| return true; | |
| } | |
| /* The different table may still fit this port in, but there | |
| * are no devices currently which have multiple tables accepting | |
| * the same tunnel type, and false positives are okay. | |
| */ | |
| __set_bit(i, &utn->missed); | |
| } | |
| return false; | |
| } | |
| static void | |
| __udp_tunnel_nic_add_port(struct net_device *dev, struct udp_tunnel_info *ti) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return; | |
| if (!netif_running(dev) && info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY) | |
| return; | |
| if (info->flags & UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN && | |
| ti->port == htons(IANA_VXLAN_UDP_PORT)) { | |
| if (ti->type != UDP_TUNNEL_TYPE_VXLAN) | |
| netdev_warn(dev, "device assumes port 4789 will be used by vxlan tunnels\n"); | |
| return; | |
| } | |
| if (!udp_tunnel_nic_is_capable(dev, utn, ti)) | |
| return; | |
| /* It may happen that a tunnel of one type is removed and different | |
| * tunnel type tries to reuse its port before the device was informed. | |
| * Rely on utn->missed to re-add this port later. | |
| */ | |
| if (udp_tunnel_nic_has_collision(dev, utn, ti)) | |
| return; | |
| if (!udp_tunnel_nic_add_existing(dev, utn, ti)) | |
| udp_tunnel_nic_add_new(dev, utn, ti); | |
| udp_tunnel_nic_device_sync(dev, utn); | |
| } | |
| static void | |
| __udp_tunnel_nic_del_port(struct net_device *dev, struct udp_tunnel_info *ti) | |
| { | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return; | |
| if (!udp_tunnel_nic_is_capable(dev, utn, ti)) | |
| return; | |
| udp_tunnel_nic_del_existing(dev, utn, ti); | |
| udp_tunnel_nic_device_sync(dev, utn); | |
| } | |
| static void __udp_tunnel_nic_reset_ntf(struct net_device *dev) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic *utn; | |
| unsigned int i, j; | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return; | |
| mutex_lock(&utn->lock); | |
| utn->need_sync = false; | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) { | |
| struct udp_tunnel_nic_table_entry *entry; | |
| entry = &utn->entries[i][j]; | |
| entry->flags &= ~(UDP_TUNNEL_NIC_ENTRY_DEL | | |
| UDP_TUNNEL_NIC_ENTRY_OP_FAIL); | |
| /* We don't release utn lock across ops */ | |
| WARN_ON(entry->flags & UDP_TUNNEL_NIC_ENTRY_FROZEN); | |
| if (!entry->use_cnt) | |
| continue; | |
| udp_tunnel_nic_entry_queue(utn, entry, | |
| UDP_TUNNEL_NIC_ENTRY_ADD); | |
| } | |
| __udp_tunnel_nic_device_sync(dev, utn); | |
| mutex_unlock(&utn->lock); | |
| } | |
| static size_t | |
| __udp_tunnel_nic_dump_size(struct net_device *dev, unsigned int table) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic *utn; | |
| unsigned int j; | |
| size_t size; | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return 0; | |
| size = 0; | |
| for (j = 0; j < info->tables[table].n_entries; j++) { | |
| if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j])) | |
| continue; | |
| size += nla_total_size(0) + /* _TABLE_ENTRY */ | |
| nla_total_size(sizeof(__be16)) + /* _ENTRY_PORT */ | |
| nla_total_size(sizeof(u32)); /* _ENTRY_TYPE */ | |
| } | |
| return size; | |
| } | |
| static int | |
| __udp_tunnel_nic_dump_write(struct net_device *dev, unsigned int table, | |
| struct sk_buff *skb) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic *utn; | |
| struct nlattr *nest; | |
| unsigned int j; | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return 0; | |
| for (j = 0; j < info->tables[table].n_entries; j++) { | |
| if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j])) | |
| continue; | |
| nest = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY); | |
| if (!nest) | |
| return -EMSGSIZE; | |
| if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT, | |
| utn->entries[table][j].port) || | |
| nla_put_u32(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_TYPE, | |
| ilog2(utn->entries[table][j].type))) | |
| goto err_cancel; | |
| nla_nest_end(skb, nest); | |
| } | |
| return 0; | |
| err_cancel: | |
| nla_nest_cancel(skb, nest); | |
| return -EMSGSIZE; | |
| } | |
| static void __udp_tunnel_nic_assert_locked(struct net_device *dev) | |
| { | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| if (utn) | |
| lockdep_assert_held(&utn->lock); | |
| } | |
| static void __udp_tunnel_nic_lock(struct net_device *dev) | |
| { | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| if (utn) | |
| mutex_lock(&utn->lock); | |
| } | |
| static void __udp_tunnel_nic_unlock(struct net_device *dev) | |
| { | |
| struct udp_tunnel_nic *utn; | |
| utn = dev->udp_tunnel_nic; | |
| if (utn) | |
| mutex_unlock(&utn->lock); | |
| } | |
| static const struct udp_tunnel_nic_ops __udp_tunnel_nic_ops = { | |
| .get_port = __udp_tunnel_nic_get_port, | |
| .set_port_priv = __udp_tunnel_nic_set_port_priv, | |
| .add_port = __udp_tunnel_nic_add_port, | |
| .del_port = __udp_tunnel_nic_del_port, | |
| .reset_ntf = __udp_tunnel_nic_reset_ntf, | |
| .dump_size = __udp_tunnel_nic_dump_size, | |
| .dump_write = __udp_tunnel_nic_dump_write, | |
| .assert_locked = __udp_tunnel_nic_assert_locked, | |
| .lock = __udp_tunnel_nic_lock, | |
| .unlock = __udp_tunnel_nic_unlock, | |
| }; | |
| static void | |
| udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| unsigned int i, j; | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) { | |
| int adj_cnt = -utn->entries[i][j].use_cnt; | |
| if (adj_cnt) | |
| udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt); | |
| } | |
| __udp_tunnel_nic_device_sync(dev, utn); | |
| for (i = 0; i < utn->n_tables; i++) | |
| memset(utn->entries[i], 0, array_size(info->tables[i].n_entries, | |
| sizeof(**utn->entries))); | |
| WARN_ON(utn->need_sync); | |
| utn->need_replay = 0; | |
| } | |
| static void | |
| udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic_shared_node *node; | |
| unsigned int i, j; | |
| /* Freeze all the ports we are already tracking so that the replay | |
| * does not double up the refcount. | |
| */ | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) | |
| udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]); | |
| utn->missed = 0; | |
| utn->need_replay = 0; | |
| if (!info->shared) { | |
| udp_tunnel_get_rx_info(dev); | |
| } else { | |
| list_for_each_entry(node, &info->shared->devices, list) | |
| udp_tunnel_get_rx_info(node->dev); | |
| } | |
| for (i = 0; i < utn->n_tables; i++) | |
| for (j = 0; j < info->tables[i].n_entries; j++) | |
| udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]); | |
| } | |
| static void udp_tunnel_nic_device_sync_work(struct work_struct *work) | |
| { | |
| struct udp_tunnel_nic *utn = | |
| container_of(work, struct udp_tunnel_nic, work.work); | |
| /* We cannot block on RTNL here, otherwise we would deadlock with | |
| * udp_tunnel_nic_unregister() calling cancel_delayed_work_sync() | |
| * while holding RTNL. Requeue with 1 jiffy delay if RTNL is contended. | |
| */ | |
| if (!rtnl_trylock()) { | |
| queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 1); | |
| return; | |
| } | |
| mutex_lock(&utn->lock); | |
| __udp_tunnel_nic_device_sync(utn->dev, utn); | |
| if (utn->need_replay) | |
| udp_tunnel_nic_replay(utn->dev, utn); | |
| mutex_unlock(&utn->lock); | |
| rtnl_unlock(); | |
| } | |
| static struct udp_tunnel_nic * | |
| udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info, | |
| unsigned int n_tables) | |
| { | |
| struct udp_tunnel_nic *utn; | |
| unsigned int i; | |
| utn = kzalloc_flex(*utn, entries, n_tables); | |
| if (!utn) | |
| return NULL; | |
| utn->n_tables = n_tables; | |
| INIT_DELAYED_WORK(&utn->work, udp_tunnel_nic_device_sync_work); | |
| mutex_init(&utn->lock); | |
| for (i = 0; i < n_tables; i++) { | |
| utn->entries[i] = kzalloc_objs(*utn->entries[i], | |
| info->tables[i].n_entries); | |
| if (!utn->entries[i]) | |
| goto err_free_prev_entries; | |
| } | |
| return utn; | |
| err_free_prev_entries: | |
| while (i--) | |
| kfree(utn->entries[i]); | |
| kfree(utn); | |
| return NULL; | |
| } | |
| static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn) | |
| { | |
| unsigned int i; | |
| for (i = 0; i < utn->n_tables; i++) | |
| kfree(utn->entries[i]); | |
| kfree(utn); | |
| } | |
| static int udp_tunnel_nic_register(struct net_device *dev) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| struct udp_tunnel_nic_shared_node *node = NULL; | |
| struct udp_tunnel_nic *utn; | |
| unsigned int n_tables, i; | |
| BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE < | |
| UDP_TUNNEL_NIC_MAX_TABLES); | |
| /* Expect use count of at most 2 (IPv4, IPv6) per device */ | |
| BUILD_BUG_ON(UDP_TUNNEL_NIC_USE_CNT_MAX < | |
| UDP_TUNNEL_NIC_MAX_SHARING_DEVICES * 2); | |
| /* Check that the driver info is sane */ | |
| if (WARN_ON(!info->set_port != !info->unset_port) || | |
| WARN_ON(!info->set_port == !info->sync_table) || | |
| WARN_ON(!info->tables[0].n_entries)) | |
| return -EINVAL; | |
| if (WARN_ON(info->shared && | |
| info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)) | |
| return -EINVAL; | |
| n_tables = 1; | |
| for (i = 1; i < UDP_TUNNEL_NIC_MAX_TABLES; i++) { | |
| if (!info->tables[i].n_entries) | |
| continue; | |
| n_tables++; | |
| if (WARN_ON(!info->tables[i - 1].n_entries)) | |
| return -EINVAL; | |
| } | |
| /* Create UDP tunnel state structures */ | |
| if (info->shared) { | |
| node = kzalloc_obj(*node); | |
| if (!node) | |
| return -ENOMEM; | |
| node->dev = dev; | |
| } | |
| if (info->shared && info->shared->udp_tunnel_nic_info) { | |
| utn = info->shared->udp_tunnel_nic_info; | |
| } else { | |
| utn = udp_tunnel_nic_alloc(info, n_tables); | |
| if (!utn) { | |
| kfree(node); | |
| return -ENOMEM; | |
| } | |
| } | |
| if (info->shared) { | |
| if (!info->shared->udp_tunnel_nic_info) { | |
| INIT_LIST_HEAD(&info->shared->devices); | |
| info->shared->udp_tunnel_nic_info = utn; | |
| } | |
| list_add_tail(&node->list, &info->shared->devices); | |
| } | |
| utn->dev = dev; | |
| dev_hold(dev); | |
| dev->udp_tunnel_nic = utn; | |
| if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)) { | |
| udp_tunnel_nic_lock(dev); | |
| udp_tunnel_get_rx_info(dev); | |
| udp_tunnel_nic_unlock(dev); | |
| } | |
| return 0; | |
| } | |
| static void | |
| udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn) | |
| { | |
| const struct udp_tunnel_nic_info *info = dev->udp_tunnel_nic_info; | |
| udp_tunnel_nic_lock(dev); | |
| /* For a shared table remove this dev from the list of sharing devices | |
| * and if there are other devices just detach. | |
| */ | |
| if (info->shared) { | |
| struct udp_tunnel_nic_shared_node *node, *first; | |
| list_for_each_entry(node, &info->shared->devices, list) | |
| if (node->dev == dev) | |
| break; | |
| if (list_entry_is_head(node, &info->shared->devices, list)) { | |
| udp_tunnel_nic_unlock(dev); | |
| return; | |
| } | |
| list_del(&node->list); | |
| kfree(node); | |
| first = list_first_entry_or_null(&info->shared->devices, | |
| typeof(*first), list); | |
| if (first) { | |
| udp_tunnel_drop_rx_info(dev); | |
| utn->dev = first->dev; | |
| udp_tunnel_nic_unlock(dev); | |
| goto release_dev; | |
| } | |
| info->shared->udp_tunnel_nic_info = NULL; | |
| } | |
| /* Flush before we check work, so we don't waste time adding entries | |
| * from the work which we will boot immediately. | |
| */ | |
| udp_tunnel_nic_flush(dev, utn); | |
| udp_tunnel_nic_unlock(dev); | |
| /* Make sure no work is running or queued before freeing @utn. | |
| * The work handler uses rtnl_trylock(), so it will not deadlock | |
| * against the RTNL we are holding here. | |
| */ | |
| cancel_delayed_work_sync(&utn->work); | |
| udp_tunnel_nic_free(utn); | |
| release_dev: | |
| dev->udp_tunnel_nic = NULL; | |
| dev_put(dev); | |
| } | |
| static int | |
| udp_tunnel_nic_netdevice_event(struct notifier_block *unused, | |
| unsigned long event, void *ptr) | |
| { | |
| struct net_device *dev = netdev_notifier_info_to_dev(ptr); | |
| const struct udp_tunnel_nic_info *info; | |
| struct udp_tunnel_nic *utn; | |
| info = dev->udp_tunnel_nic_info; | |
| if (!info) | |
| return NOTIFY_DONE; | |
| if (event == NETDEV_REGISTER) { | |
| int err; | |
| err = udp_tunnel_nic_register(dev); | |
| if (err) | |
| netdev_warn(dev, "failed to register for UDP tunnel offloads: %d", err); | |
| return notifier_from_errno(err); | |
| } | |
| /* All other events will need the udp_tunnel_nic state */ | |
| utn = dev->udp_tunnel_nic; | |
| if (!utn) | |
| return NOTIFY_DONE; | |
| if (event == NETDEV_UNREGISTER) { | |
| udp_tunnel_nic_unregister(dev, utn); | |
| return NOTIFY_OK; | |
| } | |
| /* All other events only matter if NIC has to be programmed open */ | |
| if (!(info->flags & UDP_TUNNEL_NIC_INFO_OPEN_ONLY)) | |
| return NOTIFY_DONE; | |
| if (event == NETDEV_UP) { | |
| udp_tunnel_nic_lock(dev); | |
| WARN_ON(!udp_tunnel_nic_is_empty(dev, utn)); | |
| udp_tunnel_get_rx_info(dev); | |
| udp_tunnel_nic_unlock(dev); | |
| return NOTIFY_OK; | |
| } | |
| if (event == NETDEV_GOING_DOWN) { | |
| udp_tunnel_nic_lock(dev); | |
| udp_tunnel_nic_flush(dev, utn); | |
| udp_tunnel_nic_unlock(dev); | |
| return NOTIFY_OK; | |
| } | |
| return NOTIFY_DONE; | |
| } | |
| static struct notifier_block udp_tunnel_nic_notifier_block __read_mostly = { | |
| .notifier_call = udp_tunnel_nic_netdevice_event, | |
| }; | |
| static int __init udp_tunnel_nic_init_module(void) | |
| { | |
| int err; | |
| udp_tunnel_nic_workqueue = alloc_ordered_workqueue("udp_tunnel_nic", 0); | |
| if (!udp_tunnel_nic_workqueue) | |
| return -ENOMEM; | |
| rtnl_lock(); | |
| udp_tunnel_nic_ops = &__udp_tunnel_nic_ops; | |
| rtnl_unlock(); | |
| err = register_netdevice_notifier(&udp_tunnel_nic_notifier_block); | |
| if (err) | |
| goto err_unset_ops; | |
| return 0; | |
| err_unset_ops: | |
| rtnl_lock(); | |
| udp_tunnel_nic_ops = NULL; | |
| rtnl_unlock(); | |
| destroy_workqueue(udp_tunnel_nic_workqueue); | |
| return err; | |
| } | |
| late_initcall(udp_tunnel_nic_init_module); | |
| static void __exit udp_tunnel_nic_cleanup_module(void) | |
| { | |
| unregister_netdevice_notifier(&udp_tunnel_nic_notifier_block); | |
| rtnl_lock(); | |
| udp_tunnel_nic_ops = NULL; | |
| rtnl_unlock(); | |
| destroy_workqueue(udp_tunnel_nic_workqueue); | |
| } | |
| module_exit(udp_tunnel_nic_cleanup_module); | |
| MODULE_LICENSE("GPL"); |