ofproto-dpif: Implement PACKET_IN in userspace.

In future patches, PACKET_IN messages will include meta-data which
is only available in userspace during action translation.  Either,
this data needs to be stored until it's required by a userspace
datapath action, or the PACKET_IN messages must be sent at the time
the data is available.  This patch implements the latter.

Signed-off-by: Ethan Jackson <ethan@nicira.com>

4 files changed, 77 insertions, 130 deletions

diff --git a/lib/odp-util.c b/lib/odp-util.c
index 490d35eb..bfc1937c 100644
--- a/lib/odp-util.c
+++ b/lib/odp-util.c
@@ -189,9 +189,7 @@ format_odp_userspace_action(struct ds *ds, const struct nlattr *attr)
 
         memcpy(&cookie, &userdata, sizeof cookie);
 
-        if (cookie.type == USER_ACTION_COOKIE_CONTROLLER) {
-            ds_put_format(ds, ",controller,length=%"PRIu32, cookie.data);
-        } else if (cookie.type == USER_ACTION_COOKIE_SFLOW) {
+        if (cookie.type == USER_ACTION_COOKIE_SFLOW) {
             ds_put_format(ds, ",sFlow,n_output=%"PRIu8","
                           "vid=%"PRIu16",pcp=%"PRIu8",ifindex=%"PRIu32,
                           cookie.n_output, vlan_tci_to_vid(cookie.vlan_tci),
@@ -328,7 +326,6 @@ parse_odp_action(const char *s, const struct shash *port_names,
 
     {
         unsigned long long int pid;
-        unsigned long long int length;
         unsigned long long int ifindex;
         char userdata_s[32];
         int n_output;
@@ -338,16 +335,6 @@ parse_odp_action(const char *s, const struct shash *port_names,
         if (sscanf(s, "userspace(pid=%lli)%n", &pid, &n) > 0 && n > 0) {
             odp_put_userspace_action(pid, NULL, actions);
             return n;
-        } else if (sscanf(s, "userspace(pid=%lli,controller,length=%lli)%n",
-                          &pid, &length, &n) > 0 && n > 0) {
-            struct user_action_cookie cookie;
-
-            cookie.type = USER_ACTION_COOKIE_CONTROLLER;
-            cookie.n_output = 0;
-            cookie.vlan_tci = htons(0);
-            cookie.data = length;
-            odp_put_userspace_action(pid, &cookie, actions);
-            return n;
         } else if (sscanf(s, "userspace(pid=%lli,sFlow,n_output=%i,vid=%i,"
                           "pcp=%i,ifindex=%lli)%n", &pid, &n_output,
                           &vid, &pcp, &ifindex, &n) > 0 && n > 0) {
diff --git a/lib/odp-util.h b/lib/odp-util.h
index 7c9b588c..a6f8a308 100644
--- a/lib/odp-util.h
+++ b/lib/odp-util.h
@@ -117,7 +117,6 @@ enum odp_key_fitness odp_flow_key_to_flow(const struct nlattr *, size_t,
 
 enum user_action_cookie_type {
     USER_ACTION_COOKIE_UNSPEC,
-    USER_ACTION_COOKIE_CONTROLLER,   /* Packet for controller. */
     USER_ACTION_COOKIE_SFLOW,        /* Packet for sFlow sampling. */
 };
 
diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c
index f81fbd96..97ddfedf 100644
--- a/ofproto/ofproto-dpif.c
+++ b/ofproto/ofproto-dpif.c
@@ -324,12 +324,6 @@ static struct facet *facet_lookup_valid(struct ofproto_dpif *,
                                         const struct flow *);
 static bool facet_revalidate(struct ofproto_dpif *, struct facet *);
 
-static bool execute_controller_action(struct ofproto_dpif *,
-                                      const struct flow *,
-                                      const struct nlattr *odp_actions,
-                                      size_t actions_len,
-                                      struct ofpbuf *packet);
-
 static void facet_flush_stats(struct ofproto_dpif *, struct facet *);
 
 static void facet_update_time(struct ofproto_dpif *, struct facet *,
@@ -2432,37 +2426,6 @@ send_packet_in_miss(struct ofproto_dpif *ofproto, struct ofpbuf *packet,
     connmgr_send_packet_in(ofproto->up.connmgr, &pin, flow);
 }
 
-/* Sends an OFPT_PACKET_IN message for 'packet' of type OFPR_ACTION to each
- * OpenFlow controller as necessary according to their individual
- * configurations.
- *
- * 'send_len' should be the number of bytes of 'packet' to send to the
- * controller, as specified in the action that caused the packet to be sent. */
-static void
-send_packet_in_action(struct ofproto_dpif *ofproto, struct ofpbuf *packet,
-                      uint64_t userdata, const struct flow *flow)
-{
-    struct ofputil_packet_in pin;
-    struct user_action_cookie cookie;
-
-    memcpy(&cookie, &userdata, sizeof(cookie));
-
-    pin.packet = packet->data;
-    pin.packet_len = packet->size;
-    pin.total_len = packet->size;
-    pin.reason = OFPR_ACTION;
-    pin.buffer_id = 0;          /* not yet known */
-    pin.send_len = cookie.data;
-
-    flow_get_metadata(flow, &pin.fmd);
-
-    /* Metadata may not be accurate at this time. */
-    memset(pin.fmd.reg_masks, 0, sizeof pin.fmd.reg_masks);
-    pin.fmd.tun_id_mask = 0;
-
-    connmgr_send_packet_in(ofproto->up.connmgr, &pin, flow);
-}
-
 static bool
 process_special(struct ofproto_dpif *ofproto, const struct flow *flow,
                 const struct ofpbuf *packet)
@@ -2563,6 +2526,8 @@ handle_flow_miss(struct ofproto_dpif *ofproto, struct flow_miss *miss,
 
     LIST_FOR_EACH_SAFE (packet, next_packet, list_node, &miss->packets) {
         struct dpif_flow_stats stats;
+        struct flow_miss_op *op;
+        struct dpif_execute *execute;
 
         list_remove(&packet->list_node);
         ofproto->n_matches++;
@@ -2585,38 +2550,30 @@ handle_flow_miss(struct ofproto_dpif *ofproto, struct flow_miss *miss,
             subfacet_make_actions(ofproto, subfacet, packet);
         }
 
-        /* Credit statistics to subfacet for this packet.  We must do this now
-         * because execute_controller_action() below may destroy 'packet'. */
         dpif_flow_stats_extract(&facet->flow, packet, &stats);
         subfacet_update_stats(ofproto, subfacet, &stats);
 
-        if (!execute_controller_action(ofproto, &facet->flow,
-                                       subfacet->actions,
-                                       subfacet->actions_len, packet)
-            && subfacet->actions_len > 0) {
-            struct flow_miss_op *op = &ops[(*n_ops)++];
-            struct dpif_execute *execute = &op->dpif_op.execute;
-
-            if (flow->vlan_tci != subfacet->initial_tci) {
-                /* This packet was received on a VLAN splinter port.  We added
-                 * a VLAN to the packet to make the packet resemble the flow,
-                 * but the actions were composed assuming that the packet
-                 * contained no VLAN.  So, we must remove the VLAN header from
-                 * the packet before trying to execute the actions. */
-                eth_pop_vlan(packet);
-            }
-
-            op->subfacet = subfacet;
-            execute->type = DPIF_OP_EXECUTE;
-            execute->key = miss->key;
-            execute->key_len = miss->key_len;
-            execute->actions
-                = (facet->may_install
-                   ? subfacet->actions
-                   : xmemdup(subfacet->actions, subfacet->actions_len));
-            execute->actions_len = subfacet->actions_len;
-            execute->packet = packet;
+        if (flow->vlan_tci != subfacet->initial_tci) {
+            /* This packet was received on a VLAN splinter port.  We added
+             * a VLAN to the packet to make the packet resemble the flow,
+             * but the actions were composed assuming that the packet
+             * contained no VLAN.  So, we must remove the VLAN header from
+             * the packet before trying to execute the actions. */
+            eth_pop_vlan(packet);
         }
+
+        op = &ops[(*n_ops)++];
+        execute = &op->dpif_op.execute;
+        op->subfacet = subfacet;
+        execute->type = DPIF_OP_EXECUTE;
+        execute->key = miss->key;
+        execute->key_len = miss->key_len;
+        execute->actions = (facet->may_install
+                            ? subfacet->actions
+                            : xmemdup(subfacet->actions,
+                                      subfacet->actions_len));
+        execute->actions_len = subfacet->actions_len;
+        execute->packet = packet;
     }
 
     if (facet->may_install && subfacet->key_fitness != ODP_FIT_TOO_LITTLE) {
@@ -2822,10 +2779,6 @@ handle_userspace_upcall(struct ofproto_dpif *ofproto,
             dpif_sflow_received(ofproto->sflow, upcall->packet, &flow,
                                 &cookie);
         }
-    } else if (cookie.type == USER_ACTION_COOKIE_CONTROLLER) {
-        COVERAGE_INC(ofproto_dpif_ctlr_action);
-        send_packet_in_action(ofproto, upcall->packet, upcall->userdata,
-                              &flow);
     } else {
         VLOG_WARN_RL(&rl, "invalid user cookie : 0x%"PRIx64, upcall->userdata);
     }
@@ -3162,35 +3115,6 @@ facet_free(struct facet *facet)
     free(facet);
 }
 
-/* If the 'actions_len' bytes of actions in 'odp_actions' are just a single
- * OVS_ACTION_ATTR_USERSPACE action, executes it internally and returns true.
- * Otherwise, returns false without doing anything. */
-static bool
-execute_controller_action(struct ofproto_dpif *ofproto,
-                          const struct flow *flow,
-                          const struct nlattr *odp_actions, size_t actions_len,
-                          struct ofpbuf *packet)
-{
-    if (actions_len
-        && odp_actions->nla_type == OVS_ACTION_ATTR_USERSPACE
-        && NLA_ALIGN(odp_actions->nla_len) == actions_len) {
-        /* As an optimization, avoid a round-trip from userspace to kernel to
-         * userspace.  This also avoids possibly filling up kernel packet
-         * buffers along the way.
-         *
-         * This optimization will not accidentally catch sFlow
-         * OVS_ACTION_ATTR_USERSPACE actions, since those are encapsulated
-         * inside OVS_ACTION_ATTR_SAMPLE. */
-        const struct nlattr *nla;
-
-        nla = nl_attr_find_nested(odp_actions, OVS_USERSPACE_ATTR_USERDATA);
-        send_packet_in_action(ofproto, packet, nl_attr_get_u64(nla), flow);
-        return true;
-    } else {
-        return false;
-    }
-}
-
 /* Executes, within 'ofproto', the 'n_actions' actions in 'actions' on
  * 'packet', which arrived on 'in_port'.
  *
@@ -3204,12 +3128,6 @@ execute_odp_actions(struct ofproto_dpif *ofproto, const struct flow *flow,
     struct ofpbuf key;
     int error;
 
-    if (execute_controller_action(ofproto, flow, odp_actions, actions_len,
-                                  packet)) {
-        ofpbuf_delete(packet);
-        return true;
-    }
-
     ofpbuf_use_stack(&key, &keybuf, sizeof keybuf);
     odp_flow_key_from_flow(&key, flow);
 
@@ -4359,16 +4277,59 @@ flood_packets(struct action_xlate_ctx *ctx, bool all)
 }
 
 static void
-compose_controller_action(struct action_xlate_ctx *ctx, int len)
+execute_controller_action(struct action_xlate_ctx *ctx, int len)
 {
-    struct user_action_cookie cookie;
+    struct ofputil_packet_in pin;
+    struct ofpbuf *packet;
 
-    commit_odp_actions(&ctx->flow, &ctx->base_flow, ctx->odp_actions);
-    cookie.type = USER_ACTION_COOKIE_CONTROLLER;
-    cookie.data = len;
-    cookie.n_output = 0;
-    cookie.vlan_tci = 0;
-    put_userspace_action(ctx->ofproto, ctx->odp_actions, &ctx->flow, &cookie);
+    ctx->may_set_up_flow = false;
+    if (!ctx->packet) {
+        return;
+    }
+
+    packet = ofpbuf_clone(ctx->packet);
+
+    if (packet->l2 && packet->l3) {
+        struct eth_header *eh;
+
+        eth_pop_vlan(packet);
+        eh = packet->l2;
+        assert(eh->eth_type == ctx->flow.dl_type);
+        memcpy(eh->eth_src, ctx->flow.dl_src, sizeof eh->eth_src);
+        memcpy(eh->eth_dst, ctx->flow.dl_dst, sizeof eh->eth_dst);
+
+        if (ctx->flow.vlan_tci & htons(VLAN_CFI)) {
+            eth_push_vlan(packet, ctx->flow.vlan_tci);
+        }
+
+        if (packet->l4) {
+            if (ctx->flow.dl_type == htons(ETH_TYPE_IP)) {
+                packet_set_ipv4(packet, ctx->flow.nw_src, ctx->flow.nw_dst,
+                                ctx->flow.nw_tos, ctx->flow.nw_ttl);
+            }
+
+            if (packet->l7) {
+                if (ctx->flow.nw_proto == IPPROTO_TCP) {
+                    packet_set_tcp_port(packet, ctx->flow.tp_src,
+                                        ctx->flow.tp_dst);
+                } else if (ctx->flow.nw_proto == IPPROTO_UDP) {
+                    packet_set_udp_port(packet, ctx->flow.tp_src,
+                                        ctx->flow.tp_dst);
+                }
+            }
+        }
+    }
+
+    pin.packet = packet->data;
+    pin.packet_len = packet->size;
+    pin.reason = OFPR_ACTION;
+    pin.buffer_id = 0;
+    pin.send_len = len;
+    pin.total_len = packet->size;
+    flow_get_metadata(&ctx->flow, &pin.fmd);
+
+    connmgr_send_packet_in(ctx->ofproto->up.connmgr, &pin, &ctx->flow);
+    ofpbuf_delete(packet);
 }
 
 static void
@@ -4396,7 +4357,7 @@ xlate_output_action__(struct action_xlate_ctx *ctx,
         flood_packets(ctx, true);
         break;
     case OFPP_CONTROLLER:
-        compose_controller_action(ctx, max_len);
+        execute_controller_action(ctx, max_len);
         break;
     case OFPP_LOCAL:
         compose_output_action(ctx, OFPP_LOCAL);
diff --git a/tests/ofproto-dpif.at b/tests/ofproto-dpif.at
index c90e5cb5..c33f6ac5 100644
--- a/tests/ofproto-dpif.at
+++ b/tests/ofproto-dpif.at
@@ -346,9 +346,9 @@ priority:0,tunnel:0,in_port:0000,tci(vlan:80,pcp:0) mac(80:81:81:81:81:81->82:82
 ])
 
 AT_CHECK([ovs-ofctl dump-flows br0 | STRIP_DURATION | sort], [0], [dnl
- cookie=0x0, duration=?s, table=0, n_packets=1, n_bytes=60, dl_src=10:11:11:11:11:11 actions=CONTROLLER:65535
+ cookie=0x0, duration=?s, table=0, n_packets=3, n_bytes=180, dl_src=10:11:11:11:11:11 actions=CONTROLLER:65535
  cookie=0x1, duration=?s, table=0, n_packets=2, n_bytes=120, dl_src=20:22:22:22:22:22 actions=CONTROLLER:65535,resubmit:80
- cookie=0x2, duration=?s, table=0, n_packets=1, n_bytes=60, dl_src=30:33:33:33:33:33 actions=mod_vlan_vid:15,CONTROLLER:65535
+ cookie=0x2, duration=?s, table=0, n_packets=3, n_bytes=180, dl_src=30:33:33:33:33:33 actions=mod_vlan_vid:15,CONTROLLER:65535
  cookie=0x3, duration=?s, table=0, n_packets=2, n_bytes=120, in_port=80 actions=mod_vlan_vid:80,CONTROLLER:65535,resubmit:81
  cookie=0x4, duration=?s, table=0, n_packets=2, n_bytes=120, in_port=81 actions=mod_dl_src:80:81:81:81:81:81,CONTROLLER:65535,resubmit:82
  cookie=0x5, duration=?s, table=0, n_packets=2, n_bytes=120, in_port=82 actions=mod_dl_dst:82:82:82:82:82:82,CONTROLLER:65535,resubmit:83