@@ -504,6 +504,7 @@ struct tx_port {
long long last_used;
struct hmap_node node;
struct dp_packet_batch output_pkts;
+ struct dp_netdev_rxq *output_pkts_rxqs[NETDEV_MAX_BURST];
};
/* A set of properties for the current processing loop that is not directly
@@ -515,6 +516,10 @@ struct dp_netdev_pmd_thread_ctx {
long long now;
/* Used to count cycles. See 'cycles_count_end()' */
unsigned long long last_cycles;
+ /* RX queue from which last packet was received. */
+ struct dp_netdev_rxq *last_rxq;
+ /* Indicates how should be treated last counted cycles. */
+ enum pmd_cycles_counter_type current_pmd_cycles_type;
};
/* PMD: Poll modes drivers. PMD accesses devices via polling to eliminate
@@ -3232,42 +3237,53 @@ cycles_counter(void)
/* Fake mutex to make sure that the calls to cycles_count_* are balanced */
extern struct ovs_mutex cycles_counter_fake_mutex;
-/* Start counting cycles. Must be followed by 'cycles_count_end()' */
+/* Start counting cycles. Must be followed by 'cycles_count_end()'.
+ * Counting starts from the idle type state. */
static inline void
cycles_count_start(struct dp_netdev_pmd_thread *pmd)
OVS_ACQUIRES(&cycles_counter_fake_mutex)
OVS_NO_THREAD_SAFETY_ANALYSIS
{
+ pmd->ctx.current_pmd_cycles_type = PMD_CYCLES_IDLE;
pmd->ctx.last_cycles = cycles_counter();
}
-/* Stop counting cycles and add them to the counter 'type' */
+/* Stop counting cycles and add them to the counter of the current type. */
static inline void
-cycles_count_end(struct dp_netdev_pmd_thread *pmd,
- enum pmd_stat_type type)
+cycles_count_end(struct dp_netdev_pmd_thread *pmd)
OVS_RELEASES(&cycles_counter_fake_mutex)
OVS_NO_THREAD_SAFETY_ANALYSIS
{
unsigned long long interval = cycles_counter() - pmd->ctx.last_cycles;
+ enum pmd_cycles_counter_type type = pmd->ctx.current_pmd_cycles_type;
pmd_perf_update_counter(&pmd->perf_stats, type, interval);
}
-/* Calculate the intermediate cycle result and add to the counter 'type' */
+/* Calculate the intermediate cycle result and add to the counter of
+ * the current type */
static inline void
cycles_count_intermediate(struct dp_netdev_pmd_thread *pmd,
- struct dp_netdev_rxq *rxq,
- enum pmd_stat_type type)
+ struct dp_netdev_rxq **rxqs, int n_rxqs)
OVS_NO_THREAD_SAFETY_ANALYSIS
{
unsigned long long new_cycles = cycles_counter();
unsigned long long interval = new_cycles - pmd->ctx.last_cycles;
+ enum pmd_cycles_counter_type type = pmd->ctx.current_pmd_cycles_type;
+ int i;
+
pmd->ctx.last_cycles = new_cycles;
pmd_perf_update_counter(&pmd->perf_stats, type, interval);
- if (rxq && (type == PMD_CYCLES_POLL_BUSY)) {
+ if (n_rxqs && (type == PMD_CYCLES_POLL_BUSY)) {
/* Add to the amount of current processing cycles. */
- non_atomic_ullong_add(&rxq->cycles[RXQ_CYCLES_PROC_CURR], interval);
+ interval /= n_rxqs;
+ for (i = 0; i < n_rxqs; i++) {
+ if (rxqs[i]) {
+ non_atomic_ullong_add(&rxqs[i]->cycles[RXQ_CYCLES_PROC_CURR],
+ interval);
+ }
+ }
}
}
@@ -3319,6 +3335,16 @@ dp_netdev_pmd_flush_output_on_port(struct dp_netdev_pmd_thread *pmd,
int tx_qid;
int output_cnt;
bool dynamic_txqs;
+ enum pmd_cycles_counter_type save_pmd_cycles_type;
+
+ /* In case we're in PMD_CYCLES_PROCESSING state we need to count
+ * cycles for rxq we're processing now. */
+ cycles_count_intermediate(pmd, &pmd->ctx.last_rxq, 1);
+
+ /* Save current cycles counting state to restore after accounting
+ * send cycles. */
+ save_pmd_cycles_type = pmd->ctx.current_pmd_cycles_type;
+ pmd->ctx.current_pmd_cycles_type = PMD_CYCLES_POLL_BUSY;
dynamic_txqs = p->port->dynamic_txqs;
if (dynamic_txqs) {
@@ -3336,6 +3362,10 @@ dp_netdev_pmd_flush_output_on_port(struct dp_netdev_pmd_thread *pmd,
PMD_STAT_SENT_PKTS, output_cnt);
pmd_perf_update_counter(&pmd->perf_stats,
PMD_STAT_SENT_BATCHES, 1);
+
+ /* Update send cycles for all the rx queues and restore previous state. */
+ cycles_count_intermediate(pmd, p->output_pkts_rxqs, output_cnt);
+ pmd->ctx.current_pmd_cycles_type = save_pmd_cycles_type;
}
static void
@@ -3352,7 +3382,7 @@ dp_netdev_pmd_flush_output_packets(struct dp_netdev_pmd_thread *pmd)
static int
dp_netdev_process_rxq_port(struct dp_netdev_pmd_thread *pmd,
- struct netdev_rxq *rx,
+ struct dp_netdev_rxq *rxq,
odp_port_t port_no)
{
struct pmd_perf_stats *s = &pmd->perf_stats;
@@ -3361,7 +3391,12 @@ dp_netdev_process_rxq_port(struct dp_netdev_pmd_thread *pmd,
int batch_cnt = 0;
dp_packet_batch_init(&batch);
- error = netdev_rxq_recv(rx, &batch);
+
+ cycles_count_intermediate(pmd, NULL, 0);
+ pmd->ctx.last_rxq = rxq;
+ pmd->ctx.current_pmd_cycles_type = PMD_CYCLES_POLL_BUSY;
+ error = netdev_rxq_recv(rxq->rx, &batch);
+
if (!error) {
*recirc_depth_get() = 0;
pmd_thread_ctx_time_update(pmd);
@@ -3385,14 +3420,20 @@ dp_netdev_process_rxq_port(struct dp_netdev_pmd_thread *pmd,
}
/* Process packet batch. */
dp_netdev_input(pmd, &batch, port_no);
+ cycles_count_intermediate(pmd, &rxq, 1);
+
dp_netdev_pmd_flush_output_packets(pmd);
+
} else if (error != EAGAIN && error != EOPNOTSUPP) {
static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
VLOG_ERR_RL(&rl, "error receiving data from %s: %s",
- netdev_rxq_get_name(rx), ovs_strerror(error));
+ netdev_rxq_get_name(rxq->rx), ovs_strerror(error));
}
+ pmd->ctx.current_pmd_cycles_type = PMD_CYCLES_POLL_IDLE;
+ pmd->ctx.last_rxq = NULL;
+
return batch_cnt;
}
@@ -4016,7 +4057,6 @@ dpif_netdev_run(struct dpif *dpif)
struct dp_netdev *dp = get_dp_netdev(dpif);
struct dp_netdev_pmd_thread *non_pmd;
uint64_t new_tnl_seq;
- int process_packets = 0;
ovs_mutex_lock(&dp->port_mutex);
non_pmd = dp_netdev_get_pmd(dp, NON_PMD_CORE_ID);
@@ -4028,18 +4068,13 @@ dpif_netdev_run(struct dpif *dpif)
int i;
for (i = 0; i < port->n_rxq; i++) {
- process_packets =
- dp_netdev_process_rxq_port(non_pmd,
- port->rxqs[i].rx,
- port->port_no);
- cycles_count_intermediate(non_pmd, NULL,
- process_packets
- ? PMD_CYCLES_POLL_BUSY
- : PMD_CYCLES_POLL_IDLE);
+ dp_netdev_process_rxq_port(non_pmd,
+ &port->rxqs[i],
+ port->port_no);
}
}
}
- cycles_count_end(non_pmd, PMD_CYCLES_POLL_IDLE);
+ cycles_count_end(non_pmd);
pmd_thread_ctx_time_update(non_pmd);
dpif_netdev_xps_revalidate_pmd(non_pmd, false);
ovs_mutex_unlock(&dp->non_pmd_mutex);
@@ -4190,7 +4225,6 @@ pmd_thread_main(void *f_)
bool exiting;
int poll_cnt;
int i;
- int process_packets = 0;
poll_list = NULL;
@@ -4225,14 +4259,10 @@ reload:
pmd_perf_start_iteration(s, pmd->ctx.last_cycles);
for (i = 0; i < poll_cnt; i++) {
- process_packets =
- dp_netdev_process_rxq_port(pmd, poll_list[i].rxq->rx,
- poll_list[i].port_no);
- cycles_count_intermediate(pmd, poll_list[i].rxq,
- process_packets
- ? PMD_CYCLES_POLL_BUSY
- : PMD_CYCLES_POLL_IDLE);
- iter_packets += process_packets;
+ int rxq_packets =
+ dp_netdev_process_rxq_port(pmd, poll_list[i].rxq,
+ poll_list[i].port_no);
+ iter_packets += rxq_packets;
}
if (lc++ > 1024) {
@@ -4253,14 +4283,14 @@ reload:
if (reload) {
break;
}
- cycles_count_intermediate(pmd, NULL, PMD_CYCLES_OVERHEAD);
+ cycles_count_intermediate(pmd, NULL, 0);
}
pmd_perf_end_iteration(s, pmd->ctx.last_cycles, iter_packets,
pmd_perf_metrics_enabled(pmd));
}
- cycles_count_end(pmd, PMD_CYCLES_OVERHEAD);
+ cycles_count_end(pmd);
poll_cnt = pmd_load_queues_and_ports(pmd, &poll_list);
exiting = latch_is_set(&pmd->exit_latch);
@@ -4699,6 +4729,8 @@ dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
ovs_mutex_init(&pmd->port_mutex);
cmap_init(&pmd->flow_table);
cmap_init(&pmd->classifiers);
+ pmd->ctx.last_rxq = NULL;
+ pmd->ctx.current_pmd_cycles_type = PMD_CYCLES_IDLE;
pmd_thread_ctx_time_update(pmd);
pmd->next_optimization = pmd->ctx.now + DPCLS_OPTIMIZATION_INTERVAL;
pmd->rxq_next_cycle_store = pmd->ctx.now + PMD_RXQ_INTERVAL_LEN;
@@ -5585,6 +5617,8 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
dp_netdev_pmd_flush_output_on_port(pmd, p);
}
DP_PACKET_BATCH_FOR_EACH (packet, packets_) {
+ p->output_pkts_rxqs[dp_packet_batch_size(&p->output_pkts)] =
+ pmd->ctx.last_rxq;
dp_packet_batch_add(&p->output_pkts, packet);
}
return;