[ovs-dev,RFC] Introduce "OpenFlow Controller as Shared Library"

diff --git a/Makefile.am b/Makefile.am
index ff1f94b48..7cb0a6b55 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -338,7 +338,7 @@  thread-safety-check:
 	if test -e .git && (git --version) >/dev/null 2>&1 && \
 	  grep -n -f build-aux/thread-safety-blacklist \
 	    `git ls-files | grep '\.[ch]$$' \
-	      | $(EGREP) -v '^datapath|^lib/sflow|^third-party'` /dev/null \
+	      | $(EGREP) -v '^datapath|^lib/sflow|^lib/stream-dl|^third-party'` /dev/null \
 	      | $(EGREP) -v ':[ 	]*/?\*'; \
 	then \
 	  echo "See above for list of calls to functions that are"; \
diff --git a/lib/automake.mk b/lib/automake.mk
index cc5dccf39..1e9a6eefa 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -269,6 +269,8 @@  lib_libopenvswitch_la_SOURCES = \
 	lib/sset.h \
 	lib/stp.c \
 	lib/stp.h \
+        lib/stream-dl.c \
+        lib/stream-dl.h \
 	lib/stream-fd.c \
 	lib/stream-fd.h \
 	lib/stream-provider.h \
@@ -346,6 +348,7 @@  lib_libopenvswitch_la_SOURCES += \
 	lib/signals.c \
 	lib/signals.h \
 	lib/socket-util-unix.c \
+        lib/stream-dlopen.c \
 	lib/stream-unix.c
 endif
 
diff --git a/lib/stream-dl.c b/lib/stream-dl.c
new file mode 100644
index 000000000..1ff7dfd8e
--- /dev/null
+++ b/lib/stream-dl.c
@@ -0,0 +1,161 @@ 
+/*
+ * Copyright (c) 2019 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include "stream-dl.h"
+#include <errno.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include "fatal-signal.h"
+#include "openvswitch/poll-loop.h"
+#include "socket-util.h"
+#include "util.h"
+#include "stream-provider.h"
+#include "stream.h"
+#include "openvswitch/vlog.h"
+
+#include <dlfcn.h>
+
+VLOG_DEFINE_THIS_MODULE(stream_dl);
+
+static const struct stream_class stream_dl_class;
+
+
+struct stream_dl
+{
+    struct stream stream;
+    void *handle;
+    int fd_type;
+
+    ssize_t (*recv_function)(const void *buffer, size_t n);
+    ssize_t (*send_function)(const void *buffer, size_t n);
+
+};
+
+static struct stream_dl *
+stream_dl_cast(struct stream *stream)
+{
+    stream_assert_class(stream, &stream_dl_class);
+    return CONTAINER_OF(stream, struct stream_dl, stream);
+}
+
+/* Creates a new stream named 'name' that will send and receive data on 'fd'
+ * and stores a pointer to the stream in '*streamp'.  Initial connection status
+ * 'connect_status' is interpreted as described for stream_init(). 'fd_type'
+ * tells whether the socket is TCP or Unix domain socket.
+ *
+ * Takes ownership of 'name'.
+ *
+ * Returns 0 if successful, otherwise a positive errno value.  (The current
+ * implementation never fails.) */
+int
+new_dl_stream(char *name, void *handle, struct stream **streamp)
+{
+    struct stream_dl *s;
+    char *error;
+
+    s = xmalloc(sizeof *s);
+    stream_init(&s->stream, &stream_dl_class, 0, name);
+    s->handle = handle;
+
+    s->recv_function = dlsym(s->handle, "handle_recv");
+    if ((error = dlerror()) != NULL)  {
+        VLOG_ERR("recv_function not found: %s", error);
+    }
+
+    s->send_function = dlsym(s->handle, "handle_send");
+    if ((error = dlerror()) != NULL)  {
+        VLOG_ERR("send_function not found: %s", error);
+    }
+    VLOG_INFO("%s has recv_function=%p send_function=%p", name,
+              s->recv_function, s->send_function);
+    /*TODO: validate that all functions exist. */
+    *streamp = &s->stream;
+    return 0;
+}
+
+static void
+dl_close(struct stream *stream)
+{
+    struct stream_dl *s = stream_dl_cast(stream);
+    /* TODO: figure out the right place to call dlclose() from.
+     * Since we are calling it on new_dl_stream() and not dl_connect(),
+     * then this probably is not right place to do it. */
+    free(s);
+}
+
+static int
+dl_connect(struct stream *stream OVS_UNUSED)
+{
+    return 0;
+}
+
+static ssize_t
+dl_recv(struct stream *stream, void *buffer, size_t n)
+{
+    struct stream_dl *s = stream_dl_cast(stream);
+
+    return s->recv_function(buffer, n);
+}
+
+static ssize_t
+dl_send(struct stream *stream, const void *buffer, size_t n)
+{
+    struct stream_dl *s = stream_dl_cast(stream);
+
+    return s->send_function(buffer, n);
+}
+
+static void
+dl_wait(struct stream *stream OVS_UNUSED, enum stream_wait_type wait OVS_UNUSED)
+{
+    //struct stream_dl *s = stream_dl_cast(stream);
+
+    switch (wait) {
+    case STREAM_CONNECT:
+    case STREAM_SEND:
+        // poll_fd_wait(s->fd, POLLOUT);
+        /* TODO: should pass something to event loop when
+         * shared library is ready to again receive bytes*/
+        break;
+
+    case STREAM_RECV:
+        // poll_fd_wait(s->fd, POLLIN);
+        /* TODO: should pass something to event loop when
+         * shared library is ready to send something. */
+        break;
+
+    default:
+        OVS_NOT_REACHED();
+    }
+}
+
+static const struct stream_class stream_dl_class = {
+    "dl",                       /* name */
+    false,                      /* needs_probes */
+    NULL,                       /* open */
+    dl_close,                   /* close */
+    dl_connect,                 /* connect */
+    dl_recv,                    /* recv */
+    dl_send,                    /* send */
+    NULL,                       /* run */
+    NULL,                       /* run_wait */
+    dl_wait,                    /* wait */
+};
diff --git a/lib/stream-dl.h b/lib/stream-dl.h
new file mode 100644
index 000000000..981198744
--- /dev/null
+++ b/lib/stream-dl.h
@@ -0,0 +1,28 @@ 
+/*
+ * Copyright (c) 2019 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef STREAM_DL_H
+#define STREAM_DL_H 1
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct stream;
+
+int new_dl_stream(char *name, void *handle, struct stream **streamp);
+
+#endif /* stream-dl.h */
diff --git a/lib/stream-dlopen.c b/lib/stream-dlopen.c
new file mode 100644
index 000000000..e2830c16d
--- /dev/null
+++ b/lib/stream-dlopen.c
@@ -0,0 +1,79 @@ 
+/*
+ * Copyright (c) 2019 Nicira, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include "stream.h"
+#include <errno.h>
+#include <inttypes.h>
+#include <netdb.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "ovs-atomic.h"
+#include "packets.h"
+#include "openvswitch/poll-loop.h"
+#include "socket-util.h"
+#include "dirs.h"
+#include "util.h"
+#include "stream-provider.h"
+#include "stream-dl.h"
+#include "openvswitch/vlog.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <dlfcn.h>
+VLOG_DEFINE_THIS_MODULE(stream_dlopen);
+
+/* Connection to a shared library emulating socket. */
+
+static int
+dlopen_open(const char *name, char *suffix, struct stream **streamp,
+            uint8_t dscp OVS_UNUSED)
+{
+    char *connect_path;
+    void *handle;
+
+    connect_path = abs_file_name(ovs_rundir(), suffix);
+    handle = dlopen(connect_path, RTLD_LAZY);
+    if (!handle) {
+        VLOG_INFO("%s: loading shared library failed (%s)",
+                  connect_path, dlerror());
+        free(connect_path);
+        return -EINVAL; /* TODO: translate dlerror() to errno */
+    }
+
+    free(connect_path);
+
+    return new_dl_stream(xstrdup(name), handle, streamp);
+}
+
+const struct stream_class dlopen_stream_class = {
+    "dl",                       /* name */
+    false,                      /* needs_probes */
+    dlopen_open,                /* open */
+    NULL,                       /* close */
+    NULL,                       /* connect */
+    NULL,                       /* recv */
+    NULL,                       /* send */
+    NULL,                       /* run */
+    NULL,                       /* run_wait */
+    NULL,                       /* wait */
+};
+
diff --git a/lib/stream-provider.h b/lib/stream-provider.h
index 75f4f059b..fa7603b0f 100644
--- a/lib/stream-provider.h
+++ b/lib/stream-provider.h
@@ -201,4 +201,7 @@  extern const struct stream_class ssl_stream_class;
 extern const struct pstream_class pssl_pstream_class;
 #endif
 
+/* Only active stream for dlopen class. */
+extern const struct stream_class dlopen_stream_class;
+
 #endif /* stream-provider.h */
diff --git a/lib/stream.c b/lib/stream.c
index e246b3773..89003d720 100644
--- a/lib/stream.c
+++ b/lib/stream.c
@@ -56,6 +56,7 @@  static const struct stream_class *stream_classes[] = {
     &tcp_stream_class,
 #ifndef _WIN32
     &unix_stream_class,
+    &dlopen_stream_class,
 #else
     &windows_stream_class,
 #endif
diff --git a/lib/vconn-provider.h b/lib/vconn-provider.h
index 523f26f39..e0b5caeed 100644
--- a/lib/vconn-provider.h
+++ b/lib/vconn-provider.h
@@ -194,5 +194,6 @@  extern const struct pvconn_class punix_pvconn_class;
 extern const struct vconn_class ssl_vconn_class;
 extern const struct pvconn_class pssl_pvconn_class;
 #endif
+extern const struct vconn_class dl_vconn_class;
 
 #endif /* vconn-provider.h */
diff --git a/lib/vconn-stream.c b/lib/vconn-stream.c
index 0794ecc05..5769cc0fb 100644
--- a/lib/vconn-stream.c
+++ b/lib/vconn-stream.c
@@ -395,3 +395,5 @@  const struct pvconn_class punix_pvconn_class = PSTREAM_INIT("punix");
 const struct vconn_class ssl_vconn_class = STREAM_INIT("ssl");
 const struct pvconn_class pssl_pvconn_class = PSTREAM_INIT("pssl");
 #endif
+
+const struct vconn_class dl_vconn_class = STREAM_INIT("dl");
diff --git a/lib/vconn.c b/lib/vconn.c
index 7415e6291..2e76005a7 100644
--- a/lib/vconn.c
+++ b/lib/vconn.c
@@ -67,6 +67,7 @@  static const struct vconn_class *vconn_classes[] = {
 #ifdef HAVE_OPENSSL
     &ssl_vconn_class,
 #endif
+    &dl_vconn_class,
 };
 
 static const struct pvconn_class *pvconn_classes[] = {
diff --git a/tests/automake.mk b/tests/automake.mk
index 92d56b29d..9f8edd899 100644
--- a/tests/automake.mk
+++ b/tests/automake.mk
@@ -358,6 +358,12 @@  tests_test_ovsdb_SOURCES = tests/test-ovsdb.c
 nodist_tests_test_ovsdb_SOURCES = tests/idltest.c tests/idltest.h
 tests_test_ovsdb_LDADD = ovsdb/libovsdb.la lib/libopenvswitch.la
 
+# Shoud be noinst_. However, then can't build shared
+lib_LTLIBRARIES += tests/libtest-dl.la
+tests_libtest_dl_la_SOURCES = tests/test-dl.c
+tests_libtest_dl_la_LDFLAGS = -version-info 0:0:0
+tests_libtest_dl_la_LIBADD = ovsdb/libovsdb.la lib/libopenvswitch.la
+
 noinst_PROGRAMS += tests/test-lib
 tests_test_lib_SOURCES = \
 	tests/test-lib.c
diff --git a/tests/test-dl.c b/tests/test-dl.c
new file mode 100644
index 000000000..15d1fa514
--- /dev/null
+++ b/tests/test-dl.c
@@ -0,0 +1,169 @@ 
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+
+#include "openvswitch/ofpbuf.h"
+#include "openvswitch/ofp-msgs.h"
+#include "openvswitch/ofp-util.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(libtestdl);
+
+ssize_t handle_recv(void *buffer, size_t n);
+ssize_t handle_send(const void *buffer, size_t n);
+
+/* This list contains messages that are to be sent to OVS. */
+struct ovs_list txq = OVS_LIST_INITIALIZER(&txq);
+
+static void
+handle_openflow(const void *buffer, size_t n OVS_UNUSED)
+{
+    struct ofpbuf *msg;
+    const struct ofp_header *oh = (struct ofp_header *) buffer;
+    enum ofptype type;
+
+    ofptype_decode(&type, oh);
+
+    switch (type) {
+    case OFPTYPE_HELLO:
+        msg = ofpraw_alloc(OFPRAW_OFPT_HELLO, OFP15_VERSION, 0);
+        ovs_list_push_back(&txq, &msg->list_node);
+        VLOG_INFO("Received OFPTYPE_HELLO and responded");
+        break;
+    case OFPTYPE_ECHO_REQUEST:
+        msg = ofputil_encode_echo_reply(oh);
+        ovs_list_push_back(&txq, &msg->list_node);
+        VLOG_INFO("Received OFPTYPE_ECHO_REQUEST and responded");
+        break;
+    case OFPTYPE_GET_CONFIG_REPLY:
+        break;
+    case OFPTYPE_PACKET_IN:
+        VLOG_INFO("Received OFPTYPE_PACKET_IN");
+        break;
+
+    case OFPTYPE_ERROR:
+    case OFPTYPE_ECHO_REPLY:
+    case OFPTYPE_FEATURES_REQUEST:
+    case OFPTYPE_FEATURES_REPLY:
+    case OFPTYPE_GET_CONFIG_REQUEST:
+    case OFPTYPE_SET_CONFIG:
+    case OFPTYPE_QUEUE_GET_CONFIG_REQUEST:
+    case OFPTYPE_QUEUE_GET_CONFIG_REPLY:
+    case OFPTYPE_GET_ASYNC_REQUEST:
+    case OFPTYPE_GET_ASYNC_REPLY:
+    case OFPTYPE_GROUP_STATS_REQUEST:
+    case OFPTYPE_GROUP_STATS_REPLY:
+    case OFPTYPE_GROUP_DESC_STATS_REQUEST:
+    case OFPTYPE_GROUP_DESC_STATS_REPLY:
+    case OFPTYPE_GROUP_FEATURES_STATS_REQUEST:
+    case OFPTYPE_GROUP_FEATURES_STATS_REPLY:
+    case OFPTYPE_TABLE_FEATURES_STATS_REQUEST:
+    case OFPTYPE_TABLE_FEATURES_STATS_REPLY:
+    case OFPTYPE_TABLE_DESC_REQUEST:
+    case OFPTYPE_TABLE_DESC_REPLY:
+    case OFPTYPE_FLOW_REMOVED:
+    case OFPTYPE_PORT_STATUS:
+    case OFPTYPE_PACKET_OUT:
+    case OFPTYPE_FLOW_MOD:
+    case OFPTYPE_GROUP_MOD:
+    case OFPTYPE_PORT_MOD:
+    case OFPTYPE_TABLE_MOD:
+    case OFPTYPE_METER_MOD:
+    case OFPTYPE_BARRIER_REQUEST:
+    case OFPTYPE_BARRIER_REPLY:
+    case OFPTYPE_DESC_STATS_REQUEST:
+    case OFPTYPE_DESC_STATS_REPLY:
+    case OFPTYPE_FLOW_STATS_REQUEST:
+    case OFPTYPE_FLOW_STATS_REPLY:
+    case OFPTYPE_AGGREGATE_STATS_REQUEST:
+    case OFPTYPE_AGGREGATE_STATS_REPLY:
+    case OFPTYPE_TABLE_STATS_REQUEST:
+    case OFPTYPE_TABLE_STATS_REPLY:
+    case OFPTYPE_PORT_STATS_REQUEST:
+    case OFPTYPE_PORT_STATS_REPLY:
+    case OFPTYPE_QUEUE_STATS_REQUEST:
+    case OFPTYPE_QUEUE_STATS_REPLY:
+    case OFPTYPE_PORT_DESC_STATS_REQUEST:
+    case OFPTYPE_PORT_DESC_STATS_REPLY:
+    case OFPTYPE_METER_STATS_REQUEST:
+    case OFPTYPE_METER_STATS_REPLY:
+    case OFPTYPE_METER_CONFIG_STATS_REQUEST:
+    case OFPTYPE_METER_CONFIG_STATS_REPLY:
+    case OFPTYPE_METER_FEATURES_STATS_REQUEST:
+    case OFPTYPE_METER_FEATURES_STATS_REPLY:
+    case OFPTYPE_ROLE_REQUEST:
+    case OFPTYPE_ROLE_REPLY:
+    case OFPTYPE_ROLE_STATUS:
+    case OFPTYPE_REQUESTFORWARD:
+    case OFPTYPE_TABLE_STATUS:
+    case OFPTYPE_SET_FLOW_FORMAT:
+    case OFPTYPE_FLOW_MOD_TABLE_ID:
+    case OFPTYPE_SET_PACKET_IN_FORMAT:
+    case OFPTYPE_FLOW_AGE:
+    case OFPTYPE_SET_ASYNC_CONFIG:
+    case OFPTYPE_SET_CONTROLLER_ID:
+    case OFPTYPE_FLOW_MONITOR_STATS_REQUEST:
+    case OFPTYPE_FLOW_MONITOR_STATS_REPLY:
+    case OFPTYPE_FLOW_MONITOR_CANCEL:
+    case OFPTYPE_FLOW_MONITOR_PAUSED:
+    case OFPTYPE_FLOW_MONITOR_RESUMED:
+    case OFPTYPE_BUNDLE_CONTROL:
+    case OFPTYPE_BUNDLE_ADD_MESSAGE:
+    case OFPTYPE_NXT_TLV_TABLE_MOD:
+    case OFPTYPE_NXT_TLV_TABLE_REQUEST:
+    case OFPTYPE_NXT_TLV_TABLE_REPLY:
+    case OFPTYPE_NXT_RESUME:
+    case OFPTYPE_IPFIX_BRIDGE_STATS_REQUEST:
+    case OFPTYPE_IPFIX_BRIDGE_STATS_REPLY:
+    case OFPTYPE_IPFIX_FLOW_STATS_REQUEST:
+    case OFPTYPE_IPFIX_FLOW_STATS_REPLY:
+    case OFPTYPE_CT_FLUSH_ZONE:
+    default:
+        VLOG_INFO("Unexpected message: %s", ofptype_get_name(type));
+    }
+}
+
+/* This function gets called when ovs-vswitchd attempts to get a message
+ * from controller/plugin.  The message must be put in 'buffer'.  If message
+ * exceeds size of 'n' bytes, then it should be truncated to 'n' bytes and
+ * remaining message must be sent on next invocation of this function.
+ *
+ * Return value is either negative error code. Or number of bytes put in
+ * 'buffer'.  If there is not message to be sent, then this function
+ * must return -EAGAIN. */
+ssize_t
+handle_recv(void *buffer, size_t n)
+{
+    uint8_t *b = buffer;
+    int offset = 0;
+    struct ofpbuf *cur, *next;
+
+    if (ovs_list_is_empty(&txq)) {
+        return -EAGAIN;
+    }
+
+    LIST_FOR_EACH_SAFE(cur, next, list_node, &txq) {
+        if (offset + cur->size <= n) {
+            memcpy(b+offset, cur->data, cur->size);
+            offset += cur->size;
+            ovs_list_remove(&cur->list_node);
+            VLOG_INFO("Sent whole message to Open vSwitch");
+        } else {
+            VLOG_ERR("Partial message handling not implemented yet");
+        }
+    }
+    return offset;
+}
+
+/* This function gets called when ovs-vswitchd attempts to send a message to
+ * controller/plugin.  The message is in 'buffer' and contains 'n' bytes.
+ *
+ * Return value indicates how many bytes of 'buffer' controller consumed. */
+ssize_t
+handle_send(const void *buffer, size_t n) {
+    VLOG_INFO("received %lu byte message from Open vSwitch", n);
+    handle_openflow(buffer, n);
+    return n;
+}
+