mpm/mpmd: Allow SID increment; more fixes for multi-usrp operation

4 files changed, 82 insertions, 34 deletions

diff --git a/host/lib/usrp/mpmd/mpmd_impl.cpp b/host/lib/usrp/mpmd/mpmd_impl.cpp
index f1c5efc14..04d66a5c4 100644
--- a/host/lib/usrp/mpmd/mpmd_impl.cpp
+++ b/host/lib/usrp/mpmd/mpmd_impl.cpp
@@ -203,6 +203,23 @@ size_t mpmd_impl::allocate_xbar_local_addr()
     return new_local_addr;
 }
 
+size_t mpmd_impl::identify_mboard_by_sid(const size_t remote_addr)
+{
+    for (size_t mb_index = 0; mb_index < _mb.size(); mb_index++) {
+        for (size_t xbar_index = 0;
+                xbar_index < _mb[mb_index]->num_xbars;
+                xbar_index++) {
+            if (_mb[mb_index]->get_xbar_local_addr(xbar_index) == remote_addr) {
+                return mb_index;
+            }
+        }
+    }
+    throw uhd::lookup_error(str(
+        boost::format("Cannot identify mboard for remote address %d")
+        % remote_addr
+    ));
+}
+
 
 /*****************************************************************************
  * API
@@ -239,25 +256,27 @@ uhd::device_addr_t mpmd_impl::get_rx_hints(size_t /* mb_index */)
 // }
 // Everything fake below here
 
-both_xports_t mpmd_impl::make_transport(const sid_t& address,
-                                        usrp::device3_impl::xport_type_t xport_type,
-                                        const uhd::device_addr_t& args)
-{
-    //const size_t mb_index = address.get_dst_addr();
-    size_t mb_index = 0;
+both_xports_t mpmd_impl::make_transport(
+        const sid_t& address,
+        usrp::device3_impl::xport_type_t xport_type,
+        const uhd::device_addr_t& args
+) {
+    const size_t mb_index = identify_mboard_by_sid(address.get_dst_addr());
+
+    UHD_LOGGER_TRACE("MPMD")
+        << "Creating new transport of type: "
+        << (xport_type == CTRL ? "CTRL" : (xport_type == RX_DATA ? "RX" : "TX"))
+        << " To mboard: " << mb_index
+        << " Destination address: " << address.to_pp_string_hex().substr(6)
+        << " User-defined xport args: " << args.to_string()
+    ;
 
     both_xports_t xports;
-    xports.endianness = uhd::ENDIANNESS_BIG;
     const uhd::device_addr_t& xport_args = (xport_type == CTRL) ? uhd::device_addr_t() : args;
     transport::zero_copy_xport_params default_buff_args;
 
-    /*
-    std::cout << address << std::endl;
-    std::cout << address.get_src_addr() << std::endl;
-    std::cout << address.get_dst_addr() << std::endl;
-    */
-
-    std::string interface_addr = _device_args["addr"];
+    std::string interface_addr = _mb[mb_index]->mb_args.get("addr");
+    UHD_ASSERT_THROW(not interface_addr.empty());
     const uint32_t xbar_src_addr = address.get_src_addr();
     const uint32_t xbar_src_dst = 0;
 
@@ -276,15 +295,23 @@ both_xports_t mpmd_impl::make_transport(const sid_t& address,
         xport_args);
     uint16_t port  = recv->get_local_port();
 
-    xports.send_sid = _mb[mb_index]->allocate_sid(port, address, xbar_src_addr, xbar_src_dst);
+    xports.endianness = uhd::ENDIANNESS_BIG;
+    xports.send_sid = _mb[mb_index]->allocate_sid(port,
+            address, xbar_src_addr, xbar_src_dst, _sid_framer++
+        );
     xports.recv_sid = xports.send_sid.reversed();
-
-
     xports.recv_buff_size = buff_params.recv_buff_size;
     xports.send_buff_size = buff_params.send_buff_size;
-
-    xports.recv = recv;
+    xports.recv = recv; // Note: This is a type cast!
     xports.send = xports.recv;
+    UHD_LOGGER_TRACE("MPMD")
+        << "xport info: send_sid==" << xports.send_sid.to_pp_string_hex()
+        << " recv_sid==" << xports.recv_sid.to_pp_string_hex()
+        << " endianness=="
+            << (xports.endianness == uhd::ENDIANNESS_BIG ? "BE" : "LE")
+        << " recv_buff_size==" << xports.recv_buff_size
+        << " send_buff_size==" << xports.send_buff_size
+    ;
 
     return xports;
 }
diff --git a/host/lib/usrp/mpmd/mpmd_impl.hpp b/host/lib/usrp/mpmd/mpmd_impl.hpp
index 3bf8293cd..3051ada71 100644
--- a/host/lib/usrp/mpmd/mpmd_impl.hpp
+++ b/host/lib/usrp/mpmd/mpmd_impl.hpp
@@ -73,6 +73,9 @@ class mpmd_mboard_impl
     );
 
     /*** Public attributes ***************************************************/
+    //! These are the args given by the user, with some filtering/preprocessing
+    uhd::device_addr_t mb_args;
+
     //! Device information is read back via MPM and stored here.
     uhd::device_addr_t device_info;
 
@@ -93,7 +96,9 @@ class mpmd_mboard_impl
     uhd::sid_t allocate_sid(const uint16_t port,
                             const uhd::sid_t address,
                             const uint32_t xbar_src_addr,
-                            const uint32_t xbar_src_dst);
+                            const uint32_t xbar_src_dst,
+                            const uint32_t dst_addr
+                            );
 
     //! Configure a crossbar to have a certain local address
     void set_xbar_local_addr(const size_t xbar_index, const size_t local_addr);
@@ -141,6 +146,11 @@ class mpmd_impl : public uhd::usrp::device3_impl
                                       const uhd::device_addr_t&);
 
   private:
+    uhd::device_addr_t get_rx_hints(size_t mb_index);
+
+    /*************************************************************************
+     * Private methods/helpers
+     ************************************************************************/
     /*! Initialize a single motherboard
      *
      * - See mpmd_mboard_impl ctor for details
@@ -160,8 +170,6 @@ class mpmd_impl : public uhd::usrp::device3_impl
     //! Configure all blocks that require access to an RPC client
     void setup_rpc_blocks(const uhd::device_addr_t &block_args);
 
-    uhd::device_addr_t get_rx_hints(size_t mb_index);
-
     /*! Returns a valid local address for a crossbar
      *
      * \returns Valid local address
@@ -169,7 +177,14 @@ class mpmd_impl : public uhd::usrp::device3_impl
      */
     size_t allocate_xbar_local_addr();
 
+    /*! Return the index of the motherboard carrying the crossbar at \p remote_addr
+     */
+    size_t identify_mboard_by_sid(const size_t remote_addr);
+
 
+    /*************************************************************************
+     * Private attributes
+     ************************************************************************/
     uhd::dict<std::string, std::string> recv_args;
     uhd::dict<std::string, std::string> send_args;
 
diff --git a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
index afb7570bf..82967ecad 100644
--- a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
+++ b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
@@ -33,12 +33,14 @@ using namespace uhd;
  * Structors
  ****************************************************************************/
 mpmd_mboard_impl::mpmd_mboard_impl(
-        const device_addr_t &mb_args,
-        const std::string& ip_addr
-) : rpc(uhd::rpc_client::make(ip_addr, MPM_RPC_PORT))
+        const device_addr_t &mb_args_,
+        const std::string& rpc_server_addr
+) : mb_args(mb_args_)
+  , rpc(uhd::rpc_client::make(rpc_server_addr, MPM_RPC_PORT))
 {
     UHD_LOGGER_TRACE("MPMD")
-        << "Initializing mboard, connecting to IP address: " << ip_addr
+        << "Initializing mboard, connecting to RPC server address: "
+        << rpc_server_addr
         << " mboard args: " << mb_args.to_string()
     ;
     auto device_info_dict = rpc->request<dev_info>("get_device_info");
@@ -102,14 +104,16 @@ mpmd_mboard_impl::~mpmd_mboard_impl()
 /*****************************************************************************
  * API
  ****************************************************************************/
-uhd::sid_t mpmd_mboard_impl::allocate_sid(const uint16_t port,
-                                          const uhd::sid_t address,
-                                          const uint32_t xbar_src_addr,
-                                          const uint32_t xbar_src_port)
-{
+uhd::sid_t mpmd_mboard_impl::allocate_sid(
+        const uint16_t port,
+        const uhd::sid_t address,
+        const uint32_t xbar_src_addr,
+        const uint32_t xbar_src_port,
+        const uint32_t dst_addr
+) {
     const auto sid = rpc->request_with_token<uint32_t>(
         "allocate_sid",
-        port, address.get(), xbar_src_addr, xbar_src_port
+        port, address.get(), xbar_src_addr, xbar_src_port, dst_addr
     );
     return uhd::sid_t(sid);
 }
diff --git a/mpm/python/usrp_mpm/periph_manager/n310.py b/mpm/python/usrp_mpm/periph_manager/n310.py
index f49b6d052..7b43aec38 100644
--- a/mpm/python/usrp_mpm/periph_manager/n310.py
+++ b/mpm/python/usrp_mpm/periph_manager/n310.py
@@ -149,12 +149,14 @@ class n310(PeriphManagerBase):
         for ifname, table in iteritems(self._eth_dispatchers):
             table.set_ipv4_addr(self._chdr_interfaces[ifname]['ip_addr'])
 
-    def _allocate_sid(self, sender_addr, port, sid, xbar_src_addr, xbar_src_port):
+    def _allocate_sid(self, sender_addr, port, sid, xbar_src_addr, xbar_src_port, new_ep):
         """
         Get the MAC address of the sender and store it in the FPGA ARP table
         """
         mac_addr = get_mac_addr(sender_addr)
-        new_ep = self._available_endpoints.pop(0)
+        if new_ep not in self._available_endpoints:
+            raise RuntimeError("no more sids yo")
+        self._available_endpoints.remove(new_ep)
         if mac_addr is not None:
             if sender_addr not in self.sid_endpoints:
                 self.sid_endpoints.update({sender_addr: (new_ep,)})