More extensive outbound test

Cargo.lock

@@ -29,6 +29,15 @@ dependencies = [
  "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "atty"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "autocfg"
 version = "0.1.6"
@@ -241,6 +250,18 @@ name = "either"
 version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "env_logger"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.5"
@@ -361,6 +382,14 @@ dependencies = [
  "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "humantime"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.2"
@@ -1146,6 +1175,14 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "thread_local"
 version = "0.3.6"
@@ -1489,11 +1526,28 @@ name = "winapi-i686-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "wincolor"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "wireguard-rs"
 version = "0.1.0"
@@ -1506,6 +1560,7 @@ dependencies = [
  "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures-channel 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1513,6 +1568,7 @@ dependencies = [
  "hjul 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pnet 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1574,6 +1630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5"
 "checksum arraydeque 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ffd3d69bd89910509a5d31d1f1353f38ccffdd116dd0099bbd6627f7bd8ad8"
 "checksum arrayvec 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b8d73f9beda665eaa98ab9e4f7442bd4e7de6652587de55b2525e52e29c1b0ba"
+"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
 "checksum autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b671c8fb71b457dd4ae18c4ba1e59aa81793daacc361d82fcd410cef0d491875"
 "checksum backtrace 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "1371048253fa3bac6704bfd6bbfc922ee9bdcee8881330d40f308b81cc5adc55"
 "checksum backtrace-sys 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "82a830b4ef2d1124a711c71d263c5abdc710ef8e907bd508c88be475cebc422b"
@@ -1601,6 +1658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum curve25519-dalek 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d"
 "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
 "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b"
+"checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
 "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
 "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@@ -1617,6 +1675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
 "checksum hjul 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ea9ffb9dc3e645a3fd2820b8c2f3e1c27f48586678f95cf287d75af018eba577"
 "checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
+"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114"
 "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
 "checksum ipnetwork 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3d862c86f7867f19b693ec86765e0252d82e53d4240b9b629815675a0714ad1"
 "checksum js-sys 0.3.27 (registry+https://github.com/rust-lang/crates.io-index)" = "1efc4f2a556c58e79c5500912e221dd826bec64ff4aabd8ce71ccef6da02d7d4"
@@ -1708,6 +1767,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum syntex_syntax 0.42.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7628a0506e8f9666fdabb5f265d0059b059edac9a3f810bda077abb5d826bd8d"
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1"
+"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
 "checksum tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c501eceaf96f0e1793cf26beb63da3d11c738c4a943fdf3746d81d64684c39f"
@@ -1745,7 +1805,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+"checksum wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96f5016b18804d24db43cebf3c77269e7569b8954a8464501c216cc5e070eaa9"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum x25519-dalek 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7ee1585dc1484373cbc1cee7aafda26634665cf449436fd6e24bfd1fad230538"
 "checksum zerocopy 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "992b9b31f80fd4a167f903f879b8ca43d6716cc368ea01df90538baa2dd34056"

Cargo.toml

@@ -10,6 +10,7 @@ hex = "0.3"
 spin = "0.5.0"
 rand = "0.6.5"
 blake2 = "0.8"
+log = { version = "0.4", features = ["max_level_debug", "release_max_level_info"] }
 hmac = "0.7.1"
 generic-array = "0.12.3"
 zerocopy = "0.2.7"
@@ -28,6 +29,7 @@ aead = "^0.1.1"
 clear_on_drop = "0.2.3"
 parking_lot = "^0.9"
 futures-channel = "^0.2"
+env_logger = "0.6"
 
 [dependencies.x25519-dalek]
 version = "^0.5"

src/router/device.rs

@@ -8,6 +8,8 @@ use std::sync::{Arc, Weak};
 use std::thread;
 use std::time::Instant;
 
+use log::debug;
+
 use spin;
 use treebitmap::IpLookupTable;
 
@@ -84,6 +86,8 @@ impl<C: Callbacks, T: Tun, B: Bind> Drop for Device<C, T, B> {
             }
             _ => false,
         } {}
+
+        debug!("device dropped");
     }
 }
 

src/router/peer.rs

@@ -5,6 +5,8 @@ use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::{Arc, Weak};
 use std::thread;
 
+use log::debug;
+
 use spin::Mutex;
 
 use arraydeque::{ArrayDeque, Saturating, Wrapping};
@@ -54,8 +56,8 @@ pub struct PeerInner<C: Callbacks, T: Tun, B: Bind> {
 
 pub struct Peer<C: Callbacks, T: Tun, B: Bind> {
     state: Arc<PeerInner<C, T, B>>,
-    thread_outbound: thread::JoinHandle<()>,
+    thread_outbound: Option<thread::JoinHandle<()>>,
-    thread_inbound: thread::JoinHandle<()>,
+    thread_inbound: Option<thread::JoinHandle<()>>,
 }
 
 fn treebit_list<A, E, C: Callbacks, T: Tun, B: Bind>(
@@ -109,6 +111,16 @@ impl<C: Callbacks, T: Tun, B: Bind> Drop for Peer<C, T, B> {
         let peer = &self.state;
         peer.stopped.store(true, Ordering::SeqCst);
 
+        // drop channels
+
+        mem::replace(&mut *peer.inbound.lock(), sync_channel(0).0);
+        mem::replace(&mut *peer.outbound.lock(), sync_channel(0).0);
+
+        // join with workers
+
+        mem::replace(&mut self.thread_inbound, None).map(|v| v.join());
+        mem::replace(&mut self.thread_outbound, None).map(|v| v.join());
+
         // remove from cryptkey router
 
         treebit_remove(self, &peer.device.ipv4);
@@ -130,7 +142,7 @@ impl<C: Callbacks, T: Tun, B: Bind> Drop for Peer<C, T, B> {
             }
         }
 
-        // null key-material (TODO: extend)
+        // null key-material
 
         keys.next = None;
         keys.current = None;
@@ -138,6 +150,8 @@ impl<C: Callbacks, T: Tun, B: Bind> Drop for Peer<C, T, B> {
 
         *peer.ekey.lock() = None;
         *peer.endpoint.lock() = None;
+
+        debug!("peer dropped & removed from device");
     }
 }
 
@@ -153,10 +167,10 @@ pub fn new_peer<C: Callbacks, T: Tun, B: Bind>(
         let device = device.clone();
         Arc::new(PeerInner {
             opaque,
+            device,
             inbound: Mutex::new(in_tx),
             outbound: Mutex::new(out_tx),
             stopped: AtomicBool::new(false),
-            device: device,
             ekey: spin::Mutex::new(None),
             endpoint: spin::Mutex::new(None),
             keys: spin::Mutex::new(KeyWheel {
@@ -187,8 +201,8 @@ pub fn new_peer<C: Callbacks, T: Tun, B: Bind>(
 
     Peer {
         state: peer,
-        thread_inbound,
+        thread_inbound: Some(thread_inbound),
-        thread_outbound,
+        thread_outbound: Some(thread_outbound),
     }
 }
 
@@ -212,21 +226,22 @@ impl<C: Callbacks, T: Tun, B: Bind> PeerInner<C, T, B> {
         let key = match self.ekey.lock().as_mut() {
             None => {
                 // add to staged packets (create no job)
+                debug!("execute callback: call_need_key");
                 (self.device.call_need_key)(&self.opaque);
                 self.staged_packets.lock().push_back(msg);
                 return None;
             }
             Some(mut state) => {
-                // allocate nonce
+                // avoid integer overflow in nonce
-                state.nonce += 1;
+                if state.nonce >= REJECT_AFTER_MESSAGES - 1 {
-                if state.nonce >= REJECT_AFTER_MESSAGES {
-                    state.nonce -= 1;
                     return None;
                 }
+                debug!("encryption state available, nonce = {}", state.nonce);
 
                 // set transport message fields
                 header.f_counter.set(state.nonce);
                 header.f_receiver.set(state.id);
+                state.nonce += 1;
                 state.key
             }
         };

src/router/tests.rs

@@ -1,5 +1,6 @@
 use std::error::Error;
 use std::fmt;
+use std::io;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
@@ -121,110 +122,141 @@ fn dummy_keypair(initiator: bool) -> KeyPair {
     }
 }
 
-#[test]
+#[cfg(test)]
-fn test_outbound() {
+mod tests {
-    // type for tracking events inside the router module
+    use super::*;
-    struct Flags {
+    use env_logger;
-        send: AtomicBool,
+
-        recv: AtomicBool,
+    fn init() {
-        need_key: AtomicBool,
+        let _ = env_logger::builder().is_test(true).try_init();
     }
 
-    type Opaque = Arc<Flags>;
+    #[test]
+    fn test_outbound() {
+        init();
 
-    let opaque = Arc::new(Flags {
+        // type for tracking events inside the router module
-        send: AtomicBool::new(false),
+        struct Flags {
-        recv: AtomicBool::new(false),
+            send: AtomicBool,
-        need_key: AtomicBool::new(false),
+            recv: AtomicBool,
-    });
+            need_key: AtomicBool,
-
-    // create device
-    let workers = 4;
-    let router = Device::new(
-        workers,
-        TunTest {},
-        BindTest {},
-        |t: &Opaque, data: bool, sent: bool| t.send.store(true, Ordering::SeqCst),
-        |t: &Opaque, data: bool, sent: bool| t.recv.store(true, Ordering::SeqCst),
-        |t: &Opaque| t.need_key.store(true, Ordering::SeqCst),
-    );
-
-    // create peer
-    let peer = router.new_peer(opaque.clone());
-    let tests = vec![
-        ("192.168.1.0", 24, "192.168.1.20", true),
-        ("172.133.133.133", 32, "172.133.133.133", true),
-        ("172.133.133.133", 32, "172.133.133.132", false),
-        (
-            "2001:db8::ff00:42:0000",
-            112,
-            "2001:db8::ff00:42:3242",
-            true,
-        ),
-        (
-            "2001:db8::ff00:42:8000",
-            113,
-            "2001:db8::ff00:42:0660",
-            false,
-        ),
-        (
-            "2001:db8::ff00:42:8000",
-            113,
-            "2001:db8::ff00:42:ffff",
-            true,
-        ),
-    ];
-
-    peer.add_keypair(dummy_keypair(true));
-
-    for (mask, len, ip, okay) in &tests {
-        opaque.send.store(false, Ordering::SeqCst);
-        opaque.recv.store(false, Ordering::SeqCst);
-        opaque.need_key.store(false, Ordering::SeqCst);
-
-        let mask: IpAddr = mask.parse().unwrap();
-
-        // map subnet to peer
-        peer.add_subnet(mask, *len);
-
-        // create "IP packet"
-        let mut msg = Vec::<u8>::new();
-        msg.resize(SIZE_MESSAGE_PREFIX + 1024, 0);
-        if mask.is_ipv4() {
-            let mut packet = MutableIpv4Packet::new(&mut msg[SIZE_MESSAGE_PREFIX..]).unwrap();
-            packet.set_destination(ip.parse().unwrap());
-            packet.set_version(4);
-        } else {
-            let mut packet = MutableIpv6Packet::new(&mut msg[SIZE_MESSAGE_PREFIX..]).unwrap();
-            packet.set_destination(ip.parse().unwrap());
-            packet.set_version(6);
         }
 
-        // cryptkey route the IP packet
+        type Opaque = Arc<Flags>;
-        let res = router.send(msg);
 
-        // allow some scheduling
+        // create device
-        thread::sleep(Duration::from_millis(1));
+        let workers = 4;
+        let router = Device::new(
+            workers,
+            TunTest {},
+            BindTest {},
+            |t: &Opaque, _data: bool, _sent: bool| t.send.store(true, Ordering::SeqCst),
+            |t: &Opaque, _data: bool, _sent: bool| t.recv.store(true, Ordering::SeqCst),
+            |t: &Opaque| t.need_key.store(true, Ordering::SeqCst),
+        );
 
-        if *okay {
+        // create peer
-            // cryptkey routing succeeded
+        let tests = vec![
-            assert!(res.is_ok());
+            ("192.168.1.0", 24, "192.168.1.20", true),
+            ("172.133.133.133", 32, "172.133.133.133", true),
+            ("172.133.133.133", 32, "172.133.133.132", false),
+            (
+                "2001:db8::ff00:42:0000",
+                112,
+                "2001:db8::ff00:42:3242",
+                true,
+            ),
+            (
+                "2001:db8::ff00:42:8000",
+                113,
+                "2001:db8::ff00:42:0660",
+                false,
+            ),
+            (
+                "2001:db8::ff00:42:8000",
+                113,
+                "2001:db8::ff00:42:ffff",
+                true,
+            ),
+        ];
 
-            // attempted to send message
+        for (num, (mask, len, ip, okay)) in tests.iter().enumerate() {
-            assert_eq!(opaque.need_key.load(Ordering::Acquire), false);
+            for set_key in vec![true, false] {
-            assert_eq!(opaque.send.load(Ordering::Acquire), true);
+                // add new peer
-            assert_eq!(opaque.recv.load(Ordering::Acquire), false);
+                let opaque = Arc::new(Flags {
-        } else {
+                    send: AtomicBool::new(false),
-            // no such cryptkey route
+                    recv: AtomicBool::new(false),
-            assert!(res.is_err());
+                    need_key: AtomicBool::new(false),
+                });
+                let peer = router.new_peer(opaque.clone());
+                let mask: IpAddr = mask.parse().unwrap();
 
-            // did not attempt to send message
+                if set_key {
-            assert_eq!(opaque.need_key.load(Ordering::Acquire), false);
+                    peer.add_keypair(dummy_keypair(true));
-            assert_eq!(opaque.send.load(Ordering::Acquire), false);
+                }
-            assert_eq!(opaque.recv.load(Ordering::Acquire), false);
+
+                // map subnet to peer
+                peer.add_subnet(mask, *len);
+
+                // create "IP packet"
+                let mut msg = Vec::<u8>::new();
+                msg.resize(SIZE_MESSAGE_PREFIX + 1024, 0);
+                if mask.is_ipv4() {
+                    let mut packet =
+                        MutableIpv4Packet::new(&mut msg[SIZE_MESSAGE_PREFIX..]).unwrap();
+                    packet.set_destination(ip.parse().unwrap());
+                    packet.set_version(4);
+                } else {
+                    let mut packet =
+                        MutableIpv6Packet::new(&mut msg[SIZE_MESSAGE_PREFIX..]).unwrap();
+                    packet.set_destination(ip.parse().unwrap());
+                    packet.set_version(6);
+                }
+
+                // cryptkey route the IP packet
+                let res = router.send(msg);
+
+                // allow some scheduling
+                thread::sleep(Duration::from_millis(20));
+
+                if *okay {
+                    // cryptkey routing succeeded
+                    assert!(res.is_ok(), "crypt-key routing should succeed");
+                    assert_eq!(
+                        opaque.need_key.load(Ordering::Acquire),
+                        !set_key,
+                        "should have requested a new key, if no encryption state was set"
+                    );
+                    assert_eq!(
+                        opaque.send.load(Ordering::Acquire),
+                        set_key,
+                        "transmission should have been attempted"
+                    );
+                    assert_eq!(
+                        opaque.recv.load(Ordering::Acquire),
+                        false,
+                        "no messages should have been marked as received"
+                    );
+                } else {
+                    // no such cryptkey route
+                    assert!(res.is_err(), "crypt-key routing should fail");
+                    assert_eq!(
+                        opaque.need_key.load(Ordering::Acquire),
+                        false,
+                        "should not request a new-key if crypt-key routing failed"
+                    );
+                    assert_eq!(
+                        opaque.send.load(Ordering::Acquire),
+                        false,
+                        "transmission should not have been attempted",
+                    );
+                    assert_eq!(
+                        opaque.recv.load(Ordering::Acquire),
+                        false,
+                        "no messages should have been marked as received",
+                    );
+                }
+            }
         }
-
-        // clear subnets for next test
-        peer.remove_subnets();
     }
 }

src/router/workers.rs

@@ -1,16 +1,12 @@
-use std::iter;
 use std::mem;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::Receiver;
-use std::sync::mpsc::{sync_channel, Receiver, TryRecvError};
 use std::sync::{Arc, Weak};
-use std::thread;
 
 use futures::sync::oneshot;
 use futures::*;
 
-use spin;
+use log::debug;
 
-use crossbeam_deque::{Injector, Steal, Stealer, Worker};
 use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, CHACHA20_POLY1305};
 use zerocopy::{AsBytes, LayoutVerified};
 
@@ -174,12 +170,16 @@ pub fn worker_parallel(receiver: Receiver<JobParallel>) {
 
         match buf.op {
             Operation::Encryption => {
+                debug!("worker, process encryption");
+
                 // note: extends the vector to accommodate the tag
                 key.seal_in_place_append_tag(nonce, Aad::empty(), &mut buf.msg)
                     .unwrap();
                 buf.okay = true;
             }
             Operation::Decryption => {
+                debug!("worker, process decryption");
+
                 // opening failure is signaled by fault state
                 buf.okay = match key.open_in_place(nonce, Aad::empty(), &mut buf.msg) {
                     Ok(_) => true,