outside.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537
  1. package nebula
  2. import (
  3. "encoding/binary"
  4. "errors"
  5. "fmt"
  6. "net/netip"
  7. "time"
  8. "github.com/google/gopacket/layers"
  9. "golang.org/x/net/ipv6"
  10. "github.com/sirupsen/logrus"
  11. "github.com/slackhq/nebula/firewall"
  12. "github.com/slackhq/nebula/header"
  13. "golang.org/x/net/ipv4"
  14. )
  15. const (
  16. minFwPacketLen = 4
  17. )
  18. func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
  19. err := h.Parse(packet)
  20. if err != nil {
  21. // Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
  22. if len(packet) > 1 {
  23. f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", ip, err)
  24. }
  25. return
  26. }
  27. //l.Error("in packet ", header, packet[HeaderLen:])
  28. if ip.IsValid() {
  29. _, found := f.myVpnNetworksTable.Lookup(ip.Addr())
  30. if found {
  31. if f.l.Level >= logrus.DebugLevel {
  32. f.l.WithField("udpAddr", ip).Debug("Refusing to process double encrypted packet")
  33. }
  34. return
  35. }
  36. }
  37. var hostinfo *HostInfo
  38. // verify if we've seen this index before, otherwise respond to the handshake initiation
  39. if h.Type == header.Message && h.Subtype == header.MessageRelay {
  40. hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
  41. } else {
  42. hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
  43. }
  44. var ci *ConnectionState
  45. if hostinfo != nil {
  46. ci = hostinfo.ConnectionState
  47. }
  48. switch h.Type {
  49. case header.Message:
  50. // TODO handleEncrypted sends directly to addr on error. Handle this in the tunneling case.
  51. if !f.handleEncrypted(ci, ip, h) {
  52. return
  53. }
  54. switch h.Subtype {
  55. case header.MessageNone:
  56. if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
  57. return
  58. }
  59. case header.MessageRelay:
  60. // The entire body is sent as AD, not encrypted.
  61. // The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
  62. // The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
  63. // otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
  64. // which will gracefully fail in the DecryptDanger call.
  65. signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
  66. signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
  67. out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
  68. if err != nil {
  69. return
  70. }
  71. // Successfully validated the thing. Get rid of the Relay header.
  72. signedPayload = signedPayload[header.Len:]
  73. // Pull the Roaming parts up here, and return in all call paths.
  74. f.handleHostRoaming(hostinfo, ip)
  75. // Track usage of both the HostInfo and the Relay for the received & authenticated packet
  76. f.connectionManager.In(hostinfo.localIndexId)
  77. f.connectionManager.RelayUsed(h.RemoteIndex)
  78. relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
  79. if !ok {
  80. // The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
  81. // its internal mapping. This should never happen.
  82. hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
  83. return
  84. }
  85. switch relay.Type {
  86. case TerminalType:
  87. // If I am the target of this relay, process the unwrapped packet
  88. // From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
  89. f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
  90. return
  91. case ForwardingType:
  92. // Find the target HostInfo relay object
  93. targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
  94. if err != nil {
  95. hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
  96. return
  97. }
  98. // If that relay is Established, forward the payload through it
  99. if targetRelay.State == Established {
  100. switch targetRelay.Type {
  101. case ForwardingType:
  102. // Forward this packet through the relay tunnel
  103. // Find the target HostInfo
  104. f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
  105. return
  106. case TerminalType:
  107. hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
  108. }
  109. } else {
  110. hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
  111. return
  112. }
  113. }
  114. }
  115. case header.LightHouse:
  116. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  117. if !f.handleEncrypted(ci, ip, h) {
  118. return
  119. }
  120. d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
  121. if err != nil {
  122. hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
  123. WithField("packet", packet).
  124. Error("Failed to decrypt lighthouse packet")
  125. return
  126. }
  127. lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
  128. // Fallthrough to the bottom to record incoming traffic
  129. case header.Test:
  130. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  131. if !f.handleEncrypted(ci, ip, h) {
  132. return
  133. }
  134. d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
  135. if err != nil {
  136. hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
  137. WithField("packet", packet).
  138. Error("Failed to decrypt test packet")
  139. return
  140. }
  141. if h.Subtype == header.TestRequest {
  142. // This testRequest might be from TryPromoteBest, so we should roam
  143. // to the new IP address before responding
  144. f.handleHostRoaming(hostinfo, ip)
  145. f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
  146. }
  147. // Fallthrough to the bottom to record incoming traffic
  148. // Non encrypted messages below here, they should not fall through to avoid tracking incoming traffic since they
  149. // are unauthenticated
  150. case header.Handshake:
  151. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  152. f.handshakeManager.HandleIncoming(ip, via, packet, h)
  153. return
  154. case header.RecvError:
  155. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  156. f.handleRecvError(ip, h)
  157. return
  158. case header.CloseTunnel:
  159. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  160. if !f.handleEncrypted(ci, ip, h) {
  161. return
  162. }
  163. hostinfo.logger(f.l).WithField("udpAddr", ip).
  164. Info("Close tunnel received, tearing down.")
  165. f.closeTunnel(hostinfo)
  166. return
  167. case header.Control:
  168. if !f.handleEncrypted(ci, ip, h) {
  169. return
  170. }
  171. d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
  172. if err != nil {
  173. hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
  174. WithField("packet", packet).
  175. Error("Failed to decrypt Control packet")
  176. return
  177. }
  178. f.relayManager.HandleControlMsg(hostinfo, d, f)
  179. default:
  180. f.messageMetrics.Rx(h.Type, h.Subtype, 1)
  181. hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", ip)
  182. return
  183. }
  184. f.handleHostRoaming(hostinfo, ip)
  185. f.connectionManager.In(hostinfo.localIndexId)
  186. }
  187. // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
  188. func (f *Interface) closeTunnel(hostInfo *HostInfo) {
  189. final := f.hostMap.DeleteHostInfo(hostInfo)
  190. if final {
  191. // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
  192. f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
  193. }
  194. }
  195. // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
  196. func (f *Interface) sendCloseTunnel(h *HostInfo) {
  197. f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
  198. }
  199. func (f *Interface) handleHostRoaming(hostinfo *HostInfo, vpnAddr netip.AddrPort) {
  200. if vpnAddr.IsValid() && hostinfo.remote != vpnAddr {
  201. //TODO: this is weird now that we can have multiple vpn addrs
  202. if !f.lightHouse.GetRemoteAllowList().Allow(hostinfo.vpnAddrs[0], vpnAddr.Addr()) {
  203. hostinfo.logger(f.l).WithField("newAddr", vpnAddr).Debug("lighthouse.remote_allow_list denied roaming")
  204. return
  205. }
  206. if !hostinfo.lastRoam.IsZero() && vpnAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
  207. if f.l.Level >= logrus.DebugLevel {
  208. hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", vpnAddr).
  209. Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
  210. }
  211. return
  212. }
  213. hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", vpnAddr).
  214. Info("Host roamed to new udp ip/port.")
  215. hostinfo.lastRoam = time.Now()
  216. hostinfo.lastRoamRemote = hostinfo.remote
  217. hostinfo.SetRemote(vpnAddr)
  218. }
  219. }
  220. func (f *Interface) handleEncrypted(ci *ConnectionState, addr netip.AddrPort, h *header.H) bool {
  221. // If connectionstate exists and the replay protector allows, process packet
  222. // Else, send recv errors for 300 seconds after a restart to allow fast reconnection.
  223. if ci == nil || !ci.window.Check(f.l, h.MessageCounter) {
  224. if addr.IsValid() {
  225. f.maybeSendRecvError(addr, h.RemoteIndex)
  226. return false
  227. } else {
  228. return false
  229. }
  230. }
  231. return true
  232. }
  233. // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
  234. func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
  235. if len(data) < 1 {
  236. return errors.New("packet too short")
  237. }
  238. version := int((data[0] >> 4) & 0x0f)
  239. switch version {
  240. case ipv4.Version:
  241. return parseV4(data, incoming, fp)
  242. case ipv6.Version:
  243. return parseV6(data, incoming, fp)
  244. }
  245. return fmt.Errorf("packet is an unknown ip version: %v", version)
  246. }
  247. func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
  248. dataLen := len(data)
  249. if dataLen < ipv6.HeaderLen {
  250. return fmt.Errorf("ipv6 packet is less than %v bytes", ipv4.HeaderLen)
  251. }
  252. if incoming {
  253. fp.RemoteAddr, _ = netip.AddrFromSlice(data[8:24])
  254. fp.LocalAddr, _ = netip.AddrFromSlice(data[24:40])
  255. } else {
  256. fp.LocalAddr, _ = netip.AddrFromSlice(data[8:24])
  257. fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40])
  258. }
  259. //TODO: whats a reasonable number of extension headers to attempt to parse?
  260. //https://www.ietf.org/archive/id/draft-ietf-6man-eh-limits-00.html
  261. protoAt := 6
  262. offset := 40
  263. for i := 0; i < 24; i++ {
  264. if dataLen < offset {
  265. break
  266. }
  267. proto := layers.IPProtocol(data[protoAt])
  268. //fmt.Println(proto, protoAt)
  269. switch proto {
  270. case layers.IPProtocolICMPv6:
  271. fp.Protocol = uint8(proto)
  272. fp.RemotePort = 0
  273. fp.LocalPort = 0
  274. fp.Fragment = false
  275. return nil
  276. case layers.IPProtocolTCP:
  277. if dataLen < offset+4 {
  278. return fmt.Errorf("ipv6 packet was too small")
  279. }
  280. fp.Protocol = uint8(proto)
  281. if incoming {
  282. fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
  283. fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
  284. } else {
  285. fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
  286. fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
  287. }
  288. fp.Fragment = false
  289. return nil
  290. case layers.IPProtocolUDP:
  291. if dataLen < offset+4 {
  292. return fmt.Errorf("ipv6 packet was too small")
  293. }
  294. fp.Protocol = uint8(proto)
  295. if incoming {
  296. fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
  297. fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
  298. } else {
  299. fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
  300. fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
  301. }
  302. fp.Fragment = false
  303. return nil
  304. case layers.IPProtocolIPv6Fragment:
  305. //TODO: can we determine the protocol?
  306. fp.RemotePort = 0
  307. fp.LocalPort = 0
  308. fp.Fragment = true
  309. return nil
  310. default:
  311. if dataLen < offset+1 {
  312. break
  313. }
  314. next := int(data[offset+1]) * 8
  315. if next == 0 {
  316. // each extension is at least 8 bytes
  317. next = 8
  318. }
  319. protoAt = offset
  320. offset = offset + next
  321. }
  322. }
  323. return fmt.Errorf("could not find payload in ipv6 packet")
  324. }
  325. func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
  326. // Do we at least have an ipv4 header worth of data?
  327. if len(data) < ipv4.HeaderLen {
  328. return fmt.Errorf("ipv4 packet is less than %v bytes", ipv4.HeaderLen)
  329. }
  330. // Adjust our start position based on the advertised ip header length
  331. ihl := int(data[0]&0x0f) << 2
  332. // Well formed ip header length?
  333. if ihl < ipv4.HeaderLen {
  334. return fmt.Errorf("ipv4 packet had an invalid header length: %v", ihl)
  335. }
  336. // Check if this is the second or further fragment of a fragmented packet.
  337. flagsfrags := binary.BigEndian.Uint16(data[6:8])
  338. fp.Fragment = (flagsfrags & 0x1FFF) != 0
  339. // Firewall handles protocol checks
  340. fp.Protocol = data[9]
  341. // Accounting for a variable header length, do we have enough data for our src/dst tuples?
  342. minLen := ihl
  343. if !fp.Fragment && fp.Protocol != firewall.ProtoICMP {
  344. minLen += minFwPacketLen
  345. }
  346. if len(data) < minLen {
  347. return fmt.Errorf("ipv4 packet is less than %v bytes, ip header len: %v", minLen, ihl)
  348. }
  349. // Firewall packets are locally oriented
  350. if incoming {
  351. fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16])
  352. fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20])
  353. if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
  354. fp.RemotePort = 0
  355. fp.LocalPort = 0
  356. } else {
  357. fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])
  358. fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
  359. }
  360. } else {
  361. fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16])
  362. fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
  363. if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
  364. fp.RemotePort = 0
  365. fp.LocalPort = 0
  366. } else {
  367. fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])
  368. fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
  369. }
  370. }
  371. return nil
  372. }
  373. func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
  374. var err error
  375. out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
  376. if err != nil {
  377. return nil, err
  378. }
  379. if !hostinfo.ConnectionState.window.Update(f.l, mc) {
  380. hostinfo.logger(f.l).WithField("header", h).
  381. Debugln("dropping out of window packet")
  382. return nil, errors.New("out of window packet")
  383. }
  384. return out, nil
  385. }
  386. func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
  387. var err error
  388. out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
  389. if err != nil {
  390. hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
  391. return false
  392. }
  393. err = newPacket(out, true, fwPacket)
  394. if err != nil {
  395. hostinfo.logger(f.l).WithError(err).WithField("packet", out).
  396. Warnf("Error while validating inbound packet")
  397. return false
  398. }
  399. if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
  400. hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
  401. Debugln("dropping out of window packet")
  402. return false
  403. }
  404. dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
  405. if dropReason != nil {
  406. // NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
  407. // This gives us a buffer to build the reject packet in
  408. f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
  409. if f.l.Level >= logrus.DebugLevel {
  410. hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
  411. WithField("reason", dropReason).
  412. Debugln("dropping inbound packet")
  413. }
  414. return false
  415. }
  416. f.connectionManager.In(hostinfo.localIndexId)
  417. _, err = f.readers[q].Write(out)
  418. if err != nil {
  419. f.l.WithError(err).Error("Failed to write to tun")
  420. }
  421. return true
  422. }
  423. func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
  424. if f.sendRecvErrorConfig.ShouldSendRecvError(endpoint) {
  425. f.sendRecvError(endpoint, index)
  426. }
  427. }
  428. func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) {
  429. f.messageMetrics.Tx(header.RecvError, 0, 1)
  430. b := header.Encode(make([]byte, header.Len), header.Version, header.RecvError, 0, index, 0)
  431. f.outside.WriteTo(b, endpoint)
  432. if f.l.Level >= logrus.DebugLevel {
  433. f.l.WithField("index", index).
  434. WithField("udpAddr", endpoint).
  435. Debug("Recv error sent")
  436. }
  437. }
  438. func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
  439. if f.l.Level >= logrus.DebugLevel {
  440. f.l.WithField("index", h.RemoteIndex).
  441. WithField("udpAddr", addr).
  442. Debug("Recv error received")
  443. }
  444. hostinfo := f.hostMap.QueryReverseIndex(h.RemoteIndex)
  445. if hostinfo == nil {
  446. f.l.WithField("remoteIndex", h.RemoteIndex).Debugln("Did not find remote index in main hostmap")
  447. return
  448. }
  449. if !hostinfo.RecvErrorExceeded() {
  450. return
  451. }
  452. if hostinfo.remote.IsValid() && hostinfo.remote != addr {
  453. f.l.Infoln("Someone spoofing recv_errors? ", addr, hostinfo.remote)
  454. return
  455. }
  456. f.closeTunnel(hostinfo)
  457. // We also delete it from pending hostmap to allow for fast reconnect.
  458. f.handshakeManager.DeleteHostInfo(hostinfo)
  459. }