Fix: dns lookups for database (#5998)

* Add new JavaScript framework: Just-JS

* fix bug in cached world test

* remove dependency on openssl for quicker build

* fix dns for full run

* make dns more robust

frameworks/JavaScript/just/just.dockerfile

@@ -1,13 +1,13 @@
 FROM debian:stretch-slim AS builder
 RUN apt update
 RUN apt install -y g++ curl make tar gzip libfindbin-libs-perl
-RUN curl -L -o 0.0.1.tar.gz -L https://github.com/just-js/just/archive/0.0.1.tar.gz
-RUN tar -zxvf 0.0.1.tar.gz
-WORKDIR /just-0.0.1
+RUN curl -L -o 0.0.2.tar.gz -L https://github.com/just-js/just/archive/0.0.2.tar.gz
+RUN tar -zxvf 0.0.2.tar.gz
+WORKDIR /just-0.0.2
 RUN make runtime
-RUN curl -L -o modules.tar.gz https://github.com/just-js/modules/archive/0.0.1.tar.gz
+RUN curl -L -o modules.tar.gz https://github.com/just-js/modules/archive/0.0.2.tar.gz
 RUN tar -zxvf modules.tar.gz
-RUN mv modules-0.0.1 modules
+RUN mv modules-0.0.2 modules
 RUN JUST_HOME=$(pwd) make -C modules/picohttp/ deps http.so
 RUN JUST_HOME=$(pwd) make -C modules/html/ html.so
 
@@ -16,9 +16,9 @@ WORKDIR /app
 RUN mkdir -p /app/lib
 COPY lib/stringify.js lib/connection.js lib/dns.js lib/http.js lib/lookup.js lib/pg.js lib/stats.js lib/tcp.js lib/md5.js ./lib/
 COPY techempower.js spawn.js ./
-COPY --from=builder /just-0.0.1/just ./
-COPY --from=builder /just-0.0.1/modules/picohttp/http.so ./
-COPY --from=builder /just-0.0.1/modules/html/html.so ./
+COPY --from=builder /just-0.0.2/just ./
+COPY --from=builder /just-0.0.2/modules/picohttp/http.so ./
+COPY --from=builder /just-0.0.2/modules/html/html.so ./
 ENV LD_LIBRARY_PATH=/app
 ENV PGPOOL=1
 CMD ["./just", "spawn.js", "techempower.js"]

frameworks/JavaScript/just/lib/connection.js

@@ -329,7 +329,7 @@ function setupSocket (sock, config) {
   sock.compile = compile
   sock.onData = bytes => parser.parse(bytes)
   sock.onClose = () => {
-    just.print('pg socket closed')
+    just.error('pg socket closed')
   }
   sock.getParams = () => parser.parameters
   sock.size = () => callbacks.length
@@ -339,12 +339,16 @@ function setupSocket (sock, config) {
 }
 
 function connect (config, onPGConnect) {
-  lookup(config.hostname, record => {
-    const { message } = record
-    const { ip } = message.answer[0]
-    config.address = `${ip[0]}.${ip[1]}.${ip[2]}.${ip[3]}`
+  lookup(config.hostname, (err, ip) => {
+    if (err) {
+      onPGConnect(err)
+      return
+    }
+    config.address = ip
     const sock = createClient(config.address, config.port)
-    sock.onClose = () => {}
+    sock.onClose = () => {
+      just.error('pg socket closed')
+    }
     sock.onConnect = err => {
       onPGConnect(err, setupSocket(sock, config))
       return sock.buffer

frameworks/JavaScript/just/lib/lookup.js

@@ -1,23 +1,122 @@
 const { create, parse } = require('dns.js')
 const { udp, net } = just
 const { loop } = just.factory
+const { readFile, isFile } = require('fs')
 
 const dnsServer = just.env().DNS_SERVER || '127.0.0.11'
 
+function parseLine (line) {
+  const parts = line.split(/\s+/)
+  const [address, ...hosts] = parts
+  return { address, hosts }
+}
+
+const rxipv4 = /\d+\.\d+\.\d+\.\d+/
+const rxComment = /(\s+)?#.+/
+const rxName = /nameserver\s+(.+)/
+
+function readHosts () {
+  const ipv4 = {}
+  const ipv6 = {}
+  const fileName = '/etc/hosts'
+  if (!isFile(fileName)) {
+    just.error(`${fileName} not found`)
+    return { ipv4, ipv6 }
+  }
+  const hosts = readFile(fileName)
+  just.error(`${fileName}:\n${hosts}`)
+  const lines = hosts.split('\n').filter(line => line.trim())
+  for (const line of lines) {
+    if (line.match(rxComment)) continue
+    const { address, hosts } = parseLine(line)
+    if (address.match(rxipv4)) {
+      for (const host of hosts) {
+        ipv4[host] = address
+      }
+    } else {
+      for (const host of hosts) {
+        ipv6[host] = address
+      }
+    }
+  }
+  return { ipv4, ipv6 }
+}
+
+function lookupHosts (hostname) {
+  const { ipv4 } = readHosts()
+  return ipv4[hostname]
+}
+
+function readResolv () {
+  const fileName = '/etc/resolv.conf'
+  const results = []
+  if (!isFile(fileName)) {
+    just.error(`${fileName} not found`)
+    return results
+  }
+  const resolv = readFile(fileName)
+  just.error(`${fileName}:\n${resolv}`)
+  const lines = resolv.split('\n').filter(line => line.trim())
+  for (const line of lines) {
+    const match = line.match(rxName)
+    if (match && match.length > 1) {
+      const [, ip] = match
+      if (ip.match(rxipv4)) {
+        results.push(ip)
+      }
+    }
+  }
+  return results
+}
+
 function lookup (query = 'www.google.com', onRecord = () => {}, address = dnsServer, port = 53, buf = new ArrayBuffer(65536)) {
+  const ip = lookupHosts(query)
+  if (ip) {
+    just.error(`found ${ip} for ${query} in /etc/hosts`)
+    onRecord(null, ip)
+    return
+  }
+  const ips = readResolv()
+  if (ips.length) {
+    address = ips[0]
+    just.error(`dns server ${address} found in /etc/resolv.conf`)
+  }
   const fd = net.socket(net.AF_INET, net.SOCK_DGRAM | net.SOCK_NONBLOCK, 0)
   net.bind(fd, address, port)
   loop.add(fd, (fd, event) => {
+    just.clearTimeout(timer)
     const answer = []
     const len = udp.recvmsg(fd, buf, answer)
-    const [address, port] = answer
-    const message = { length: len, address, port, message: parse(buf, len) }
+    if (len <= 0) {
+      onRecord(new Error('Bad Message Length'))
+      return
+    }
+    const message = parse(buf, len)
+    if (!message.answer.length) {
+      onRecord(new Error(`Address Not Found for ${query}`))
+      return
+    }
+    const { ip } = message.answer[0]
+    const result = `${ip[0]}.${ip[1]}.${ip[2]}.${ip[3]}`
+    just.error(`got ${result} for ${query} from ${address}`)
     loop.remove(fd)
     net.close(fd)
-    onRecord(message)
+    onRecord(null, result)
   })
   const len = create(query, buf, 1)
-  udp.sendmsg(fd, buf, address, port, len)
+  const rc = udp.sendmsg(fd, buf, address, port, len)
+  if (rc === -1) {
+    const errno = just.sys.errno()
+    onRecord(new Error(`Error sending ${query} to ${address}: ${just.sys.strerror(errno)} (${errno})`))
+    loop.remove(fd)
+    net.close(fd)
+    return
+  }
+  const timer = just.setTimeout(() => {
+    onRecord(new Error(`Request timed out for ${query} at ${address}`))
+    loop.remove(fd)
+    net.close(fd)
+  }, 1000)
 }
 
 module.exports = { lookup }

frameworks/JavaScript/just/spawn.js

@@ -16,7 +16,7 @@ function onTimer () {
   for (const thread of threads) {
     total += Atomics.load(thread.u32, 0)
   }
-  just.print(`threads ${threads.length} total ${total} mem ${rss} cpu (${user.toFixed(2)}/${system.toFixed(2)}) ${(user + system).toFixed(2)} qps/core ${(total / (user + system)).toFixed(2)}`)
+  just.error(`threads ${threads.length} total ${total} mem ${rss} cpu (${user.toFixed(2)}/${system.toFixed(2)}) ${(user + system).toFixed(2)} qps/core ${(total / (user + system)).toFixed(2)}`)
 }
 
 const source = readFile(just.args[2] || 'test.js')

frameworks/JavaScript/just/techempower.js

@@ -161,7 +161,10 @@ function onPGConnect (err, sock) {
     just.setTimeout(() => connect(tfb, onPGConnect), 1000)
     return
   }
-  sock.onClose = () => just.print('pg.close')
+  sock.onClose = () => {
+    // todo: remove from pool and reconnect?
+    just.error('pg.close')
+  }
   sock.start(err => {
     if (err) return just.error(err.stack)
     sock.authenticate(err => {
@@ -425,7 +428,6 @@ const stats = start()
 let i = poolSize
 const sJSON = sjs({ message: attr('string') })
 const sDB = sjs({ id: attr('number'), randomNumber: attr('number') })
-const sQUERY = sjs(attr('array', sjs({ id: attr('number'), randomNumber: attr('number') })))
 while (i--) connect(tfb, onPGConnect)
 const { loop } = just.factory
 let microtasks = true
@@ -440,6 +442,6 @@ just.setInterval(() => {
   r404 = `HTTP/1.1 404 Not Found\r\nServer: j\r\nDate: ${stats.time}\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n`
 }, 100)
 while (1) {
-  loop.poll(10)
+  loop.poll(-1)
   if (microtasks) just.sys.runMicroTasks()
 }