Graduate Program KB

Matthew's Digging Into Node Notes

Table of Contents

Command Line Scripts

When creating command line scripts, we first set the top line in program, to allow it to utilise node (see below).

#! /usr/bin/env node
'use strict'

var args = require('minimist')(process.argv.slice(2), {
  boolean: ['help'],
  string: ['file'],
})

What this does, is take the first two arguments provided when the file is executed and place them into a variable. Additionally it sets some precedents for these arguments. The first one is a boolean (ie is it present or not, true if given, false if omitted). The second being a string of type file, so that the user knows what it should be.

Useful to Know:

  • Minimist: A library for processing arguments. It has no dependencies, but almost every other package depends on it. It returns an object with structure.
  • process.argv contains all arguments passed to a script
  • __dirname is a variable which contains the absolute path of the currently executing script
  • process.cwd() returns pwd

At the end of the exercise, our file looks like this:

bin/env node
"use strict";

var fs = require("fs");
var path = require("path");

var getStdin = require("get-stdin");

var args = require("minimist")(process.argv.slice(2), {
  boolean: ["help", "in"],
  string: ["file"],
});

var BASE_PATH = path.resolve(process.env.BASE_PATH || __dirname);

if (process.env.HELLO) {
  console.log(process.env.HELLO);
}

if (args.help) {
  printHelp();
} else if (args.in || args._.includes("-")) {
  getStdin().then(processFile).catch(error);
} else if (args.file) {
  fs.readFile(
    path.join(BASE_PATH, args.file),
    function onContents(err, contents) {
      if (err) {
        error(err.toString);
      } else {
        processFile(contents.toString());
      }
    }
  );
} else {
  error("Incorrect usage", true);
}

function processFile(contents) {
  contents = contents.toUpperCase();
  process.stdout.write(contents);
}
function error(msg, includeHelp = false) {
  console.error(msg);
  if (includeHelp) {
    console.log("");
    printHelp();
  }
}

function printHelp() {
  console.log("Hello! This prints when help is required");
}

Streams

Streams are unidirectional. They can be either readable or writable but not both. Duplex streams are those which are both readable and writable.

let path = require('path')
let fs = require('fs')

let readStream = fs.createReadStream(filePath)
let writeStream = fs.createWriteStream(filePath)

let outStream = readStream
outStream.pipe(process.stdout) // Writes input to stdout
  • Transform allows us to intercept the pipe and process items/chunks one at a time
    • In this example, chunk is a buffer from the stream
    • The callback function is used to notify when the transform is complete
    • We are converting chunks to string and then making them uppercase
let Transform = require('stream').Transform

let upperStream = new Transform({
  transform(chunk, encoding, callback) {
    this.push(chunk.toString().toUpperCase())
    callback()
  },
})

let outStream = outStream.pipe(upperStream)
outStream.pipe(process.stdout)
  • Zipping and Unzipping
    • We can create zipping and unzipping streams
    • We change the file type of the compressed file to .gz as this is the normal convention
let zlib = require('zlib')

let gzipStream = zlib.createGzip()
let outStream = outStream.pipe(gzipStream)
outputFile = `{outputFile}.gz`

let gunzipStream = zlib.createGunzip()
let outStream = outStream.pipe(gunzipStream)
  • To determine the end of a stream, use a promise
function streamComplete(stream) {
  return new Promise(function callback(response) {
    stream.on('end', response)
  })
}

outStream.pipe(process.stdout)
await streamComplete(outStream)

Database

For this demonstration, sqlite3 was used. This type can be maintained directly from the program itself.

let sqlite3 = require('sqlite3')
let path = require('path')

const DB_PATH = path.join(__dirname, 'my.db')
const DB_SQL_PATH = path.join(__dirname, 'mydb.sql')

var myDB = new sqlite3.Database(DB_PATH)
SQL3 = {
  run(...args) {
    return new Promise(function c(resolve, reject) {
      myDB.run(...args, function onResult(err) {
        if (err) reject(err)
        else resolve(this)
      })
    })
  },
  // Define a promise interface for each callback
  get: util.promisify(myDB.get.bind(myDB)),
  all: util.promisify(myDB.all.bind(myDB)),
  exec: util.promisify(myDB.exec.bind(myDB)),
}

We can then create some functions to directly interact with the database. An example below is getAllRecords, which as the name suggests, takes all the records in the database and returns them back to the main function which outputs them (provided records are found).

async function getAllRecords() {
  let result = await SQL3.all(
    `
			SELECT
				Other.data AS 'other',
				Something.data AS 'something'
			FROM 
				Something JOIN Other 
				ON (Something.otherID = Other.id)
			ORDER BY
				Other.id DESC, Something.data ASC 	
		`,
  )

  if (result && result.length > 0) {
    return result
  }
}

Webservers

We can setup simple webservers to serve static pages or content. An example of setting up a simple web server is depicted below:

const http = require('http')
// Create server
const httpserve = http.createServer(handleRequest)
const HTTP_PORT = 8039

main()

function main() {
  httpserv.listen(HTTP_PORT)
  console.log(`Listening on http://localhost:${HTTP_PORT}...`)
}

async function handleRequest(req, res) {
  if (req.url == '/hello') {
    // Write to header
    res.writeHead(200, { 'Content-Type': 'text/plain' })
    res.write('Hello World')
    res.end() // Close stream
  } else {
    res.writeHead(401)
    res.end()
  }
}

We can also configure a simple static alias server to allow access under the web root directory only. This file server uses regex to check incoming requests and serve different pages dependant on the outcome.

let path = require("path");
let http = require("http");
let staticAlias = require("node-static-alias");

const WEB_PATH = path.join(__dirname, "web");
const HTTP_PORT = 8039;

let fileServer = new staticAlias.Server(WEB_PATH, {
    // Time to cache files in seconds
    cache: 100,
    serverInfo: "Server name",
    alias: [
        {
            // If the URL contains index after '/' or nothing (excludes HTTP portion at the start)
            match: /^\/(?:index\/?)?(?:[?#].*$)?$/,
            serve: "index.html",
            force: true
        },
        {
            // If the URL contains .js
            match: /^\/js\/.+$/,
            serve: "<% absPath %>",
            force: true
        },
        {
            // If the URL contains words or numbers followed by separators, serve it and append .html
            match: /^\/(?:[\w\d]+)(?:[\/?#].*$)?$/,
            serve: function onMath(params) {
                return `${params.basename}.html`;
            }
            force: true
        },
        {
          // 404 Pages for non matching
            match: /[^]/,
            serve: "404.html"
        }
    ]
});

async function handleRequest(request, response) {
    fileServer.serve(request, response);
}

// Instantiate server
let httpServer = http.createServer(handleRequest);

httpServer.listen(HTTP_PORT);

We can then update our previous handleRequest function to check for api requests to our database.

async function handleRequest(req, res) {
  // Api Request
  if (req.url == '/get-records') {
    // Get all records from DB
    let records = await getAllRecords()

    res.writeHead(200, {
      'Content-Type': 'application/json',
      'Cache-Control': 'no-cache',
    })
    res.write(JSON.stringify(records))
    res.end()
  } else {
    // Non-api request
    fileServer.serve(req, res)
  }
}

Instead of manually routing like before using regex, we could instead use Express.js. We setup some middleware (this is a function that gets called if the incoming requests passes some criteria). Middleware is executed in order, so placement matters. We can use next() within a middleware function to execute the next middleware function other than the first.

let path = require('path')
let http = require('http')
let express = require('express')

const WEB_PATH = path.join(__dirname, 'web')
let app = express()

function defineRoutes() {
  // GET requests
  app.get('/get-records', async function (request, response) {
    let records = await getAllRecords()

    response.writeHead(200, {
      'Content-Type': 'application/json',
      'Cache-Control': 'no-cache',
    })
    response.end(JSON.stringify(records))
  })

  // Next example
  app.use(function rewriter(req, res, next) {
    if (/^\/(?:index\/?)?(?:[?#].*$)?$/.test(req.url)) {
      req.url = '/index.html'
    } else if (/^\/js\/.+$/.test(req.url)) {
      next()
      return
    } else if (/^\/(?:[\w\d]+)(?:[\/?#].*$)?$/.test(req.url)) {
      let [, basename] = req.url.match(/^\/([\w\d]+)(?:[\/?#].*$)?$/)
      req.url = `${basename}.html`
    }

    next()
  })

  //All incoming requests
  app.use(
    express.static(WEB_PATH, {
      maxAge: 100,
      setHeaders: function setHeaders(res) {
        res.setHeader('Server', 'Server name')
      },
    }),
  )
}

defineRoutes()
let httpServer = http.createServer(app)

Child Processes

We can create a child process, and then check its status.

let util = require("util");
let childProcess = require("child_process");

let child = childProcess.spawn("node", [ "file" ]);
child.on("exit", function(code) {
    console.log("Child finished", code);
})

async function main() {
    try {
        let response = await fetch(`http://localhost:{HTTP_PORT}/get-records`);

        if (response && response.ok) {
            let records = await response.json();

            if (records && records.length > 0) {
                process.excitCode = 0;

                return;
            }
        }
    }
    catch(error) {
        ...
    }

    process.exitCode = -1
}

main();

Debugging NodeJS

We can open chrome://inspect. This listens to the port on localhost. From here we can observe the server activity. We should run our programs with the --inspect flag to broadcast the port on the localhost so we can see changes from within the inspect.