Partition ZDNS into library and CLI tool

[paul-pearce]

trafficstars

We built it as if it could be a library, but the configuration overhead is a mess (sorry @kumarde).

I think the necessary steps are to move most of the configuration setup inside of the "library" so that the invocation of the library more closely resembles passing command line parameters, instead of mimicking our logic from main.go

[mlazzje]

+1.

It will be easier for me to integrate zdns as a library inside my code :)

[wolveix]

Hey guys, has there been any movement on this? I've got a project that ZDNS would be perfect for, but it's near impossible to utilize it right now without invoking it via CLI. I wrote a simplistic library interface which works, but the internal logging via Logrus disrupts this. Here's a simple example:

package scanner

import (
	"errors"
	"github.com/zmap/zdns/iohandlers"
	"io/ioutil"
	"math/rand"
	"net"
	"runtime"
	"strings"
	"time"

	"github.com/miekg/dns"
	"github.com/zmap/zdns"
	_ "github.com/zmap/zdns/modules/alookup"
	_ "github.com/zmap/zdns/modules/axfr"
	_ "github.com/zmap/zdns/modules/bindversion"
	_ "github.com/zmap/zdns/modules/dmarc"
	_ "github.com/zmap/zdns/modules/miekg"
	_ "github.com/zmap/zdns/modules/mxlookup"
	_ "github.com/zmap/zdns/modules/nslookup"
	_ "github.com/zmap/zdns/modules/spf"
)

var (
	config zdns.GlobalConf
)

func defaultConfig() zdns.GlobalConf {
	return zdns.GlobalConf{
		AlexaFormat:           false,
		CacheSize:             10000,
		Class:                 dns.ClassINET,
		GoMaxProcs:            0,
		IncludeInOutput:       "",
		InputFilePath:         "-",
		InputHandler:          nil,
		IterationTimeout:      time.Second * time.Duration(0),
		IterativeResolution:   false,
		LocalAddrs:            nil,
		LocalAddrSpecified:    false,
		LogFilePath:           "",
		MaxDepth:              10,
		MetadataFilePath:      "",
		Module:                "",
		NamePrefix:            "",
		NameOverride:          "",
		NameServers:           nil,
		NameServerInputFormat: false,
		NameServerMode:        false,
		NameServersSpecified:  false,
		OutputFilePath:        "-",
		OutputGroups:          nil,
		OutputHandler:         nil,
		PassedName:            "",
		ResultVerbosity:       "normal",
		Retries:               1,
		TCPOnly:               false,
		Threads:               1000,
		TimeFormat:            time.RFC3339,
		Timeout:               time.Second * time.Duration(0),
		UDPOnly:               false,
		Verbosity:             3,
	}
}

func Scan(dnsClass string, nameservers string, recordType string, domains ...string) error {
	config = defaultConfig()

	if dnsClass != "" {
		switch strings.ToUpper(dnsClass) {
		case "INET", "IN":
			config.Class = dns.ClassINET
		case "CSNET", "CS":
			config.Class = dns.ClassCSNET
		case "CHAOS", "CH":
			config.Class = dns.ClassCHAOS
		case "HESIOD", "HS":
			config.Class = dns.ClassHESIOD
		case "NONE":
			config.Class = dns.ClassNONE
		case "ANY":
			config.Class = dns.ClassANY
		default:
			return errors.New("unknown dns class specified. Valid values are INET (default), CSNET, CHAOS, HESIOD, NONE, ANY")
		}
	} else {
		config.Class = dns.ClassINET
	}

	if nameservers == "" {
		if config.IterativeResolution {
			config.NameServers = zdns.RootServers[:]
		} else {
			ns, err := zdns.GetDNSServers("/etc/resolv.conf")
			if err != nil {
				return errors.New("unable to fetch correct nameservers")
			}
			config.NameServers = ns
		}
		config.NameServersSpecified = false
	} else {
		if config.NameServerMode {
			return errors.New("name servers cannot be specified if nameserver mode is enabled")
		}

		var ns []string
		if (nameservers)[0] == '@' {
			filepath := (nameservers)[1:]
			f, err := ioutil.ReadFile(filepath)
			if err != nil {
				return errors.New("unable to read file for nameservers")
			}

			if len(f) == 0 {
				return errors.New("nameserver input file empty")
			}

			ns = strings.Split(strings.Trim(string(f), "\n"), "\n")
		} else {
			ns = strings.Split(nameservers, ",")
		}
		for i, s := range ns {
			ns[i] = zdns.AddDefaultPortToDNSServerName(s)
		}
		config.NameServers = ns
		config.NameServersSpecified = true
	}

	// TODO: make this configurable
	maxProcesses := 0
	if maxProcesses != 0 && maxProcesses > 0 {
		config.GoMaxProcs = maxProcesses
		runtime.GOMAXPROCS(maxProcesses)
	}

	if conn, err := net.Dial("udp", "8.8.8.8:53"); err != nil {
		return errors.New("unable to determine default IP address for inbound UDP sockets")
	} else {
		config.LocalAddrs = append(config.LocalAddrs, conn.LocalAddr().(*net.UDPAddr).IP)
	}

	rand.Seed(time.Now().UnixNano())

	if recordType == "" {
		recordType = "A"
	}

	config.Module = strings.ToUpper(recordType)
	factory := zdns.GetLookup(config.Module)
	if factory == nil {
		return errors.New("invalid record type specified")
	}

	var domainString string
	for _, domain := range domains {
		if domain == "" {
			return errors.New("empty domain provided")
		}

		if len(domainString) == 0 {
			domainString = domain
		} else {
			domainString = domainString + "\n" + domain
		}
	}

	config.InputHandler = iohandlers.NewStreamInputHandler(strings.NewReader(domainString))
	config.OutputHandler = iohandlers.NewFileOutputHandler(config.OutputFilePath)

	if err := factory.Initialize(&config); err != nil {
		return errors.New("factory could not be initialized:" + err.Error())
	}

	if err := zdns.DoLookups(factory, &config); err != nil {
		return errors.New("unable to process lookups:" + err.Error())
	}

	if err := factory.Finalize(); err != nil {
		return errors.New("factory could not be finalized:" + err.Error())
	}

	return nil
}

[zakird]

There hasn't, not due to lack of interest, but just lack of time. I've started to slowly move pieces into libraries, particularly our caching and recursive logic (e.g., https://github.com/zmap/zdns/pull/239). I think it'll happen over the next months to year, but not weeks.

[wolveix]

Glad to hear it. This is an incredible tool, so I look forward to being able to use its later revisions. Are PRs welcome for this issue? I may take a stab at dividing up some of the CLI code into a cmd directory (as is typically the norm with Go projects).

[zakird]

Absolutely. Would love the help.