dao
dao copied to clipboard
Published
20 hours ago •
daokoder
daokoder
The package manager
Here is a very preliminary sketch of the core daopkg stuff. Had to bring it on early to resolve SIGSEGV issue.
load os
load os.fs
class PackageManager {
protected
const name = 'daopkg'
const pkgExtension = 'dpk'
const registryUrl = '<URL of the package registry>'
const fossil = (
data = '_FOSSIL_',
clone = 'fossil clone $url',
cloneAs = 'fossil clone $url $name.fossil',
open = 'fossil open $name',
update = 'fossil update',
updateTo = 'fossil update $rev'
)
const daomake = 'daomake --platform $os makefile.dao'
const make = (
install = '$make && $make install',
uninstall = '$make uninstall'
)
const maxDepLevel = 100
invar sysInfo: tuple<name: string, version: string>
invar showCommands: bool
var dir: fs::Dir
var out: io::Stream
var depLevel = 0
public
const ShellError = Error::define('Error::Shell')
const PackageError = Error::define('Error::Package')
const ParamError = Error::define('Error::Param')
type Package = tuple<name: string, author: string, description: string, license: string, repository: string, revision: string,
compatibility: list<string>, dependencies: list<Package>>
protected
routine info(...: string as msg){
if (showCommands)
out.write(msg, ...)
}
routine run(cmdline: string, ...: tuple<enum,string> as args) => string {
var pipe = os.popen(cmdline, 'r')
var res = ''
var success = false
var cmd = cmdline
for (item in args)
cmd = cmd.replace((string)item[1][0], item[1][1])
info(cmd, '\n')
while (not pipe.check($eof)){
invar buf = pipe.read()
info(buf)
res += buf
}
if (os.pclose(pipe) != 0)
std.error(ShellError, 'shell command failed', (command = cmd, output = res))
return res
}
routine openDir(path: string) => fs::Dir { dir.exists(path)? (fs::Dir)dir[path] : dir.mkdir(path) }
routine deleteTree(dirTree: fs::Dir){
for (entry in dirTree.entries()){
if (entry ?< fs::Dir)
deleteTree((fs::Dir)entry)
entry.delete()
}
}
routine getPackageProperty(pkgName: string, invar properties: map<string, string>, propName: string) => string {
const required = {'description', 'repository'}
if (invar res = properties.find(propName); res != none)
return res.value
else {
if (propName in required)
std.error(PackageError, 'required property not specified', (package = pkgName, cause = propName))
return ''
}
}
routine checkPackageSystem(pkgName: string, sysName: string) => bool {
invar sname = sysName.convert($lower)
return (sysInfo.name == sname) or (sname == 'unix' and sysInfo.name != 'windows') or
(sysInfo.name + ' ' + sysInfo.version == sname)
}
routine processPackageEntry(name: string, entry: string, recursively: bool) => Package {
invar props = entry.split('\n').associate {
if (invar line = X.trim(); %line){
if (invar prop = line.match('^%w+ %s* : %s*'); prop != none)
return (line.fetch('^%w+'), line[prop.end + 1 :])
else
std.error(PackageError, 'invalid property definition', (package = name, cause = line))
}
}
invar getprop = getPackageProperty.{self, name, props}
invar checksys = checkPackageSystem.{self, name}
return (
name = name,
author = getprop('author'),
description = getprop('description'),
license = getprop('license'),
repository = getprop('repository'),
revision = getprop('revision'),
compatibility = getprop('compatibility').split(',').collect {
if (invar pname = X.trim(); %pname)
return pname
},
dependencies = getprop('dependencies').extract('%s* %S+ %s* %b()? %s* ,?', $both).collect(){
if (invar item = X.trim(); item != ''){
invar lst = item.capture('^ (%S+) %s* (%b()?) %s* ,? $')
if (%lst < 1)
std.error(PackageError, 'invalid dependency format', (package = name, cause = item))
else {
invar dep = lst[0]
invar pnames = (%lst < 2)? '' : lst[1].trim()
invar req = (pnames == '')? true : (pnames.split(',').find { checksys(X.trim()) } != none)
if (req){
if (dep == name)
std.error(PackageError, 'recursive dependency', (package = name, cause = dep))
else if (recursively){
if (invar pkg = findPackage(dep); pkg != none)
return (Package)pkg
else
std.error(PackageError, 'unknown dependency', (package = name, cause = dep))
}
else
return (name = dep, author = '', description = '', license = '', repository = '', revision = '',
compatibility = (list<string>){}, dependencies = (list<Package>){})
}
}
}
}
)
}
routine checkPackageInstalled(pkg: Package) => bool {
openDir('installed').files('*.' + pkgExtension).find { X.basename == pkg.name } != none
}
public
routine PackageManager(output = io.stdio, showShellCommands = false){
const dirp = '.' + name
invar sinfo = os.uname()
var home = fs.home()
if (not output.check($writable))
std.error(ParamError, 'output stream not writable')
out = output
showCommands = showShellCommands
sysInfo = (name = sinfo.system.convert($lower), version = sinfo.version.convert($lower))
dir = home.exists(dirp)? (fs::Dir)home[dirp] : home.mkdir(dirp)
}
routine updateRegistry(){
invar rdir = openDir('registry')
fs.cd(rdir)
if (not rdir.exists(fossil.data)){
run(fossil.clone, url = registryUrl)
if (invar files = rdir.files('*.fossil'); %files)
run(fossil.open, name = files[0].name)
}
else
run(fossil.update)
}
routine clearCache(){
if (dir.exists('registry'))
deleteTree((fs::Dir)dir['registry'])
if (dir.exists('cache')){
invar installed = openDir('installed').files('*.' + pkgExtension)
for (pkgdir in ((fs::Dir)dir['cache']).dirs())
if (installed.find { X.name == pkgdir.name } == none)
deleteTree(pkgdir)
}
}
routine findPackage(name: string, getTree = true) => Package|none {
if (++depLevel > maxDepLevel){
depLevel = 0
std.error(PackageError, 'dependency level too high', (name = name, cause = ''))
}
defer { --depLevel }
if (invar fname = fs.ls(openDir('registry')).find { X == name + '.' + pkgExtension }; fname != none)
return processPackageEntry(name, io.read(fname.value), getTree)
else
return none
}
routine downloadPackage(pkg: Package){
invar repname = pkg.repository.fetch('/ ([^/]+) $', 1)
var cdir = openDir('cache')
if (not cdir.exists(repname)){
fs.cd(cdir.mkdir(repname))
out.write('Downloading ', pkg.name, ' from ', pkg.repository, '...')
run(fossil.cloneAs, url = pkg.repository, name = repname)
run(fossil.open, name = repname)
out.writeln(' done')
}
else if (pkg.revision == ''){
out.write('Updating cache for ', pkg.name, '...')
fs.cd((fs::Dir)cdir[name])
run(fossil.update)
out.writeln(' done')
}
for (dep in pkg.dependencies)
downloadPackage(dep)
}
routine installPackage(pkg: Package){
downloadPackage(pkg)
fs.cd((fs::Dir)dir['cache'/pkg.name])
if (pkg.revision != ''){
out.write('Updating to revision ', pkg.revision, '...')
run(fossil.updateTo, rev = pkg.revision)
out.writeln(' done')
}
if (not fs.exists('makefile.dao'))
std.error(PackageError, 'missing makefile.dao', (name = pkg.name, cause = 'makefile.dao'))
out.write('Executing makefile.dao...')
run(daomake, os = (sysInfo.name == 'windows')? 'mingw' : sysInfo.name)
out.writeln(' done')
if (fs.exists('Makefile')){
out.write('Building ', pkg.name, '...')
run(make.install, make = (sysInfo.name == 'windows')? 'mingw32-make' : 'make')
out.writeln(' done')
}
((fs::File)(dir['registry'/pkg.name])).copy(openDir('installed'))
out.writeln('Package installed')
for (dep in pkg.dependencies)
installPackage(dep)
}
routine listPackages(status: enum<installed,all>) => list<Package> {
openDir(status == $installed? 'installed' : 'registry').files('*.' + pkgExtension).collect {
processPackageEntry(X.basename, io.read(X.path), false)
}
}
routine findDependentPackages(name: string) => list<Package> {
listPackages($installed).select { X.dependencies.find { [dep] dep.name == name } != none }
}
routine uninstallPackage(name: string){
if (var pkgfile = openDir('installed').files('*.' + pkgExtension).find { X.basename == name }; pkgfile != none){
if (var pkgdir = openDir('cache').dirs().find { X.name == name }; pkgdir != none){
if (pkgdir.value.exists('Makefile')){
out.write('Uninstalling ', name, '...')
run(make.uninstall, make = (sysInfo.name == 'windows')? 'mingw32-make' : 'make')
pkgfile.value.delete()
out.writeln(' done')
}
else
std.error(PackageError, 'Makefile not found', (name = name, cause = 'Makefile'))
}
else
std.error(PackageError, 'Package cache is missing', (name = name, cause = 'cache'))
}
else
std.error(PackageError, 'Package not installed', (name = name, cause = 'installed'))
}
routine searchPackages(keywords: string) => list<Package> {
invar words = keywords.convert($lower).split().collect {
invar word = X.change('%W+', '')
if (word != '')
return word
}.associate { (X, 1) }.keys()
invar pkgs = listPackages($all).select { [pkg] words.find { X in pkg.name.convert($lower) or X in pkg.description.convert($lower) } != none }
invar ranks = pkgs.associate { [pkg] (pkg, words.reduce(0){ Y + (X in pkg.name.convert($lower) or X in pkg.description.convert($lower)? 1 : 0)) }
return pkgs.sort {
invar (xrank, yrank) = (ranks[X], ranks[Y])
return xrank == yrank? X.name < Y.name : xrank > yrank
}
}
}
const pkgSample =
@[pkg]
author: Someone
description: some package
license: license list
compatibility: sys1, sys2
repository: url
revision: 1234
dependencies: pkg1, pkg2 (sys1), pkg3
@[pkg]
So, a SIGSEGV:
Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7b80709 in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
1624 bl |= DaoType_CheckTypeHolder( self->nested->items.pType[i], tht );
(gdb) bt
#0 0x00007ffff7b80709 in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#1 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#2 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#3 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#4 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#5 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#6 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#7 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#8 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#9 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#10 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#11 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#12 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#13 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#14 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#15 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#16 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#17 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#18 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#19 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#20 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#21 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#22 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#23 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#24 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#25 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#26 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#27 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
#28 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cf1d0, tht=0x60f820) at kernel/daoType.c:1624
#29 0x00007ffff7b807e4 in DaoType_CheckTypeHolder (self=0x6cfe10, tht=0x60f820) at kernel/daoType.c:1634
#30 0x00007ffff7b8070e in DaoType_CheckTypeHolder (self=0x6cfd60, tht=0x60f820) at kernel/daoType.c:1624
All entries up the stack (at least the first 500) are the same, apparently stack overflow takes place.
This infinite recursion bug was caused by recursive type, now fixed.
Nice to see some code for the package manager:)
I get package manager instance outputted via .info(), e.g. PackageManager[0x202cc40]. Test code to reproduce:
...
class PMTester: PackageManager {
routine PMTester(): PackageManager(io.stdio, true){}
routine test(){
info('\n')
}
}
var pm = PMTester()
pm.test()
Looks like ... captures self as well, quite unexpectedly.
Now I can't even use .run(cmdline: string, ...: tuple<enum,string> as args) -- self gets into args even here.
Looks like ... captures self as well, quite unexpectedly.
Yes, it looks a bit unexpected, but actually reasonable, because the self is implicitly the first parameter for instance methods. But to avoid confusion, as now captures only the explicit parameters.
[[ERROR]] in file "/home/danilov/Downloads/dao/daopkg.dao":
At line 4 : Invalid class definition --- " class PackageManager { protected ... ";
At line 4 : Tokens not paired --- " ( ) ";
Dunno what it's talking about.
Recently I've attended talks about Go lang and Ceylon on DevConf 2015 and the most interesting question from the audience (asked in both talks by different person) was about signature and overall security of fetching of the modules/packages. No doubt we'll need it at some point and we should think in advance how to deal with that.
First of all, we should forbid the use of unsecure transmission protocols (no http etc. in favour of https etc.). Then in the future, we should use some trusted source with certificate and always check it and use only signed packages. E.g. maven is not doing it and in the last years it's a great topic among guys dealing with Java deployments with auto-installation of dependencies (which is needed by pretty much any Java application today).
First of all, we should forbid the use of unsecure transmission protocols (no http etc. in favour of https etc.). Then in the future, we should use some trusted source with certificate and always check it. E.g. maven is not doing it and in the last years it's a great topic among guys dealing with Java deployments with auto-installation of dependencies (which is needed by pretty much any Java application today).
This should concern us when the need comes. But thanks for bring it up, we should keep it in mind.