From b054828aa8d648810bef6dadf51f7e9290365ee7 Mon Sep 17 00:00:00 2001 From: jguer Date: Mon, 22 Aug 2022 23:28:53 +0200 Subject: [PATCH] wip --- cmd.go | 2 +- go.mod | 3 +- go.sum | 2 + install.go | 12 +- local_install.go | 150 ++++++++++++++++++++++ pkg/dep/depCheck.go | 14 +- pkg/dep/depOrder.go | 14 +- pkg/dep/depPool.go | 16 +-- pkg/query/aur_info.go | 2 +- pkg/topo/dep.go | 292 ++++++++++++++++++++++++++++++++++++++++++ pkg/topo/errors.go | 7 + 11 files changed, 479 insertions(+), 35 deletions(-) create mode 100644 pkg/topo/dep.go create mode 100644 pkg/topo/errors.go diff --git a/cmd.go b/cmd.go index 5b784bbb..6c8e6883 100644 --- a/cmd.go +++ b/cmd.go @@ -330,7 +330,7 @@ func handleUpgrade(ctx context.Context, config *settings.Configuration, dbExecutor db.Executor, cmdArgs *parser.Arguments, ) error { if cmdArgs.ExistsArg("i", "install") { - return installLocalPKGBUILD(ctx, cmdArgs, dbExecutor, false) + return installLocalPKGBUILD(ctx, cmdArgs, dbExecutor, config.Runtime.AURClient, false) } return config.Runtime.CmdBuilder.Show(config.Runtime.CmdBuilder.BuildPacmanCmd(ctx, diff --git a/go.mod b/go.mod index e6fc8d22..26ef933d 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,9 @@ require ( github.com/adrg/strutil v0.3.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect + github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.17 +go 1.19 diff --git a/go.sum b/go.sum index b5bf3843..25346a34 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/leonelquinteros/gotext v1.5.0 h1:ODY7LzLpZWWSJdAHnzhreOr6cwLXTAmc914F github.com/leonelquinteros/gotext v1.5.0/go.mod h1:OCiUVHuhP9LGFBQ1oAmdtNCHJCiHiQA8lf4nAifHkr0= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/install.go b/install.go index 76ceb92d..584b09f5 100644 --- a/install.go +++ b/install.go @@ -634,14 +634,12 @@ func buildInstallPkgbuilds( satisfied := true all: for _, pkg := range base { - for _, deps := range dep.ComputeCombinedDepList(pkg, noDeps, noCheck) { - for _, dep := range deps { - if !dp.AlpmExecutor.LocalSatisfierExists(dep) { - satisfied = false - text.Warnln(gotext.Get("%s not satisfied, flushing install queue", dep)) + for _, dep := range dep.ComputeCombinedDepList(pkg, noDeps, noCheck) { + if !dp.AlpmExecutor.LocalSatisfierExists(dep) { + satisfied = false + text.Warnln(gotext.Get("%s not satisfied, flushing install queue", dep)) - break all - } + break all } } } diff --git a/local_install.go b/local_install.go index c48db3cc..b4a5a3d1 100644 --- a/local_install.go +++ b/local_install.go @@ -2,16 +2,166 @@ package main import ( "context" + "fmt" + "os" + "path/filepath" + "strings" + "github.com/Jguer/aur" "github.com/Jguer/yay/v11/pkg/db" + "github.com/Jguer/yay/v11/pkg/dep" + "github.com/Jguer/yay/v11/pkg/query" "github.com/Jguer/yay/v11/pkg/settings/parser" + "github.com/Jguer/yay/v11/pkg/topo" + gosrc "github.com/Morganamilo/go-srcinfo" + "github.com/leonelquinteros/gotext" + "github.com/pkg/errors" ) +func archStringToString(alpmArches []string, archString []gosrc.ArchString) []string { + pkgs := make([]string, 0, len(archString)) + + for _, arch := range archString { + if alpmArchIsSupported(alpmArches, arch.Arch) { + pkgs = append(pkgs, arch.Value) + } + } + + return pkgs +} + +func makeAURPKGFromSrcinfo(dbExecutor db.Executor, srcInfo *gosrc.Srcinfo) ([]aur.Pkg, error) { + pkgs := make([]aur.Pkg, 0, 1) + + alpmArch, err := dbExecutor.AlpmArchitectures() + if err != nil { + return nil, err + } + + alpmArch = append(alpmArch, "") // srcinfo assumes no value as "" + + for _, pkg := range srcInfo.Packages { + pkgs = append(pkgs, aur.Pkg{ + ID: 0, + Name: pkg.Pkgname, + PackageBaseID: 0, + PackageBase: srcInfo.Pkgbase, + Version: srcInfo.Version(), + Description: pkg.Pkgdesc, + URL: pkg.URL, + Depends: append(archStringToString(alpmArch, pkg.Depends), archStringToString(alpmArch, srcInfo.Package.Depends)...), + MakeDepends: archStringToString(alpmArch, srcInfo.PackageBase.MakeDepends), + CheckDepends: archStringToString(alpmArch, srcInfo.PackageBase.CheckDepends), + Conflicts: append(archStringToString(alpmArch, pkg.Conflicts), archStringToString(alpmArch, srcInfo.Package.Conflicts)...), + Provides: append(archStringToString(alpmArch, pkg.Provides), archStringToString(alpmArch, srcInfo.Package.Provides)...), + Replaces: append(archStringToString(alpmArch, pkg.Replaces), archStringToString(alpmArch, srcInfo.Package.Replaces)...), + OptDepends: []string{}, + Groups: pkg.Groups, + License: pkg.License, + Keywords: []string{}, + }) + } + + return pkgs, nil +} + +func splitDep(dep string) (pkg, mod, ver string) { + split := strings.FieldsFunc(dep, func(c rune) bool { + match := c == '>' || c == '<' || c == '=' + + if match { + mod += string(c) + } + + return match + }) + + if len(split) == 0 { + return "", "", "" + } + + if len(split) == 1 { + return split[0], "", "" + } + + return split[0], mod, split[1] +} + func installLocalPKGBUILD( ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Executor, + aurClient aur.ClientInterface, ignoreProviders bool, ) error { + wd, err := os.Getwd() + if err != nil { + return errors.Wrap(err, gotext.Get("failed to retrieve working directory")) + } + + if len(cmdArgs.Targets) > 1 { + return errors.New(gotext.Get("only one target is allowed")) + } + + if len(cmdArgs.Targets) == 1 { + wd = cmdArgs.Targets[0] + } + + pkgbuild, err := gosrc.ParseFile(filepath.Join(wd, ".SRCINFO")) + if err != nil { + return errors.Wrap(err, gotext.Get("failed to parse .SRCINFO")) + } + + aurPkgs, err := makeAURPKGFromSrcinfo(dbExecutor, pkgbuild) + if err != nil { + return err + } + + graph := topo.New[string]() + + for _, pkg := range aurPkgs { + depSlice := dep.ComputeCombinedDepList(&pkg, false, false) + addNodes(dbExecutor, aurClient, pkg.Name, pkg.PackageBase, depSlice, graph) + } + + fmt.Println(graph) + return nil } + +func addNodes(dbExecutor db.Executor, aurClient aur.ClientInterface, pkgName string, pkgBase string, deps []string, graph *topo.Graph[string]) { + graph.AddNode(pkgBase) + graph.Alias(pkgBase, pkgName) + + for _, depString := range deps { + depName, _, _ := splitDep(depString) + + if dbExecutor.LocalSatisfierExists(depString) { + continue + } + + graph.DependOn(depName, pkgBase) + + if alpmPkg := dbExecutor.SyncSatisfier(depString); alpmPkg != nil { + newDeps := alpmPkg.Depends().Slice() + newDepsSlice := make([]string, 0, len(newDeps)) + + for _, newDep := range newDeps { + newDepsSlice = append(newDepsSlice, newDep.Name) + } + + addNodes(dbExecutor, aurClient, alpmPkg.Name(), alpmPkg.Base(), newDepsSlice, graph) + } + + warnings := query.AURWarnings{} + if aurPkgs, _ := query.AURInfo(context.TODO(), aurClient, []string{depName}, &warnings, 1); len(aurPkgs) != 0 { + pkg := aurPkgs[0] + newDeps := dep.ComputeCombinedDepList(pkg, false, false) + newDepsSlice := make([]string, 0, len(newDeps)) + + addNodes(dbExecutor, aurClient, pkg.PackageBase, pkg.Name, newDepsSlice, graph) + } + } + + return +} diff --git a/pkg/dep/depCheck.go b/pkg/dep/depCheck.go index 02aab7a8..16ea0a73 100644 --- a/pkg/dep/depCheck.go +++ b/pkg/dep/depCheck.go @@ -230,15 +230,13 @@ func (dp *Pool) _checkMissing(dep string, stack []string, missing *missing, noDe missing.Good.Set(dep) combinedDepList := ComputeCombinedDepList(aurPkg, noDeps, noCheckDeps) - for _, deps := range combinedDepList { - for _, aurDep := range deps { - if dp.AlpmExecutor.LocalSatisfierExists(aurDep) { - missing.Good.Set(aurDep) - continue - } - - dp._checkMissing(aurDep, append(stack, aurPkg.Name), missing, noDeps, noCheckDeps) + for _, aurDep := range combinedDepList { + if dp.AlpmExecutor.LocalSatisfierExists(aurDep) { + missing.Good.Set(aurDep) + continue } + + dp._checkMissing(aurDep, append(stack, aurPkg.Name), missing, noDeps, noCheckDeps) } return diff --git a/pkg/dep/depOrder.go b/pkg/dep/depOrder.go index 6550b040..69233407 100644 --- a/pkg/dep/depOrder.go +++ b/pkg/dep/depOrder.go @@ -48,15 +48,13 @@ func (do *Order) orderPkgAur(pkg *aur.Pkg, dp *Pool, runtime, noDeps, noCheckDep delete(dp.Aur, pkg.Name) - for i, deps := range ComputeCombinedDepList(pkg, noDeps, noCheckDeps) { - for _, dep := range deps { - if aurPkg := dp.findSatisfierAur(dep); aurPkg != nil { - do.orderPkgAur(aurPkg, dp, runtime && i == 0, noDeps, noCheckDeps) - } + for i, dep := range ComputeCombinedDepList(pkg, noDeps, noCheckDeps) { + if aurPkg := dp.findSatisfierAur(dep); aurPkg != nil { + do.orderPkgAur(aurPkg, dp, runtime && i == 0, noDeps, noCheckDeps) + } - if repoPkg := dp.findSatisfierRepo(dep); repoPkg != nil { - do.orderPkgRepo(repoPkg, dp, runtime && i == 0) - } + if repoPkg := dp.findSatisfierRepo(dep); repoPkg != nil { + do.orderPkgRepo(repoPkg, dp, runtime && i == 0) } } diff --git a/pkg/dep/depPool.go b/pkg/dep/depPool.go index 2848c6c3..f19e9da0 100644 --- a/pkg/dep/depPool.go +++ b/pkg/dep/depPool.go @@ -275,17 +275,17 @@ func (dp *Pool) cacheAURPackages(ctx context.Context, _pkgs stringset.StringSet, // Compute dependency lists used in Package dep searching and ordering. // Order sensitive TOFIX. -func ComputeCombinedDepList(pkg *aur.Pkg, noDeps, noCheckDeps bool) [][]string { - combinedDepList := make([][]string, 0, 3) +func ComputeCombinedDepList(pkg *aur.Pkg, noDeps, noCheckDeps bool) []string { + combinedDepList := make([]string, 0, len(pkg.Depends)+len(pkg.MakeDepends)+len(pkg.CheckDepends)) if !noDeps { - combinedDepList = append(combinedDepList, pkg.Depends) + combinedDepList = append(combinedDepList, pkg.Depends...) } - combinedDepList = append(combinedDepList, pkg.MakeDepends) + combinedDepList = append(combinedDepList, pkg.MakeDepends...) if !noCheckDeps { - combinedDepList = append(combinedDepList, pkg.CheckDepends) + combinedDepList = append(combinedDepList, pkg.CheckDepends...) } return combinedDepList @@ -326,10 +326,8 @@ func (dp *Pool) resolveAURPackages(ctx context.Context, dp.Aur[pkg.Name] = pkg combinedDepList := ComputeCombinedDepList(pkg, noDeps, noCheckDeps) - for _, deps := range combinedDepList { - for _, dep := range deps { - newPackages.Set(dep) - } + for _, dep := range combinedDepList { + newPackages.Set(dep) } } diff --git a/pkg/query/aur_info.go b/pkg/query/aur_info.go index d9b83ca1..d6a239a2 100644 --- a/pkg/query/aur_info.go +++ b/pkg/query/aur_info.go @@ -19,7 +19,7 @@ type Pkg = aur.Pkg // of packages exceeds the number set in config.RequestSplitN. // If the number does exceed config.RequestSplitN multiple aur requests will be // performed concurrently. -func AURInfo(ctx context.Context, aurClient *aur.Client, names []string, warnings *AURWarnings, splitN int) ([]*Pkg, error) { +func AURInfo(ctx context.Context, aurClient aur.ClientInterface, names []string, warnings *AURWarnings, splitN int) ([]*Pkg, error) { info := make([]*Pkg, 0, len(names)) seen := make(map[string]int) diff --git a/pkg/topo/dep.go b/pkg/topo/dep.go new file mode 100644 index 00000000..7e4f9f6b --- /dev/null +++ b/pkg/topo/dep.go @@ -0,0 +1,292 @@ +package topo + +import ( + "fmt" + "strings" +) + +type Mapable interface { + Key() string +} + +type ( + AliasMap[T comparable] map[T]T + NodeSet[T comparable] map[T]bool + DepMap[T comparable] map[T]NodeSet[T] +) + +type Graph[T comparable] struct { + alias AliasMap[T] + nodes NodeSet[T] + + // `dependencies` tracks child -> parents. + dependencies DepMap[T] + // `dependents` tracks parent -> children. + dependents DepMap[T] + // Keep track of the nodes of the graph themselves. +} + +func New[T comparable]() *Graph[T] { + return &Graph[T]{ + nodes: make(NodeSet[T]), + dependencies: make(DepMap[T]), + dependents: make(DepMap[T]), + alias: make(AliasMap[T]), + } +} + +func (g *Graph[T]) Alias(node, alias T) error { + if alias == node { + return ErrSelfReferential + } + + // add node + g.nodes[node] = true + + // add alias + if _, ok := g.alias[alias]; ok { + return ErrConflictingAlias + } + g.alias[alias] = node + + return nil +} + +func (g *Graph[T]) AddNode(node T) { + // check aliases + if aliasNode, ok := g.alias[node]; ok { + node = aliasNode + } + + g.nodes[node] = true +} + +func (g *Graph[T]) DependOn(child, parent T) error { + if child == parent { + return ErrSelfReferential + } + + if g.DependsOn(parent, child) { + return ErrCircular + } + + g.AddNode(parent) + g.AddNode(child) + + // Add nodes. + g.nodes[parent] = true + g.nodes[child] = true + + // Add edges. + g.dependents.addNodeToNodeset(parent, child) + g.dependencies.addNodeToNodeset(child, parent) + + return nil +} + +func (g *Graph[T]) String() string { + var sb strings.Builder + sb.WriteString("digraph {\n") + // sb.WriteString("rankdir=LR;\n") + sb.WriteString("node [shape = record, ordering=out];\n") + for node := range g.nodes { + sb.WriteString(fmt.Sprintf("\t\"%v\";\n", node)) + } + for parent, children := range g.dependencies { + for child := range children { + sb.WriteString(fmt.Sprintf("\t\"%v\" -> \"%v\";\n", parent, child)) + } + } + sb.WriteString("}") + return sb.String() +} + +func (g *Graph[T]) DependsOn(child, parent T) bool { + deps := g.Dependencies(child) + _, ok := deps[parent] + + return ok +} + +func (g *Graph[T]) HasDependent(parent, child T) bool { + deps := g.Dependents(parent) + _, ok := deps[child] + + return ok +} + +func (g *Graph[T]) Leaves() []T { + leaves := make([]T, 0) + + for node := range g.nodes { + if _, ok := g.dependencies[node]; !ok { + leaves = append(leaves, node) + } + } + + return leaves +} + +// TopoSortedLayers returns a slice of all of the graph nodes in topological sort order. That is, +// if `B` depends on `A`, then `A` is guaranteed to come before `B` in the sorted output. +// The graph is guaranteed to be cycle-free because cycles are detected while building the +// graph. Additionally, the output is grouped into "layers", which are guaranteed to not have +// any dependencies within each layer. This is useful, e.g. when building an execution plan for +// some DAG, in which case each element within each layer could be executed in parallel. If you +// do not need this layered property, use `Graph.TopoSorted()`, which flattens all elements. +func (g *Graph[T]) TopoSortedLayers() [][]T { + layers := [][]T{} + + // Copy the graph + shrinkingGraph := g.clone() + + for { + leaves := shrinkingGraph.Leaves() + if len(leaves) == 0 { + break + } + + layers = append(layers, leaves) + + for _, leafNode := range leaves { + shrinkingGraph.remove(leafNode) + } + } + + return layers +} + +func (dm DepMap[T]) removeFromDepmap(key, node T) { + if nodes := dm[key]; len(nodes) == 1 { + // The only element in the nodeset must be `node`, so we + // can delete the entry entirely. + delete(dm, key) + } else { + // Otherwise, remove the single node from the nodeset. + delete(nodes, node) + } +} + +func (g *Graph[T]) remove(node T) { + // Remove edges from things that depend on `node`. + for dependent := range g.dependents[node] { + g.dependencies.removeFromDepmap(dependent, node) + } + + delete(g.dependents, node) + + // Remove all edges from node to the things it depends on. + for dependency := range g.dependencies[node] { + g.dependents.removeFromDepmap(dependency, node) + } + + delete(g.dependencies, node) + + // Finally, remove the node itself. + delete(g.nodes, node) +} + +// TopoSorted returns all the nodes in the graph is topological sort order. +// See also `Graph.TopoSortedLayers()`. +func (g *Graph[T]) TopoSorted() []T { + nodeCount := 0 + layers := g.TopoSortedLayers() + + for _, layer := range layers { + nodeCount += len(layer) + } + + allNodes := make([]T, 0, nodeCount) + + for _, layer := range layers { + allNodes = append(allNodes, layer...) + } + + return allNodes +} + +func (g *Graph[T]) Dependencies(child T) NodeSet[T] { + return g.buildTransitive(child, g.immediateDependencies) +} + +func (g *Graph[T]) immediateDependencies(node T) NodeSet[T] { + return g.dependencies[node] +} + +func (g *Graph[T]) Dependents(parent T) NodeSet[T] { + return g.buildTransitive(parent, g.immediateDependents) +} + +func (g *Graph[T]) immediateDependents(node T) NodeSet[T] { + return g.dependents[node] +} + +func (g *Graph[T]) clone() *Graph[T] { + return &Graph[T]{ + dependencies: g.dependencies.copy(), + dependents: g.dependents.copy(), + nodes: g.nodes.copy(), + } +} + +// buildTransitive starts at `root` and continues calling `nextFn` to keep discovering more nodes until +// the graph cannot produce any more. It returns the set of all discovered nodes. +func (g *Graph[T]) buildTransitive(root T, nextFn func(T) NodeSet[T]) NodeSet[T] { + if _, ok := g.nodes[root]; !ok { + return nil + } + + out := make(NodeSet[T]) + searchNext := []T{root} + + for len(searchNext) > 0 { + // List of new nodes from this layer of the dependency graph. This is + // assigned to `searchNext` at the end of the outer "discovery" loop. + discovered := []T{} + + for _, node := range searchNext { + // For each node to discover, find the next nodes. + for nextNode := range nextFn(node) { + // If we have not seen the node before, add it to the output as well + // as the list of nodes to traverse in the next iteration. + if _, ok := out[nextNode]; !ok { + out[nextNode] = true + + discovered = append(discovered, nextNode) + } + } + } + + searchNext = discovered + } + + return out +} + +func (s NodeSet[T]) copy() NodeSet[T] { + out := make(NodeSet[T], len(s)) + for k, v := range s { + out[k] = v + } + + return out +} + +func (m DepMap[T]) copy() DepMap[T] { + out := make(DepMap[T], len(m)) + for k, v := range m { + out[k] = v.copy() + } + + return out +} + +func (dm DepMap[T]) addNodeToNodeset(key, node T) { + nodes, ok := dm[key] + if !ok { + nodes = make(NodeSet[T]) + dm[key] = nodes + } + + nodes[node] = true +} diff --git a/pkg/topo/errors.go b/pkg/topo/errors.go new file mode 100644 index 00000000..f5a30b10 --- /dev/null +++ b/pkg/topo/errors.go @@ -0,0 +1,7 @@ +package topo + +import "errors" + +var ErrSelfReferential = errors.New("self-referential dependencies not allowed") +var ErrConflictingAlias = errors.New("alias already defined") +var ErrCircular = errors.New("circular dependencies not allowed")