jmarya/pacco-makepkg
jmarya/pacco-makepkg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

update

Browse source
This commit is contained in:
JMARyA 2025-06-28 03:08:16 +02:00
parent 69e1f5d458
commit 669b3724e1
Signed by: jmarya
GPG key ID: 901B2ADDF27C2263
8 changed files with 804 additions and 345 deletions
Download patch file Download diff file Expand all files Collapse all files

789
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

5
Cargo.toml
View file

@ -5,11 +5,16 @@ edition = "2024"
[dependencies]
based = { git = "https://git.hydrar.de/jmarya/based" }
cmd_lib = "1.9.5"
comrade = { git = "https://git.hydrar.de/jmarya/comrade" }
env_logger = "0.11.7"
globals = "1.0.5"
log = "0.4.26"
reqwest = { version = "0.12.13", features = ["blocking", "multipart"] }
serde = { version = "1.0.219", features = ["derive"] }
sqlx = { version = "0.8.3", features = ["postgres"] }
tokio = { version = "1.44.0", features = ["full"] }
uuid = "1.15.1"
owl = { git = "https://git.hydrar.de/red/owl" }
argh = "0.1.13"
yansi = "1.0.1"

7
Dockerfile → Containerfile
View file

@ -5,15 +5,14 @@ RUN rustup default nightly
COPY ./Cargo.toml /app/Cargo.toml
COPY ./Cargo.lock /app/Cargo.lock
COPY ./src /app/src
COPY ./migrations /app/migrations
WORKDIR /app
RUN cargo build --release
FROM archlinux
FROM git.hydrar.de/navos/navos:latest
RUN pacman -Syu --noconfirm base-devel openssl-1.1 git curl rsync systemd arch-install-scripts
RUN pacman-key --init && pacman-key --populate archlinux && pacman-key --populate archlinuxarm && pacman -Syu --noconfirm base-devel openssl-1.1 git curl rsync systemd arch-install-scripts podman
COPY ./pacman.conf /etc/pacman.conf
@ -21,4 +20,4 @@ COPY --from=builder /app/target/release/pacco-makepkg /pacco-makepkg
WORKDIR /
CMD ["/pacco-makepkg"]
CMD ["/pacco-makepkg", "build"]

24
docker-compose.yml
View file

@ -1,26 +1,14 @@
services:
pacco:
pacco-makepkg:
build: .
ports:
- "8080:8000"
privileged: true
environment:
- RUST_LOG=info
- PACCO_HOST=example.com
- PACCO_TOKEN=token
volumes:
- ./owl_db:/owl_db
- ./packages:/packages
- ./repositories:/repositories
- type: tmpfs
target: /build
tmpfs:
size: 42949672960 # 40GB
env_file: .env
postgres:
image: timescale/timescaledb:latest-pg16
restart: always
ports:
- 5432:5432
volumes:
- ./db:/var/lib/postgresql/data/
environment:
- POSTGRES_USER=user
- POSTGRES_PASSWORD=pass
- POSTGRES_DB=pacco_makepkg

8
migrations/0001_init.sql
View file

@ -1,8 +0,0 @@
CREATE TABLE package (
repo TEXT NOT NULL,
pkg TEXT NOT NULL,
last_commit TEXT,
last_pkgver TEXT,
PRIMARY KEY (repo, pkg)
)

90
src/builder.rs
View file

@ -1,11 +1,13 @@
use std::process::{Command, Stdio};
use based::get_pg;
use cmd_lib::run_cmd;
use owl::prelude::*;
use crate::{
PackageIndex,
git::get_commit_hash,
pacco_push,
package::{PackageIndex, find_pkgs, get_pkgver},
package::{IndexEntry, find_package_files, find_pkgs, get_pkgver},
};
pub struct BuildEnv(String, bool);
@ -136,18 +138,18 @@ impl Drop for BuildEnv {
}
}
pub async fn build(pkg: &PackageIndex) -> (String, Vec<(String, Vec<u8>)>) {
let base_env = BuildEnv::new();
let commit = get_commit_hash(pkg.path().to_str().unwrap());
let pkgver = get_pkgver(pkg.path().to_str().unwrap());
base_env.copy_build_env_dir(&pkg.build_dir());
pub async fn build(pkg: &Model<IndexEntry>) -> (String, Vec<String>) {
let build = pkg.read().path().display().to_string();
let commit = get_commit_hash(pkg.read().path().to_str().unwrap());
let pkgver = get_pkgver(pkg.read().path().to_str().unwrap());
let repo = &pkg.read().repo;
let old_ver = &pkg.read().last_pkgver;
let old_commit = &pkg.read().last_commit;
log::info!(
"Building {} / {} @ {}{}",
pkg.repo,
pkg.pkg,
pkg.read().repo,
pkg.read().pkg,
pkgver.as_ref().unwrap_or(&String::new()),
if let Some(c) = &commit {
format!(" [#{c}]")
@ -156,62 +158,42 @@ pub async fn build(pkg: &PackageIndex) -> (String, Vec<(String, Vec<u8>)>) {
}
);
let success = base_env.run_command_inherit(
r#"cd /build;pacman -Syu --noconfirm;useradd -m build;echo "ALL ALL=(ALL) NOPASSWD: ALL"|tee -a /etc/sudoers;chown -R build /build;su -c "makepkg -c -C -s --noconfirm --skippgpcheck" build"#);
let our_dir = std::env::current_dir()
.unwrap()
.join("packages")
.join("our")
.display()
.to_string();
let success = run_cmd!(cd "$build";pacco build --push $repo --out $our_dir).is_ok();
if success {
sqlx::query(
"UPDATE package SET last_commit = $1, last_pkgver = $2 WHERE repo = $3 AND pkg = $4",
)
.bind(&commit)
.bind(&pkgver)
.bind(&pkg.repo)
.bind(&pkg.pkg)
.execute(get_pg!())
.await
.unwrap();
let mut index = PackageIndex;
index.set_last(
commit.clone(),
pkgver.clone(),
&pkg.read().repo,
&pkg.read().pkg,
);
drop(index);
let packages = find_pkgs(&base_env.build_dir());
let packages = find_package_files(&build);
for package in &packages {
log::info!(
"Successfully built {} / {} @ {}{}",
pkg.repo,
pkg.pkg,
pkgver.as_ref().unwrap_or(&String::new()),
if let Some(c) = &commit {
format!(" [#{c}]")
} else {
String::new()
}
"Successfully built {package} - VER: {:?} -> {:?} | GIT: {:?} -> {:?}",
old_ver,
pkgver,
old_commit,
commit
);
std::fs::create_dir_all(std::path::Path::new("./packages").join(&pkg.repo)).unwrap();
std::fs::write(
std::path::Path::new("./packages")
.join(&pkg.repo)
.join(&package.0),
&package.1,
)
.unwrap();
pacco_push(
std::path::Path::new("./packages")
.join(&pkg.repo)
.join(&package.0)
.to_str()
.unwrap(),
&pkg.repo,
)
.await;
}
(String::new(), packages)
} else {
log::error!(
"Error building {} / {} @ {}{}",
pkg.repo,
pkg.pkg,
pkg.read().repo,
pkg.read().pkg,
pkgver.as_ref().unwrap_or(&String::new()),
if let Some(c) = &commit {
format!(" [#{c}]")

132
src/main.rs
View file

@ -1,48 +1,120 @@
use argh::FromArgs;
use based::get_pg;
use comrade::defer;
use comrade::service::ServiceManager;
use package::{rebuild_pkgs, reindex_pkg};
use comrade::{defer, serde_json};
use owl::{prelude::*, query, save, set_global_db, update};
use package::{IndexEntry, rebuild_pkgs, reindex_pkg};
use serde::{Deserialize, Serialize};
use yansi::{Color, Paint};
use std::process::Command;
pub mod builder;
pub mod git;
pub mod package;
#[tokio::main]
async fn main() {
env_logger::init();
pub struct PackageIndex;
sqlx::migrate!().run(get_pg!()).await.unwrap();
impl PackageIndex {
pub fn init() {
let db = Database::filesystem("./owl_db");
set_global_db!(db);
}
reindex_pkg(&std::path::Path::new("./repositories")).await;
pub fn is_present(&self, repo: &str, pkg: &str) -> bool {
!query!(|x: &IndexEntry| x.repo == repo && x.pkg == pkg).is_empty()
}
rebuild_pkgs().await;
pub fn insert_if_not_present(&mut self, repo: &str, pkg: &str) {
if !self.is_present(repo, pkg) {
let entry = IndexEntry {
repo: repo.to_string(),
pkg: pkg.to_string(),
last_commit: None,
last_pkgver: None,
id: Id::String(format!("{repo}-{pkg}")),
};
save!(entry);
}
}
let s = ServiceManager::new();
let s = s.spawn();
defer!(move || s.join().unwrap());
pub fn set_last(
&mut self,
commit: Option<String>,
pkgver: Option<String>,
repo: &str,
pkg: &str,
) {
println!("setting last");
let mut entry = query!(|x: &IndexEntry| x.repo == repo && x.pkg == pkg);
println!("have entry {}", entry.len());
update!(&mut entry, |entry| {
entry.last_commit = commit.clone();
entry.last_pkgver = pkgver.clone();
});
}
}
async fn pacco_push(pkg_path: &str, repo: &str) {
if let Ok(pacco) = std::env::var("PACCO_URL") {
log::info!("Pushing package to pacco at {pacco}");
let upload_url = format!("{}/pkg/{repo}/upload", pacco);
pub fn list_pkgs() {
let pkgs: Vec<Model<IndexEntry>> = query!(|_| true);
let token = std::env::var("PACCO_TOKEN").unwrap();
let output = Command::new("curl")
.arg("-X")
.arg("POST")
.arg("-F")
.arg(format!("pkg=@{}", pkg_path))
.arg("-H")
.arg(format!("Token: {token}"))
.arg(upload_url)
.output()
.unwrap();
if !output.status.success() {
log::error!("Failed to push to pacco");
for pkg in pkgs {
let pkg = pkg.read();
if pkg.last_commit.is_none() && pkg.last_pkgver.is_none() {
println!("{}", format!("- {} / {} @ NONE", pkg.repo, pkg.pkg).paint(Color::Red));
} else {
println!(
"- {} / {} @ {} [{}]",
pkg.repo, pkg.pkg, pkg.last_pkgver.as_ref().unwrap(), pkg.last_commit.as_ref().unwrap()
);
}
}
}
#[derive(FromArgs)]
/// pacco makepkg
pub struct PaccoMakePkgArgs {
#[argh(subcommand)]
pub cmd: PaccoMakePkgCmds,
}
#[derive(FromArgs)]
#[argh(subcommand)]
pub enum PaccoMakePkgCmds {
List(ListCommand),
Build(BuildCommand),
}
#[derive(FromArgs)]
#[argh(subcommand, name = "list")]
/// list packages
pub struct ListCommand {}
#[derive(FromArgs)]
#[argh(subcommand, name = "build")]
/// build packages
pub struct BuildCommand {}
#[tokio::main]
async fn main() {
env_logger::init();
PackageIndex::init();
let args: PaccoMakePkgArgs = argh::from_env();
match args.cmd {
PaccoMakePkgCmds::List(_) => list_pkgs(),
PaccoMakePkgCmds::Build(_) => {
reindex_pkg(&std::path::Path::new("./repositories")).await;
// TODO : summary ui
rebuild_pkgs().await;
log::info!("Writing index");
let s = ServiceManager::new();
let s = s.spawn();
defer!(move || s.join().unwrap());
}
}
}

94
src/package.rs
View file

@ -1,30 +1,34 @@
use based::get_pg;
use cmd_lib::{FunResult, run_fun};
use owl::{prelude::*, query};
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
use std::fs::read_dir;
use std::io::Read;
use std::ops::Deref;
use std::path::Path;
use std::process::Command;
use std::{fs, path::PathBuf};
use crate::PackageIndex;
use crate::builder::{BuildEnv, build};
use crate::git::{is_git, pull};
#[derive(Deserialize, Serialize, FromRow)]
pub struct PackageIndex {
#[model]
pub struct IndexEntry {
pub id: Id,
pub repo: String,
pub pkg: String,
pub last_commit: Option<String>,
pub last_pkgver: Option<String>,
}
impl PackageIndex {
pub async fn all() -> Vec<Self> {
sqlx::query_as("SELECT * FROM package")
.fetch_all(get_pg!())
.await
.unwrap()
impl IndexEntry {
pub async fn all() -> Vec<Model<Self>> {
query!(|_| true)
}
/// The path to the indexed pkgs build environment
pub fn path(&self) -> PathBuf {
std::path::Path::new("./repositories")
.join(&self.repo)
@ -38,19 +42,18 @@ impl PackageIndex {
pub async fn rebuild_pkgs() {
log::info!("Start checking for package rebuilds");
let all_pkgs = PackageIndex::all().await;
for pkg in &all_pkgs {
for pkg in &IndexEntry::all().await {
let mut should_rebuild = false;
if is_git(pkg.path().to_str().unwrap()) {
let git_commit = pull(pkg.path().to_str().unwrap()).unwrap_or_default();
if let Some(old_commit) = &pkg.last_commit {
if is_git(pkg.read().path().to_str().unwrap()) {
let git_commit = pull(pkg.read().path().to_str().unwrap()).unwrap_or_default();
if let Some(old_commit) = &pkg.read().last_commit {
if *old_commit != git_commit {
log::info!(
"Package {} / {} should rebuild: Commit {} -> {}",
pkg.repo,
pkg.pkg,
pkg.read().repo,
pkg.read().pkg,
old_commit,
git_commit
);
@ -59,21 +62,21 @@ pub async fn rebuild_pkgs() {
} else {
log::info!(
"Package {} / {} should rebuild: No last commit",
pkg.repo,
pkg.pkg
pkg.read().repo,
pkg.read().pkg
);
should_rebuild = true;
}
}
let pkgver = get_pkgver(pkg.path().to_str().unwrap()).unwrap_or_default();
let pkgver = get_pkgver(pkg.read().path().to_str().unwrap()).unwrap_or_default();
if let Some(old_pkgver) = &pkg.last_pkgver {
if let Some(old_pkgver) = &pkg.read().last_pkgver {
if *old_pkgver != pkgver {
log::info!(
"Package {} / {} should rebuild: pkgver {} -> {}",
pkg.repo,
pkg.pkg,
pkg.read().repo,
pkg.read().pkg,
old_pkgver,
pkgver
);
@ -82,8 +85,8 @@ pub async fn rebuild_pkgs() {
} else {
log::info!(
"Package {} / {} should rebuild: No last pkgver",
pkg.repo,
pkg.pkg
pkg.read().repo,
pkg.read().pkg
);
should_rebuild = true;
}
@ -95,16 +98,10 @@ pub async fn rebuild_pkgs() {
log::info!("Done checking for package rebuilds");
}
pub fn get_pkgver(repo: &str) -> Option<String> {
// let base_env = BuildEnv::new_from("./build/srcinfo");
let base_env = BuildEnv::new();
pub fn get_pkgver(path: &str) -> Option<String> {
let out: FunResult = run_fun!(cd $path;pacco pkg info);
base_env.copy_build_env_dir(repo);
let (success, out) = base_env.run_command(
r#"cd /build;useradd -m build;echo "ALL ALL=(ALL) NOPASSWD: ALL"|tee -a /etc/sudoers;chown -R build /build;su -c "makepkg --printsrcinfo" build"#);
if success {
if let Ok(out) = out {
for line in out.lines() {
let trimmed = line.trim();
if let Some(pkgver) = trimmed.strip_prefix("pkgver = ") {
@ -116,6 +113,29 @@ pub fn get_pkgver(repo: &str) -> Option<String> {
None
}
pub fn find_package_files(dir: &str) -> Vec<String> {
let output = Command::new("sh")
.arg("-c")
.arg(&format!("cd {dir};ls -1 *.pkg.tar.*"))
.output()
.expect("failed to execute process");
let res = String::from_utf8_lossy(&output.stdout).to_string();
res.split("\n")
.filter_map(|x| {
let x = x.trim();
if x.is_empty() {
None
} else {
if x.ends_with(".sig") {
return None;
}
Some(x.to_string())
}
})
.collect()
}
pub fn find_pkgs(dir: &str) -> Vec<(String, Vec<u8>)> {
let mut pkgs = Vec::new();
@ -150,6 +170,7 @@ pub fn find_pkgs(dir: &str) -> Vec<(String, Vec<u8>)> {
pub async fn reindex_pkg(repo_dir: &Path) {
log::info!("Indexing package builds");
let mut dir_entries = read_dir(repo_dir).unwrap();
let mut index = PackageIndex;
while let Some(Ok(repo_entry)) = dir_entries.next() {
if repo_entry.file_type().unwrap().is_dir() {
@ -161,14 +182,7 @@ pub async fn reindex_pkg(repo_dir: &Path) {
let pkg_name = pkg_entry.file_name().into_string().unwrap();
log::info!("Found package {repo_name} / {pkg_name}");
sqlx::query(
"INSERT INTO package (repo, pkg) VALUES ($1, $2) ON CONFLICT DO NOTHING",
)
.bind(&repo_name)
.bind(&pkg_name)
.execute(get_pg!())
.await
.unwrap();
index.insert_if_not_present(&repo_name, &pkg_name);
}
}
}