jmarya/pacco
jmarya/pacco
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages 1 Wiki Activity

smart mirroring

Browse source
This commit is contained in:
JMARyA 2025-01-13 11:39:00 +01:00
parent 68cb32f07b
commit 7647616242
Signed by: jmarya
GPG key ID: 901B2ADDF27C2263
13 changed files with 701 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

30
src/config.rs Normal file
View file

@ -0,0 +1,30 @@
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize, Default)]
pub struct Config {
pub mirror: Option<MirrorConfig>,
}
#[derive(Debug, Clone, Deserialize, Default)]
pub struct MirrorConfig {
repos: Vec<String>,
mirrorlist: Vec<String>,
}
impl Config {
pub fn is_mirrored_repo(&self, repo: &str) -> bool {
if let Some(mirrorc) = &self.mirror {
return mirrorc.repos.iter().any(|x| x == repo);
}
false
}
pub fn mirrorlist(&self) -> Option<&[String]> {
if let Some(mirror) = &self.mirror {
return Some(mirror.mirrorlist.as_slice());
}
None
}
}

38
src/main.rs
View file

@ -1,7 +1,9 @@
use based::auth::User;
use based::get_pg;
use config::Config;
use rocket::routes;
pub mod config;
pub mod routes;
#[rocket::launch]
@ -11,21 +13,27 @@ async fn launch() -> _ {
let pg = get_pg!();
sqlx::migrate!("./migrations").run(pg).await.unwrap();
let config: Config =
toml::from_str(&std::fs::read_to_string("config.toml").unwrap_or_default())
.unwrap_or_default();
let _ = User::create("admin".to_string(), "admin", based::auth::UserRole::Admin).await;
rocket::build().mount("/", routes![
based::htmx::htmx_script_route,
routes::index_page,
routes::pkg_route,
routes::push::upload_pkg,
routes::user::login,
routes::user::login_post,
routes::user::account_page,
routes::ui::pkg_ui,
routes::ui::repo_ui,
routes::user::new_api_key,
routes::user::end_session,
routes::user::change_password,
routes::user::change_password_post
])
rocket::build()
.mount("/", routes![
based::htmx::htmx_script_route,
routes::index_page,
routes::pkg_route,
routes::push::upload_pkg,
routes::user::login,
routes::user::login_post,
routes::user::account_page,
routes::ui::pkg_ui,
routes::ui::repo_ui,
routes::user::new_api_key,
routes::user::end_session,
routes::user::change_password,
routes::user::change_password_post
])
.manage(config)
}

120
src/pkg/mirror.rs Normal file
View file

@ -0,0 +1,120 @@
use std::path::PathBuf;
use super::{Package, Repository, arch::Architecture};
pub struct MirrorRepository {
pub inner: Repository,
}
impl MirrorRepository {
pub fn new(repo: &str) -> Self {
std::fs::create_dir_all(format!("./data/{repo}")).unwrap();
Self {
inner: Repository::new(repo).unwrap(),
}
}
pub async fn download_file(
&self,
url: &str,
file_path: PathBuf,
mirrorlist: &[String],
arch: Architecture,
) {
log::info!("Downloading {url} to {}", file_path.to_str().unwrap());
for mirror in mirrorlist {
let mirror = mirror
.replace("$repo", &self.inner.name)
.replace("$arch", &arch.to_string());
let url = format!("{mirror}/{url}");
log::info!("Trying mirror {url}");
let client = reqwest::Client::new();
if let Ok(resp) = client.get(url).send().await {
if resp.status().is_success() {
let data = resp.bytes().await.unwrap().to_vec();
let parent = file_path.parent().unwrap();
std::fs::create_dir_all(parent).unwrap();
std::fs::write(file_path, data).unwrap();
return;
} else {
log::warn!("Mirror {mirror} failed [{}]", resp.status().as_u16());
}
}
}
}
/// Get the `.db.tar.gz` content for the repository of `arch`
pub async fn db_content(&self, arch: Architecture, mirrorlist: &[String]) -> Option<Vec<u8>> {
if let Some(content) = self.inner.db_content(arch.clone()) {
return Some(content);
}
self.download_file(
&format!("{}.db.tar.gz", self.inner.name),
self.inner
.base_path(arch.clone())
.join(format!("{}.db.tar.gz", self.inner.name)),
mirrorlist,
arch.clone(),
)
.await;
self.inner.db_content(arch)
}
/// Get the `.db.tar.gz.sig` content for the repository of `arch`
pub async fn sig_content(&self, arch: Architecture, mirrorlist: &[String]) -> Option<Vec<u8>> {
if let Some(content) = self.inner.sig_content(arch.clone()) {
return Some(content);
}
self.download_file(
&format!("{}.db.tar.gz.sig", self.inner.name),
self.inner
.base_path(arch.clone())
.join(format!("{}.db.tar.gz.sig", self.inner.name)),
mirrorlist,
arch.clone(),
)
.await;
self.inner.sig_content(arch)
}
pub async fn get_pkg(&self, pkg_name: &str, mirrorlist: &[String]) -> Option<Package> {
if let Some(pkg) = self.inner.get_pkg(pkg_name) {
return Some(pkg);
}
// PKG
let (name, _, _, arch) = Package::extract_pkg_name(pkg_name).unwrap();
self.download_file(
pkg_name,
self.inner
.base_path(arch.clone())
.join(&name)
.join(pkg_name),
mirrorlist,
arch.clone(),
)
.await;
// SIG
self.download_file(
&format!("{pkg_name}.sig"),
self.inner
.base_path(arch.clone())
.join(&name)
.join(format!("{pkg_name}.sig")),
mirrorlist,
arch.clone(),
)
.await;
self.inner.get_pkg(pkg_name)
}
}

16
src/pkg/mod.rs
View file

@ -1,8 +1,20 @@
// TODO : Read DB Info
pub mod repo;
pub use repo::Repository;
pub mod package;
pub use package::Package;
pub mod arch;
pub mod db;
pub mod mirror;
pub fn find_package_by_name(pkg: &str) -> Option<Package> {
for repo in Repository::list() {
let repo = Repository::new(&repo).unwrap();
let pkgs = repo.list_pkg();
if let Some(res) = pkgs.iter().find(|x| *x == pkg) {
return repo.get_pkg_by_name(res);
}
}
None
}

17
src/pkg/package.rs
View file

@ -18,18 +18,21 @@ pub struct Package {
/// Name of the package
pub name: String,
/// Version of the package
version: Option<String>,
pub version: Option<String>,
}
impl Package {
/// Create a new package
pub fn new(repo: &str, arch: Architecture, pkg_name: &str, version: &str) -> Self {
Package {
let pkg = Package {
repo: repo.to_string(),
arch,
name: pkg_name.to_string(),
version: Some(version.to_string()),
}
};
std::fs::create_dir_all(pkg.base_path()).unwrap();
pkg
}
pub fn install_script(&self) -> Option<String> {
@ -186,7 +189,7 @@ impl Package {
pub fn pacman_hooks(&self) -> Vec<(String, String)> {
let pkg_file = self.base_path().join(self.file_name());
let files = list_tar_file(&pkg_file).unwrap_or_default();
let files = self.file_list();
files
.into_iter()
.filter(|x| {
@ -237,12 +240,10 @@ impl Package {
fn base_path(&self) -> PathBuf {
// <repo>/<arch>/<pkg>/
let p = Path::new("./data")
Path::new("./data")
.join(&self.repo)
.join(self.arch.to_string())
.join(&self.name);
std::fs::create_dir_all(&p).unwrap();
p
.join(&self.name)
}
/// Switch the `Architecture` of the package

1
src/pkg/repo.rs
View file

@ -18,6 +18,7 @@ impl Repository {
repos.push(file_name);
}
repos.sort();
repos
}

85
src/routes/mod.rs
View file

@ -2,18 +2,25 @@ use based::auth::MaybeUser;
use based::page::{Shell, htmx_link, render_page};
use based::request::{RawResponse, RequestContext, StringResponse, respond_with};
use maud::{PreEscaped, html};
use rocket::get;
use pacco::pkg::mirror::MirrorRepository;
use rocket::http::{ContentType, Status};
use rocket::{State, get};
use pacco::pkg::Repository;
use pacco::pkg::arch::Architecture;
use crate::config::Config;
pub mod push;
pub mod ui;
pub mod user;
#[get("/")]
pub async fn index_page(ctx: RequestContext, user: MaybeUser) -> StringResponse {
pub async fn index_page(
ctx: RequestContext,
user: MaybeUser,
config: &State<Config>,
) -> StringResponse {
let repos: Vec<String> = Repository::list();
let content = html!(
@ -26,8 +33,12 @@ pub async fn index_page(ctx: RequestContext, user: MaybeUser) -> StringResponse
div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-6" {
@for repo in repos {
(htmx_link(&format!("/{repo}"), "flex items-center gap-4 p-4 bg-gray-100 dark:bg-gray-700 rounded-lg shadow-md transition hover:bg-gray-200 dark:hover:bg-gray-600", "", html! {
p class="font-medium flex-1 text-gray-800 dark:text-gray-100" {
p class="font-medium text-gray-800 dark:text-gray-100" {
(repo)
@if config.is_mirrored_repo(&repo) {
div class="inline-block px-3 py-1 text-sm font-medium text-white bg-blue-500 rounded-full" { "Mirrored" };
};
};
}))
}
@ -38,15 +49,64 @@ pub async fn index_page(ctx: RequestContext, user: MaybeUser) -> StringResponse
}
#[get("/pkg/<repo>/<arch>/<pkg_name>")]
pub async fn pkg_route(repo: &str, arch: &str, pkg_name: &str) -> RawResponse {
pub async fn pkg_route(
repo: &str,
arch: &str,
pkg_name: &str,
config: &State<Config>,
) -> RawResponse {
let arch = Architecture::parse(arch).unwrap();
if config.is_mirrored_repo(repo) {
let repo = MirrorRepository::new(repo);
if is_repo_db(pkg_name) {
if pkg_name.ends_with("sig") {
return respond_with(
Status::Ok,
ContentType::new("application", "pgp-signature"),
repo.sig_content(arch, config.mirrorlist().unwrap())
.await
.unwrap(),
);
} else {
return respond_with(
Status::Ok,
ContentType::new("application", "tar"),
repo.db_content(arch, config.mirrorlist().unwrap())
.await
.unwrap(),
);
}
}
let pkg = repo
.get_pkg(pkg_name, config.mirrorlist().unwrap())
.await
.unwrap();
if pkg_name.ends_with("pkg.tar.zst") {
return respond_with(
Status::Ok,
ContentType::new("application", "tar"),
pkg.pkg_content().unwrap(),
);
} else if pkg_name.ends_with("pkg.tar.zst.sig") {
return respond_with(
Status::Ok,
ContentType::new("application", "pgp-signature"),
pkg.sig_content().unwrap(),
);
}
return respond_with(
Status::Ok,
ContentType::Plain,
"Not found".as_bytes().to_vec(),
);
}
if let Some(repo) = Repository::new(repo) {
if pkg_name.ends_with("db.tar.gz")
|| pkg_name.ends_with("db")
|| pkg_name.ends_with("db.tar.gz.sig")
|| pkg_name.ends_with("db.sig")
{
if is_repo_db(pkg_name) {
if pkg_name.ends_with("sig") {
return respond_with(
Status::Ok,
@ -107,3 +167,10 @@ pub async fn render(
)
.await
}
pub fn is_repo_db(pkg_name: &str) -> bool {
pkg_name.ends_with("db.tar.gz")
|| pkg_name.ends_with("db")
|| pkg_name.ends_with("db.tar.gz.sig")
|| pkg_name.ends_with("db.sig")
}

9
src/routes/push.rs
View file

@ -1,6 +1,6 @@
use based::request::api::{FallibleApiResponse, api_error};
use rocket::tokio::io::AsyncReadExt;
use rocket::{FromForm, post};
use rocket::{FromForm, State, post};
use serde_json::json;
use pacco::pkg::Package;
@ -12,6 +12,8 @@ use pacco::pkg::arch::Architecture;
use rocket::form::Form;
use rocket::fs::TempFile;
use crate::config::Config;
#[derive(FromForm)]
pub struct PkgUpload<'r> {
name: String,
@ -37,12 +39,17 @@ pub async fn upload_pkg(
repo: &str,
upload: Form<PkgUpload<'_>>,
user: based::auth::APIUser,
config: &State<Config>,
) -> FallibleApiResponse {
// TODO : Permission System
if !user.0.is_admin() {
return Err(api_error("Forbidden"));
}
if config.is_mirrored_repo(repo) {
return Err(api_error("This repository is a mirror."));
}
let pkg = Package::new(
repo,
Architecture::parse(&upload.arch).ok_or_else(|| api_error("Invalid architecture"))?,

85
src/routes/ui.rs
View file

@ -3,18 +3,30 @@ use based::{
request::{RequestContext, StringResponse},
};
use maud::{PreEscaped, html};
use rocket::get;
use rocket::{State, get};
use pacco::pkg::{Repository, arch::Architecture};
use pacco::pkg::{Repository, arch::Architecture, find_package_by_name};
use crate::config::Config;
use super::render;
// TODO : API
#[get("/<repo>/<pkg_name>")]
pub async fn pkg_ui(repo: &str, pkg_name: &str, ctx: RequestContext) -> Option<StringResponse> {
#[get("/<repo>/<pkg_name>?<ver>")]
pub async fn pkg_ui(
repo: &str,
pkg_name: &str,
ctx: RequestContext,
ver: Option<&str>,
) -> Option<StringResponse> {
let repo = Repository::new(repo).unwrap();
let pkg = repo.get_pkg_by_name(pkg_name)?;
let mut pkg = repo.get_pkg_by_name(pkg_name)?;
if let Some(ver) = ver {
pkg = pkg.get_version(ver);
}
let versions = pkg.versions();
let arch = pkg.arch();
let install_script = pkg.install_script();
@ -88,16 +100,17 @@ pub async fn pkg_ui(repo: &str, pkg_name: &str, ctx: RequestContext) -> Option<S
h2 class="text-xl font-semibold text-gray-700 dark:text-gray-300" { "Versions" }
ul class="space-y-1" {
@for version in versions {
// TODO : Implement page per version ?version=
li class="text-gray-800 dark:text-gray-100 hover:text-blue-500 dark:hover:text-blue-400 transition" {
(version)
(htmx_link(&format!("/{}/{}?ver={version}", repo.name, &pkg.name), if pkg.version.as_ref().map(|x| *x == version).unwrap_or_default() { "text-blue-500" } else { "" }, "", html! {
(version)
}))
};
}
}
}
};
div class="flex flex-wrap pt-6" {
div class="flex flex-wrap pt-6" {
div class="space-y-2" {
h2 class="text-xl font-bold text-gray-700 dark:text-gray-300" { "Content" }
@ -166,8 +179,12 @@ pub async fn pkg_ui(repo: &str, pkg_name: &str, ctx: RequestContext) -> Option<S
}
#[get("/<repo>?<arch>")]
pub async fn repo_ui(repo: &str, ctx: RequestContext, arch: Option<&str>) -> StringResponse {
// TODO : permissions
pub async fn repo_ui(
repo: &str,
ctx: RequestContext,
arch: Option<&str>,
config: &State<Config>,
) -> StringResponse {
let arch = arch.map(|x| Architecture::parse(x).unwrap_or(Architecture::any));
let repo = Repository::new(repo).unwrap();
@ -181,13 +198,16 @@ pub async fn repo_ui(repo: &str, ctx: RequestContext, arch: Option<&str>) -> Str
let content = html! {
// Repository name and architectures
div class="flex flex-wrap items-center justify-center mb-6" {
h1 class="text-3xl font-bold text-gray-800 dark:text-gray-100 pr-4" {
h1 class="text-3xl font-bold text-gray-800 dark:text-gray-100" {
(repo.name)
};
div class="flex gap-2 mt-2 md:mt-0" {
@if config.is_mirrored_repo(&repo.name) {
div class="inline-block px-3 py-1 text-sm font-medium text-white bg-blue-500 rounded-full ml-4" { "Mirrored" };
};
div class="flex gap-2 mt-2 md:mt-0 ml-4" {
@for a in architectures {
// TODO : Filter per arch with ?arch=
@if let Some(arch) = arch.as_ref() {
@if arch.to_string() == a {
(htmx_link(&format!("/{}", repo.name), "px-3 py-1 text-sm font-medium bg-blue-400 dark:bg-blue-500 text-gray-600 dark:text-gray-300 rounded-full", "", html! {
@ -258,12 +278,10 @@ pub fn build_info(key: String, value: String) -> PreEscaped<String> {
);
}
"depend" => {
// TODO : Find package link
return key_value("Depends on".to_string(), value);
return pkg_list_info("Depends on", &value);
}
"makedepend" => {
// TODO : Find package link
return key_value("Build Dependencies".to_string(), value);
return pkg_list_info("Build Dependencies", &value);
}
"license" => {
return key_value("License".to_string(), value);
@ -307,3 +325,36 @@ pub fn take_out<T>(v: &mut Vec<T>, f: impl Fn(&T) -> bool) -> (&mut Vec<T>, Opti
(v, None)
}
pub fn find_pkg_url(pkg: &str) -> Option<String> {
if let Some(pkg) = find_package_by_name(pkg) {
return Some(format!("/{}/{}", pkg.repo, pkg.name));
}
None
}
pub fn pkg_list_info(key: &str, value: &str) -> PreEscaped<String> {
let pkgs = value.split_whitespace().map(|pkg| {
if let Some(pkg_url) = find_pkg_url(pkg) {
html! {
a href=(pkg_url) class="ml-2 text-blue-400" { (pkg) };
}
} else {
html! {
span class="ml-2" { (pkg) };
}
}
});
html! {
div class="flex items-center" {
span class="font-bold w-32" { (format!("{key}: ")) };
div class="flex flex-wrap" {
@for pkg in pkgs {
(pkg)
};
};
};
}
}