git: dc4581589a32 - main - pkg: clean support for repositories
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 14 Jan 2025 11:21:43 UTC
The branch main has been updated by bapt:
URL: https://cgit.FreeBSD.org/src/commit/?id=dc4581589a3256667fafd46a30c67abdfd86618f
commit dc4581589a3256667fafd46a30c67abdfd86618f
Author: Baptiste Daroussin <bapt@FreeBSD.org>
AuthorDate: 2025-01-14 11:12:00 +0000
Commit: Baptiste Daroussin <bapt@FreeBSD.org>
CommitDate: 2025-01-14 11:21:40 +0000
pkg: clean support for repositories
Rework the way the bootstrap fetches pkg, by implementing a full support
for the repositories, the boostrap will now loop over all available repo
and try to fetch the full package from there. It will at the first valid
package found.
Fallback to packagesite (which has been deprecated for a while) if needed, by
transforming it into a repo, if no repo is found.
MFC After: 3 weeks
---
usr.sbin/pkg/config.c | 158 ++++++++++++++++++++++++++++++++++++++------------
usr.sbin/pkg/config.h | 24 ++++++++
usr.sbin/pkg/pkg.c | 149 +++++++++++++++++++++++------------------------
3 files changed, 216 insertions(+), 115 deletions(-)
diff --git a/usr.sbin/pkg/config.c b/usr.sbin/pkg/config.c
index d8e15d44da3d..23bccb7eb96e 100644
--- a/usr.sbin/pkg/config.c
+++ b/usr.sbin/pkg/config.c
@@ -59,6 +59,8 @@ struct config_entry {
bool main_only; /* Only set in pkg.conf. */
};
+static struct repositories repositories = STAILQ_HEAD_INITIALIZER(repositories);
+
static struct config_entry c[] = {
[PACKAGESITE] = {
PKG_CONFIG_STRING,
@@ -211,7 +213,7 @@ boolstr_to_bool(const char *str)
}
static void
-config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
+config_parse(const ucl_object_t *obj)
{
FILE *buffp;
char *buf = NULL;
@@ -238,29 +240,9 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
memset(buf, 0, bufsz);
rewind(buffp);
- if (conftype == CONFFILE_PKG) {
for (j = 0; j < strlen(key); ++j)
fputc(toupper(key[j]), buffp);
fflush(buffp);
- } else if (conftype == CONFFILE_REPO) {
- if (strcasecmp(key, "url") == 0)
- fputs("PACKAGESITE", buffp);
- else if (strcasecmp(key, "mirror_type") == 0)
- fputs("MIRROR_TYPE", buffp);
- else if (strcasecmp(key, "signature_type") == 0)
- fputs("SIGNATURE_TYPE", buffp);
- else if (strcasecmp(key, "fingerprints") == 0)
- fputs("FINGERPRINTS", buffp);
- else if (strcasecmp(key, "pubkey") == 0)
- fputs("PUBKEY", buffp);
- else if (strcasecmp(key, "enabled") == 0) {
- if ((cur->type != UCL_BOOLEAN) ||
- !ucl_object_toboolean(cur))
- goto cleanup;
- } else
- continue;
- fflush(buffp);
- }
for (i = 0; i < CONFIG_SIZE; i++) {
if (strcmp(buf, c[i].key) == 0)
@@ -324,7 +306,7 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
if (c[i].envset)
continue;
/* Prevent overriding ABI, ASSUME_ALWAYS_YES, etc. */
- if (conftype != CONFFILE_PKG && c[i].main_only == true)
+ if (c[i].main_only == true)
continue;
switch (c[i].type) {
case PKG_CONFIG_LIST:
@@ -336,12 +318,91 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
}
}
-cleanup:
free(temp_config);
fclose(buffp);
free(buf);
}
+
+static void
+parse_mirror_type(struct repository *r, const char *mt)
+{
+ if (strcasecmp(mt, "srv") == 0)
+ r->mirror_type = MIRROR_SRV;
+ r->mirror_type = MIRROR_NONE;
+}
+
+static bool
+parse_signature_type(struct repository *repo, const char *st)
+{
+ if (strcasecmp(st, "FINGERPRINTS") == 0)
+ repo->signature_type = SIGNATURE_FINGERPRINT;
+ else if (strcasecmp(st, "PUBKEY") == 0)
+ repo->signature_type = SIGNATURE_PUBKEY;
+ else if (strcasecmp(st, "NONE") == 0)
+ repo->signature_type = SIGNATURE_NONE;
+ else {
+ warnx("Signature type %s is not supported for bootstraping,"
+ " ignoring repository %s", st, repo->name);
+ free(repo->url);
+ free(repo->name);
+ free(repo->fingerprints);
+ free(repo->pubkey);
+ free(repo);
+ return false;
+ }
+ return (true);
+}
+
+static void
+parse_repo(const ucl_object_t *o)
+{
+ const ucl_object_t *cur;
+ const char *key;
+ ucl_object_iter_t it = NULL;
+
+ struct repository *repo = calloc(1, sizeof(struct repository));
+ if (repo == NULL)
+ err(EXIT_FAILURE, "calloc");
+
+ repo->name = strdup(ucl_object_key(o));
+ if (repo->name == NULL)
+ err(EXIT_FAILURE, "strdup");
+ while ((cur = ucl_iterate_object(o, &it, true))) {
+ key = ucl_object_key(cur);
+ if (key == NULL)
+ continue;
+ if (strcasecmp(key, "url") == 0) {
+ repo->url = strdup(ucl_object_tostring(cur));
+ if (repo->url == NULL)
+ err(EXIT_FAILURE, "strdup");
+ } else if (strcasecmp(key, "mirror_type") == 0) {
+ parse_mirror_type(repo, ucl_object_tostring(cur));
+ } else if (strcasecmp(key, "signature_type") == 0) {
+ if (!parse_signature_type(repo, ucl_object_tostring(cur)))
+ return;
+ } else if (strcasecmp(key, "fingerprints") == 0) {
+ repo->fingerprints = strdup(ucl_object_tostring(cur));
+ if (repo->fingerprints == NULL)
+ err(EXIT_FAILURE, "strdup");
+ } else if (strcasecmp(key, "pubkey") == 0) {
+ repo->pubkey = strdup(ucl_object_tostring(cur));
+ if (repo->pubkey == NULL)
+ err(EXIT_FAILURE, "strdup");
+ } else if (strcasecmp(key, "enabled") == 0) {
+ if ((cur->type != UCL_BOOLEAN) ||
+ !ucl_object_toboolean(cur)) {
+ free(repo->url);
+ free(repo->name);
+ free(repo);
+ return;
+ }
+ }
+ }
+ STAILQ_INSERT_TAIL(&repositories, repo, next);
+ return;
+}
+
/*-
* Parse new repo style configs in style:
* Name:
@@ -367,8 +428,7 @@ parse_repo_file(ucl_object_t *obj, const char *requested_repo)
if (requested_repo != NULL && strcmp(requested_repo, key) != 0)
continue;
-
- config_parse(cur, CONFFILE_REPO);
+ parse_repo(cur);
}
}
@@ -379,8 +439,12 @@ read_conf_file(const char *confpath, const char *requested_repo,
{
struct ucl_parser *p;
ucl_object_t *obj = NULL;
+ const char *abi = pkg_get_myabi();
+ if (abi == NULL)
+ errx(EXIT_FAILURE, "Fail do determine ABI");
p = ucl_parser_new(0);
+ ucl_parser_register_variable(p, "ABI", abi);
if (!ucl_parser_add_file(p, confpath)) {
if (errno != ENOENT)
@@ -397,7 +461,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
"configuration file %s", confpath);
else {
if (conftype == CONFFILE_PKG)
- config_parse(obj, conftype);
+ config_parse(obj);
else if (conftype == CONFFILE_REPO)
parse_repo_file(obj, requested_repo);
}
@@ -408,7 +472,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
return (0);
}
-static int
+static void
load_repositories(const char *repodir, const char *requested_repo)
{
struct dirent *ent;
@@ -416,12 +480,9 @@ load_repositories(const char *repodir, const char *requested_repo)
char *p;
size_t n;
char path[MAXPATHLEN];
- int ret;
-
- ret = 0;
if ((d = opendir(repodir)) == NULL)
- return (1);
+ return;
while ((ent = readdir(d))) {
/* Trim out 'repos'. */
@@ -437,7 +498,6 @@ load_repositories(const char *repodir, const char *requested_repo)
continue;
if (read_conf_file(path, requested_repo,
CONFFILE_REPO)) {
- ret = 1;
goto cleanup;
}
}
@@ -445,8 +505,6 @@ load_repositories(const char *repodir, const char *requested_repo)
cleanup:
closedir(d);
-
- return (ret);
}
int
@@ -508,8 +566,7 @@ config_init(const char *requested_repo)
}
STAILQ_FOREACH(cv, c[REPOS_DIR].list, next)
- if (load_repositories(cv->value, requested_repo))
- goto finalize;
+ load_repositories(cv->value, requested_repo);
finalize:
if (c[ABI].val == NULL && c[ABI].value == NULL) {
@@ -520,8 +577,6 @@ finalize:
c[ABI].val = abi;
}
- subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
-
return (0);
}
@@ -560,6 +615,33 @@ config_bool(pkg_config_key k, bool *val)
return (0);
}
+struct repositories *
+config_get_repositories(void)
+{
+ if (STAILQ_EMPTY(&repositories)) {
+ /* Fall back to PACKAGESITE - deprecated - */
+ struct repository *r = calloc(1, sizeof(r));
+ if (r == NULL)
+ err(EXIT_FAILURE, "calloc");
+ r->name = strdup("fallback");
+ if (r->name == NULL)
+ err(EXIT_FAILURE, "strdup");
+ subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
+ r->url = c[PACKAGESITE].value;
+ if (c[SIGNATURE_TYPE].value != NULL)
+ if (!parse_signature_type(r, c[SIGNATURE_TYPE].value))
+ exit(EXIT_FAILURE);
+ if (c[MIRROR_TYPE].value != NULL)
+ parse_mirror_type(r, c[MIRROR_TYPE].value);
+ if (c[PUBKEY].value != NULL)
+ r->pubkey = c[PUBKEY].value;
+ if (c[FINGERPRINTS].value != NULL)
+ r->fingerprints = c[FINGERPRINTS].value;
+ STAILQ_INSERT_TAIL(&repositories, r, next);
+ }
+ return (&repositories);
+}
+
void
config_finish(void) {
int i;
diff --git a/usr.sbin/pkg/config.h b/usr.sbin/pkg/config.h
index 21d115e41db2..26f3ff79541b 100644
--- a/usr.sbin/pkg/config.h
+++ b/usr.sbin/pkg/config.h
@@ -30,6 +30,7 @@
#define _PKG_CONFIG_H
#include <paths.h>
+#include <sys/queue.h>
#define URL_SCHEME_PREFIX "pkg+"
@@ -58,9 +59,32 @@ typedef enum {
CONFFILE_REPO,
} pkg_conf_file_t;
+typedef enum {
+ SIGNATURE_NONE = 0,
+ SIGNATURE_FINGERPRINT,
+ SIGNATURE_PUBKEY,
+} signature_t;
+
+typedef enum {
+ MIRROR_NONE = 0,
+ MIRROR_SRV,
+} mirror_t;
+
+struct repository {
+ char *name;
+ char *url;
+ mirror_t mirror_type;
+ signature_t signature_type;
+ char *fingerprints;
+ char *pubkey;
+ STAILQ_ENTRY(repository) next;
+};
+STAILQ_HEAD(repositories, repository);
+
int config_init(const char *);
void config_finish(void);
int config_string(pkg_config_key, const char **);
int config_bool(pkg_config_key, bool *);
+struct repositories *config_get_repositories(void);
#endif
diff --git a/usr.sbin/pkg/pkg.c b/usr.sbin/pkg/pkg.c
index 54884dbe0b23..0b46d2831e9e 100644
--- a/usr.sbin/pkg/pkg.c
+++ b/usr.sbin/pkg/pkg.c
@@ -255,7 +255,7 @@ install_pkg_static(const char *path, const char *pkgpath, bool force)
}
static int
-fetch_to_fd(const char *url, char *path, const char *fetchOpts)
+fetch_to_fd(struct repository *repo, const char *url, char *path, const char *fetchOpts)
{
struct url *u;
struct dns_srvinfo *mirrors, *current;
@@ -267,18 +267,11 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
ssize_t r;
char buf[10240];
char zone[MAXHOSTNAMELEN + 13];
- static const char *mirror_type = NULL;
max_retry = 3;
current = mirrors = NULL;
remote = NULL;
- if (mirror_type == NULL && config_string(MIRROR_TYPE, &mirror_type)
- != 0) {
- warnx("No MIRROR_TYPE defined");
- return (-1);
- }
-
if ((fd = mkstemp(path)) == -1) {
warn("mkstemp()");
return (-1);
@@ -294,7 +287,7 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
while (remote == NULL) {
if (retry == max_retry) {
if (strcmp(u->scheme, "file") != 0 &&
- strcasecmp(mirror_type, "srv") == 0) {
+ repo->mirror_type == MIRROR_SRV) {
snprintf(zone, sizeof(zone),
"_%s._tcp.%s", u->scheme, u->host);
mirrors = dns_getsrvinfo(zone);
@@ -653,23 +646,31 @@ parse_cert(int fd) {
}
static bool
-verify_pubsignature(int fd_pkg, int fd_sig)
+verify_pubsignature(int fd_pkg, int fd_sig, struct repository *r)
{
struct pubkey *pk;
- const char *pubkey;
char *data;
struct pkgsign_ctx *sctx;
size_t datasz;
bool ret;
+ const char *pubkey;
pk = NULL;
- pubkey = NULL;
sctx = NULL;
data = NULL;
ret = false;
- if (config_string(PUBKEY, &pubkey) != 0) {
- warnx("No CONFIG_PUBKEY defined");
- goto cleanup;
+
+ if (r != NULL) {
+ if (r->pubkey == NULL) {
+ warnx("No CONFIG_PUBKEY defined for %s", r->name);
+ goto cleanup;
+ }
+ pubkey = r->pubkey;
+ } else {
+ if (config_string(PUBKEY, &pubkey) != 0) {
+ warnx("No CONFIG_PUBKEY defined for %s", r->name);
+ goto cleanup;
+ }
}
if ((pk = read_pubkey(fd_sig)) == NULL) {
@@ -703,8 +704,8 @@ verify_pubsignature(int fd_pkg, int fd_sig)
}
/* Verify the signature. */
- printf("Verifying signature with public key %s... ", pubkey);
- if (pkgsign_verify_data(sctx, data, datasz, pubkey, NULL, 0, pk->sig,
+ printf("Verifying signature with public key %s.a.. ", r->pubkey);
+ if (pkgsign_verify_data(sctx, data, datasz, r->pubkey, NULL, 0, pk->sig,
pk->siglen) == false) {
fprintf(stderr, "Signature is not valid\n");
goto cleanup;
@@ -723,7 +724,7 @@ cleanup:
}
static bool
-verify_signature(int fd_pkg, int fd_sig)
+verify_signature(int fd_pkg, int fd_sig, struct repository *r)
{
struct fingerprint_list *trusted, *revoked;
struct fingerprint *fingerprint;
@@ -742,9 +743,17 @@ verify_signature(int fd_pkg, int fd_sig)
ret = false;
/* Read and parse fingerprints. */
- if (config_string(FINGERPRINTS, &fingerprints) != 0) {
- warnx("No CONFIG_FINGERPRINTS defined");
- goto cleanup;
+ if (r != NULL) {
+ if (r->fingerprints == NULL) {
+ warnx("No FINGERPRINTS defined for %s", r->name);
+ goto cleanup;
+ }
+ fingerprints = r->fingerprints;
+ } else {
+ if (config_string(FINGERPRINTS, &fingerprints) != 0) {
+ warnx("No FINGERPRINTS defined");
+ goto cleanup;
+ }
}
snprintf(path, MAXPATHLEN, "%s/trusted", fingerprints);
@@ -833,7 +842,7 @@ cleanup:
}
static int
-bootstrap_pkg(bool force, const char *fetchOpts)
+bootstrap_pkg(bool force, const char *fetchOpts, struct repository *repo)
{
int fd_pkg, fd_sig;
int ret;
@@ -841,28 +850,18 @@ bootstrap_pkg(bool force, const char *fetchOpts)
char tmppkg[MAXPATHLEN];
char tmpsig[MAXPATHLEN];
const char *packagesite;
- const char *signature_type;
char pkgstatic[MAXPATHLEN];
const char *bootstrap_name;
fd_sig = -1;
ret = -1;
- if (config_string(PACKAGESITE, &packagesite) != 0) {
- warnx("No PACKAGESITE defined");
- return (-1);
- }
-
- if (config_string(SIGNATURE_TYPE, &signature_type) != 0) {
- warnx("Error looking up SIGNATURE_TYPE");
- return (-1);
- }
-
- printf("Bootstrapping pkg from %s, please wait...\n", packagesite);
+ printf("Bootstrapping pkg from %s, please wait...\n", repo->url);
/* Support pkg+http:// for PACKAGESITE which is the new format
in 1.2 to avoid confusion on why http://pkg.FreeBSD.org has
no A record. */
+ packagesite = repo->url;
if (strncmp(URL_SCHEME_PREFIX, packagesite,
strlen(URL_SCHEME_PREFIX)) == 0)
packagesite += strlen(URL_SCHEME_PREFIX);
@@ -873,53 +872,44 @@ bootstrap_pkg(bool force, const char *fetchOpts)
snprintf(tmppkg, MAXPATHLEN, "%s/%s.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
- if ((fd_pkg = fetch_to_fd(url, tmppkg, fetchOpts)) != -1)
+ if ((fd_pkg = fetch_to_fd(repo, url, tmppkg, fetchOpts)) != -1)
break;
bootstrap_name = NULL;
}
if (bootstrap_name == NULL)
goto fetchfail;
- if (signature_type != NULL &&
- strcasecmp(signature_type, "NONE") != 0) {
- if (strcasecmp(signature_type, "FINGERPRINTS") == 0) {
-
- snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
- getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
- bootstrap_name);
- snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
- packagesite, bootstrap_name);
-
- if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
- fprintf(stderr, "Signature for pkg not "
- "available.\n");
- goto fetchfail;
- }
+ if (repo->signature_type == SIGNATURE_FINGERPRINT) {
+ snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
+ bootstrap_name);
+ snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
+ packagesite, bootstrap_name);
- if (verify_signature(fd_pkg, fd_sig) == false)
- goto cleanup;
- } else if (strcasecmp(signature_type, "PUBKEY") == 0) {
+ if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
+ fprintf(stderr, "Signature for pkg not "
+ "available.\n");
+ goto fetchfail;
+ }
- snprintf(tmpsig, MAXPATHLEN,
- "%s/%s.pubkeysig.XXXXXX",
- getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
- bootstrap_name);
- snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
- packagesite, bootstrap_name);
+ if (verify_signature(fd_pkg, fd_sig, repo) == false)
+ goto cleanup;
+ } else if (repo->signature_type == SIGNATURE_PUBKEY) {
+ snprintf(tmpsig, MAXPATHLEN,
+ "%s/%s.pubkeysig.XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
+ bootstrap_name);
+ snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
+ repo->url, bootstrap_name);
- if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
- fprintf(stderr, "Signature for pkg not "
- "available.\n");
- goto fetchfail;
- }
+ if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
+ fprintf(stderr, "Signature for pkg not "
+ "available.\n");
+ goto fetchfail;
+ }
- if (verify_pubsignature(fd_pkg, fd_sig) == false)
- goto cleanup;
- } else {
- warnx("Signature type %s is not supported for "
- "bootstrapping.", signature_type);
+ if (verify_pubsignature(fd_pkg, fd_sig, repo) == false)
goto cleanup;
- }
}
if ((ret = extract_pkg_static(fd_pkg, pkgstatic, MAXPATHLEN)) == 0)
@@ -929,18 +919,15 @@ bootstrap_pkg(bool force, const char *fetchOpts)
fetchfail:
for (int j = 0; bootstrap_names[j] != NULL; j++) {
- warnx("Attempted to fetch %s/Latest/%s", packagesite,
+ warnx("Attempted to fetch %s/Latest/%s", repo->url,
bootstrap_names[j]);
}
warnx("Error: %s", fetchLastErrString);
if (fetchLastErrCode == FETCH_RESOLV) {
fprintf(stderr, "Address resolution failed for %s.\n", packagesite);
- fprintf(stderr, "Consider changing PACKAGESITE.\n");
} else {
fprintf(stderr, "A pre-built version of pkg could not be found for "
"your system.\n");
- fprintf(stderr, "Consider changing PACKAGESITE or installing it from "
- "ports: 'ports-mgmt/pkg'.\n");
}
cleanup:
@@ -1024,7 +1011,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
goto cleanup;
}
- if (verify_signature(fd_pkg, fd_sig) == false)
+ if (verify_signature(fd_pkg, fd_sig, NULL) == false)
goto cleanup;
} else if (strcasecmp(signature_type, "PUBKEY") == 0) {
@@ -1037,7 +1024,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
goto cleanup;
}
- if (verify_pubsignature(fd_pkg, fd_sig) == false)
+ if (verify_pubsignature(fd_pkg, fd_sig, NULL) == false)
goto cleanup;
} else {
@@ -1105,6 +1092,7 @@ main(int argc, char *argv[])
signed char ch;
const char *fetchOpts;
char *command;
+ struct repositories *repositories;
activation_test = false;
add_pkg = false;
@@ -1232,6 +1220,8 @@ main(int argc, char *argv[])
fetchDebug = 1;
if ((bootstrap_only && force) || access(pkgpath, X_OK) == -1) {
+ struct repository *repo;
+ int ret = 0;
/*
* To allow 'pkg -N' to be used as a reliable test for whether
* a system is configured to use pkg, don't bootstrap pkg
@@ -1272,7 +1262,12 @@ main(int argc, char *argv[])
if (pkg_query_yes_no() == 0)
exit(EXIT_FAILURE);
}
- if (bootstrap_pkg(force, fetchOpts) != 0)
+ repositories = config_get_repositories();
+ STAILQ_FOREACH(repo, repositories, next) {
+ if ((ret = bootstrap_pkg(force, fetchOpts, repo)) == 0)
+ break;
+ }
+ if (ret != 0)
exit(EXIT_FAILURE);
config_finish();