git: 0ce9a414adc3 - main - release: add cloudware oracle targets to package and upload .oci files
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 04 Mar 2025 23:56:06 UTC
The branch main has been updated by dch:
URL: https://cgit.FreeBSD.org/src/commit/?id=0ce9a414adc33af29607adbd81e0760e014fcd76
commit 0ce9a414adc33af29607adbd81e0760e014fcd76
Author: Dave Cottlehuber <dch@FreeBSD.org>
AuthorDate: 2025-03-04 23:56:00 +0000
Commit: Dave Cottlehuber <dch@FreeBSD.org>
CommitDate: 2025-03-04 23:56:00 +0000
release: add cloudware oracle targets to package and upload .oci files
- requires base tar & flua, qemu-tools & curl from ports
- set ORACLE_PAR_URL to upload to local file:/// dir or cloud buckets
Reviewed by: emaste
Approved by: cperciva
Differential Revision: https://reviews.freebsd.org/D48382
Sponsored by: SkunkWerks, GmbH
---
release/Makefile.oracle | 101 +++++++++++++++++++++
release/Makefile.vm | 1 +
.../oracle/arm64_shape_compatibilities.json | 24 +++++
.../oracle/default_shape_compatibilities.json | 1 +
release/scripts/oracle/generate_metadata.lua | 74 +++++++++++++++
release/scripts/oracle/image_capability_data.json | 96 ++++++++++++++++++++
release/scripts/oracle/image_metadata.json | 21 +++++
7 files changed, 318 insertions(+)
diff --git a/release/Makefile.oracle b/release/Makefile.oracle
new file mode 100644
index 000000000000..b4f3bbdf86f2
--- /dev/null
+++ b/release/Makefile.oracle
@@ -0,0 +1,101 @@
+#D48382
+# Makefile for preparing & uploading Oracle Cloud images from existing
+# .raw files created by cloudware-release.
+#
+# Overview:
+#
+# The base image is already created by cloudware-release.
+#
+# Construct the custom OCI metadata, derived from exported official OCI images.
+# It is architecture-specific but appears mostly stable over time.
+# Compress the raw image and place it in the same directory as the metadata.
+# Make a GNU format tarball of these files.
+# Upload the tarball to Oracle Cloud via a pre-approved curl URI, into
+# the FreeBSD Foundation's Oracle Cloud account.
+#
+# These images go into the "re" bucket in us-ashburn-1 region, which
+# is mounted into the FreeBSD Foundation Oracle Marketplace account.
+# Once uploaded, a manual step is needed to import the images as local
+# custom images. These can then be tested within the us-ashburn-1 region.
+# Once tested, follow the manual Oracle Marketplace import process to
+# create a new FreeBSD version, attach the images, and initiate validation
+# by Oracle. This can take up to 5 working days. Once complete, a final
+# manual step is needed to mark the currently private images, public.
+# Syncing to all sites should take 2-3 hours after this final step.
+
+ORACLE_BASENAME= ${OSRELEASE}-${BUILDDATE}${GITREV:C/^(.+)/-\1/}
+ORACLE_PORTS_LIST= ftp/curl emulators/qemu@tools
+CLEANFILES+= cw-oracle-portinstall
+
+cw-oracle-portinstall: .PHONY
+.if !exists(/usr/local/bin/curl) || !exists(/usr/local/bin/qemu-img)
+. if !exists(${PORTSDIR}/Makefile)
+. if !exists(/usr/local/sbin/pkg-static)
+ env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf
+. endif
+ env ASSUME_ALWAYS_YES=yes pkg install -y ${ORACLE_PORTS_LIST}
+. else
+ env UNAME_r=${UNAME_r} make -C \
+ ${PORTSDIR}/ftp/curl \
+ BATCH=1 WRKDIRPREFIX=/tmp/ports DISTDIR=/tmp/distfiles \
+ all install clean
+ env UNAME_r=${UNAME_r} FLAVOR=tools make -C \
+ ${PORTSDIR}/emulators/qemu \
+ BATCH=1 WRKDIRPREFIX=/tmp/ports DISTDIR=/tmp/distfiles \
+ all install clean
+. endif
+.endif
+
+.for _FS in ${ORACLE_FSLIST}
+ORACLE_OCI_LIST+= cw-oracle-${_FS}.oci
+ORACLE_UPLOAD_LIST+= cw-oracle-upload-${_FS}
+CLEANFILES+= cw-oracle-${_FS}.oci
+ORACLE_TMP_${_FS}= cw-oracle-${_FS}.oci.tmpdir
+CLEANDIRS+= ${ORACLE_TMP_${_FS}}
+ORACLE_METADATA= ${.CURDIR}/scripts/oracle
+ORACLE_CAPABILITY= ${.CURDIR}/scripts/oracle/image_capability_data.json
+ORACLE_TEMPLATE= ${.CURDIR}/scripts/oracle/image_metadata.json
+ORACLE_OUTPUT_${_FS}= ${ORACLE_TMP_${_FS}}/image_metadata.json
+.if ${TARGET} == "arm64"
+ORACLE_SHAPES= ${ORACLE_METADATA}/arm64_shape_compatibilities.json
+.else
+ORACLE_SHAPES= ${ORACLE_METADATA}/default_shape_compatibilities.json
+.endif
+
+cw-oracle-${_FS}.oci: cw-oracle-portinstall cw-oracle-${_FS}-raw
+ mkdir -p ${ORACLE_TMP_${_FS}}
+ # create architecture-specific JSON metadata
+ env TYPE="${TYPE}" \
+ OSRELEASE="${OSRELEASE}" \
+ ORACLE_CAPABILITY="${ORACLE_CAPABILITY}" \
+ ORACLE_SHAPES="${ORACLE_SHAPES}" \
+ ORACLE_TEMPLATE="${ORACLE_TEMPLATE}" \
+ ORACLE_OUTPUT="${ORACLE_OUTPUT_${_FS}}" \
+ ${ORACLE_METADATA}/generate_metadata.lua
+
+ # convert raw to native qcow2 for zstd compression, saves ~ 8GiB
+ qemu-img convert -S 512b -p -O qcow2 -c -o compression_type=zstd \
+ ${.OBJDIR}/${ORACLE${_FS:tu}RAWIMAGE} \
+ ${ORACLE_TMP_${_FS}}/output.QCOW2
+
+ # Create GNU-compatible tarball using BSD tar
+ tar --format=gnutar -cf ${.TARGET} -C ${ORACLE_TMP_${_FS}} \
+ image_metadata.json output.QCOW2
+
+ echo "Oracle image ${.TARGET} is ready for upload."
+
+cw-oracle-upload-${_FS}: cw-oracle-${_FS}.oci
+.if !defined(ORACLE_PAR_URL) || empty(ORACLE_PAR_URL)
+ @echo "--------------------------------------------------------------"
+ @echo ">>> ORACLE_PAR_URL must be set for Oracle image upload"
+ @echo ">>> for testing, use a file:/// URL to a local directory"
+ @echo "--------------------------------------------------------------"
+ @false
+.endif
+ echo "Please wait ... uploading cw-oracle-${_FS}.oci to ${ORACLE_BASENAME}-${_FS}.oci"
+ curl -s ${ORACLE_PAR_URL}/${ORACLE_BASENAME}-${_FS}.oci --upload-file cw-oracle-${_FS}.oci
+ echo "Uploaded cw-oracle-${_FS}.oci as ${ORACLE_BASENAME}-${_FS}.oci"
+ touch ${.TARGET}
+.endfor
+
+cw-oracle-upload: cw-oracle-portinstall ${ORACLE_UPLOAD_LIST}
diff --git a/release/Makefile.vm b/release/Makefile.vm
index 5aa506ff6787..ec3e734285ce 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -274,5 +274,6 @@ cloudware-release:
.include "${.CURDIR}/Makefile.ec2"
.include "${.CURDIR}/Makefile.firecracker"
.include "${.CURDIR}/Makefile.gce"
+.include "${.CURDIR}/Makefile.oracle"
.include "${.CURDIR}/Makefile.vagrant"
.include "${.CURDIR}/Makefile.inc1"
diff --git a/release/scripts/oracle/arm64_shape_compatibilities.json b/release/scripts/oracle/arm64_shape_compatibilities.json
new file mode 100644
index 000000000000..dfd066b5474f
--- /dev/null
+++ b/release/scripts/oracle/arm64_shape_compatibilities.json
@@ -0,0 +1,24 @@
+[
+ {
+ "internalShapeName": "VM.Standard.A1.Flex",
+ "ocpuConstraints": {
+ "min": 1,
+ "max": 80
+ },
+ "memoryConstraints": {
+ "minInGBs": 1,
+ "maxInGBs": 512
+ }
+ },
+ {
+ "internalShapeName": "VM.Standard.A2.Flex",
+ "ocpuConstraints": {
+ "min": 1,
+ "max": 78
+ },
+ "memoryConstraints": {
+ "minInGBs": 1,
+ "maxInGBs": 946
+ }
+ }
+]
diff --git a/release/scripts/oracle/default_shape_compatibilities.json b/release/scripts/oracle/default_shape_compatibilities.json
new file mode 100644
index 000000000000..fe51488c7066
--- /dev/null
+++ b/release/scripts/oracle/default_shape_compatibilities.json
@@ -0,0 +1 @@
+[]
diff --git a/release/scripts/oracle/generate_metadata.lua b/release/scripts/oracle/generate_metadata.lua
new file mode 100755
index 000000000000..751b9680cc29
--- /dev/null
+++ b/release/scripts/oracle/generate_metadata.lua
@@ -0,0 +1,74 @@
+#!/usr/libexec/flua
+
+local ucl = require("ucl")
+
+-- read from environment variables
+local os_type = os.getenv("TYPE")
+local os_version = os.getenv("OSRELEASE")
+-- the raw file
+local capability_file = os.getenv("ORACLE_CAPABILITY")
+-- the platform-specific file
+local shapes_file = os.getenv("ORACLE_SHAPES")
+-- base template
+local template_file = os.getenv("ORACLE_TEMPLATE")
+local output_file = os.getenv("ORACLE_OUTPUT")
+
+if not os_type or not os_version or not capability_file or
+ not shapes_file or not template_file or not output_file then
+ io.stderr:write("Error: Oracle metadata script is missing required environment variables:\n")
+ io.stderr:write("TYPE, OSRELEASE, ORACLE_CAPABILITY, ORACLE_SHAPES, ORACLE_TEMPLATE, ORACLE_OUTPUT\n")
+ os.exit(1)
+end
+
+-- read files
+local function read_file(path)
+ local f = io.open(path, "r")
+ if not f then
+ io.stderr:write("Error: Oracle metadata script cannot open file: " .. path .. "\n")
+ os.exit(1)
+ end
+ local content = f:read("*a")
+ f:close()
+ return content
+end
+
+-- parse the template
+local template = read_file(template_file)
+local metadata = ucl.parser()
+metadata:parse_string(template)
+local data = metadata:get_object()
+
+-- update the simple fields
+data.operatingSystem = os_type
+data.operatingSystemVersion = os_version
+
+-- capability data is actually JSON, but needs to be inserted as a raw blob
+local caps = read_file(capability_file)
+-- remove all newlines and preceding spaces to match Oracle's format
+caps = caps:gsub("\n", "")
+caps = caps:gsub("%s+", "")
+-- is it still valid JSON?
+local caps_parser = ucl.parser()
+if not caps_parser:parse_string(caps) then
+ io.stderr:write("Error: Oracle metadata script found invalid JSON in capability file\n")
+ os.exit(1)
+end
+-- insert as a raw blob
+data.imageCapabilityData = caps
+
+-- parse and insert architecture-dependent shape compatibilities data
+local shapes_data = read_file(shapes_file)
+local shapes = ucl.parser()
+shapes:parse_string(shapes_data)
+data.additionalMetadata.shapeCompatibilities = shapes:get_object()
+
+-- save the metadata file
+local dir = os.getenv("PWD")
+local out = io.open(output_file, "w")
+if not out then
+ io.stderr:write("Error: Oracle metadata script cannot create output file: "
+ .. dir .. "/" .. output_file .. "\n")
+ os.exit(1)
+end
+out:write(ucl.to_format(data, "json", {pretty = true}))
+out:close()
diff --git a/release/scripts/oracle/image_capability_data.json b/release/scripts/oracle/image_capability_data.json
new file mode 100644
index 000000000000..01af71f73031
--- /dev/null
+++ b/release/scripts/oracle/image_capability_data.json
@@ -0,0 +1,96 @@
+{
+ "capabilities": {
+ "Compute.AMD_SecureEncryptedVirtualization": {
+ "descriptorType": "boolean",
+ "defaultValue": false
+ },
+ "Storage.BootVolumeType": {
+ "descriptorType": "enumstring",
+ "values": [
+ "ISCSI",
+ "PARAVIRTUALIZED",
+ "SCSI",
+ "IDE",
+ "NVME"
+ ],
+ "defaultValue": "PARAVIRTUALIZED"
+ },
+ "Storage.Iscsi.MultipathDeviceSupported": {
+ "descriptorType": "boolean",
+ "defaultValue": false
+ },
+ "Storage.ParaVirtualization.EncryptionInTransit": {
+ "descriptorType": "boolean",
+ "defaultValue": true
+ },
+ "Storage.ConsistentVolumeNaming": {
+ "descriptorType": "boolean",
+ "defaultValue": true
+ },
+ "Compute.SecureBoot": {
+ "descriptorType": "boolean",
+ "defaultValue": false
+ },
+ "Storage.ParaVirtualization.AttachmentVersion": {
+ "descriptorType": "enuminteger",
+ "values": [
+ 1,
+ 2
+ ],
+ "defaultValue": 2
+ },
+ "Storage.LocalDataVolumeType": {
+ "descriptorType": "enumstring",
+ "values": [
+ "ISCSI",
+ "PARAVIRTUALIZED",
+ "SCSI",
+ "IDE",
+ "NVME"
+ ],
+ "defaultValue": "PARAVIRTUALIZED"
+ },
+ "Network.AttachmentType": {
+ "descriptorType": "enumstring",
+ "values": [
+ "PARAVIRTUALIZED",
+ "VDPA"
+ ],
+ "defaultValue": "PARAVIRTUALIZED"
+ },
+ "Storage.RemoteDataVolumeType": {
+ "descriptorType": "enumstring",
+ "values": [
+ "ISCSI",
+ "PARAVIRTUALIZED",
+ "SCSI",
+ "IDE",
+ "NVME"
+ ],
+ "defaultValue": "PARAVIRTUALIZED"
+ },
+ "Compute.LaunchMode": {
+ "descriptorType": "enumstring",
+ "values": [
+ "NATIVE",
+ "EMULATED",
+ "VDPA",
+ "PARAVIRTUALIZED",
+ "CUSTOM"
+ ],
+ "defaultValue": "PARAVIRTUALIZED"
+ },
+ "Network.IPv6Only": {
+ "descriptorType": "boolean",
+ "defaultValue": false
+ },
+ "Compute.Firmware": {
+ "descriptorType": "enumstring",
+ "values": [
+ "BIOS",
+ "UEFI_64"
+ ],
+ "defaultValue": "UEFI_64"
+ }
+ }
+}
diff --git a/release/scripts/oracle/image_metadata.json b/release/scripts/oracle/image_metadata.json
new file mode 100644
index 000000000000..eaea3dd1cad2
--- /dev/null
+++ b/release/scripts/oracle/image_metadata.json
@@ -0,0 +1,21 @@
+{
+ "version": 2,
+ "externalLaunchOptions": {
+ "firmware": "UEFI_64",
+ "networkType": "PARAVIRTUALIZED",
+ "bootVolumeType": "PARAVIRTUALIZED",
+ "remoteDataVolumeType": "PARAVIRTUALIZED",
+ "localDataVolumeType": "PARAVIRTUALIZED",
+ "launchOptionsSource": "PARAVIRTUALIZED",
+ "pvAttachmentVersion": 2,
+ "pvEncryptionInTransitEnabled": false,
+ "consistentVolumeNamingEnabled": false
+ },
+ "imageCapabilityData": "REPLACE",
+ "imageCapsFormatVersion": "23cfd738-ad9c-4f56-9281-67be6c8cd14c",
+ "operatingSystem": "REPLACE",
+ "operatingSystemVersion": "REPLACE",
+ "additionalMetadata": {
+ "shapeCompatibilities": "REPLACE"
+ }
+}