#!/usr/bin/env python3 """从依赖镜像下载锁定发行文件,校验后仅提取必需的普通 ELF 文件。""" import argparse import gzip import hashlib import json import os from pathlib import Path import re import shutil import struct import subprocess import sys import tarfile import tempfile from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parent.parent DEFAULT_MIRROR = 'https://release.monkeycode-ai.com/public/monkeyai/engine/dependencies' NAMES = {'gvisor_sentry', 'runsc-metric-server', 'ctr', 'docker-init', 'runsc-fd-parking', 'runsc', 'containerd', 'dockerd', 'containerd-shim-runc-v2', 'runc', 'docker', 'checkpointgofer', 'docker-proxy', 'gvisor-proc-umounter', 'gvisor-sentry-prewarmer'} def digest(path): with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() def extract(archive, members, destination, arch): with tarfile.open(archive, 'r:*') as package: for member, name in members.items(): if name not in NAMES: raise ValueError('清单包含未知组件') matches = [item for item in package.getmembers() if item.name == member] if len(matches) != 1 or not matches[0].isfile() or not 0 < matches[0].size <= 256 << 20: raise ValueError(f'{member} 缺失、重复或不是受限普通文件') with package.extractfile(matches[0]) as source: header = source.read(20) machine = {'amd64': 62, 'arm64': 183}[arch] if len(header) != 20 or header[:6] != b'\x7fELF\x02\x01' or struct.unpack(' 10240 or any(trailer): raise ValueError('依赖包尾部无效') if seen != set(expected): raise ValueError('依赖包缺少锁定归档') for entry in entries: os.replace(stage / entry['sha256'], cache / entry['sha256']) def ensure_archives(arch, entries, cache, lockfile, mirror): version = dependency_version(lockfile) parsed = urlsplit(mirror) if parsed.scheme != 'https' or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment: raise ValueError('依赖镜像必须为无凭据的 HTTPS 地址') present = [cached(entry, cache) is not None for entry in entries] if all(present): print(f'{arch} 全部依赖归档缓存校验通过,跳过下载', flush=True) return print(f'下载依赖包 {version}/{arch}/dependencies.tgz', flush=True) with tempfile.TemporaryDirectory(prefix='.download-', dir=cache) as work: path = Path(work) / 'dependencies.tgz' download(f'{mirror.rstrip("/")}/{version}/{arch}/dependencies.tgz', path, len(entries) * (512 << 20)) unpack_bundle(path, entries, cache) def prepare(arch, output, cache, lockfile, reuse=False, mirror=DEFAULT_MIRROR): entries = locked_entries(arch, lockfile) if output.is_symlink(): raise ValueError('依赖输出目录不能是符号链接') output = output.resolve() if output.exists() and reuse: validate_existing(arch, output, lockfile) print(f'{arch} 已有依赖校验通过,复用:{output}') return if output.exists(): raise ValueError('输出目录已存在,请使用新目录,避免覆盖已审核依赖') output.parent.mkdir(parents=True, exist_ok=True) cache.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='.dependencies-', dir=output.parent) as work: stage = Path(work) ensure_archives(arch, entries, cache, lockfile, mirror) for entry in entries: extract(cache / entry['sha256'], entry['members'], stage, arch) provenance = {'architecture': arch, 'lock_sha256': digest(lockfile), 'sources': entries, 'files': {name: digest(stage / name) for name in sorted(NAMES)}} (stage / 'provenance.json').write_text(json.dumps(provenance, ensure_ascii=False, indent=2) + '\n') stage.rename(output) print(f'{arch} 依赖已按固定摘要准备:{output}') if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--reuse', action='store_true', help='校验并复用已有完整依赖目录') parser.add_argument('--arch', choices=['amd64', 'arm64'], required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--cache', type=Path, default=ROOT / 'dist/cache') parser.add_argument('--lock', type=Path, default=ROOT / 'dependencies.lock.json') parser.add_argument('--mirror', default=os.environ.get('ENGINE_DEPENDENCY_URL') or DEFAULT_MIRROR, help='依赖镜像地址;缺失时直接失败,不回退官方上游') args = parser.parse_args() try: prepare(args.arch, args.output, args.cache, args.lock, args.reuse, args.mirror) except (ValueError, OSError, EOFError, tarfile.TarError, subprocess.CalledProcessError) as error: print(f'依赖准备失败:{error};请检查缓存或先同步锁定依赖到镜像源', file=sys.stderr) sys.exit(1)